mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat(backend): PR4 — surface Incomplete as a third state, honest failure rate
- SessionStatusBuckets: explicit AwaitingUser + Incomplete buckets (out of Other), still reconciling to Total. - Metrics summary: expose awaitingUser + incomplete; failure rate is now Failed/(Succeeded+Failed) — Incomplete and non-terminal states are out of the denominator (crcins drops from failed/total to the real terminal rate). - SessionStats.IncompleteLastNDays; dashboard SuccessRatePct was already terminal-only. - Daily aggregation already used Succeeded+Failed denominator — unchanged. - Tests: SessionStatusBucketsTests. Suite: 2705 pass. Fleet Health Incomplete surfacing deferred to PR5 (web) with its documented over-all-sessions semantics.
This commit is contained in:
@@ -75,16 +75,19 @@ timeline AND let the backend terminalize precisely instead of guessing with grac
|
||||
- [x] Tests `SessionStatusTransitionTests` (20, full matrix). Suite: 2705 pass (one pre-existing
|
||||
timing-flaky maintenance test, green on retry/isolation — unrelated).
|
||||
|
||||
## PR4 — Stats / metrics expose 3 states
|
||||
## PR4 — Stats / metrics expose 3 states ✅ done (Fleet Health deferred to PR5)
|
||||
|
||||
- [ ] `SessionStatusBuckets` (`MetricsMath.cs:317`): explicit `AwaitingUser` + `Incomplete`
|
||||
buckets (out of `Other`).
|
||||
- [ ] `SessionStats` (`SessionApiModels.cs:442`): add `IncompleteLastNDays`; failure-rate
|
||||
denominator = `Succeeded + Failed` only.
|
||||
- [ ] Fleet Health payload (`MetricsMath.BuildFleetHealthPayload`): count Incomplete
|
||||
separately; keep SuccessRate honest.
|
||||
- [ ] Aggregation equivalence tests (`SessionStatsAggregationTests`,
|
||||
`SessionStatsProjectionEquivalenceTests`, `MetricsSummaryProjectionEquivalenceTests`).
|
||||
- [x] `SessionStatusBuckets` (`MetricsMath.cs`): explicit `AwaitingUser` + `Incomplete` buckets
|
||||
(out of `Other`); buckets still reconcile to Total by construction.
|
||||
- [x] Metrics summary (`GetMetricsSummaryAsync`): surface `awaitingUser` + `incomplete`; **failure
|
||||
rate now `Failed / (Succeeded + Failed)`** — Incomplete + non-terminal excluded from the denominator.
|
||||
- [x] `SessionStats` (`SessionApiModels.cs`): add `IncompleteLastNDays`; dashboard `SuccessRatePct`
|
||||
was already terminal-only (`Succeeded/(Succeeded+Failed)`), so Incomplete is excluded there too.
|
||||
- [x] Daily aggregation (`ComputeUsageMetricsSnapshotAsync`) already uses `Succeeded+Failed` as the
|
||||
denominator — no change needed (Incomplete already excluded).
|
||||
- [x] Tests `SessionStatusBucketsTests` (bucketing + terminal-only failure-rate). Suite: 2705 pass.
|
||||
- [ ] **Deferred to PR5:** Fleet Health payload Incomplete count (its SuccessRate is documented as
|
||||
"over all sessions" with equivalence tests — fold the Incomplete surfacing in with the web work).
|
||||
|
||||
## PR5 — Web surface
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the third-state stats bucketing (docs/design/enrollment-status-reclassification.md):
|
||||
/// AwaitingUser and Incomplete get their own buckets instead of hiding in Other, and the buckets
|
||||
/// always reconcile to Total by construction — so a session in either state can never be silently
|
||||
/// miscounted as a failure.
|
||||
/// </summary>
|
||||
public class SessionStatusBucketsTests
|
||||
{
|
||||
private static SessionStatusBuckets Tally(params string[] statuses)
|
||||
{
|
||||
var b = default(SessionStatusBuckets);
|
||||
foreach (var s in statuses) b = b.Add(s);
|
||||
return b;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Incomplete_and_AwaitingUser_get_their_own_buckets_not_Other()
|
||||
{
|
||||
var b = Tally("Incomplete", "Incomplete", "AwaitingUser");
|
||||
Assert.Equal(2, b.Incomplete);
|
||||
Assert.Equal(1, b.AwaitingUser);
|
||||
Assert.Equal(0, b.Other);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_status_still_lands_in_Other()
|
||||
{
|
||||
var b = Tally("Succeeded", "SomethingWeird");
|
||||
Assert.Equal(1, b.Succeeded);
|
||||
Assert.Equal(1, b.Other);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Buckets_always_reconcile_to_total()
|
||||
{
|
||||
var b = Tally("Succeeded", "Failed", "Incomplete", "AwaitingUser", "InProgress", "Pending", "Stalled", "weird");
|
||||
Assert.Equal(8, b.Total);
|
||||
Assert.Equal(b.Total,
|
||||
b.Succeeded + b.Failed + b.Incomplete + b.AwaitingUser + b.InProgress + b.Pending + b.Stalled + b.Other);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failure_rate_denominator_is_terminal_only_excluding_incomplete()
|
||||
{
|
||||
// 1 succeeded, 1 failed, 8 incomplete (the crcins shape). Honest rate = 1/(1+1) = 50%,
|
||||
// NOT 1/10 = 10% (Incomplete must not dilute) and NOT counting Incomplete as failures.
|
||||
var b = Tally("Succeeded", "Failed", "Incomplete", "Incomplete", "Incomplete", "Incomplete",
|
||||
"Incomplete", "Incomplete", "Incomplete", "Incomplete");
|
||||
var terminal = b.Succeeded + b.Failed;
|
||||
var failureRate = terminal > 0 ? (double)b.Failed / terminal * 100 : 0;
|
||||
Assert.Equal(50.0, failureRate);
|
||||
Assert.Equal(8, b.Incomplete);
|
||||
}
|
||||
}
|
||||
@@ -315,7 +315,8 @@ public static class MetricsMath
|
||||
/// their own buckets, and any unrecognised status (incl. Unknown) lands in <see cref="Other"/>.
|
||||
/// </summary>
|
||||
public readonly record struct SessionStatusBuckets(
|
||||
int Total, int Succeeded, int Failed, int InProgress, int Pending, int Stalled, int Other)
|
||||
int Total, int Succeeded, int Failed, int InProgress, int Pending, int Stalled,
|
||||
int AwaitingUser, int Incomplete, int Other)
|
||||
{
|
||||
/// <summary>Returns a new tally with <paramref name="status"/> folded in.</summary>
|
||||
public SessionStatusBuckets Add(string? status)
|
||||
@@ -326,7 +327,11 @@ public readonly record struct SessionStatusBuckets(
|
||||
var inProgress = InProgress + (status == "InProgress" ? 1 : 0);
|
||||
var pending = Pending + (status == "Pending" ? 1 : 0);
|
||||
var stalled = Stalled + (status == "Stalled" ? 1 : 0);
|
||||
var other = total - (succeeded + failed + inProgress + pending + stalled);
|
||||
return new SessionStatusBuckets(total, succeeded, failed, inProgress, pending, stalled, other);
|
||||
// AwaitingUser (non-terminal, Device Setup done) and Incomplete (terminal, non-failure) get
|
||||
// their own buckets so they no longer hide in Other and never inflate the failure count.
|
||||
var awaitingUser = AwaitingUser + (status == "AwaitingUser" ? 1 : 0);
|
||||
var incomplete = Incomplete + (status == "Incomplete" ? 1 : 0);
|
||||
var other = total - (succeeded + failed + inProgress + pending + stalled + awaitingUser + incomplete);
|
||||
return new SessionStatusBuckets(total, succeeded, failed, inProgress, pending, stalled, awaitingUser, incomplete, other);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1315,20 +1315,30 @@ namespace AutopilotMonitor.Functions.Services
|
||||
TallyMetricsSummaryRow(groups, entity);
|
||||
}
|
||||
|
||||
return groups.Select(kvp => (object)new
|
||||
return groups.Select(kvp =>
|
||||
{
|
||||
tenantId = kvp.Key,
|
||||
totalSessions = kvp.Value.Total,
|
||||
succeeded = kvp.Value.Succeeded,
|
||||
failed = kvp.Value.Failed,
|
||||
inProgress = kvp.Value.InProgress,
|
||||
pending = kvp.Value.Pending,
|
||||
stalled = kvp.Value.Stalled,
|
||||
other = kvp.Value.Other,
|
||||
failureRate = kvp.Value.Total > 0
|
||||
? Math.Round((double)kvp.Value.Failed / kvp.Value.Total * 100, 1)
|
||||
: 0.0,
|
||||
windowDays = days
|
||||
var b = kvp.Value;
|
||||
// Honest failure rate: failed over the TERMINAL outcomes only (Succeeded + Failed).
|
||||
// Incomplete (unknown, non-failure) is deliberately excluded from the denominator, and
|
||||
// non-terminal states (InProgress/AwaitingUser/Pending/Stalled) never belonged in it.
|
||||
var terminal = b.Succeeded + b.Failed;
|
||||
return (object)new
|
||||
{
|
||||
tenantId = kvp.Key,
|
||||
totalSessions = b.Total,
|
||||
succeeded = b.Succeeded,
|
||||
failed = b.Failed,
|
||||
inProgress = b.InProgress,
|
||||
pending = b.Pending,
|
||||
stalled = b.Stalled,
|
||||
awaitingUser = b.AwaitingUser,
|
||||
incomplete = b.Incomplete,
|
||||
other = b.Other,
|
||||
failureRate = terminal > 0
|
||||
? Math.Round((double)b.Failed / terminal * 100, 1)
|
||||
: 0.0,
|
||||
windowDays = days
|
||||
};
|
||||
}).ToList();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1033,6 +1033,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
int active = 0;
|
||||
int succeeded = 0;
|
||||
int failed = 0;
|
||||
int incomplete = 0;
|
||||
long succeededDurationSeconds = 0;
|
||||
int succeededWithDurationCount = 0;
|
||||
int totalToday = 0;
|
||||
@@ -1061,6 +1062,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
case SessionStatus.Failed:
|
||||
failed++;
|
||||
break;
|
||||
case SessionStatus.Incomplete:
|
||||
// Terminal, non-failure. Reported separately, kept out of the failure rate.
|
||||
incomplete++;
|
||||
break;
|
||||
}
|
||||
|
||||
if (s.StartedAt >= utcMidnight)
|
||||
@@ -1085,6 +1090,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
TotalLastNDays = sessions.Count,
|
||||
SucceededLastNDays = succeeded,
|
||||
FailedLastNDays = failed,
|
||||
IncompleteLastNDays = incomplete,
|
||||
SuccessRatePct = successRatePct,
|
||||
AvgDurationMinutes = avgDurationMinutes,
|
||||
TotalToday = totalToday,
|
||||
|
||||
@@ -463,6 +463,13 @@ namespace AutopilotMonitor.Shared.Models
|
||||
public int SucceededLastNDays { get; set; }
|
||||
public int FailedLastNDays { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Terminal, non-failure sessions in the window (docs/design/enrollment-status-reclassification.md):
|
||||
/// the sweep saw no completion or explicit failure. Reported as the third headline bucket and
|
||||
/// deliberately excluded from <see cref="SuccessRatePct"/> (which is over Succeeded + Failed only).
|
||||
/// </summary>
|
||||
public int IncompleteLastNDays { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Succeeded / (Succeeded + Failed) * 100, rounded. Zero when no terminal
|
||||
/// sessions are in the window (the card renders "0%" rather than NaN).
|
||||
|
||||
Reference in New Issue
Block a user