feat(backend): PR4 — surface Incomplete as a third state, honest failure rate

- SessionStatusBuckets: explicit AwaitingUser + Incomplete buckets (out of Other),
  still reconciling to Total.
- Metrics summary: expose awaitingUser + incomplete; failure rate is now
  Failed/(Succeeded+Failed) — Incomplete and non-terminal states are out of the
  denominator (crcins drops from failed/total to the real terminal rate).
- SessionStats.IncompleteLastNDays; dashboard SuccessRatePct was already terminal-only.
- Daily aggregation already used Succeeded+Failed denominator — unchanged.
- Tests: SessionStatusBucketsTests. Suite: 2705 pass. Fleet Health Incomplete surfacing
  deferred to PR5 (web) with its documented over-all-sessions semantics.
This commit is contained in:
Claude
2026-07-08 15:17:35 +00:00
parent c0585133be
commit 96ad65f715
6 changed files with 114 additions and 25 deletions
@@ -75,16 +75,19 @@ timeline AND let the backend terminalize precisely instead of guessing with grac
- [x] Tests `SessionStatusTransitionTests` (20, full matrix). Suite: 2705 pass (one pre-existing
timing-flaky maintenance test, green on retry/isolation — unrelated).
## PR4 — Stats / metrics expose 3 states
## PR4 — Stats / metrics expose 3 states ✅ done (Fleet Health deferred to PR5)
- [ ] `SessionStatusBuckets` (`MetricsMath.cs:317`): explicit `AwaitingUser` + `Incomplete`
buckets (out of `Other`).
- [ ] `SessionStats` (`SessionApiModels.cs:442`): add `IncompleteLastNDays`; failure-rate
denominator = `Succeeded + Failed` only.
- [ ] Fleet Health payload (`MetricsMath.BuildFleetHealthPayload`): count Incomplete
separately; keep SuccessRate honest.
- [ ] Aggregation equivalence tests (`SessionStatsAggregationTests`,
`SessionStatsProjectionEquivalenceTests`, `MetricsSummaryProjectionEquivalenceTests`).
- [x] `SessionStatusBuckets` (`MetricsMath.cs`): explicit `AwaitingUser` + `Incomplete` buckets
(out of `Other`); buckets still reconcile to Total by construction.
- [x] Metrics summary (`GetMetricsSummaryAsync`): surface `awaitingUser` + `incomplete`; **failure
rate now `Failed / (Succeeded + Failed)`** — Incomplete + non-terminal excluded from the denominator.
- [x] `SessionStats` (`SessionApiModels.cs`): add `IncompleteLastNDays`; dashboard `SuccessRatePct`
was already terminal-only (`Succeeded/(Succeeded+Failed)`), so Incomplete is excluded there too.
- [x] Daily aggregation (`ComputeUsageMetricsSnapshotAsync`) already uses `Succeeded+Failed` as the
denominator — no change needed (Incomplete already excluded).
- [x] Tests `SessionStatusBucketsTests` (bucketing + terminal-only failure-rate). Suite: 2705 pass.
- [ ] **Deferred to PR5:** Fleet Health payload Incomplete count (its SuccessRate is documented as
"over all sessions" with equivalence tests — fold the Incomplete surfacing in with the web work).
## PR5 — Web surface
@@ -0,0 +1,58 @@
using AutopilotMonitor.Functions.Helpers;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Pins the third-state stats bucketing (docs/design/enrollment-status-reclassification.md):
/// AwaitingUser and Incomplete get their own buckets instead of hiding in Other, and the buckets
/// always reconcile to Total by construction — so a session in either state can never be silently
/// miscounted as a failure.
/// </summary>
public class SessionStatusBucketsTests
{
private static SessionStatusBuckets Tally(params string[] statuses)
{
var b = default(SessionStatusBuckets);
foreach (var s in statuses) b = b.Add(s);
return b;
}
[Fact]
public void Incomplete_and_AwaitingUser_get_their_own_buckets_not_Other()
{
var b = Tally("Incomplete", "Incomplete", "AwaitingUser");
Assert.Equal(2, b.Incomplete);
Assert.Equal(1, b.AwaitingUser);
Assert.Equal(0, b.Other);
}
[Fact]
public void Unknown_status_still_lands_in_Other()
{
var b = Tally("Succeeded", "SomethingWeird");
Assert.Equal(1, b.Succeeded);
Assert.Equal(1, b.Other);
}
[Fact]
public void Buckets_always_reconcile_to_total()
{
var b = Tally("Succeeded", "Failed", "Incomplete", "AwaitingUser", "InProgress", "Pending", "Stalled", "weird");
Assert.Equal(8, b.Total);
Assert.Equal(b.Total,
b.Succeeded + b.Failed + b.Incomplete + b.AwaitingUser + b.InProgress + b.Pending + b.Stalled + b.Other);
}
[Fact]
public void Failure_rate_denominator_is_terminal_only_excluding_incomplete()
{
// 1 succeeded, 1 failed, 8 incomplete (the crcins shape). Honest rate = 1/(1+1) = 50%,
// NOT 1/10 = 10% (Incomplete must not dilute) and NOT counting Incomplete as failures.
var b = Tally("Succeeded", "Failed", "Incomplete", "Incomplete", "Incomplete", "Incomplete",
"Incomplete", "Incomplete", "Incomplete", "Incomplete");
var terminal = b.Succeeded + b.Failed;
var failureRate = terminal > 0 ? (double)b.Failed / terminal * 100 : 0;
Assert.Equal(50.0, failureRate);
Assert.Equal(8, b.Incomplete);
}
}
@@ -315,7 +315,8 @@ public static class MetricsMath
/// their own buckets, and any unrecognised status (incl. Unknown) lands in <see cref="Other"/>.
/// </summary>
public readonly record struct SessionStatusBuckets(
int Total, int Succeeded, int Failed, int InProgress, int Pending, int Stalled, int Other)
int Total, int Succeeded, int Failed, int InProgress, int Pending, int Stalled,
int AwaitingUser, int Incomplete, int Other)
{
/// <summary>Returns a new tally with <paramref name="status"/> folded in.</summary>
public SessionStatusBuckets Add(string? status)
@@ -326,7 +327,11 @@ public readonly record struct SessionStatusBuckets(
var inProgress = InProgress + (status == "InProgress" ? 1 : 0);
var pending = Pending + (status == "Pending" ? 1 : 0);
var stalled = Stalled + (status == "Stalled" ? 1 : 0);
var other = total - (succeeded + failed + inProgress + pending + stalled);
return new SessionStatusBuckets(total, succeeded, failed, inProgress, pending, stalled, other);
// AwaitingUser (non-terminal, Device Setup done) and Incomplete (terminal, non-failure) get
// their own buckets so they no longer hide in Other and never inflate the failure count.
var awaitingUser = AwaitingUser + (status == "AwaitingUser" ? 1 : 0);
var incomplete = Incomplete + (status == "Incomplete" ? 1 : 0);
var other = total - (succeeded + failed + inProgress + pending + stalled + awaitingUser + incomplete);
return new SessionStatusBuckets(total, succeeded, failed, inProgress, pending, stalled, awaitingUser, incomplete, other);
}
}
@@ -1315,20 +1315,30 @@ namespace AutopilotMonitor.Functions.Services
TallyMetricsSummaryRow(groups, entity);
}
return groups.Select(kvp => (object)new
return groups.Select(kvp =>
{
tenantId = kvp.Key,
totalSessions = kvp.Value.Total,
succeeded = kvp.Value.Succeeded,
failed = kvp.Value.Failed,
inProgress = kvp.Value.InProgress,
pending = kvp.Value.Pending,
stalled = kvp.Value.Stalled,
other = kvp.Value.Other,
failureRate = kvp.Value.Total > 0
? Math.Round((double)kvp.Value.Failed / kvp.Value.Total * 100, 1)
: 0.0,
windowDays = days
var b = kvp.Value;
// Honest failure rate: failed over the TERMINAL outcomes only (Succeeded + Failed).
// Incomplete (unknown, non-failure) is deliberately excluded from the denominator, and
// non-terminal states (InProgress/AwaitingUser/Pending/Stalled) never belonged in it.
var terminal = b.Succeeded + b.Failed;
return (object)new
{
tenantId = kvp.Key,
totalSessions = b.Total,
succeeded = b.Succeeded,
failed = b.Failed,
inProgress = b.InProgress,
pending = b.Pending,
stalled = b.Stalled,
awaitingUser = b.AwaitingUser,
incomplete = b.Incomplete,
other = b.Other,
failureRate = terminal > 0
? Math.Round((double)b.Failed / terminal * 100, 1)
: 0.0,
windowDays = days
};
}).ToList();
}
}
@@ -1033,6 +1033,7 @@ namespace AutopilotMonitor.Functions.Services
int active = 0;
int succeeded = 0;
int failed = 0;
int incomplete = 0;
long succeededDurationSeconds = 0;
int succeededWithDurationCount = 0;
int totalToday = 0;
@@ -1061,6 +1062,10 @@ namespace AutopilotMonitor.Functions.Services
case SessionStatus.Failed:
failed++;
break;
case SessionStatus.Incomplete:
// Terminal, non-failure. Reported separately, kept out of the failure rate.
incomplete++;
break;
}
if (s.StartedAt >= utcMidnight)
@@ -1085,6 +1090,7 @@ namespace AutopilotMonitor.Functions.Services
TotalLastNDays = sessions.Count,
SucceededLastNDays = succeeded,
FailedLastNDays = failed,
IncompleteLastNDays = incomplete,
SuccessRatePct = successRatePct,
AvgDurationMinutes = avgDurationMinutes,
TotalToday = totalToday,
@@ -463,6 +463,13 @@ namespace AutopilotMonitor.Shared.Models
public int SucceededLastNDays { get; set; }
public int FailedLastNDays { get; set; }
/// <summary>
/// Terminal, non-failure sessions in the window (docs/design/enrollment-status-reclassification.md):
/// the sweep saw no completion or explicit failure. Reported as the third headline bucket and
/// deliberately excluded from <see cref="SuccessRatePct"/> (which is over Succeeded + Failed only).
/// </summary>
public int IncompleteLastNDays { get; set; }
/// <summary>
/// Succeeded / (Succeeded + Failed) * 100, rounded. Zero when no terminal
/// sessions are in the window (the card renders "0%" rather than NaN).