From 61e53a2e9b5db709286f2dbf56c33eccb515514d Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Fri, 21 Aug 2026 12:49:42 +0200 Subject: [PATCH] feat: lead duration cards with median + P90 instead of outlier-skewed average Enrollment durations are heavily right-skewed; a handful of overnight/WhiteGlove outliers made the Fleet Health and Dashboard average cards read like hours (541 min) and answer no real question. Both cards now lead with the median (robust 'typical enrollment' answer) and keep the tail visible via a P90 subtext. AvgDurationMinutes stays in both DTOs for API compatibility. Co-Authored-By: Claude Fable 5 --- .../FleetHealthPayloadTests.cs | 24 ++++++++++++++++++ .../SessionStatsAggregationTests.cs | 25 +++++++++++++++++++ .../SessionStatsProjectionEquivalenceTests.cs | 4 ++- .../Helpers/MetricsMath.cs | 25 +++++++++++-------- .../Services/TableStorageService.Sessions.cs | 15 ++++++----- .../Models/FleetHealthModels.cs | 11 +++++++- .../Models/SessionApiModels.cs | 10 +++++++- .../app/dashboard/hooks/useDashboardStats.ts | 4 +++ .../app/dashboard/page.tsx | 19 +++++++++++--- .../app/fleet-health/hooks/useFleetHealth.ts | 4 +++ .../app/fleet-health/page.tsx | 20 ++++++++++----- 11 files changed, 131 insertions(+), 30 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs index dc438688..9a8ba1f0 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs @@ -46,6 +46,8 @@ public class FleetHealthPayloadTests Assert.Equal(0, payload.Stats.Total); Assert.Equal(0d, payload.Stats.SuccessRate); Assert.Equal(0, payload.Stats.AvgDurationMinutes); + Assert.Equal(0, payload.Stats.MedianDurationMinutes); + Assert.Equal(0, payload.Stats.P90DurationMinutes); Assert.Empty(payload.FailureReasons); Assert.Empty(payload.ModelHealth); Assert.Empty(payload.SlowestModels); @@ -109,6 +111,28 @@ public class FleetHealthPayloadTests // (600 + 1200) / 2 / 60 = 15 min. Assert.Equal(15, payload.Stats.AvgDurationMinutes); + // Same population feeds the percentile stats: sorted [600, 1200]. + Assert.Equal(10, payload.Stats.MedianDurationMinutes); + Assert.Equal(20, payload.Stats.P90DurationMinutes); + } + + [Fact] + public void MedianDuration_IsRobustAgainstOutliers_ThatDominateTheAverage() + { + // The card's raison d'être: 8 typical 10-min enrollments plus a 2 h and a 24 h + // outlier (overnight ESP / WhiteGlove late user phase). The mean explodes to ~2.7 h + // while the median still answers "how long does a typical enrollment take". + var sessions = new List(); + for (int i = 0; i < 8; i++) + sessions.Add(S(SessionStatus.Succeeded, durationSeconds: 600)); + sessions.Add(S(SessionStatus.Succeeded, durationSeconds: 7200)); + sessions.Add(S(SessionStatus.Succeeded, durationSeconds: 86400)); + + var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7); + + Assert.Equal(164, payload.Stats.AvgDurationMinutes); // (8*600 + 7200 + 86400) / 10 / 60 + Assert.Equal(10, payload.Stats.MedianDurationMinutes); // unmoved by the tail + Assert.Equal(120, payload.Stats.P90DurationMinutes); // tail stays visible } [Fact] diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionStatsAggregationTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionStatsAggregationTests.cs index dbc6ea60..69275a3f 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionStatsAggregationTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionStatsAggregationTests.cs @@ -36,6 +36,8 @@ public class SessionStatsAggregationTests Assert.Equal(0, stats.FailedLastNDays); Assert.Equal(0, stats.SuccessRatePct); Assert.Equal(0, stats.AvgDurationMinutes); + Assert.Equal(0, stats.MedianDurationMinutes); + Assert.Equal(0, stats.P90DurationMinutes); Assert.Equal(0, stats.TotalToday); Assert.Equal(0, stats.FailedToday); } @@ -116,6 +118,29 @@ public class SessionStatsAggregationTests // (30 + 10) / 2 = 20 minutes Assert.Equal(20, stats.AvgDurationMinutes); + // Same Succeeded population feeds the percentile stats: sorted [600, 1800]. + Assert.Equal(10, stats.MedianDurationMinutes); + Assert.Equal(30, stats.P90DurationMinutes); + } + + [Fact] + public void Median_duration_is_robust_against_outliers_that_dominate_the_average() + { + // 8 typical 10-min enrollments plus a 2 h and a 24 h outlier: the mean explodes + // to ~2.7 h while the median still answers "how long does a typical enrollment + // take" — the reason the dashboard card leads with the median. + var now = DateTime.UtcNow; + var sessions = new List(); + for (int i = 0; i < 8; i++) + sessions.Add(MakeSession(SessionStatus.Succeeded, now.AddHours(-1), 600)); + sessions.Add(MakeSession(SessionStatus.Succeeded, now.AddHours(-2), 7200)); + sessions.Add(MakeSession(SessionStatus.Succeeded, now.AddHours(-3), 86400)); + + var stats = TableStorageService.AggregateSessionStats(sessions, days: 7); + + Assert.Equal(164, stats.AvgDurationMinutes); // (8*600 + 7200 + 86400) / 10 / 60 + Assert.Equal(10, stats.MedianDurationMinutes); // unmoved by the tail + Assert.Equal(120, stats.P90DurationMinutes); // tail stays visible } [Fact] diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionStatsProjectionEquivalenceTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionStatsProjectionEquivalenceTests.cs index 1ee9762d..f85b2396 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionStatsProjectionEquivalenceTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionStatsProjectionEquivalenceTests.cs @@ -172,7 +172,9 @@ public class SessionStatsProjectionEquivalenceTests Assert.Equal(full.SucceededLastNDays, projected.SucceededLastNDays); Assert.Equal(full.FailedLastNDays, projected.FailedLastNDays); Assert.Equal(full.SuccessRatePct, projected.SuccessRatePct); - Assert.Equal(full.AvgDurationMinutes, projected.AvgDurationMinutes); // the duration-sensitive card + Assert.Equal(full.AvgDurationMinutes, projected.AvgDurationMinutes); // duration-sensitive + Assert.Equal(full.MedianDurationMinutes, projected.MedianDurationMinutes); // the headline card + Assert.Equal(full.P90DurationMinutes, projected.P90DurationMinutes); Assert.Equal(full.TotalToday, projected.TotalToday); Assert.Equal(full.FailedToday, projected.FailedToday); } diff --git a/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs b/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs index 4d78421a..37044656 100644 --- a/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs +++ b/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs @@ -168,14 +168,15 @@ public static class MetricsMath /// drained up to 200k raw sessions into the browser and ran these aggregations on the main /// thread. Success rate follows the SLA convention: Succeeded / (Succeeded + Failed) — finished /// enrollments only, so in-flight sessions and Incomplete (terminal, non-failure) never dilute - /// it. Average duration counts every non-in-progress session that carries a positive duration - /// (failures included). + /// it. Duration stats (average, median, P90) count every non-in-progress session that carries a + /// positive duration (failures included); the cards lead with the median because enrollment + /// durations are heavily right-skewed and a handful of multi-hour outliers make the mean + /// meaningless as a "typical enrollment" answer. /// public static FleetHealthMetrics BuildFleetHealthPayload(IReadOnlyList sessions, int days) { int succeeded = 0, failed = 0, inProgress = 0, incomplete = 0; - long completedDurationSeconds = 0; - int completedWithDurationCount = 0; + var completedDurationSeconds = new List(); foreach (var s in sessions) { @@ -188,12 +189,10 @@ public static class MetricsMath } if (s.Status != SessionStatus.InProgress && s.DurationSeconds is int d && d > 0) - { - completedDurationSeconds += d; - completedWithDurationCount++; - } + completedDurationSeconds.Add(d); } + completedDurationSeconds.Sort(); // Percentile() expects ascending order int total = sessions.Count; int finished = succeeded + failed; var stats = new FleetHealthStats @@ -204,9 +203,11 @@ public static class MetricsMath InProgress = inProgress, Incomplete = incomplete, SuccessRate = finished > 0 ? Math.Round((double)succeeded / finished * 100, 1) : 0, - AvgDurationMinutes = completedWithDurationCount > 0 - ? (int)Math.Round((double)completedDurationSeconds / completedWithDurationCount / 60.0, MidpointRounding.AwayFromZero) + AvgDurationMinutes = completedDurationSeconds.Count > 0 + ? (int)Math.Round(completedDurationSeconds.Average() / 60.0, MidpointRounding.AwayFromZero) : 0, + MedianDurationMinutes = SecondsToMinutes(Percentile(completedDurationSeconds, 50)), + P90DurationMinutes = SecondsToMinutes(Percentile(completedDurationSeconds, 90)), }; return new FleetHealthMetrics @@ -369,6 +370,10 @@ public static class MetricsMath /// rounded to one decimal place. Callers MUST pass values pre-sorted ascending. /// Returns 0 for an empty list. /// + /// Rounds a duration in seconds to whole minutes (away from zero, matching the avg). + public static int SecondsToMinutes(double seconds) + => (int)Math.Round(seconds / 60.0, MidpointRounding.AwayFromZero); + public static double Percentile(List sortedValues, int percentile) { if (sortedValues.Count == 0) return 0; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index 8fc6872f..df5def9f 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -1230,8 +1230,7 @@ namespace AutopilotMonitor.Functions.Services int succeeded = 0; int failed = 0; int incomplete = 0; - long succeededDurationSeconds = 0; - int succeededWithDurationCount = 0; + var succeededDurationSeconds = new List(); int totalToday = 0; int failedToday = 0; @@ -1250,10 +1249,7 @@ namespace AutopilotMonitor.Functions.Services case SessionStatus.Succeeded: succeeded++; if (s.DurationSeconds is int d && d > 0) - { - succeededDurationSeconds += d; - succeededWithDurationCount++; - } + succeededDurationSeconds.Add(d); break; case SessionStatus.Failed: failed++; @@ -1275,8 +1271,9 @@ namespace AutopilotMonitor.Functions.Services int successRatePct = terminal > 0 ? (int)Math.Round((double)succeeded / terminal * 100.0) : 0; - int avgDurationMinutes = succeededWithDurationCount > 0 - ? (int)Math.Round((double)succeededDurationSeconds / succeededWithDurationCount / 60.0) + succeededDurationSeconds.Sort(); // MetricsMath.Percentile expects ascending order + int avgDurationMinutes = succeededDurationSeconds.Count > 0 + ? (int)Math.Round(succeededDurationSeconds.Average() / 60.0) : 0; return new SessionStats @@ -1289,6 +1286,8 @@ namespace AutopilotMonitor.Functions.Services IncompleteLastNDays = incomplete, SuccessRatePct = successRatePct, AvgDurationMinutes = avgDurationMinutes, + MedianDurationMinutes = MetricsMath.SecondsToMinutes(MetricsMath.Percentile(succeededDurationSeconds, 50)), + P90DurationMinutes = MetricsMath.SecondsToMinutes(MetricsMath.Percentile(succeededDurationSeconds, 90)), TotalToday = totalToday, FailedToday = failedToday, ComputedAt = DateTime.UtcNow, diff --git a/src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs index d4bb7405..02be53ac 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs @@ -39,8 +39,17 @@ namespace AutopilotMonitor.Shared.Models /// Succeeded / (Succeeded + Failed) * 100 (one decimal) — finished enrollments only, /// matching the SLA convention. 0 when nothing has finished yet (clients render "—"). public double SuccessRate { get; set; } - /// Average duration in minutes over non-in-progress sessions that carry a duration. + /// Average duration in minutes over non-in-progress sessions that carry a duration. + /// Kept for API compatibility; the cards lead with the median, which a handful of + /// multi-hour outliers (overnight ESP, WhiteGlove late user phase) cannot drag around. public int AvgDurationMinutes { get; set; } + + /// Median duration in minutes over the same population as . + public int MedianDurationMinutes { get; set; } + + /// 90th-percentile duration in minutes over the same population — the tail signal + /// the median alone would hide. + public int P90DurationMinutes { get; set; } } public sealed class FleetDailyPoint diff --git a/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs index 06cc9452..076c0cb7 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs @@ -576,9 +576,17 @@ namespace AutopilotMonitor.Shared.Models /// public int SuccessRatePct { get; set; } - /// Average duration of Succeeded sessions in the window, in minutes (rounded). + /// Average duration of Succeeded sessions in the window, in minutes (rounded). + /// Kept for API compatibility; the dashboard card leads with the median, which + /// multi-hour outliers cannot drag around. public int AvgDurationMinutes { get; set; } + /// Median duration in minutes over the same Succeeded population. + public int MedianDurationMinutes { get; set; } + + /// 90th-percentile duration in minutes over the same population — the tail signal. + public int P90DurationMinutes { get; set; } + /// Sessions whose StartedAt is on or after UTC midnight of the current day. public int TotalToday { get; set; } public int FailedToday { get; set; } diff --git a/src/Web/autopilot-monitor-web/app/dashboard/hooks/useDashboardStats.ts b/src/Web/autopilot-monitor-web/app/dashboard/hooks/useDashboardStats.ts index 9a5cbc49..dc9099de 100644 --- a/src/Web/autopilot-monitor-web/app/dashboard/hooks/useDashboardStats.ts +++ b/src/Web/autopilot-monitor-web/app/dashboard/hooks/useDashboardStats.ts @@ -18,6 +18,10 @@ export interface DashboardStats { failedLastNDays: number; successRatePct: number; avgDurationMinutes: number; + /** Median over the same Succeeded population — the card's headline value (outlier-robust). */ + medianDurationMinutes: number; + /** 90th percentile — the tail signal the median alone would hide. */ + p90DurationMinutes: number; totalToday: number; failedToday: number; computedAt: string; diff --git a/src/Web/autopilot-monitor-web/app/dashboard/page.tsx b/src/Web/autopilot-monitor-web/app/dashboard/page.tsx index 1b694211..c9b0b8fe 100644 --- a/src/Web/autopilot-monitor-web/app/dashboard/page.tsx +++ b/src/Web/autopilot-monitor-web/app/dashboard/page.tsx @@ -27,6 +27,7 @@ import { useDashboardSessions } from "./hooks/useDashboardSessions"; import { useDashboardStats } from "./hooks/useDashboardStats"; import { api } from "@/lib/api"; import { authenticatedFetch } from "@/lib/authenticatedFetch"; +import { formatDuration } from "@/lib/formatting"; import { hasTenantReadScope } from "@/lib/tenantScope"; import { DOCS_URL } from "@/utils/config"; import { TableSkeleton } from "@/components/skeletons/TableSkeleton"; @@ -463,10 +464,22 @@ function HomeContent() { description="Last 7 days" color="green" /> + {/* Median, not mean — a few overnight/WhiteGlove outliers dominate the average + of the right-skewed duration distribution; P90 keeps the tail visible. */} 0 + ? `${dashboardStats.medianDurationMinutes} min` + : "—" + : "..." + } + description={ + dashboardStats && dashboardStats.p90DurationMinutes > 0 + ? `P90 ${formatDuration(dashboardStats.p90DurationMinutes * 60)} · last 7 days` + : "Last 7 days" + } color="purple" /> + {/* Median, not mean: enrollment durations are heavily right-skewed, so a few + overnight/WhiteGlove outliers made the old average card read like hours. + The P90 subtitle keeps the tail visible — that skew is the diagnostic signal. */} 0 ? `${stats.medianDuration} min` : "—"} + subtitle={ + stats.p90Duration > 0 + ? `P90 ${formatDuration(stats.p90Duration * 60)} · completed enrollments` + : "Completed enrollments" + } color="blue" />