diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs index c8b970e1..b7498ee8 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs @@ -10,9 +10,10 @@ namespace AutopilotMonitor.Functions.Tests; /// /// Covers the server-side Fleet Health aggregation in /// , the single source of truth shared by the -/// tenant (metrics/fleet-health) and global (global/metrics/fleet-health) functions. These assert -/// the semantics ported verbatim from the previous client-side aggregation — notably success rate -/// over ALL sessions and average duration over non-in-progress sessions (failures included). +/// tenant (metrics/fleet-health) and global (global/metrics/fleet-health) functions. Success and +/// failure rates follow the SLA convention — over finished enrollments (Succeeded + Failed) only, +/// so in-flight sessions never dilute them. Average duration stays over non-in-progress sessions +/// that carry a duration (failures included). /// public class FleetHealthPayloadTests { @@ -55,7 +56,7 @@ public class FleetHealthPayloadTests } [Fact] - public void SuccessRate_IsOverAllSessions_NotJustTerminal() + public void SuccessRate_IsOverFinishedSessions_InFlightAndIncompleteExcluded() { var sessions = new List { @@ -63,16 +64,34 @@ public class FleetHealthPayloadTests S(SessionStatus.Succeeded), S(SessionStatus.Failed), S(SessionStatus.InProgress), + S(SessionStatus.Pending), + S(SessionStatus.Incomplete), // terminal but non-failure — stays out of the rate }; var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7); - Assert.Equal(4, payload.Stats.Total); + Assert.Equal(6, payload.Stats.Total); Assert.Equal(2, payload.Stats.Succeeded); Assert.Equal(1, payload.Stats.Failed); Assert.Equal(1, payload.Stats.InProgress); - // 2 / 4 = 50% (over total), NOT 2 / 3 = 66.7% (over terminal). - Assert.Equal(50d, payload.Stats.SuccessRate); + Assert.Equal(1, payload.Stats.Incomplete); + // 2 / (2 + 1) = 66.7% (over finished), NOT 2 / 6 = 33.3% (over total). + Assert.Equal(66.7d, payload.Stats.SuccessRate); + } + + [Fact] + public void SuccessRate_IsZero_WhenNothingFinishedYet() + { + var sessions = new List + { + S(SessionStatus.InProgress), + S(SessionStatus.Pending), + }; + + var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7); + + // No finished enrollments → no rate; clients render "—" off Succeeded + Failed == 0. + Assert.Equal(0d, payload.Stats.SuccessRate); } [Fact] @@ -111,6 +130,27 @@ public class FleetHealthPayloadTests Assert.DoesNotContain(payload.ModelHealth, m => m.Model == "Contoso Model1"); } + [Fact] + public void ModelHealth_CountsFailedSeparately_InFlightInTotalOnly() + { + var sessions = new List + { + S(SessionStatus.Succeeded, model: "X"), + S(SessionStatus.Failed, model: "X"), + S(SessionStatus.InProgress, model: "X"), + }; + + var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7); + + var m = Assert.Single(payload.ModelHealth); + // Total keeps the full device count; the client derives the rate from + // Succeeded / (Succeeded + Failed), so the in-flight session must land + // in Total but in neither outcome bucket. + Assert.Equal(3, m.Total); + Assert.Equal(1, m.Succeeded); + Assert.Equal(1, m.Failed); + } + [Fact] public void ModelKey_FallsBackToUnknown_WhenManufacturerAndModelBlank() { @@ -147,10 +187,13 @@ public class FleetHealthPayloadTests { var sessions = new List { - // Model A: 1 failed of 2 → 50% + // Model A: 1 failed / 2 finished → 50%; the 2 in-flight sessions widen Total + // but must not dilute the rate (over-total it would read 25%). S(SessionStatus.Failed, model: "A"), S(SessionStatus.Succeeded, model: "A"), - // Model B: 2 failed of 2 → 100% + S(SessionStatus.InProgress, model: "A"), + S(SessionStatus.Pending, model: "A"), + // Model B: 2 failed of 2 finished → 100% S(SessionStatus.Failed, model: "B"), S(SessionStatus.Failed, model: "B"), // Model C: no failures → excluded entirely @@ -164,6 +207,7 @@ public class FleetHealthPayloadTests Assert.Equal(2, payload.TopFailingModels[0].Failed); Assert.Equal(100, payload.TopFailingModels[0].FailureRate); Assert.Equal("Contoso A", payload.TopFailingModels[1].Model); + Assert.Equal(4, payload.TopFailingModels[1].Total); Assert.Equal(50, payload.TopFailingModels[1].FailureRate); Assert.DoesNotContain(payload.TopFailingModels, m => m.Model == "Contoso C"); } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsSuccessRateTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsSuccessRateTests.cs new file mode 100644 index 00000000..ad757cf7 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsSuccessRateTests.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Functions.Functions.Metrics; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Pins the success-rate semantics of the geographic aggregation +/// (): the rate is an outcome +/// quota over finished enrollments (Succeeded + Failed) following the SLA convention. In-flight +/// sessions (InProgress/Pending/Stalled/AwaitingUser) and Incomplete (terminal, non-failure) sit +/// in SessionCount but never dilute the rate — the customer-visible symptom this pins against was +/// a site mid-rollout showing a poor rate despite zero actual failures. +/// +public class GeographicMetricsSuccessRateTests +{ + private static SessionSummary S(SessionStatus status, string city = "Frankfurt") + => new() + { + TenantId = "00000000-0000-0000-0000-000000000fe0", + SessionId = Guid.NewGuid().ToString(), + Status = status, + StartedAt = DateTime.UtcNow.AddHours(-2), + GeoCountry = "DE", + GeoRegion = "Hessen", + GeoCity = city, + GeoLoc = "50.11,8.68", + }; + + [Fact] + public void SuccessRate_IsOverFinishedSessions_InFlightAndIncompleteExcluded() + { + var sessions = new List + { + S(SessionStatus.Succeeded), + S(SessionStatus.Succeeded), + S(SessionStatus.Succeeded), + S(SessionStatus.Failed), + S(SessionStatus.InProgress), + S(SessionStatus.Pending), + S(SessionStatus.AwaitingUser), + S(SessionStatus.Incomplete), // terminal but non-failure — stays out of the rate + }; + + var result = GetGeographicMetricsFunction.ComputeGeographicMetrics( + sessions, new List(), "city"); + + var loc = Assert.Single(result.Locations); + Assert.Equal(8, loc.SessionCount); + Assert.Equal(3, loc.Succeeded); + Assert.Equal(1, loc.Failed); + // 3 / (3 + 1) = 75% (over finished), NOT 3 / 8 = 37.5% (over all sessions). + Assert.Equal(75d, loc.SuccessRate); + } + + [Fact] + public void SuccessRate_IsZero_WhenNothingFinishedYet() + { + var sessions = new List + { + S(SessionStatus.InProgress), + S(SessionStatus.Pending), + }; + + var result = GetGeographicMetricsFunction.ComputeGeographicMetrics( + sessions, new List(), "city"); + + var loc = Assert.Single(result.Locations); + // No finished enrollments → no rate; the frontend renders "—" off Succeeded + Failed == 0. + Assert.Equal(0d, loc.SuccessRate); + Assert.Equal(0, loc.Succeeded + loc.Failed); + Assert.Equal(2, loc.SessionCount); + } + + [Fact] + public void SuccessRate_IsPerLocation_NotGlobal() + { + var sessions = new List + { + // Frankfurt: 1 / 1 finished → 100% despite the in-flight session. + S(SessionStatus.Succeeded, city: "Frankfurt"), + S(SessionStatus.InProgress, city: "Frankfurt"), + // Berlin: 1 / 2 finished → 50%. + S(SessionStatus.Succeeded, city: "Berlin"), + S(SessionStatus.Failed, city: "Berlin"), + }; + + var result = GetGeographicMetricsFunction.ComputeGeographicMetrics( + sessions, new List(), "city"); + + var frankfurt = result.Locations.Single(l => l.City == "Frankfurt"); + var berlin = result.Locations.Single(l => l.City == "Berlin"); + Assert.Equal(100d, frankfurt.SuccessRate); + Assert.Equal(50d, berlin.SuccessRate); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs index 085eefda..9808700d 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs @@ -106,6 +106,13 @@ namespace AutopilotMonitor.Functions.Functions.Metrics var first = group.First(); var succeeded = group.Where(s => s.Status == SessionStatus.Succeeded).ToList(); var failed = group.Where(s => s.Status == SessionStatus.Failed).ToList(); + // Success rate is an outcome quota over finished enrollments only + // (Succeeded + Failed), matching the SLA convention. In-flight sessions + // (InProgress/Pending/Stalled/AwaitingUser) and Incomplete (terminal, + // non-failure) must not dilute it — a site mid-rollout would otherwise + // show a poor rate despite zero actual failures. 0 finished → 0; the + // frontend renders "—" off Succeeded+Failed instead of a 0% badge. + var finished = succeeded.Count + failed.Count; // Duration stats from succeeded sessions with valid duration var durations = succeeded @@ -156,7 +163,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics SessionCount = group.Count(), Succeeded = succeeded.Count, Failed = failed.Count, - SuccessRate = group.Count() > 0 ? Math.Round((double)succeeded.Count / group.Count() * 100, 1) : 0, + SuccessRate = finished > 0 ? Math.Round((double)succeeded.Count / finished * 100, 1) : 0, AvgDurationMinutes = Math.Round(avgDuration, 1), MedianDurationMinutes = Math.Round(medianDuration, 1), P95DurationMinutes = Math.Round(p95Duration, 1), diff --git a/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs b/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs index f3fe1833..0559efd3 100644 --- a/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs +++ b/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs @@ -93,9 +93,10 @@ public static class MetricsMath /// Single source of truth for the tenant (metrics/fleet-health) and global /// (global/metrics/fleet-health) functions. Replaces the previous client-side path that /// drained up to 200k raw sessions into the browser and ran these aggregations on the main - /// thread. Semantics are preserved verbatim from that client code, notably: success rate is - /// over ALL sessions (not just terminal ones), and average duration counts every - /// non-in-progress session that carries a positive duration (failures included). + /// thread. Success rate follows the SLA convention: Succeeded / (Succeeded + Failed) — finished + /// enrollments only, so in-flight sessions and Incomplete (terminal, non-failure) never dilute + /// it. Average duration counts every non-in-progress session that carries a positive duration + /// (failures included). /// public static FleetHealthMetrics BuildFleetHealthPayload(IReadOnlyList sessions, int days) { @@ -121,6 +122,7 @@ public static class MetricsMath } int total = sessions.Count; + int finished = succeeded + failed; var stats = new FleetHealthStats { Total = total, @@ -128,7 +130,7 @@ public static class MetricsMath Failed = failed, InProgress = inProgress, Incomplete = incomplete, - SuccessRate = total > 0 ? Math.Round((double)succeeded / total * 100, 1) : 0, + SuccessRate = finished > 0 ? Math.Round((double)succeeded / finished * 100, 1) : 0, AvgDurationMinutes = completedWithDurationCount > 0 ? (int)Math.Round((double)completedDurationSeconds / completedWithDurationCount / 60.0, MidpointRounding.AwayFromZero) : 0, @@ -208,6 +210,7 @@ public static class MetricsMath } m.Total++; if (s.Status == SessionStatus.Succeeded) m.Succeeded++; + else if (s.Status == SessionStatus.Failed) m.Failed++; } return models.Values .OrderByDescending(m => m.Total) @@ -240,15 +243,18 @@ public static class MetricsMath private static List BuildFleetTopFailingModels(IReadOnlyList sessions) { - var acc = new Dictionary(); + var acc = new Dictionary(); foreach (var s in sessions) { var key = FleetModelKey(s); acc.TryGetValue(key, out var cur); cur.Total++; if (s.Status == SessionStatus.Failed) cur.Failed++; + else if (s.Status == SessionStatus.Succeeded) cur.Succeeded++; acc[key] = cur; } + // FailureRate over finished enrollments only (mirror of the success-rate convention); + // the Where(Failed > 0) guard also keeps the denominator non-zero. return acc .Where(kv => kv.Value.Failed > 0) .Select(kv => new FleetFailingModel @@ -256,7 +262,8 @@ public static class MetricsMath Model = kv.Key, Failed = kv.Value.Failed, Total = kv.Value.Total, - FailureRate = (int)Math.Round((double)kv.Value.Failed / kv.Value.Total * 100, MidpointRounding.AwayFromZero), + FailureRate = (int)Math.Round( + (double)kv.Value.Failed / (kv.Value.Failed + kv.Value.Succeeded) * 100, MidpointRounding.AwayFromZero), }) .OrderByDescending(m => m.Failed) .Take(5) diff --git a/src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs index 06b08da7..d4bb7405 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs @@ -36,7 +36,8 @@ namespace AutopilotMonitor.Shared.Models public int InProgress { get; set; } /// Terminal, non-failure sessions (timeout reclassification). Surfaced as its own count; not a failure. public int Incomplete { get; set; } - /// Succeeded / total * 100 (one decimal). Note: over all sessions, not just terminal. + /// Succeeded / (Succeeded + Failed) * 100 (one decimal) — finished enrollments only, + /// matching the SLA convention. 0 when nothing has finished yet (clients render "—"). public double SuccessRate { get; set; } /// Average duration in minutes over non-in-progress sessions that carry a duration. public int AvgDurationMinutes { get; set; } @@ -58,8 +59,11 @@ namespace AutopilotMonitor.Shared.Models public sealed class FleetModelHealth { public string Model { get; set; } = default!; + /// All sessions on this model in the window, including in-flight ones. public int Total { get; set; } public int Succeeded { get; set; } + /// Clients derive the per-model rate as Succeeded / (Succeeded + Failed). + public int Failed { get; set; } } public sealed class FleetSlowModel @@ -73,7 +77,9 @@ namespace AutopilotMonitor.Shared.Models { public string Model { get; set; } = default!; public int Failed { get; set; } + /// All sessions on this model in the window, including in-flight ones. public int Total { get; set; } + /// Failed / (Failed + Succeeded) * 100, rounded — finished enrollments only. public int FailureRate { get; set; } } } diff --git a/src/Web/autopilot-monitor-web/app/fleet-health/hooks/useFleetHealth.ts b/src/Web/autopilot-monitor-web/app/fleet-health/hooks/useFleetHealth.ts index f969e4cb..b2ea5813 100644 --- a/src/Web/autopilot-monitor-web/app/fleet-health/hooks/useFleetHealth.ts +++ b/src/Web/autopilot-monitor-web/app/fleet-health/hooks/useFleetHealth.ts @@ -16,6 +16,7 @@ export interface FleetHealthStats { inProgress: number; /** Terminal, non-failure (timeout reclassification). Surfaced separately; not counted as a failure. */ incomplete: number; + /** Succeeded / (succeeded + failed) — finished enrollments only. 0 when nothing finished yet. */ successRate: number; avgDurationMinutes: number; } @@ -30,8 +31,10 @@ export interface FleetFailureReason { } export interface FleetModelHealth { model: string; + /** All sessions on this model in the window, including in-flight ones. */ total: number; succeeded: number; + failed: number; } export interface FleetSlowModel { model: string; diff --git a/src/Web/autopilot-monitor-web/app/fleet-health/page.tsx b/src/Web/autopilot-monitor-web/app/fleet-health/page.tsx index 9b960cec..7c6c88f8 100644 --- a/src/Web/autopilot-monitor-web/app/fleet-health/page.tsx +++ b/src/Web/autopilot-monitor-web/app/fleet-health/page.tsx @@ -129,6 +129,9 @@ export default function FleetHealthPage() { successRate: s?.successRate ?? 0, avgDuration: s?.avgDurationMinutes ?? 0, }; + // Success rate is an outcome quota over finished enrollments (succeeded + failed); + // with none finished there is no rate to show — the card renders "—" instead of 0%. + const finishedCount = stats.succeeded + stats.failed; const modelHealth = data?.modelHealth ?? []; const slowestModels = data?.slowestModels ?? []; @@ -238,10 +241,16 @@ export default function FleetHealthPage() {
0 ? `${stats.successRate.toFixed(1)}%` : "—"} + subtitle={ + finishedCount > 0 + ? `${stats.succeeded} of ${finishedCount} finished enrollments` + : "No finished enrollments yet" + } color={ - stats.successRate >= 95 + finishedCount === 0 + ? "slate" + : stats.successRate >= 95 ? "green" : stats.successRate >= 80 ? "yellow" @@ -631,10 +640,13 @@ export default function FleetHealthPage() { ) : (
{modelHealth.map((m) => { + // Rate over finished enrollments only; devices still enrolling don't count + // against the model. No finished enrollments → no rate ("—", neutral bar). + const finished = m.succeeded + m.failed; const successRate = - m.total > 0 - ? Math.round((m.succeeded / m.total) * 100) - : 0; + finished > 0 + ? Math.round((m.succeeded / finished) * 100) + : null; return ( {m.model} - {successRate}% ({m.total} devices) + {successRate !== null ? `${successRate}%` : "—"} ({m.total} devices)
= 95 + successRate === null + ? "bg-gray-300" + : successRate >= 95 ? "bg-green-500" : successRate >= 80 ? "bg-yellow-500" : "bg-red-500" }`} - style={{ width: `${successRate}%` }} + style={{ width: `${successRate ?? 0}%` }} />
diff --git a/src/Web/autopilot-monitor-web/app/geographic-performance/GeoMap.tsx b/src/Web/autopilot-monitor-web/app/geographic-performance/GeoMap.tsx index 257a0e73..d3b4fc89 100644 --- a/src/Web/autopilot-monitor-web/app/geographic-performance/GeoMap.tsx +++ b/src/Web/autopilot-monitor-web/app/geographic-performance/GeoMap.tsx @@ -11,8 +11,11 @@ interface LocationMetric { city: string; loc: string; sessionCount: number; + succeeded: number; + failed: number; avgDurationMinutes: number; appLoadScore: number; + /** Over finished enrollments (succeeded + failed) only; 0 when nothing finished yet. */ successRate: number; avgThroughputBytesPerSec: number; isOutlier: boolean; @@ -160,7 +163,10 @@ export default function GeoMap({ locations, globalAvgDuration, selectedLocation,
{loc.locationKey}
-
Sessions: {loc.sessionCount} ({loc.successRate}% success)
+
+ Sessions: {loc.sessionCount} + {loc.succeeded + loc.failed > 0 ? ` (${loc.successRate}% success)` : " (none finished yet)"} +
Avg Duration: {Math.round(loc.avgDurationMinutes)} min
{loc.appLoadScore > 0 && (
App-Load-Score: {Math.round(loc.appLoadScore)}
diff --git a/src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx b/src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx index a082ad7b..b59fb760 100644 --- a/src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx +++ b/src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx @@ -507,17 +507,28 @@ export default function GeographicPerformancePage() { {loc.sessionCount} - = 90 - ? "bg-green-100 text-green-800" - : loc.successRate >= 70 - ? "bg-yellow-100 text-yellow-800" - : "bg-red-100 text-red-800" - }`} - > - {loc.successRate}% - + {/* Rate is over finished enrollments (succeeded + failed) only; a + location where everything is still in flight has no rate yet. */} + {loc.succeeded + loc.failed > 0 ? ( + = 90 + ? "bg-green-100 text-green-800" + : loc.successRate >= 70 + ? "bg-yellow-100 text-yellow-800" + : "bg-red-100 text-red-800" + }`} + > + {loc.successRate}% + + ) : ( + + — + + )}