gsadmin/AutopilotMonitor
1
0
Fork 0
mirror of https://github.com/okieselbach/AutopilotMonitor.git synced 2026-10-24 12:42:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(metrics): success rate over finished enrollments only (geo + fleet health)

Browse Source 
Customer feedback: the geographic performance pane showed a poor success
rate for a site mid-rollout because in-progress/pending sessions counted
against the rate. Success rate is an outcome quota — align Geo and Fleet
Health with the SLA convention already used by dashboard stats, usage
snapshots and the fleet rollup: Succeeded / (Succeeded + Failed).
Incomplete (terminal, non-failure) stays excluded.

- Geo: per-location SuccessRate over finished sessions; table badge and
  map tooltip render an em dash instead of a red 0% when nothing has
  finished yet
- Fleet Health: Stats.SuccessRate terminal-based; card shows "X of Y
  finished enrollments" and a neutral em dash state; per-model health
  gets a Failed count (additive wire field) so the client rate excludes
  in-flight devices; TopFailingModels FailureRate now failed/finished
- Tests: FleetHealthPayloadTests flipped to the new convention plus
  zero-finished, model-failed and in-flight-dilution cases; new
  GeographicMetricsSuccessRateTests pin the geo semantics

Backend 3216/3216, web tsc clean, vitest 602/602.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-07-24 09:47:31 +02:00
parent 11339c9868
commit 1cfc32c9aa
9 changed files with 235 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Backend/AutopilotMonitor.Functions.Tests/FleetHealthPayloadTests.cs
+53 -9
View File
@@ -10,9 +10,10 @@ namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Covers the server-side Fleet Health aggregation in
/// <see cref="MetricsMath.BuildFleetHealthPayload"/>, the single source of truth shared by the
/// tenant (metrics/fleet-health) and global (global/metrics/fleet-health) functions. These assert
/// the semantics ported verbatim from the previous client-side aggregation — notably success rate
/// over ALL sessions and average duration over non-in-progress sessions (failures included).
/// tenant (metrics/fleet-health) and global (global/metrics/fleet-health) functions. Success and
/// failure rates follow the SLA convention — over finished enrollments (Succeeded + Failed) only,
/// so in-flight sessions never dilute them. Average duration stays over non-in-progress sessions
/// that carry a duration (failures included).
/// </summary>
public class FleetHealthPayloadTests
{
@@ -55,7 +56,7 @@ public class FleetHealthPayloadTests
}
[Fact]
public void SuccessRate_IsOverAllSessions_NotJustTerminal()
public void SuccessRate_IsOverFinishedSessions_InFlightAndIncompleteExcluded()
{
var sessions = new List<SessionSummary>
{
@@ -63,16 +64,34 @@ public class FleetHealthPayloadTests
S(SessionStatus.Succeeded),
S(SessionStatus.Failed),
S(SessionStatus.InProgress),
S(SessionStatus.Pending),
S(SessionStatus.Incomplete), // terminal but non-failure — stays out of the rate
};
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
Assert.Equal(4, payload.Stats.Total);
Assert.Equal(6, payload.Stats.Total);
Assert.Equal(2, payload.Stats.Succeeded);
Assert.Equal(1, payload.Stats.Failed);
Assert.Equal(1, payload.Stats.InProgress);
// 2 / 4 = 50% (over total), NOT 2 / 3 = 66.7% (over terminal).
Assert.Equal(50d, payload.Stats.SuccessRate);
Assert.Equal(1, payload.Stats.Incomplete);
// 2 / (2 + 1) = 66.7% (over finished), NOT 2 / 6 = 33.3% (over total).
Assert.Equal(66.7d, payload.Stats.SuccessRate);
}
[Fact]
public void SuccessRate_IsZero_WhenNothingFinishedYet()
{
var sessions = new List<SessionSummary>
{
S(SessionStatus.InProgress),
S(SessionStatus.Pending),
};
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
// No finished enrollments → no rate; clients render "—" off Succeeded + Failed == 0.
Assert.Equal(0d, payload.Stats.SuccessRate);
}
[Fact]
@@ -111,6 +130,27 @@ public class FleetHealthPayloadTests
Assert.DoesNotContain(payload.ModelHealth, m => m.Model == "Contoso Model1");
}
[Fact]
public void ModelHealth_CountsFailedSeparately_InFlightInTotalOnly()
{
var sessions = new List<SessionSummary>
{
S(SessionStatus.Succeeded, model: "X"),
S(SessionStatus.Failed, model: "X"),
S(SessionStatus.InProgress, model: "X"),
};
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
var m = Assert.Single(payload.ModelHealth);
// Total keeps the full device count; the client derives the rate from
// Succeeded / (Succeeded + Failed), so the in-flight session must land
// in Total but in neither outcome bucket.
Assert.Equal(3, m.Total);
Assert.Equal(1, m.Succeeded);
Assert.Equal(1, m.Failed);
}
[Fact]
public void ModelKey_FallsBackToUnknown_WhenManufacturerAndModelBlank()
{
@@ -147,10 +187,13 @@ public class FleetHealthPayloadTests
{
var sessions = new List<SessionSummary>
{
// Model A: 1 failed of 2 → 50%
// Model A: 1 failed / 2 finished → 50%; the 2 in-flight sessions widen Total
// but must not dilute the rate (over-total it would read 25%).
S(SessionStatus.Failed, model: "A"),
S(SessionStatus.Succeeded, model: "A"),
// Model B: 2 failed of 2 → 100%
S(SessionStatus.InProgress, model: "A"),
S(SessionStatus.Pending, model: "A"),
// Model B: 2 failed of 2 finished → 100%
S(SessionStatus.Failed, model: "B"),
S(SessionStatus.Failed, model: "B"),
// Model C: no failures → excluded entirely
@@ -164,6 +207,7 @@ public class FleetHealthPayloadTests
Assert.Equal(2, payload.TopFailingModels[0].Failed);
Assert.Equal(100, payload.TopFailingModels[0].FailureRate);
Assert.Equal("Contoso A", payload.TopFailingModels[1].Model);
Assert.Equal(4, payload.TopFailingModels[1].Total);
Assert.Equal(50, payload.TopFailingModels[1].FailureRate);
Assert.DoesNotContain(payload.TopFailingModels, m => m.Model == "Contoso C");
}
src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsSuccessRateTests.cs
+99
View File
@@ -0,0 +1,99 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AutopilotMonitor.Functions.Functions.Metrics;
using AutopilotMonitor.Shared.Models;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Pins the success-rate semantics of the geographic aggregation
/// (<see cref="GetGeographicMetricsFunction.ComputeGeographicMetrics"/>): the rate is an outcome
/// quota over finished enrollments (Succeeded + Failed) following the SLA convention. In-flight
/// sessions (InProgress/Pending/Stalled/AwaitingUser) and Incomplete (terminal, non-failure) sit
/// in SessionCount but never dilute the rate — the customer-visible symptom this pins against was
/// a site mid-rollout showing a poor rate despite zero actual failures.
/// </summary>
public class GeographicMetricsSuccessRateTests
{
private static SessionSummary S(SessionStatus status, string city = "Frankfurt")
=> new()
{
TenantId = "00000000-0000-0000-0000-000000000fe0",
SessionId = Guid.NewGuid().ToString(),
Status = status,
StartedAt = DateTime.UtcNow.AddHours(-2),
GeoCountry = "DE",
GeoRegion = "Hessen",
GeoCity = city,
GeoLoc = "50.11,8.68",
};
[Fact]
public void SuccessRate_IsOverFinishedSessions_InFlightAndIncompleteExcluded()
{
var sessions = new List<SessionSummary>
{
S(SessionStatus.Succeeded),
S(SessionStatus.Succeeded),
S(SessionStatus.Succeeded),
S(SessionStatus.Failed),
S(SessionStatus.InProgress),
S(SessionStatus.Pending),
S(SessionStatus.AwaitingUser),
S(SessionStatus.Incomplete), // terminal but non-failure — stays out of the rate
};
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
sessions, new List<AppInstallSummary>(), "city");
var loc = Assert.Single(result.Locations);
Assert.Equal(8, loc.SessionCount);
Assert.Equal(3, loc.Succeeded);
Assert.Equal(1, loc.Failed);
// 3 / (3 + 1) = 75% (over finished), NOT 3 / 8 = 37.5% (over all sessions).
Assert.Equal(75d, loc.SuccessRate);
}
[Fact]
public void SuccessRate_IsZero_WhenNothingFinishedYet()
{
var sessions = new List<SessionSummary>
{
S(SessionStatus.InProgress),
S(SessionStatus.Pending),
};
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
sessions, new List<AppInstallSummary>(), "city");
var loc = Assert.Single(result.Locations);
// No finished enrollments → no rate; the frontend renders "—" off Succeeded + Failed == 0.
Assert.Equal(0d, loc.SuccessRate);
Assert.Equal(0, loc.Succeeded + loc.Failed);
Assert.Equal(2, loc.SessionCount);
}
[Fact]
public void SuccessRate_IsPerLocation_NotGlobal()
{
var sessions = new List<SessionSummary>
{
// Frankfurt: 1 / 1 finished → 100% despite the in-flight session.
S(SessionStatus.Succeeded, city: "Frankfurt"),
S(SessionStatus.InProgress, city: "Frankfurt"),
// Berlin: 1 / 2 finished → 50%.
S(SessionStatus.Succeeded, city: "Berlin"),
S(SessionStatus.Failed, city: "Berlin"),
};
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
sessions, new List<AppInstallSummary>(), "city");
var frankfurt = result.Locations.Single(l => l.City == "Frankfurt");
var berlin = result.Locations.Single(l => l.City == "Berlin");
Assert.Equal(100d, frankfurt.SuccessRate);
Assert.Equal(50d, berlin.SuccessRate);
}
}
src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs
+8 -1
View File
@@ -106,6 +106,13 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
var first = group.First();
var succeeded = group.Where(s => s.Status == SessionStatus.Succeeded).ToList();
var failed = group.Where(s => s.Status == SessionStatus.Failed).ToList();
// Success rate is an outcome quota over finished enrollments only
// (Succeeded + Failed), matching the SLA convention. In-flight sessions
// (InProgress/Pending/Stalled/AwaitingUser) and Incomplete (terminal,
// non-failure) must not dilute it — a site mid-rollout would otherwise
// show a poor rate despite zero actual failures. 0 finished → 0; the
// frontend renders "—" off Succeeded+Failed instead of a 0% badge.
var finished = succeeded.Count + failed.Count;
// Duration stats from succeeded sessions with valid duration
var durations = succeeded
@@ -156,7 +163,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
SessionCount = group.Count(),
Succeeded = succeeded.Count,
Failed = failed.Count,
SuccessRate = group.Count() > 0 ? Math.Round((double)succeeded.Count / group.Count() * 100, 1) : 0,
SuccessRate = finished > 0 ? Math.Round((double)succeeded.Count / finished * 100, 1) : 0,
AvgDurationMinutes = Math.Round(avgDuration, 1),
MedianDurationMinutes = Math.Round(medianDuration, 1),
P95DurationMinutes = Math.Round(p95Duration, 1),
src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs
+13 -6
View File
@@ -93,9 +93,10 @@ public static class MetricsMath
/// Single source of truth for the tenant (<c>metrics/fleet-health</c>) and global
/// (<c>global/metrics/fleet-health</c>) functions. Replaces the previous client-side path that
/// drained up to 200k raw sessions into the browser and ran these aggregations on the main
/// thread. Semantics are preserved verbatim from that client code, notably: success rate is
/// over ALL sessions (not just terminal ones), and average duration counts every
/// non-in-progress session that carries a positive duration (failures included).
/// thread. Success rate follows the SLA convention: Succeeded / (Succeeded + Failed) — finished
/// enrollments only, so in-flight sessions and Incomplete (terminal, non-failure) never dilute
/// it. Average duration counts every non-in-progress session that carries a positive duration
/// (failures included).
/// </summary>
public static FleetHealthMetrics BuildFleetHealthPayload(IReadOnlyList<SessionSummary> sessions, int days)
{
@@ -121,6 +122,7 @@ public static class MetricsMath
}
int total = sessions.Count;
int finished = succeeded + failed;
var stats = new FleetHealthStats
{
Total = total,
@@ -128,7 +130,7 @@ public static class MetricsMath
Failed = failed,
InProgress = inProgress,
Incomplete = incomplete,
SuccessRate = total > 0 ? Math.Round((double)succeeded / total * 100, 1) : 0,
SuccessRate = finished > 0 ? Math.Round((double)succeeded / finished * 100, 1) : 0,
AvgDurationMinutes = completedWithDurationCount > 0
? (int)Math.Round((double)completedDurationSeconds / completedWithDurationCount / 60.0, MidpointRounding.AwayFromZero)
: 0,
@@ -208,6 +210,7 @@ public static class MetricsMath
}
m.Total++;
if (s.Status == SessionStatus.Succeeded) m.Succeeded++;
else if (s.Status == SessionStatus.Failed) m.Failed++;
}
return models.Values
.OrderByDescending(m => m.Total)
@@ -240,15 +243,18 @@ public static class MetricsMath
private static List<FleetFailingModel> BuildFleetTopFailingModels(IReadOnlyList<SessionSummary> sessions)
{
var acc = new Dictionary<string, (int Failed, int Total)>();
var acc = new Dictionary<string, (int Failed, int Succeeded, int Total)>();
foreach (var s in sessions)
{
var key = FleetModelKey(s);
acc.TryGetValue(key, out var cur);
cur.Total++;
if (s.Status == SessionStatus.Failed) cur.Failed++;
else if (s.Status == SessionStatus.Succeeded) cur.Succeeded++;
acc[key] = cur;
}
// FailureRate over finished enrollments only (mirror of the success-rate convention);
// the Where(Failed > 0) guard also keeps the denominator non-zero.
return acc
.Where(kv => kv.Value.Failed > 0)
.Select(kv => new FleetFailingModel
@@ -256,7 +262,8 @@ public static class MetricsMath
Model = kv.Key,
Failed = kv.Value.Failed,
Total = kv.Value.Total,
FailureRate = (int)Math.Round((double)kv.Value.Failed / kv.Value.Total * 100, MidpointRounding.AwayFromZero),
FailureRate = (int)Math.Round(
(double)kv.Value.Failed / (kv.Value.Failed + kv.Value.Succeeded) * 100, MidpointRounding.AwayFromZero),
})
.OrderByDescending(m => m.Failed)
.Take(5)
src/Shared/AutopilotMonitor.Shared/Models/FleetHealthModels.cs
+7 -1
View File
@@ -36,7 +36,8 @@ namespace AutopilotMonitor.Shared.Models
public int InProgress { get; set; }
/// <summary>Terminal, non-failure sessions (timeout reclassification). Surfaced as its own count; not a failure.</summary>
public int Incomplete { get; set; }
/// <summary>Succeeded / total * 100 (one decimal). Note: over all sessions, not just terminal.</summary>
/// <summary>Succeeded / (Succeeded + Failed) * 100 (one decimal) — finished enrollments only,
/// matching the SLA convention. 0 when nothing has finished yet (clients render "—").</summary>
public double SuccessRate { get; set; }
/// <summary>Average duration in minutes over non-in-progress sessions that carry a duration.</summary>
public int AvgDurationMinutes { get; set; }
@@ -58,8 +59,11 @@ namespace AutopilotMonitor.Shared.Models
public sealed class FleetModelHealth
{
public string Model { get; set; } = default!;
/// <summary>All sessions on this model in the window, including in-flight ones.</summary>
public int Total { get; set; }
public int Succeeded { get; set; }
/// <summary>Clients derive the per-model rate as Succeeded / (Succeeded + Failed).</summary>
public int Failed { get; set; }
}
public sealed class FleetSlowModel
@@ -73,7 +77,9 @@ namespace AutopilotMonitor.Shared.Models
{
public string Model { get; set; } = default!;
public int Failed { get; set; }
/// <summary>All sessions on this model in the window, including in-flight ones.</summary>
public int Total { get; set; }
/// <summary>Failed / (Failed + Succeeded) * 100, rounded — finished enrollments only.</summary>
public int FailureRate { get; set; }
}
}
src/Web/autopilot-monitor-web/app/fleet-health/hooks/useFleetHealth.ts
+3
View File
@@ -16,6 +16,7 @@ export interface FleetHealthStats {
inProgress: number;
/** Terminal, non-failure (timeout reclassification). Surfaced separately; not counted as a failure. */
incomplete: number;
/** Succeeded / (succeeded + failed) — finished enrollments only. 0 when nothing finished yet. */
successRate: number;
avgDurationMinutes: number;
}
@@ -30,8 +31,10 @@ export interface FleetFailureReason {
}
export interface FleetModelHealth {
model: string;
/** All sessions on this model in the window, including in-flight ones. */
total: number;
succeeded: number;
failed: number;
}
export interface FleetSlowModel {
model: string;
src/Web/autopilot-monitor-web/app/fleet-health/page.tsx
+23 -9
View File
@@ -129,6 +129,9 @@ export default function FleetHealthPage() {
successRate: s?.successRate ?? 0,
avgDuration: s?.avgDurationMinutes ?? 0,
};
// Success rate is an outcome quota over finished enrollments (succeeded + failed);
// with none finished there is no rate to show — the card renders "—" instead of 0%.
const finishedCount = stats.succeeded + stats.failed;
const modelHealth = data?.modelHealth ?? [];
const slowestModels = data?.slowestModels ?? [];
@@ -238,10 +241,16 @@ export default function FleetHealthPage() {
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-5 gap-5 mb-8">
<FleetStatCard
title="Success Rate"
value={`${stats.successRate.toFixed(1)}%`}
subtitle={`${stats.succeeded} of ${stats.total} enrollments`}
value={finishedCount > 0 ? `${stats.successRate.toFixed(1)}%` : "—"}
subtitle={
finishedCount > 0
? `${stats.succeeded} of ${finishedCount} finished enrollments`
: "No finished enrollments yet"
}
color={
stats.successRate >= 95
finishedCount === 0
? "slate"
: stats.successRate >= 95
? "green"
: stats.successRate >= 80
? "yellow"
@@ -631,10 +640,13 @@ export default function FleetHealthPage() {
) : (
<div className="space-y-3">
{modelHealth.map((m) => {
// Rate over finished enrollments only; devices still enrolling don't count
// against the model. No finished enrollments → no rate ("—", neutral bar).
const finished = m.succeeded + m.failed;
const successRate =
m.total > 0
? Math.round((m.succeeded / m.total) * 100)
: 0;
finished > 0
? Math.round((m.succeeded / finished) * 100)
: null;
return (
<Link
key={m.model}
@@ -645,19 +657,21 @@ export default function FleetHealthPage() {
<div className="flex items-baseline justify-between mb-1">
<span className="text-sm text-gray-700 group-hover:text-blue-600 break-words leading-snug">{m.model}</span>
<span className="ml-3 flex-shrink-0 text-sm font-medium text-gray-900">
{successRate}% <span className="text-xs font-normal text-gray-400">({m.total} devices)</span>
{successRate !== null ? `${successRate}%` : "—"} <span className="text-xs font-normal text-gray-400">({m.total} devices)</span>
</span>
</div>
<div className="w-full h-2 bg-gray-100 rounded-full overflow-hidden">
<div
className={`h-full rounded-full transition-all ${
successRate >= 95
successRate === null
? "bg-gray-300"
: successRate >= 95
? "bg-green-500"
: successRate >= 80
? "bg-yellow-500"
: "bg-red-500"
}`}
style={{ width: `${successRate}%` }}
style={{ width: `${successRate ?? 0}%` }}
/>
</div>
</Link>
src/Web/autopilot-monitor-web/app/geographic-performance/GeoMap.tsx
+7 -1
View File
@@ -11,8 +11,11 @@ interface LocationMetric {
city: string;
loc: string;
sessionCount: number;
succeeded: number;
failed: number;
avgDurationMinutes: number;
appLoadScore: number;
/** Over finished enrollments (succeeded + failed) only; 0 when nothing finished yet. */
successRate: number;
avgThroughputBytesPerSec: number;
isOutlier: boolean;
@@ -160,7 +163,10 @@ export default function GeoMap({ locations, globalAvgDuration, selectedLocation,
<Tooltip direction="top" offset={[0, -10]} opacity={0.95}>
<div className="text-xs">
<div className="font-bold text-sm mb-1">{loc.locationKey}</div>
<div><strong>Sessions:</strong> {loc.sessionCount} ({loc.successRate}% success)</div>
<div>
<strong>Sessions:</strong> {loc.sessionCount}
{loc.succeeded + loc.failed > 0 ? ` (${loc.successRate}% success)` : " (none finished yet)"}
</div>
<div><strong>Avg Duration:</strong> {Math.round(loc.avgDurationMinutes)} min</div>
{loc.appLoadScore > 0 && (
<div><strong>App-Load-Score:</strong> {Math.round(loc.appLoadScore)}</div>
src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx
+22 -11
View File
@@ -507,17 +507,28 @@ export default function GeographicPerformancePage() {
</td>
<td className="px-4 py-3 text-sm text-gray-700 font-medium">{loc.sessionCount}</td>
<td className="px-4 py-3 text-sm">
<span
className={`inline-flex items-center px-2 py-0.5 rounded text-xs font-medium ${
loc.successRate >= 90
? "bg-green-100 text-green-800"
: loc.successRate >= 70
? "bg-yellow-100 text-yellow-800"
: "bg-red-100 text-red-800"
}`}
>
{loc.successRate}%
</span>
{/* Rate is over finished enrollments (succeeded + failed) only; a
location where everything is still in flight has no rate yet. */}
{loc.succeeded + loc.failed > 0 ? (
<span
className={`inline-flex items-center px-2 py-0.5 rounded text-xs font-medium ${
loc.successRate >= 90
? "bg-green-100 text-green-800"
: loc.successRate >= 70
? "bg-yellow-100 text-yellow-800"
: "bg-red-100 text-red-800"
}`}
>
{loc.successRate}%
</span>
) : (
<span
className="inline-flex items-center px-2 py-0.5 rounded text-xs font-medium bg-gray-100 text-gray-500"
title="No finished enrollments yet"
>
—
</span>
)}
</td>
<td className="px-4 py-3 text-sm">
<span