diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsApiLatencyTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsApiLatencyTests.cs index 6d4b1c52..1e7145a9 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsApiLatencyTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsApiLatencyTests.cs @@ -9,11 +9,13 @@ namespace AutopilotMonitor.Functions.Tests; /// /// Pins the API-latency aggregation of -/// (2026-07-28): both the -/// per-location and the global figure are REQUEST-WEIGHTED averages — -/// Σ(sessionAvg·requests)/Σ(requests) — not averages of session averages, so a long session -/// with many uploads counts proportionally. Sessions without latency data (agents predating -/// AvgApiLatencyMs) drop out of the weighting entirely. +/// : the avg figures are +/// REQUEST-WEIGHTED averages — Σ(sessionAvg·requests)/Σ(requests) — not averages of session +/// averages, so a long session with many uploads counts proportionally. The median figures +/// (upper median over per-session averages, duration convention) are the robust display +/// statistic and the baseline for ApiLatencyVsGlobalPct: one corrupt session average +/// (a request spanning a sleep/hibernate gap) must not drag a whole location (2026-08-15). +/// Sessions without latency data (agents predating AvgApiLatencyMs) drop out entirely. /// public class GeographicMetricsApiLatencyTests { @@ -47,9 +49,37 @@ public class GeographicMetricsApiLatencyTests var loc = Assert.Single(result.Locations); // (1000·300 + 500·100) / 400 = 875 — an unweighted mean would say 750. Assert.Equal(875d, loc.AvgApiLatencyMs); + // Upper median of [500, 1000]. + Assert.Equal(1000d, loc.MedianApiLatencyMs); Assert.Equal(2, loc.ApiLatencySessionCount); } + [Fact] + public void MedianLatency_IsRobustAgainstCorruptSessionAverage() + { + // Real-world shape (Amersfoort 2026-08): one session whose average was inflated to + // ~23 minutes per request by a sleep-spanning sample, among ordinary ~150-200 ms peers. + var sessions = new List + { + S("NL", "Amersfoort", 150, 100), + S("NL", "Amersfoort", 170, 80), + S("NL", "Amersfoort", 200, 120), + S("NL", "Amersfoort", 1_382_551.5, 115), + }; + + var result = GetGeographicMetricsFunction.ComputeGeographicMetrics( + sessions, new List(), "country"); + + var loc = Assert.Single(result.Locations); + // The weighted mean is wrecked by the corrupt session… + Assert.True(loc.AvgApiLatencyMs > 100_000); + // …the median is not: upper median of [150, 170, 200, 1382551.5] = 200. + Assert.Equal(200d, loc.MedianApiLatencyMs); + Assert.Equal(200d, result.GlobalAverages.MedianApiLatencyMs); + // Median vs global median (same single location) → no deviation. + Assert.Equal(0d, loc.ApiLatencyVsGlobalPct); + } + [Fact] public void SessionsWithoutLatencyData_DropOutOfWeighting_ButStayInSessionCount() { @@ -67,6 +97,7 @@ public class GeographicMetricsApiLatencyTests Assert.Equal(3, loc.SessionCount); Assert.Equal(1, loc.ApiLatencySessionCount); Assert.Equal(200d, loc.AvgApiLatencyMs); + Assert.Equal(200d, loc.MedianApiLatencyMs); } [Fact] @@ -79,9 +110,11 @@ public class GeographicMetricsApiLatencyTests var loc = Assert.Single(result.Locations); Assert.Equal(0d, loc.AvgApiLatencyMs); + Assert.Equal(0d, loc.MedianApiLatencyMs); Assert.Equal(0, loc.ApiLatencySessionCount); Assert.Equal(0d, loc.ApiLatencyVsGlobalPct); Assert.Equal(0d, result.GlobalAverages.AvgApiLatencyMs); + Assert.Equal(0d, result.GlobalAverages.MedianApiLatencyMs); } [Fact] @@ -98,11 +131,13 @@ public class GeographicMetricsApiLatencyTests // Equal request weights → global = (200 + 800) / 2 = 500. Assert.Equal(500d, result.GlobalAverages.AvgApiLatencyMs); + // Upper median of [200, 800]. + Assert.Equal(800d, result.GlobalAverages.MedianApiLatencyMs); var de = result.Locations.Single(l => l.Country == "DE"); var id = result.Locations.Single(l => l.Country == "ID"); - Assert.Equal(-60d, de.ApiLatencyVsGlobalPct); // 200 vs 500 - Assert.Equal(60d, id.ApiLatencyVsGlobalPct); // 800 vs 500 + Assert.Equal(-75d, de.ApiLatencyVsGlobalPct); // median 200 vs global median 800 + Assert.Equal(0d, id.ApiLatencyVsGlobalPct); // median 800 vs global median 800 } [Fact] @@ -119,6 +154,7 @@ public class GeographicMetricsApiLatencyTests var fr = result.Locations.Single(l => l.Country == "FR"); Assert.Equal(0d, fr.AvgApiLatencyMs); + Assert.Equal(0d, fr.MedianApiLatencyMs); Assert.Equal(0d, fr.ApiLatencyVsGlobalPct); // no data ≠ "0 ms fast" } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs index a546f2d5..b859665c 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs @@ -157,6 +157,10 @@ namespace AutopilotMonitor.Functions.Functions.Metrics // count so the location figure equals Σ(latency·requests)/Σ(requests) — the exact // whole-population average, not an average of session averages. Sessions from // agents predating the feature carry no data and drop out. + // The median (upper median, same convention as durations) is the display statistic: + // a single corrupt session average — e.g. one request spanning a sleep/hibernate + // gap counts hours of wall clock — inflates the weighted mean by orders of + // magnitude, while the median is unaffected. var latencySessions = group .Where(s => s.AvgApiLatencyMs is > 0 && s.ApiRequestCount is > 0) .ToList(); @@ -164,6 +168,11 @@ namespace AutopilotMonitor.Functions.Functions.Metrics var avgApiLatency = latencyRequestSum > 0 ? latencySessions.Sum(s => s.AvgApiLatencyMs!.Value * s.ApiRequestCount!.Value) / latencyRequestSum : 0; + var sortedLatencies = latencySessions + .Select(s => s.AvgApiLatencyMs!.Value) + .OrderBy(v => v) + .ToList(); + var medianApiLatency = sortedLatencies.Count > 0 ? sortedLatencies[sortedLatencies.Count / 2] : 0; locations.Add(new LocationMetrics { @@ -184,6 +193,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics AvgThroughputBytesPerSec = Math.Round(avgThroughput, 0), TotalDownloadBytes = totalBytes, AvgApiLatencyMs = Math.Round(avgApiLatency, 0), + MedianApiLatencyMs = Math.Round(medianApiLatency, 0), ApiLatencySessionCount = latencySessions.Count, // Delivery Optimization DoSessionCount = doSessionCount, @@ -218,7 +228,8 @@ namespace AutopilotMonitor.Functions.Functions.Metrics // Global API latency: request-weighted over ALL geo sessions (not location averages), // so big sites weigh in proportionally — same Σ(latency·requests)/Σ(requests) math as - // the per-location figure. + // the per-location figure. The median over the same session set is the robust display + // statistic and the comparison baseline for ApiLatencyVsGlobalPct. var globalLatencySessions = geoSessions .Where(s => s.AvgApiLatencyMs is > 0 && s.ApiRequestCount is > 0) .ToList(); @@ -226,6 +237,13 @@ namespace AutopilotMonitor.Functions.Functions.Metrics var globalAvgApiLatency = globalLatencyRequestSum > 0 ? globalLatencySessions.Sum(s => s.AvgApiLatencyMs!.Value * s.ApiRequestCount!.Value) / globalLatencyRequestSum : 0; + var globalSortedLatencies = globalLatencySessions + .Select(s => s.AvgApiLatencyMs!.Value) + .OrderBy(v => v) + .ToList(); + var globalMedianApiLatency = globalSortedLatencies.Count > 0 + ? globalSortedLatencies[globalSortedLatencies.Count / 2] + : 0; // Global DO metrics (weighted by bytes, not simple average) var globalDoPeers = locations.Sum(l => l.TotalDoBytesFromPeers); @@ -244,9 +262,10 @@ namespace AutopilotMonitor.Functions.Functions.Metrics loc.ThroughputVsGlobalPct = globalAvgThroughput > 0 ? Math.Round((loc.AvgThroughputBytesPerSec - globalAvgThroughput) / globalAvgThroughput * 100, 1) : 0; - // API latency vs global (positive = slower/farther from the backend region) - loc.ApiLatencyVsGlobalPct = globalAvgApiLatency > 0 && loc.AvgApiLatencyMs > 0 - ? Math.Round((loc.AvgApiLatencyMs - globalAvgApiLatency) / globalAvgApiLatency * 100, 1) : 0; + // API latency vs global (positive = slower/farther from the backend region) — + // median vs global median, so an outlier session skews neither side. + loc.ApiLatencyVsGlobalPct = globalMedianApiLatency > 0 && loc.MedianApiLatencyMs > 0 + ? Math.Round((loc.MedianApiLatencyMs - globalMedianApiLatency) / globalMedianApiLatency * 100, 1) : 0; // AppLoadScore: normalize minutesPerApp to global median = 100 loc.AppLoadScore = globalMedianMinutesPerApp > 0 @@ -273,6 +292,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics AvgThroughputBytesPerSec = Math.Round(globalAvgThroughput, 0), StdDevDurationMinutes = Math.Round(globalStdDev, 1), AvgApiLatencyMs = Math.Round(globalAvgApiLatency, 0), + MedianApiLatencyMs = Math.Round(globalMedianApiLatency, 0), AvgDoPercentPeerCaching = Math.Round(globalDoPct, 1), TotalDoBytesFromPeers = globalDoPeers, TotalDoBytesFromHttp = globalDoHttp diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts index 9ee7e7c1..3b09c78e 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts @@ -296,8 +296,9 @@ export function registerAdminTools(server: McpServer, ga: boolean, strictGa: boo description: 'Get geographic distribution of enrollments — where devices are enrolling from, with performance comparisons. ' + 'Shows per-location: session counts, success rates, avg/median/p95 duration, throughput, outlier detection, and ' + - 'avgApiLatencyMs (agent→backend HTTP round-trip, request-weighted — the metric for judging backend-region distance, ' + - 'e.g. when evaluating a new backend region; apiLatencyVsGlobalPct compares against the global weighted average). ' + + 'API latency (agent→backend HTTP round-trip): medianApiLatencyMs is the robust display metric for judging ' + + 'backend-region distance (apiLatencyVsGlobalPct compares against the global median); avgApiLatencyMs is the ' + + 'request-weighted mean, which a single corrupt session average can inflate by orders of magnitude. ' + (ga ? 'Omit tenantId for cross-tenant view (Global Admin). ' : '') + 'Use get_geographic_sessions to drill into a specific location. ' + 'days accepts any value 1-365 (e.g. 5, 7, 12, 30, 90).', diff --git a/src/Shared/AutopilotMonitor.Shared/Models/GeographicApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/GeographicApiModels.cs index 76f558f0..854f4728 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/GeographicApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/GeographicApiModels.cs @@ -57,11 +57,18 @@ namespace AutopilotMonitor.Shared.Models /// /// Average agent→backend HTTP round-trip (ms) at this location, weighted per session by /// its request count. 0 when no session here carries latency data (pre-feature agents). + /// A single corrupt session average (e.g. a request spanning a sleep/hibernate) can + /// dominate this figure — use as the display statistic. /// public double AvgApiLatencyMs { get; set; } + /// + /// Median of the per-session average agent→backend round-trips (ms) at this location — + /// robust against outlier sessions. 0 when no session here carries latency data. + /// + public double MedianApiLatencyMs { get; set; } /// Sessions at this location that carry API-latency data public int ApiLatencySessionCount { get; set; } - /// Percentage difference from global avg API latency (positive = slower/farther) + /// Percentage difference from global median API latency (positive = slower/farther) public double ApiLatencyVsGlobalPct { get; set; } public bool IsOutlier { get; set; } @@ -189,6 +196,8 @@ namespace AutopilotMonitor.Shared.Models public double StdDevDurationMinutes { get; set; } /// Global request-weighted average agent→backend API latency (ms); 0 = no data yet public double AvgApiLatencyMs { get; set; } + /// Global median of per-session average API latency (ms), robust against outliers; 0 = no data yet + public double MedianApiLatencyMs { get; set; } /// Global weighted average peer caching percentage public double AvgDoPercentPeerCaching { get; set; } /// Total peer bytes across all locations with DO data diff --git a/src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx b/src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx index 5bbc2c43..4b1567f6 100644 --- a/src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx +++ b/src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx @@ -43,6 +43,7 @@ interface LocationMetric { durationVsGlobalPct: number; throughputVsGlobalPct: number; avgApiLatencyMs: number; + medianApiLatencyMs: number; apiLatencySessionCount: number; apiLatencyVsGlobalPct: number; isOutlier: boolean; @@ -64,6 +65,7 @@ interface GlobalAverages { avgThroughputBytesPerSec: number; stdDevDurationMinutes: number; avgApiLatencyMs: number; + medianApiLatencyMs: number; avgDoPercentPeerCaching: number; totalDoBytesFromPeers: number; totalDoBytesFromHttp: number; @@ -80,7 +82,7 @@ interface GeographicMetricsResponse { } type GroupBy = "city" | "region" | "country"; -type SortBy = "sessionCount" | "avgDurationMinutes" | "appLoadScore" | "avgThroughputBytesPerSec" | "avgApiLatencyMs" | "avgDoPercentPeerCaching"; +type SortBy = "sessionCount" | "avgDurationMinutes" | "appLoadScore" | "avgThroughputBytesPerSec" | "medianApiLatencyMs" | "avgDoPercentPeerCaching"; type TimeRange = "7d" | "30d" | "90d"; @@ -402,11 +404,11 @@ export default function GeographicPerformancePage() { : "No data"} -
+
API Latency:{" "} - {geoMetrics.globalAverages.avgApiLatencyMs > 0 - ? `${Math.round(geoMetrics.globalAverages.avgApiLatencyMs)} ms` + {geoMetrics.globalAverages.medianApiLatencyMs > 0 + ? `${Math.round(geoMetrics.globalAverages.medianApiLatencyMs)} ms` : "No data"}
@@ -464,10 +466,10 @@ export default function GeographicPerformancePage() { handleSort("avgApiLatencyMs")} - title="Agent→backend HTTP round-trip measured on the devices — high values indicate network distance to the backend region" + onClick={() => handleSort("medianApiLatencyMs")} + title="Agent→backend HTTP round-trip measured on the devices (median per location) — high values indicate network distance to the backend region" > - API Latency + API Latency - {loc.avgApiLatencyMs > 0 ? ( -
0 ? "+" : ""}${loc.apiLatencyVsGlobalPct.toFixed(0)}% vs global` : ""}`}> - - {Math.round(loc.avgApiLatencyMs)} ms + {loc.medianApiLatencyMs > 0 ? ( +
0 ? "+" : ""}${loc.apiLatencyVsGlobalPct.toFixed(0)}% vs global` : ""}`}> + + {Math.round(loc.medianApiLatencyMs)} ms ({loc.apiLatencySessionCount})
diff --git a/src/Web/autopilot-monitor-web/app/geographic-performance/sessions/page.tsx b/src/Web/autopilot-monitor-web/app/geographic-performance/sessions/page.tsx index 1f177756..67dbbea4 100644 --- a/src/Web/autopilot-monitor-web/app/geographic-performance/sessions/page.tsx +++ b/src/Web/autopilot-monitor-web/app/geographic-performance/sessions/page.tsx @@ -177,13 +177,15 @@ function LocationSessionsContent() { .map((s) => s.durationSeconds!); return durations.length > 0 ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length / 60) : 0; })(), - // Request-weighted API latency across sessions with latency data (matches region aggregate formula) + // Median API latency across sessions with latency data (matches region aggregate: robust + // against a single corrupt session average, unlike the request-weighted mean) latencySessions: data.sessions.filter((s) => (s.avgApiLatencyMs ?? 0) > 0 && (s.apiRequestCount ?? 0) > 0).length, - avgApiLatency: (() => { - const withLatency = data.sessions.filter((s) => (s.avgApiLatencyMs ?? 0) > 0 && (s.apiRequestCount ?? 0) > 0); - const totalRequests = withLatency.reduce((a, s) => a + s.apiRequestCount!, 0); - const weightedSum = withLatency.reduce((a, s) => a + s.avgApiLatencyMs! * s.apiRequestCount!, 0); - return totalRequests > 0 ? Math.round(weightedSum / totalRequests) : 0; + medianApiLatency: (() => { + const sorted = data.sessions + .filter((s) => (s.avgApiLatencyMs ?? 0) > 0 && (s.apiRequestCount ?? 0) > 0) + .map((s) => s.avgApiLatencyMs!) + .sort((a, b) => a - b); + return sorted.length > 0 ? Math.round(sorted[Math.floor(sorted.length / 2)]) : 0; })(), // Weighted peer-caching % across sessions with DO telemetry (matches region aggregate formula) doSessions: data.sessions.filter((s) => s.hasDoTelemetry).length, @@ -254,10 +256,10 @@ function LocationSessionsContent() {
Sessions
{stats.total}
-
+
API Latency
-
0 ? "text-gray-900" : "text-gray-400"}`}> - {stats.avgApiLatency > 0 ? `${stats.avgApiLatency} ms` : "—"} +
0 ? "text-gray-900" : "text-gray-400"}`}> + {stats.medianApiLatency > 0 ? `${stats.medianApiLatency} ms` : "—"}
{stats.latencySessions > 0 ? `${stats.latencySessions} session(s) with data` : "requires latency-reporting agent"}