geo metrics: median API latency as display statistic

A single corrupt session average (request spanning sleep/hibernate counts
wall-clock hours) inflated the request-weighted location mean by orders of
magnitude (Amersfoort: 73286 ms from one 1.38M-ms session). Add
MedianApiLatencyMs per location + global (upper median over per-session
averages, duration convention), base ApiLatencyVsGlobalPct on medians, and
switch web display/sort to the median with the weighted avg in the tooltip.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/Backend/AutopilotMonitor.Functions.Tests/GeographicMetricsApiLatencyTests.cs

@@ -9,11 +9,13 @@ namespace AutopilotMonitor.Functions.Tests;
 
 /// <summary>
 /// Pins the API-latency aggregation of
-/// <see cref="GetGeographicMetricsFunction.ComputeGeographicMetrics"/> (2026-07-28): both the
-/// per-location and the global figure are REQUEST-WEIGHTED averages —
-/// Σ(sessionAvg·requests)/Σ(requests) — not averages of session averages, so a long session
-/// with many uploads counts proportionally. Sessions without latency data (agents predating
-/// <c>AvgApiLatencyMs</c>) drop out of the weighting entirely.
+/// <see cref="GetGeographicMetricsFunction.ComputeGeographicMetrics"/>: the avg figures are
+/// REQUEST-WEIGHTED averages — Σ(sessionAvg·requests)/Σ(requests) — not averages of session
+/// averages, so a long session with many uploads counts proportionally. The median figures
+/// (upper median over per-session averages, duration convention) are the robust display
+/// statistic and the baseline for <c>ApiLatencyVsGlobalPct</c>: one corrupt session average
+/// (a request spanning a sleep/hibernate gap) must not drag a whole location (2026-08-15).
+/// Sessions without latency data (agents predating <c>AvgApiLatencyMs</c>) drop out entirely.
 /// </summary>
 public class GeographicMetricsApiLatencyTests
 {
@@ -47,9 +49,37 @@ public class GeographicMetricsApiLatencyTests
         var loc = Assert.Single(result.Locations);
         // (1000·300 + 500·100) / 400 = 875 — an unweighted mean would say 750.
         Assert.Equal(875d, loc.AvgApiLatencyMs);
+        // Upper median of [500, 1000].
+        Assert.Equal(1000d, loc.MedianApiLatencyMs);
         Assert.Equal(2, loc.ApiLatencySessionCount);
     }
 
+    [Fact]
+    public void MedianLatency_IsRobustAgainstCorruptSessionAverage()
+    {
+        // Real-world shape (Amersfoort 2026-08): one session whose average was inflated to
+        // ~23 minutes per request by a sleep-spanning sample, among ordinary ~150-200 ms peers.
+        var sessions = new List<SessionSummary>
+        {
+            S("NL", "Amersfoort", 150, 100),
+            S("NL", "Amersfoort", 170, 80),
+            S("NL", "Amersfoort", 200, 120),
+            S("NL", "Amersfoort", 1_382_551.5, 115),
+        };
+
+        var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
+            sessions, new List<AppInstallSummary>(), "country");
+
+        var loc = Assert.Single(result.Locations);
+        // The weighted mean is wrecked by the corrupt session…
+        Assert.True(loc.AvgApiLatencyMs > 100_000);
+        // …the median is not: upper median of [150, 170, 200, 1382551.5] = 200.
+        Assert.Equal(200d, loc.MedianApiLatencyMs);
+        Assert.Equal(200d, result.GlobalAverages.MedianApiLatencyMs);
+        // Median vs global median (same single location) → no deviation.
+        Assert.Equal(0d, loc.ApiLatencyVsGlobalPct);
+    }
+
     [Fact]
     public void SessionsWithoutLatencyData_DropOutOfWeighting_ButStayInSessionCount()
     {
@@ -67,6 +97,7 @@ public class GeographicMetricsApiLatencyTests
         Assert.Equal(3, loc.SessionCount);
         Assert.Equal(1, loc.ApiLatencySessionCount);
         Assert.Equal(200d, loc.AvgApiLatencyMs);
+        Assert.Equal(200d, loc.MedianApiLatencyMs);
     }
 
     [Fact]
@@ -79,9 +110,11 @@ public class GeographicMetricsApiLatencyTests
 
         var loc = Assert.Single(result.Locations);
         Assert.Equal(0d, loc.AvgApiLatencyMs);
+        Assert.Equal(0d, loc.MedianApiLatencyMs);
         Assert.Equal(0, loc.ApiLatencySessionCount);
         Assert.Equal(0d, loc.ApiLatencyVsGlobalPct);
         Assert.Equal(0d, result.GlobalAverages.AvgApiLatencyMs);
+        Assert.Equal(0d, result.GlobalAverages.MedianApiLatencyMs);
     }
 
     [Fact]
@@ -98,11 +131,13 @@ public class GeographicMetricsApiLatencyTests
 
         // Equal request weights → global = (200 + 800) / 2 = 500.
         Assert.Equal(500d, result.GlobalAverages.AvgApiLatencyMs);
+        // Upper median of [200, 800].
+        Assert.Equal(800d, result.GlobalAverages.MedianApiLatencyMs);
 
         var de = result.Locations.Single(l => l.Country == "DE");
         var id = result.Locations.Single(l => l.Country == "ID");
-        Assert.Equal(-60d, de.ApiLatencyVsGlobalPct); // 200 vs 500
-        Assert.Equal(60d, id.ApiLatencyVsGlobalPct);  // 800 vs 500
+        Assert.Equal(-75d, de.ApiLatencyVsGlobalPct); // median 200 vs global median 800
+        Assert.Equal(0d, id.ApiLatencyVsGlobalPct);   // median 800 vs global median 800
     }
 
     [Fact]
@@ -119,6 +154,7 @@ public class GeographicMetricsApiLatencyTests
 
         var fr = result.Locations.Single(l => l.Country == "FR");
         Assert.Equal(0d, fr.AvgApiLatencyMs);
+        Assert.Equal(0d, fr.MedianApiLatencyMs);
         Assert.Equal(0d, fr.ApiLatencyVsGlobalPct); // no data ≠ "0 ms fast"
     }
 }

src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs

@@ -157,6 +157,10 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
                 // count so the location figure equals Σ(latency·requests)/Σ(requests) — the exact
                 // whole-population average, not an average of session averages. Sessions from
                 // agents predating the feature carry no data and drop out.
+                // The median (upper median, same convention as durations) is the display statistic:
+                // a single corrupt session average — e.g. one request spanning a sleep/hibernate
+                // gap counts hours of wall clock — inflates the weighted mean by orders of
+                // magnitude, while the median is unaffected.
                 var latencySessions = group
                     .Where(s => s.AvgApiLatencyMs is > 0 && s.ApiRequestCount is > 0)
                     .ToList();
@@ -164,6 +168,11 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
                 var avgApiLatency = latencyRequestSum > 0
                     ? latencySessions.Sum(s => s.AvgApiLatencyMs!.Value * s.ApiRequestCount!.Value) / latencyRequestSum
                     : 0;
+                var sortedLatencies = latencySessions
+                    .Select(s => s.AvgApiLatencyMs!.Value)
+                    .OrderBy(v => v)
+                    .ToList();
+                var medianApiLatency = sortedLatencies.Count > 0 ? sortedLatencies[sortedLatencies.Count / 2] : 0;
 
                 locations.Add(new LocationMetrics
                 {
@@ -184,6 +193,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
                     AvgThroughputBytesPerSec = Math.Round(avgThroughput, 0),
                     TotalDownloadBytes = totalBytes,
                     AvgApiLatencyMs = Math.Round(avgApiLatency, 0),
+                    MedianApiLatencyMs = Math.Round(medianApiLatency, 0),
                     ApiLatencySessionCount = latencySessions.Count,
                     // Delivery Optimization
                     DoSessionCount = doSessionCount,
@@ -218,7 +228,8 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
 
             // Global API latency: request-weighted over ALL geo sessions (not location averages),
             // so big sites weigh in proportionally — same Σ(latency·requests)/Σ(requests) math as
-            // the per-location figure.
+            // the per-location figure. The median over the same session set is the robust display
+            // statistic and the comparison baseline for ApiLatencyVsGlobalPct.
             var globalLatencySessions = geoSessions
                 .Where(s => s.AvgApiLatencyMs is > 0 && s.ApiRequestCount is > 0)
                 .ToList();
@@ -226,6 +237,13 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
             var globalAvgApiLatency = globalLatencyRequestSum > 0
                 ? globalLatencySessions.Sum(s => s.AvgApiLatencyMs!.Value * s.ApiRequestCount!.Value) / globalLatencyRequestSum
                 : 0;
+            var globalSortedLatencies = globalLatencySessions
+                .Select(s => s.AvgApiLatencyMs!.Value)
+                .OrderBy(v => v)
+                .ToList();
+            var globalMedianApiLatency = globalSortedLatencies.Count > 0
+                ? globalSortedLatencies[globalSortedLatencies.Count / 2]
+                : 0;
 
             // Global DO metrics (weighted by bytes, not simple average)
             var globalDoPeers = locations.Sum(l => l.TotalDoBytesFromPeers);
@@ -244,9 +262,10 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
                 loc.ThroughputVsGlobalPct = globalAvgThroughput > 0
                     ? Math.Round((loc.AvgThroughputBytesPerSec - globalAvgThroughput) / globalAvgThroughput * 100, 1) : 0;
 
-                // API latency vs global (positive = slower/farther from the backend region)
-                loc.ApiLatencyVsGlobalPct = globalAvgApiLatency > 0 && loc.AvgApiLatencyMs > 0
-                    ? Math.Round((loc.AvgApiLatencyMs - globalAvgApiLatency) / globalAvgApiLatency * 100, 1) : 0;
+                // API latency vs global (positive = slower/farther from the backend region) —
+                // median vs global median, so an outlier session skews neither side.
+                loc.ApiLatencyVsGlobalPct = globalMedianApiLatency > 0 && loc.MedianApiLatencyMs > 0
+                    ? Math.Round((loc.MedianApiLatencyMs - globalMedianApiLatency) / globalMedianApiLatency * 100, 1) : 0;
 
                 // AppLoadScore: normalize minutesPerApp to global median = 100
                 loc.AppLoadScore = globalMedianMinutesPerApp > 0
@@ -273,6 +292,7 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
                     AvgThroughputBytesPerSec = Math.Round(globalAvgThroughput, 0),
                     StdDevDurationMinutes = Math.Round(globalStdDev, 1),
                     AvgApiLatencyMs = Math.Round(globalAvgApiLatency, 0),
+                    MedianApiLatencyMs = Math.Round(globalMedianApiLatency, 0),
                     AvgDoPercentPeerCaching = Math.Round(globalDoPct, 1),
                     TotalDoBytesFromPeers = globalDoPeers,
                     TotalDoBytesFromHttp = globalDoHttp

src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts

@@ -296,8 +296,9 @@ export function registerAdminTools(server: McpServer, ga: boolean, strictGa: boo
       description:
         'Get geographic distribution of enrollments — where devices are enrolling from, with performance comparisons. ' +
         'Shows per-location: session counts, success rates, avg/median/p95 duration, throughput, outlier detection, and ' +
-        'avgApiLatencyMs (agent→backend HTTP round-trip, request-weighted — the metric for judging backend-region distance, ' +
-        'e.g. when evaluating a new backend region; apiLatencyVsGlobalPct compares against the global weighted average). ' +
+        'API latency (agent→backend HTTP round-trip): medianApiLatencyMs is the robust display metric for judging ' +
+        'backend-region distance (apiLatencyVsGlobalPct compares against the global median); avgApiLatencyMs is the ' +
+        'request-weighted mean, which a single corrupt session average can inflate by orders of magnitude. ' +
         (ga ? 'Omit tenantId for cross-tenant view (Global Admin). ' : '') +
         'Use get_geographic_sessions to drill into a specific location. ' +
         'days accepts any value 1-365 (e.g. 5, 7, 12, 30, 90).',

src/Shared/AutopilotMonitor.Shared/Models/GeographicApiModels.cs

@@ -57,11 +57,18 @@ namespace AutopilotMonitor.Shared.Models
         /// <summary>
         /// Average agent→backend HTTP round-trip (ms) at this location, weighted per session by
         /// its request count. 0 when no session here carries latency data (pre-feature agents).
+        /// A single corrupt session average (e.g. a request spanning a sleep/hibernate) can
+        /// dominate this figure — use <see cref="MedianApiLatencyMs"/> as the display statistic.
         /// </summary>
         public double AvgApiLatencyMs { get; set; }
+        /// <summary>
+        /// Median of the per-session average agent→backend round-trips (ms) at this location —
+        /// robust against outlier sessions. 0 when no session here carries latency data.
+        /// </summary>
+        public double MedianApiLatencyMs { get; set; }
         /// <summary>Sessions at this location that carry API-latency data</summary>
         public int ApiLatencySessionCount { get; set; }
-        /// <summary>Percentage difference from global avg API latency (positive = slower/farther)</summary>
+        /// <summary>Percentage difference from global median API latency (positive = slower/farther)</summary>
         public double ApiLatencyVsGlobalPct { get; set; }
 
         public bool IsOutlier { get; set; }
@@ -189,6 +196,8 @@ namespace AutopilotMonitor.Shared.Models
         public double StdDevDurationMinutes { get; set; }
         /// <summary>Global request-weighted average agent→backend API latency (ms); 0 = no data yet</summary>
         public double AvgApiLatencyMs { get; set; }
+        /// <summary>Global median of per-session average API latency (ms), robust against outliers; 0 = no data yet</summary>
+        public double MedianApiLatencyMs { get; set; }
         /// <summary>Global weighted average peer caching percentage</summary>
         public double AvgDoPercentPeerCaching { get; set; }
         /// <summary>Total peer bytes across all locations with DO data</summary>

src/Web/autopilot-monitor-web/app/geographic-performance/page.tsx

@@ -43,6 +43,7 @@ interface LocationMetric {
   durationVsGlobalPct: number;
   throughputVsGlobalPct: number;
   avgApiLatencyMs: number;
+  medianApiLatencyMs: number;
   apiLatencySessionCount: number;
   apiLatencyVsGlobalPct: number;
   isOutlier: boolean;
@@ -64,6 +65,7 @@ interface GlobalAverages {
   avgThroughputBytesPerSec: number;
   stdDevDurationMinutes: number;
   avgApiLatencyMs: number;
+  medianApiLatencyMs: number;
   avgDoPercentPeerCaching: number;
   totalDoBytesFromPeers: number;
   totalDoBytesFromHttp: number;
@@ -80,7 +82,7 @@ interface GeographicMetricsResponse {
 }
 
 type GroupBy = "city" | "region" | "country";
-type SortBy = "sessionCount" | "avgDurationMinutes" | "appLoadScore" | "avgThroughputBytesPerSec" | "avgApiLatencyMs" | "avgDoPercentPeerCaching";
+type SortBy = "sessionCount" | "avgDurationMinutes" | "appLoadScore" | "avgThroughputBytesPerSec" | "medianApiLatencyMs" | "avgDoPercentPeerCaching";
 
 type TimeRange = "7d" | "30d" | "90d";
 
@@ -402,11 +404,11 @@ export default function GeographicPerformancePage() {
                       : "No data"}
                   </span>
                 </div>
-                <div title="Agent→backend HTTP round-trip, request-weighted across sessions with latency data">
+                <div title="Agent→backend HTTP round-trip: median across sessions with latency data (robust against outlier sessions)">
                   <span className="text-blue-600 font-medium">API Latency:</span>{" "}
                   <span className="text-blue-900">
-                    {geoMetrics.globalAverages.avgApiLatencyMs > 0
-                      ? `${Math.round(geoMetrics.globalAverages.avgApiLatencyMs)} ms`
+                    {geoMetrics.globalAverages.medianApiLatencyMs > 0
+                      ? `${Math.round(geoMetrics.globalAverages.medianApiLatencyMs)} ms`
                       : "No data"}
                   </span>
                 </div>
@@ -464,10 +466,10 @@ export default function GeographicPerformancePage() {
                         </th>
                         <th
                           className="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider cursor-pointer hover:text-gray-700"
-                          onClick={() => handleSort("avgApiLatencyMs")}
-                          title="Agent→backend HTTP round-trip measured on the devices — high values indicate network distance to the backend region"
+                          onClick={() => handleSort("medianApiLatencyMs")}
+                          title="Agent→backend HTTP round-trip measured on the devices (median per location) — high values indicate network distance to the backend region"
                         >
-                          API Latency <SortIcon col="avgApiLatencyMs" sortBy={sortBy} sortDesc={sortDesc} />
+                          API Latency <SortIcon col="medianApiLatencyMs" sortBy={sortBy} sortDesc={sortDesc} />
                         </th>
                         <th
                           className="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider cursor-pointer hover:text-gray-700"
@@ -570,10 +572,10 @@ export default function GeographicPerformancePage() {
                             {formatThroughput(loc.avgThroughputBytesPerSec)}
                           </td>
                           <td className="px-4 py-3 text-sm">
-                            {loc.avgApiLatencyMs > 0 ? (
-                              <div title={`${loc.apiLatencySessionCount} session(s) with latency data${loc.apiLatencyVsGlobalPct !== 0 ? ` · ${loc.apiLatencyVsGlobalPct > 0 ? "+" : ""}${loc.apiLatencyVsGlobalPct.toFixed(0)}% vs global` : ""}`}>
-                                <span className={`inline-flex items-center px-2 py-0.5 rounded text-xs font-medium ${latencyColor(loc.avgApiLatencyMs)}`}>
-                                  {Math.round(loc.avgApiLatencyMs)} ms
+                            {loc.medianApiLatencyMs > 0 ? (
+                              <div title={`Median across ${loc.apiLatencySessionCount} session(s) with latency data · weighted avg ${Math.round(loc.avgApiLatencyMs)} ms${loc.apiLatencyVsGlobalPct !== 0 ? ` · ${loc.apiLatencyVsGlobalPct > 0 ? "+" : ""}${loc.apiLatencyVsGlobalPct.toFixed(0)}% vs global` : ""}`}>
+                                <span className={`inline-flex items-center px-2 py-0.5 rounded text-xs font-medium ${latencyColor(loc.medianApiLatencyMs)}`}>
+                                  {Math.round(loc.medianApiLatencyMs)} ms
                                 </span>
                                 <span className="text-xs text-gray-400 ml-1">({loc.apiLatencySessionCount})</span>
                               </div>

src/Web/autopilot-monitor-web/app/geographic-performance/sessions/page.tsx

@@ -177,13 +177,15 @@ function LocationSessionsContent() {
             .map((s) => s.durationSeconds!);
           return durations.length > 0 ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length / 60) : 0;
         })(),
-        // Request-weighted API latency across sessions with latency data (matches region aggregate formula)
+        // Median API latency across sessions with latency data (matches region aggregate: robust
+        // against a single corrupt session average, unlike the request-weighted mean)
         latencySessions: data.sessions.filter((s) => (s.avgApiLatencyMs ?? 0) > 0 && (s.apiRequestCount ?? 0) > 0).length,
-        avgApiLatency: (() => {
-          const withLatency = data.sessions.filter((s) => (s.avgApiLatencyMs ?? 0) > 0 && (s.apiRequestCount ?? 0) > 0);
-          const totalRequests = withLatency.reduce((a, s) => a + s.apiRequestCount!, 0);
-          const weightedSum = withLatency.reduce((a, s) => a + s.avgApiLatencyMs! * s.apiRequestCount!, 0);
-          return totalRequests > 0 ? Math.round(weightedSum / totalRequests) : 0;
+        medianApiLatency: (() => {
+          const sorted = data.sessions
+            .filter((s) => (s.avgApiLatencyMs ?? 0) > 0 && (s.apiRequestCount ?? 0) > 0)
+            .map((s) => s.avgApiLatencyMs!)
+            .sort((a, b) => a - b);
+          return sorted.length > 0 ? Math.round(sorted[Math.floor(sorted.length / 2)]) : 0;
         })(),
         // Weighted peer-caching % across sessions with DO telemetry (matches region aggregate formula)
         doSessions: data.sessions.filter((s) => s.hasDoTelemetry).length,
@@ -254,10 +256,10 @@ function LocationSessionsContent() {
                 <div className="text-sm font-medium text-gray-500">Sessions</div>
                 <div className="text-2xl font-bold text-gray-900">{stats.total}</div>
               </div>
-              <div className="bg-white rounded-lg shadow p-4" title="Agent→backend HTTP round-trip, request-weighted across sessions with latency data">
+              <div className="bg-white rounded-lg shadow p-4" title="Agent→backend HTTP round-trip: median across sessions with latency data (robust against outlier sessions)">
                 <div className="text-sm font-medium text-gray-500">API Latency</div>
-                <div className={`text-2xl font-bold ${stats.avgApiLatency > 0 ? "text-gray-900" : "text-gray-400"}`}>
-                  {stats.avgApiLatency > 0 ? `${stats.avgApiLatency} ms` : "—"}
+                <div className={`text-2xl font-bold ${stats.medianApiLatency > 0 ? "text-gray-900" : "text-gray-400"}`}>
+                  {stats.medianApiLatency > 0 ? `${stats.medianApiLatency} ms` : "—"}
                 </div>
                 <div className="text-xs text-gray-400">
                   {stats.latencySessions > 0 ? `${stats.latencySessions} session(s) with data` : "requires latency-reporting agent"}