added agent crash detection, added platform metrics

2026-10-24 20:53:06 +00:00 · 2026-03-29 21:15:01 +02:00
parent 815dc47886
commit 8654cbbf42
11 changed files with 456 additions and 34 deletions
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.Threading;
 using AutopilotMonitor.Agent.Core.Logging;
+using AutopilotMonitor.Agent.Core.Monitoring.Core;
 using AutopilotMonitor.Agent.Core.Monitoring.Network;
 using AutopilotMonitor.Shared.Models;

@@ -17,6 +18,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
    {
        private readonly string _agentVersion;
        private readonly NetworkMetrics _networkMetrics;
+        private readonly EventSpool _spool;

        // Previous sample for delta calculations
        private TimeSpan _prevCpuTime;
@@ -28,12 +30,14 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
            string tenantId,
            Action<EnrollmentEvent> onEventCollected,
            NetworkMetrics networkMetrics,
+            EventSpool spool,
            AgentLogger logger,
            string agentVersion = "unknown",
            int intervalSeconds = 60)
            : base(sessionId, tenantId, onEventCollected, logger, intervalSeconds)
        {
            _networkMetrics = networkMetrics ?? throw new ArgumentNullException(nameof(networkMetrics));
+            _spool = spool;
            _agentVersion = string.IsNullOrWhiteSpace(agentVersion) ? "unknown" : agentVersion;
        }

@@ -97,6 +101,13 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
                Logger.Debug($"Process metrics read failed: {ex.Message}");
            }

+            // --- Spool queue depth ---
+            if (_spool != null)
+            {
+                try { data["spool_queue_depth"] = _spool.GetCount(); }
+                catch { }
+            }
+
            // --- Network delta ---
            try
            {
@@ -53,6 +53,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
                        _configuration.TenantId,
                        EmitEvent,
                        _apiClient.NetworkMetrics,
+                        _spool,
                        _logger,
                        _agentVersion,
                        selfMetricsInterval
@@ -373,6 +374,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
                        _configuration.TenantId,
                        EmitEvent,
                        _apiClient.NetworkMetrics,
+                        _spool,
                        _logger,
                        _agentVersion,
                        selfMetricsInterval
@@ -82,6 +82,11 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
        // Admin override detected during session registration (agent restart after admin action)
        private string _pendingAdminAction;

+        // Previous exit classification (crash detection)
+        private readonly string _previousExitType;     // "clean" | "exception_crash" | "hard_kill" | "first_run"
+        private readonly string _previousCrashException; // exception type name (only for exception_crash)
+        private long _persistedSequenceAtStartup;      // for detecting spool ceiling recovery
+
        // Auth failure circuit breaker
        private int _consecutiveAuthFailures = 0;
        private DateTime? _firstAuthFailureTime = null;
@@ -94,11 +99,14 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
        private bool _isFirstConfigApply = true;
        private int? _deferredSecurityAuditConfigVersion; // deferred until after agent_started

-        public MonitoringService(AgentConfiguration configuration, AgentLogger logger, string agentVersion)
+        public MonitoringService(AgentConfiguration configuration, AgentLogger logger, string agentVersion,
+            string previousExitType = "first_run", string previousCrashException = null)
        {
            _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
            _agentVersion = string.IsNullOrWhiteSpace(agentVersion) ? "unknown" : agentVersion;
+            _previousExitType = previousExitType ?? "first_run";
+            _previousCrashException = previousCrashException;

            if (!_configuration.IsValid())
            {
@@ -115,6 +123,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
            var dataDirectory = Environment.ExpandEnvironmentVariables(Constants.AgentDataDirectory);
            _sessionPersistence = new SessionPersistence(dataDirectory);
            _eventSequence = _sessionPersistence.LoadSequence();
+            _persistedSequenceAtStartup = _eventSequence;
            var spoolMax = _spool.GetMaxSequence();
            if (spoolMax > _eventSequence)
            {
@@ -193,6 +202,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
            _logger.Info("FileSystemWatcher started for efficient event upload");

            // Emit agent_started event with startup context so the portal shows how the agent was launched
+            var spoolCeilingTriggered = _eventSequence > _persistedSequenceAtStartup;
            var startupData = new Dictionary<string, object>
            {
                { "agentVersion", _agentVersion },
@@ -202,8 +212,13 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
                { "selfDestructOnComplete", _configuration.SelfDestructOnComplete },
                { "certAuth", _configuration.UseClientCertAuth },
                { "agentMaxLifetimeMinutes", _configuration.AgentMaxLifetimeMinutes },
-                { "diagnosticsUploadMode", _configuration.DiagnosticsUploadMode ?? "Off" }
+                { "diagnosticsUploadMode", _configuration.DiagnosticsUploadMode ?? "Off" },
+                { "previousExitType", _previousExitType },
+                { "unsentSpoolEvents", _spool.GetCount() },
+                { "spoolCeilingRecovery", spoolCeilingTriggered }
            };
+            if (_previousCrashException != null)
+                startupData["previousCrashException"] = _previousCrashException;

            EmitEvent(new EnrollmentEvent
            {
@@ -42,6 +42,16 @@ namespace AutopilotMonitor.Agent
                return;
            }

+            // Register ProcessExit handler BEFORE RunAgent so we can distinguish
+            // OS shutdown/reboot (clean exit) from crashes (no marker written).
+            var stateDir = Environment.ExpandEnvironmentVariables(Constants.AgentDataDirectory);
+            var cleanExitMarkerPath = Path.Combine(stateDir, "clean-exit.marker");
+            AppDomain.CurrentDomain.ProcessExit += (s, e) =>
+            {
+                try { File.WriteAllText(cleanExitMarkerPath, DateTime.UtcNow.ToString("O")); }
+                catch { }
+            };
+
            // Always run directly - no ServiceBase.Run.
            // The agent is started by the Scheduled Task (SYSTEM) or manually with --console.
            // Both paths land here and run identically; --console just enables console output.
@@ -205,7 +215,62 @@ namespace AutopilotMonitor.Agent
                    }
                }

-                using (var service = new MonitoringService(config, logger, agentVersion))
+                // Evaluate previous exit signals before starting the monitoring service.
+                // clean-exit.marker present  → previous run exited cleanly (OS shutdown, reboot, normal exit)
+                // crash_*.log present        → unhandled exception crash (with stack trace)
+                // neither                    → hard kill (power loss, BSOD, taskkill /F) or first run
+                var cleanExitMarker = Path.Combine(
+                    Environment.ExpandEnvironmentVariables(Constants.AgentDataDirectory),
+                    "clean-exit.marker");
+                var crashLogDir = Environment.ExpandEnvironmentVariables(Constants.LogDirectory);
+                var hadCleanExit = File.Exists(cleanExitMarker);
+                var crashLogs = Directory.Exists(crashLogDir)
+                    ? Directory.GetFiles(crashLogDir, "crash_*.log")
+                    : Array.Empty<string>();
+
+                string previousExitType;
+                string previousCrashException = null;
+                if (hadCleanExit)
+                {
+                    previousExitType = "clean";
+                }
+                else if (crashLogs.Length > 0)
+                {
+                    previousExitType = "exception_crash";
+                    // Extract exception type from most recent crash log
+                    try
+                    {
+                        var mostRecent = crashLogs[crashLogs.Length - 1]; // sorted by name = chronological
+                        var crashContent = File.ReadAllText(mostRecent);
+                        // Format: "[timestamp] FATAL: ExceptionType: message..."
+                        var fatalIdx = crashContent.IndexOf("FATAL: ");
+                        if (fatalIdx >= 0)
+                        {
+                            var afterFatal = crashContent.Substring(fatalIdx + 7);
+                            var colonIdx = afterFatal.IndexOf(':');
+                            if (colonIdx > 0)
+                                previousCrashException = afterFatal.Substring(0, colonIdx).Trim();
+                        }
+                    }
+                    catch { }
+                }
+                else
+                {
+                    // No marker and no crash log — either first run ever or hard kill
+                    var sessionFile = Path.Combine(
+                        Environment.ExpandEnvironmentVariables(Constants.AgentDataDirectory), "session.id");
+                    previousExitType = File.Exists(sessionFile) ? "hard_kill" : "first_run";
+                }
+
+                // Clean up: delete marker and crash logs so next cycle starts fresh
+                try { File.Delete(cleanExitMarker); } catch { }
+                foreach (var crashLog in crashLogs)
+                    try { File.Delete(crashLog); } catch { }
+
+                if (previousExitType != "first_run")
+                    logger.Info($"Previous exit: {previousExitType}{(previousCrashException != null ? $" ({previousCrashException})" : "")}");
+
+                using (var service = new MonitoringService(config, logger, agentVersion, previousExitType, previousCrashException))
                {
                    service.Start();

@@ -1 +1 @@
-702
+704
@@ -4,6 +4,7 @@ using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
 using AutopilotMonitor.Shared.DataAccess;
+using AutopilotMonitor.Shared.Models;
 using Microsoft.Extensions.Logging;

 namespace AutopilotMonitor.Functions.Services
@@ -101,6 +102,7 @@ namespace AutopilotMonitor.Functions.Services
                    var wsValues = snapshots.Select(s => GetDouble(s, "agent_working_set_mb")).ToList();
                    var pbValues = snapshots.Select(s => GetDouble(s, "agent_private_bytes_mb")).ToList();
                    var latValues = snapshots.Select(s => GetDouble(s, "net_avg_latency_ms")).Where(v => v > 0).ToList();
+                    var spoolValues = snapshots.Select(s => GetDouble(s, "spool_queue_depth")).ToList();

                    var lastSnapshot = snapshots.Last();

@@ -130,7 +132,9 @@ namespace AutopilotMonitor.Functions.Services
                        AvgWorkingSet = wsValues.Count > 0 ? wsValues.Average() : 0,
                        MaxWorkingSet = wsValues.Count > 0 ? wsValues.Max() : 0,
                        AvgPrivateBytes = pbValues.Count > 0 ? pbValues.Average() : 0,
-                        AvgLatency = latValues.Count > 0 ? latValues.Average() : 0
+                        AvgLatency = latValues.Count > 0 ? latValues.Average() : 0,
+                        AvgSpoolDepth = spoolValues.Count > 0 ? spoolValues.Average() : 0,
+                        MaxSpoolDepth = spoolValues.Count > 0 ? spoolValues.Max() : 0
                    };
                }
                catch (Exception ex)
@@ -143,12 +147,159 @@ namespace AutopilotMonitor.Functions.Services
            var results = await Task.WhenAll(tasks);
            sessionMetrics = results.Where(r => r != null).ToList()!;

+            // Compute delivery latency from all events across sessions
+            var deliveryLatency = ComputeDeliveryLatency(allSessions, _sessionRepo);
+            var crashRate = ComputeCrashRate(allSessions, _sessionRepo);
+
+            // Run both in parallel
+            await Task.WhenAll(deliveryLatency, crashRate);
+
            return new PlatformAgentMetricsResponse
            {
-                Sessions = sessionMetrics
+                Sessions = sessionMetrics,
+                DeliveryLatency = deliveryLatency.Result,
+                CrashRate = crashRate.Result
            };
        }

+        private async Task<DeliveryLatencyMetrics> ComputeDeliveryLatency(
+            List<AutopilotMonitor.Shared.Models.SessionSummary> sessions,
+            ISessionRepository sessionRepo)
+        {
+            try
+            {
+                // Sample up to 20 recent sessions for latency data
+                var sampleSessions = sessions.Take(20).ToList();
+                var allDeltas = new List<double>();
+
+                var latencyTasks = sampleSessions.Select(async session =>
+                {
+                    try
+                    {
+                        var events = await sessionRepo.GetSessionEventsAsync(session.TenantId, session.SessionId);
+                        return events
+                            .Where(e => e.ReceivedAt.HasValue && e.Timestamp != default)
+                            .Select(e => (e.ReceivedAt!.Value - e.Timestamp).TotalMilliseconds)
+                            .ToList();
+                    }
+                    catch { return new List<double>(); }
+                }).ToList();
+
+                var latencyResults = await Task.WhenAll(latencyTasks);
+                foreach (var deltas in latencyResults)
+                    allDeltas.AddRange(deltas);
+
+                if (allDeltas.Count == 0)
+                    return new DeliveryLatencyMetrics();
+
+                var negativeCount = allDeltas.Count(d => d < 0);
+                var validDeltas = allDeltas.Where(d => d >= 0).OrderBy(d => d).ToList();
+
+                if (validDeltas.Count == 0)
+                    return new DeliveryLatencyMetrics
+                    {
+                        SampleCount = allDeltas.Count,
+                        ClockSkewPercent = 100.0
+                    };
+
+                return new DeliveryLatencyMetrics
+                {
+                    P50Ms = Math.Round(Percentile(validDeltas, 0.50), 0),
+                    P95Ms = Math.Round(Percentile(validDeltas, 0.95), 0),
+                    P99Ms = Math.Round(Percentile(validDeltas, 0.99), 0),
+                    AvgMs = Math.Round(validDeltas.Average(), 0),
+                    SampleCount = allDeltas.Count,
+                    ClockSkewPercent = Math.Round((double)negativeCount / allDeltas.Count * 100, 1)
+                };
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Failed to compute delivery latency");
+                return new DeliveryLatencyMetrics();
+            }
+        }
+
+        private async Task<CrashRateMetrics> ComputeCrashRate(
+            List<AutopilotMonitor.Shared.Models.SessionSummary> sessions,
+            ISessionRepository sessionRepo)
+        {
+            try
+            {
+                var metrics = new CrashRateMetrics();
+                var exceptionCounts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
+
+                var crashTasks = sessions.Select(async session =>
+                {
+                    try
+                    {
+                        var events = await sessionRepo.GetSessionEventsAsync(session.TenantId, session.SessionId);
+                        return events
+                            .Where(e => e.EventType == "agent_started" && e.Data != null)
+                            .Select(e => e.Data)
+                            .ToList();
+                    }
+                    catch { return new List<Dictionary<string, object>>(); }
+                }).ToList();
+
+                var crashResults = await Task.WhenAll(crashTasks);
+                foreach (var startEvents in crashResults)
+                {
+                    foreach (var data in startEvents)
+                    {
+                        metrics.TotalStarts++;
+                        var exitType = GetString(data, "previousExitType");
+                        switch (exitType)
+                        {
+                            case "clean":
+                                metrics.CleanExits++;
+                                break;
+                            case "exception_crash":
+                                metrics.ExceptionCrashes++;
+                                var exType = GetString(data, "previousCrashException");
+                                if (!string.IsNullOrEmpty(exType))
+                                {
+                                    exceptionCounts.TryGetValue(exType, out var count);
+                                    exceptionCounts[exType] = count + 1;
+                                }
+                                break;
+                            case "hard_kill":
+                                metrics.HardKills++;
+                                break;
+                            default:
+                                metrics.FirstRuns++;
+                                break;
+                        }
+                    }
+                }
+
+                var crashCount = metrics.ExceptionCrashes + metrics.HardKills;
+                var nonFirstRuns = metrics.TotalStarts - metrics.FirstRuns;
+                metrics.CrashRatePercent = nonFirstRuns > 0
+                    ? Math.Round((double)crashCount / nonFirstRuns * 100, 1)
+                    : 0;
+
+                metrics.TopExceptions = exceptionCounts
+                    .OrderByDescending(kv => kv.Value)
+                    .Take(5)
+                    .Select(kv => new CrashExceptionSummary { ExceptionType = kv.Key, Count = kv.Value })
+                    .ToList();
+
+                return metrics;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Failed to compute crash rate");
+                return new CrashRateMetrics();
+            }
+        }
+
+        private static double Percentile(List<double> sortedValues, double percentile)
+        {
+            if (sortedValues.Count == 0) return 0;
+            var index = (int)Math.Ceiling(percentile * sortedValues.Count) - 1;
+            return sortedValues[Math.Max(0, Math.Min(index, sortedValues.Count - 1))];
+        }
+
        private static double GetDouble(Dictionary<string, object> data, string key)
        {
            if (data.TryGetValue(key, out var value))
@@ -177,6 +328,8 @@ namespace AutopilotMonitor.Functions.Services
    public class PlatformAgentMetricsResponse
    {
        public List<SessionAgentMetric> Sessions { get; set; } = new();
+        public DeliveryLatencyMetrics? DeliveryLatency { get; set; }
+        public CrashRateMetrics? CrashRate { get; set; }
        public DateTime ComputedAt { get; set; }
        public int ComputeDurationMs { get; set; }
        public bool FromCache { get; set; }
@@ -202,5 +355,34 @@ namespace AutopilotMonitor.Functions.Services
        public double MaxWorkingSet { get; set; }
        public double AvgPrivateBytes { get; set; }
        public double AvgLatency { get; set; }
+        public double AvgSpoolDepth { get; set; }
+        public double MaxSpoolDepth { get; set; }
+    }
+
+    public class DeliveryLatencyMetrics
+    {
+        public double P50Ms { get; set; }
+        public double P95Ms { get; set; }
+        public double P99Ms { get; set; }
+        public double AvgMs { get; set; }
+        public int SampleCount { get; set; }
+        public double ClockSkewPercent { get; set; }
+    }
+
+    public class CrashRateMetrics
+    {
+        public int TotalStarts { get; set; }
+        public int CleanExits { get; set; }
+        public int ExceptionCrashes { get; set; }
+        public int HardKills { get; set; }
+        public int FirstRuns { get; set; }
+        public double CrashRatePercent { get; set; }
+        public List<CrashExceptionSummary> TopExceptions { get; set; } = new();
+    }
+
+    public class CrashExceptionSummary
+    {
+        public string ExceptionType { get; set; } = string.Empty;
+        public int Count { get; set; }
    }
 }
@@ -5,7 +5,7 @@ export function MetricsSidebar({ children }: { children: React.ReactNode }) {
    <>
      <header className="bg-white shadow">
        <div className="py-6 px-4 sm:px-6 lg:px-8">
-          <h1 className="text-2xl font-normal text-gray-900">Metrics</h1>
+          <h1 className="text-2xl font-normal text-gray-900">Platform Metrics</h1>
          <p className="text-sm text-gray-500 mt-1">Platform-wide analytics</p>
        </div>
      </header>
@@ -1,5 +1,5 @@
 export const METRICS_NAV_SECTIONS = [
-  { id: "agent-metrics", label: "Agent Metrics", description: "Agent version distribution, footprints, and detailed breakdown" },
+  { id: "agent-metrics", label: "Platform Metrics", description: "Agent performance, delivery latency, crash rates, and platform health" },
  { id: "usage", label: "Platform Usage", description: "Platform usage statistics across all tenants" },
  { id: "mcp-usage", label: "MCP Usage", description: "MCP API usage metrics across all users" },
 ] as const;
@@ -28,10 +28,38 @@ interface SessionAgentMetricDTO {
  maxWorkingSet: number;
  avgPrivateBytes: number;
  avgLatency: number;
+  avgSpoolDepth: number;
+  maxSpoolDepth: number;
+}
+
+interface DeliveryLatencyMetricsDTO {
+  p50Ms: number;
+  p95Ms: number;
+  p99Ms: number;
+  avgMs: number;
+  sampleCount: number;
+  clockSkewPercent: number;
+}
+
+interface CrashExceptionSummaryDTO {
+  exceptionType: string;
+  count: number;
+}
+
+interface CrashRateMetricsDTO {
+  totalStarts: number;
+  cleanExits: number;
+  exceptionCrashes: number;
+  hardKills: number;
+  firstRuns: number;
+  crashRatePercent: number;
+  topExceptions: CrashExceptionSummaryDTO[];
 }

 interface PlatformMetricsResponse {
  sessions: SessionAgentMetricDTO[];
+  deliveryLatency?: DeliveryLatencyMetricsDTO;
+  crashRate?: CrashRateMetricsDTO;
  computedAt: string;
  computeDurationMs: number;
  fromCache: boolean;
@@ -49,6 +77,8 @@ interface SessionAgentMetrics {
  maxWorkingSet: number;
  avgPrivateBytes: number;
  avgLatency: number;
+  avgSpoolDepth: number;
+  maxSpoolDepth: number;
 }

 // ── Helpers ────────────────────────────────────────────────────────────────────
@@ -61,6 +91,12 @@ function formatBytes(bytes: number): string {
  return `${val.toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
 }

+function formatLatency(ms: number): string {
+  if (ms < 1000) return `${ms.toFixed(0)} ms`;
+  if (ms < 60000) return `${(ms / 1000).toFixed(1)} s`;
+  return `${(ms / 60000).toFixed(1)} min`;
+}
+
 function formatVersion(version: string): string {
  // "1.0.189.0+95a2ea7abc..." → "1.0.189+95a2ea7"
  const plusIdx = version.indexOf('+');
@@ -103,6 +139,8 @@ export function SectionAgentMetrics() {
  const [error, setError] = useState<string | null>(null);
  const [versionFilter, setVersionFilter] = useState<string>('all');
  const [cacheInfo, setCacheInfo] = useState<{ fromCache: boolean; computeDurationMs: number; computedAt: string } | null>(null);
+  const [deliveryLatency, setDeliveryLatency] = useState<DeliveryLatencyMetricsDTO | null>(null);
+  const [crashRate, setCrashRate] = useState<CrashRateMetricsDTO | null>(null);

  // ── Fetch pre-computed metrics from backend (with 5-min cache) ────────────

@@ -124,6 +162,8 @@ export function SectionAgentMetrics() {

      const data: PlatformMetricsResponse = await res.json();
      setCacheInfo({ fromCache: data.fromCache, computeDurationMs: data.computeDurationMs, computedAt: data.computedAt });
+      setDeliveryLatency(data.deliveryLatency || null);
+      setCrashRate(data.crashRate || null);

      const mapped: SessionAgentMetrics[] = (data.sessions || []).map((s) => ({
        session: {
@@ -146,6 +186,8 @@ export function SectionAgentMetrics() {
        maxWorkingSet: s.maxWorkingSet,
        avgPrivateBytes: s.avgPrivateBytes,
        avgLatency: s.avgLatency,
+        avgSpoolDepth: s.avgSpoolDepth,
+        maxSpoolDepth: s.maxSpoolDepth,
      }));

      setSessionMetrics(mapped);
@@ -540,33 +582,138 @@ export function SectionAgentMetrics() {
              </div>
            </div>

-            {/* ── Ideas for future metrics ──────────────────────────── */}
+            {/* ── Platform Health Metrics ───────────────────────────── */}
            <div className="bg-white shadow rounded-lg p-6">
-              <h3 className="text-sm font-semibold text-gray-900 mb-3">Future Platform Metrics Ideas</h3>
-              <div className="grid grid-cols-1 md:grid-cols-3 gap-4 text-xs text-gray-500">
-                <div className="flex items-start gap-2">
-                  <span className="text-gray-300 mt-0.5">&#9679;</span>
-                  <span>Agent crash rate &amp; restart frequency</span>
+              <h2 className="text-lg font-semibold text-gray-900 mb-4">Platform Health</h2>
+              <div className="grid grid-cols-1 md:grid-cols-3 gap-6">
+
+                {/* Event Delivery Latency */}
+                <div>
+                  <h3 className="text-sm font-medium text-gray-700 mb-3">Event Delivery Latency</h3>
+                  {deliveryLatency && deliveryLatency.sampleCount > 0 ? (
+                    <div className="space-y-2">
+                      <div className="flex justify-between text-sm">
+                        <span className="text-gray-500">p50</span>
+                        <span className="font-mono text-gray-900">{formatLatency(deliveryLatency.p50Ms)}</span>
+                      </div>
+                      <div className="flex justify-between text-sm">
+                        <span className="text-gray-500">p95</span>
+                        <span className={`font-mono ${deliveryLatency.p95Ms > 30000 ? 'text-yellow-600' : 'text-gray-900'}`}>
+                          {formatLatency(deliveryLatency.p95Ms)}
+                        </span>
+                      </div>
+                      <div className="flex justify-between text-sm">
+                        <span className="text-gray-500">p99</span>
+                        <span className={`font-mono ${deliveryLatency.p99Ms > 60000 ? 'text-red-600' : 'text-gray-900'}`}>
+                          {formatLatency(deliveryLatency.p99Ms)}
+                        </span>
+                      </div>
+                      <div className="flex justify-between text-sm border-t border-gray-100 pt-2">
+                        <span className="text-gray-500">avg</span>
+                        <span className="font-mono text-gray-900">{formatLatency(deliveryLatency.avgMs)}</span>
+                      </div>
+                      <p className="text-xs text-gray-400 mt-1">
+                        {deliveryLatency.sampleCount.toLocaleString()} events sampled
+                        {deliveryLatency.clockSkewPercent > 0 && (
+                          <span className="text-yellow-600 ml-1">
+                            ({deliveryLatency.clockSkewPercent.toFixed(1)}% clock skew)
+                          </span>
+                        )}
+                      </p>
+                    </div>
+                  ) : (
+                    <p className="text-sm text-gray-400">No latency data yet</p>
+                  )}
                </div>
-                <div className="flex items-start gap-2">
-                  <span className="text-gray-300 mt-0.5">&#9679;</span>
-                  <span>Event delivery latency (emit &rarr; backend)</span>
+
+                {/* Crash Rate */}
+                <div>
+                  <h3 className="text-sm font-medium text-gray-700 mb-3">Agent Crash Rate</h3>
+                  {crashRate && crashRate.totalStarts > 0 ? (
+                    <div className="space-y-2">
+                      <div className="flex items-baseline gap-2">
+                        <span className={`text-2xl font-bold ${crashRate.crashRatePercent === 0 ? 'text-green-600' : crashRate.crashRatePercent < 5 ? 'text-yellow-600' : 'text-red-600'}`}>
+                          {crashRate.crashRatePercent.toFixed(1)}%
+                        </span>
+                        <span className="text-xs text-gray-500">crash rate</span>
+                      </div>
+                      <div className="space-y-1 text-sm">
+                        <div className="flex justify-between">
+                          <span className="text-gray-500">Clean exits</span>
+                          <span className="font-mono text-green-600">{crashRate.cleanExits}</span>
+                        </div>
+                        <div className="flex justify-between">
+                          <span className="text-gray-500">Exception crashes</span>
+                          <span className={`font-mono ${crashRate.exceptionCrashes > 0 ? 'text-red-600' : 'text-gray-400'}`}>{crashRate.exceptionCrashes}</span>
+                        </div>
+                        <div className="flex justify-between">
+                          <span className="text-gray-500">Hard kills</span>
+                          <span className={`font-mono ${crashRate.hardKills > 0 ? 'text-orange-600' : 'text-gray-400'}`}>{crashRate.hardKills}</span>
+                        </div>
+                        <div className="flex justify-between">
+                          <span className="text-gray-500">First runs</span>
+                          <span className="font-mono text-gray-400">{crashRate.firstRuns}</span>
+                        </div>
+                      </div>
+                      {crashRate.topExceptions.length > 0 && (
+                        <div className="border-t border-gray-100 pt-2 mt-2">
+                          <p className="text-xs text-gray-500 mb-1">Top exceptions:</p>
+                          {crashRate.topExceptions.map((e) => (
+                            <div key={e.exceptionType} className="flex justify-between text-xs">
+                              <span className="text-red-600 font-mono truncate mr-2">{e.exceptionType}</span>
+                              <span className="text-gray-500">{e.count}x</span>
+                            </div>
+                          ))}
+                        </div>
+                      )}
+                      <p className="text-xs text-gray-400 mt-1">
+                        {crashRate.totalStarts} total starts across {globalStats.sessionsAnalyzed} sessions
+                      </p>
+                    </div>
+                  ) : (
+                    <p className="text-sm text-gray-400">No crash data yet (requires agent with clean-exit marker)</p>
+                  )}
                </div>
-                <div className="flex items-start gap-2">
-                  <span className="text-gray-300 mt-0.5">&#9679;</span>
-                  <span>Spool queue depth over time</span>
-                </div>
-                <div className="flex items-start gap-2">
-                  <span className="text-gray-300 mt-0.5">&#9679;</span>
-                  <span>Backend Function execution time &amp; cost</span>
-                </div>
-                <div className="flex items-start gap-2">
-                  <span className="text-gray-300 mt-0.5">&#9679;</span>
-                  <span>SignalR connection health &amp; message throughput</span>
-                </div>
-                <div className="flex items-start gap-2">
-                  <span className="text-gray-300 mt-0.5">&#9679;</span>
-                  <span>Agent footprint trend over releases</span>
+
+                {/* Spool Queue Depth */}
+                <div>
+                  <h3 className="text-sm font-medium text-gray-700 mb-3">Spool Queue Depth</h3>
+                  {(() => {
+                    const spoolAvgs = filteredMetrics.map(s => s.avgSpoolDepth);
+                    const spoolMaxes = filteredMetrics.map(s => s.maxSpoolDepth);
+                    const hasData = spoolAvgs.some(v => v > 0) || spoolMaxes.some(v => v > 0);
+                    const overallAvg = avg(spoolAvgs);
+                    const overallMax = max(spoolMaxes);
+                    return hasData ? (
+                      <div className="space-y-2">
+                        <div className="flex items-baseline gap-2">
+                          <span className={`text-2xl font-bold ${overallAvg < 5 ? 'text-green-600' : overallAvg < 20 ? 'text-yellow-600' : 'text-red-600'}`}>
+                            {overallAvg.toFixed(1)}
+                          </span>
+                          <span className="text-xs text-gray-500">avg depth</span>
+                        </div>
+                        <div className="space-y-1 text-sm">
+                          <div className="flex justify-between">
+                            <span className="text-gray-500">Avg across sessions</span>
+                            <span className="font-mono text-gray-900">{overallAvg.toFixed(1)}</span>
+                          </div>
+                          <div className="flex justify-between">
+                            <span className="text-gray-500">Peak max</span>
+                            <span className={`font-mono ${overallMax > 50 ? 'text-red-600' : 'text-gray-900'}`}>{overallMax.toFixed(0)}</span>
+                          </div>
+                          <div className="flex justify-between">
+                            <span className="text-gray-500">p95 max</span>
+                            <span className="font-mono text-gray-900">{pN(spoolMaxes, 95).toFixed(0)}</span>
+                          </div>
+                        </div>
+                        <p className="text-xs text-gray-400 mt-1">
+                          Events queued in spool before upload (0 = real-time delivery)
+                        </p>
+                      </div>
+                    ) : (
+                      <p className="text-sm text-gray-400">No spool data yet (requires updated agent)</p>
+                    );
+                  })()}
                </div>
              </div>
            </div>
@@ -224,7 +224,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [
      {
        id: "ga-metrics", label: "Metrics", icon: <ChartBarIcon />,
        items: [
-          { id: "ga-agent-metrics", label: "Agent Metrics", href: "/admin/metrics/agent-metrics" },
+          { id: "ga-agent-metrics", label: "Platform Metrics", href: "/admin/metrics/agent-metrics" },
          { id: "ga-usage", label: "Platform Usage", href: "/admin/metrics/usage" },
          { id: "ga-mcp-usage", label: "MCP Usage", href: "/admin/metrics/mcp-usage" },
        ],