From 939bc2ebc7a7d5ef5498c995b9d46bfdcd5804ce Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 7 May 2026 09:14:37 +0200 Subject: [PATCH] fixed platform metrics, ops and session reports fetching --- .../PlatformMetricsServiceWindowTests.cs | 81 ++- .../GetGlobalPlatformMetricsFunction.cs | 21 +- .../Services/PlatformMetricsService.cs | 482 ++++++++++-------- .../app/admin/components/OpsEventsSection.tsx | 10 +- .../components/SessionReportsSection.tsx | 10 +- .../metrics/sections/SectionAgentMetrics.tsx | 53 +- .../autopilot-monitor-web/app/audit/page.tsx | 13 +- src/Web/autopilot-monitor-web/lib/api.ts | 6 +- 8 files changed, 429 insertions(+), 247 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/PlatformMetricsServiceWindowTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/PlatformMetricsServiceWindowTests.cs index 34f01721..491f66cd 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/PlatformMetricsServiceWindowTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/PlatformMetricsServiceWindowTests.cs @@ -1,23 +1,27 @@ using AutopilotMonitor.Functions.Services; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Pagination; using Microsoft.Extensions.Logging.Abstractions; using Moq; namespace AutopilotMonitor.Functions.Tests; /// -/// Verifies PlatformMetricsService passes the requested window through to GetAllSessionsAsync -/// and surfaces it on the response. +/// Verifies PlatformMetricsService passes the requested window + session limit +/// through to GetAllSessionsPageAsync and surfaces both on the response. /// public class PlatformMetricsServiceWindowTests { + private static RawPage EmptyPage() => + new RawPage(new List(), null); + private static (PlatformMetricsService Service, Mock Repo) CreateService() { var sessionRepo = new Mock(); sessionRepo - .Setup(r => r.GetAllSessionsAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new List()); + .Setup(r => r.GetAllSessionsPageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(EmptyPage()); sessionRepo .Setup(r => r.GetSessionEventsAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new List()); @@ -37,16 +41,41 @@ public class PlatformMetricsServiceWindowTests int? capturedDays = null; repo - .Setup(r => r.GetAllSessionsAsync(It.IsAny(), It.IsAny())) - .Callback((_, d) => capturedDays = d) - .ReturnsAsync(new List()); + .Setup(r => r.GetAllSessionsPageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, d, _, _) => capturedDays = d) + .ReturnsAsync(EmptyPage()); - var result = await service.ComputePlatformMetricsAsync(days); + var result = await service.ComputePlatformMetricsAsync(days, limit: 50); Assert.Equal(days, capturedDays); Assert.Equal(days, result.WindowDays); } + [Theory] + // Use days=200 to avoid colliding with the static cache populated by + // the per-(days,limit) cache test below. Each row picks a unique limit + // so cache hits across rows never short-circuit the storage callback. + [InlineData(200, 21)] + [InlineData(200, 101)] + [InlineData(200, 501)] + [InlineData(200, 1001)] + [InlineData(200, 2000)] + public async Task ComputePlatformMetrics_passes_limit_through_to_session_repo_pageSize(int days, int limit) + { + var (service, repo) = CreateService(); + + int? capturedPageSize = null; + repo + .Setup(r => r.GetAllSessionsPageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, _, ps, _) => capturedPageSize = ps) + .ReturnsAsync(EmptyPage()); + + var result = await service.ComputePlatformMetricsAsync(days: days, limit: limit); + + Assert.Equal(limit, capturedPageSize); + Assert.Equal(limit, result.SessionLimit); + } + [Fact] public async Task ComputePlatformMetrics_clamps_zero_to_one() { @@ -54,9 +83,9 @@ public class PlatformMetricsServiceWindowTests int? capturedDays = null; repo - .Setup(r => r.GetAllSessionsAsync(It.IsAny(), It.IsAny())) - .Callback((_, d) => capturedDays = d) - .ReturnsAsync(new List()); + .Setup(r => r.GetAllSessionsPageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, d, _, _) => capturedDays = d) + .ReturnsAsync(EmptyPage()); var result = await service.ComputePlatformMetricsAsync(0); @@ -65,10 +94,38 @@ public class PlatformMetricsServiceWindowTests } [Fact] - public async Task ComputePlatformMetrics_clamps_excessive_to_365() + public async Task ComputePlatformMetrics_clamps_excessive_days_to_365() { var (service, _) = CreateService(); var result = await service.ComputePlatformMetricsAsync(99999); Assert.Equal(365, result.WindowDays); } + + [Fact] + public async Task ComputePlatformMetrics_clamps_excessive_limit_to_2000() + { + var (service, _) = CreateService(); + var result = await service.ComputePlatformMetricsAsync(days: 30, limit: 99999); + Assert.Equal(2000, result.SessionLimit); + } + + [Fact] + public async Task ComputePlatformMetrics_caches_per_days_limit_pair() + { + // Different (days, limit) combinations must hit storage independently; + // a shared cache slot would yield the wrong sample size. + var (service, repo) = CreateService(); + int callCount = 0; + repo + .Setup(r => r.GetAllSessionsPageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback(() => callCount++) + .ReturnsAsync(EmptyPage()); + + await service.ComputePlatformMetricsAsync(days: 30, limit: 20); + await service.ComputePlatformMetricsAsync(days: 30, limit: 20); // cache hit + await service.ComputePlatformMetricsAsync(days: 30, limit: 100); // miss — different limit + await service.ComputePlatformMetricsAsync(days: 60, limit: 20); // miss — different days + + Assert.Equal(3, callCount); + } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGlobalPlatformMetricsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGlobalPlatformMetricsFunction.cs index 8e02a1a2..a5940d76 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGlobalPlatformMetricsFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGlobalPlatformMetricsFunction.cs @@ -34,7 +34,8 @@ namespace AutopilotMonitor.Functions.Functions.Metrics // Authentication + GlobalAdminOnly authorization enforced by PolicyEnforcementMiddleware var days = ParseDays(req); - var metrics = await _platformMetricsService.ComputePlatformMetricsAsync(days); + var limit = ParseLimit(req); + var metrics = await _platformMetricsService.ComputePlatformMetricsAsync(days, limit); var response = req.CreateResponse(HttpStatusCode.OK); await response.WriteAsJsonAsync(metrics); @@ -67,5 +68,23 @@ namespace AutopilotMonitor.Functions.Functions.Metrics if (days > 365) days = 365; return days; } + + // Drives the per-session work the service performs (each session + // triggers its own GetSessionEventsAsync call). The UI dropdown sets + // this explicitly so the user controls the analysis depth (e.g. + // 90 days × 1000 sessions). The service uses bounded concurrency + // internally so even large limits don't blow the Function timeout. + // Hard ceiling 2000 protects against runaway query strings. + private static int ParseLimit(HttpRequestData req) + { + var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query); + var raw = query["limit"]; + var limit = 100; + if (!string.IsNullOrEmpty(raw) && int.TryParse(raw, out var parsed) && parsed > 0) + limit = parsed; + if (limit < 1) limit = 1; + if (limit > 2000) limit = 2000; + return limit; + } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/PlatformMetricsService.cs b/src/Backend/AutopilotMonitor.Functions/Services/PlatformMetricsService.cs index 6552fddd..f4240c92 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/PlatformMetricsService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/PlatformMetricsService.cs @@ -18,12 +18,23 @@ namespace AutopilotMonitor.Functions.Services private readonly ISessionRepository _sessionRepo; private readonly ILogger _logger; - // In-memory per-window cache - private static readonly Dictionary _cachedByDays = new(); + // In-memory per-(days, limit) cache. Different `limit` values produce + // different aggregate stats (different N feeds the averages/percentiles) + // so they must not share a cache slot. + private static readonly Dictionary<(int days, int limit), (PlatformAgentMetricsResponse metrics, DateTime expiry)> _cachedByKey = new(); private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5); private static readonly object _cacheLock = new object(); private const int DefaultWindowDays = 90; + private const int DefaultSessionLimit = 100; + private const int MaxSessionLimit = 2000; + + // Bounded concurrency for the per-session event-fetch fan-out. Without + // a cap, a 1000-session limit fires 1000 parallel storage calls — that + // both throttles Azure Tables and balloons memory transient. 32 is a + // gentle middle ground: enough overlap to hide latency, low enough to + // avoid 503s on busy installs. + private const int PerSessionFetchConcurrency = 32; public PlatformMetricsService( ISessionRepository sessionRepo, @@ -34,39 +45,45 @@ namespace AutopilotMonitor.Functions.Services } /// - /// Computes platform agent metrics over the last days (5-minute per-window cache). + /// Computes platform agent metrics over the last days, + /// limited to the newest sessions (5-minute per-key cache). /// - public async Task ComputePlatformMetricsAsync(int days = DefaultWindowDays) + public async Task ComputePlatformMetricsAsync( + int days = DefaultWindowDays, + int limit = DefaultSessionLimit) { days = ClampDays(days); + limit = ClampLimit(limit); + var key = (days, limit); lock (_cacheLock) { - if (_cachedByDays.TryGetValue(days, out var entry) && DateTime.UtcNow < entry.expiry) + if (_cachedByKey.TryGetValue(key, out var entry) && DateTime.UtcNow < entry.expiry) { - _logger.LogInformation("Returning cached platform metrics for days={Days} (expires in {Seconds}s)", - days, (entry.expiry - DateTime.UtcNow).TotalSeconds); + _logger.LogInformation("Returning cached platform metrics for days={Days} limit={Limit} (expires in {Seconds}s)", + days, limit, (entry.expiry - DateTime.UtcNow).TotalSeconds); entry.metrics.FromCache = true; return entry.metrics; } } - _logger.LogInformation("Computing fresh platform agent metrics for days={Days}...", days); + _logger.LogInformation("Computing fresh platform agent metrics for days={Days} limit={Limit}...", days, limit); var stopwatch = Stopwatch.StartNew(); - var metrics = await ComputePlatformMetricsInternalAsync(days); + var metrics = await ComputePlatformMetricsInternalAsync(days, limit); stopwatch.Stop(); metrics.ComputeDurationMs = (int)stopwatch.ElapsedMilliseconds; metrics.ComputedAt = DateTime.UtcNow; metrics.FromCache = false; metrics.WindowDays = days; + metrics.SessionLimit = limit; - _logger.LogInformation("Platform agent metrics computed in {Ms}ms (days={Days})", metrics.ComputeDurationMs, days); + _logger.LogInformation("Platform agent metrics computed in {Ms}ms (days={Days} limit={Limit})", metrics.ComputeDurationMs, days, limit); lock (_cacheLock) { - _cachedByDays[days] = (metrics, DateTime.UtcNow.Add(CacheDuration)); + _cachedByKey[key] = (metrics, DateTime.UtcNow.Add(CacheDuration)); } return metrics; @@ -79,245 +96,281 @@ namespace AutopilotMonitor.Functions.Services return days; } - private async Task ComputePlatformMetricsInternalAsync(int days) + private static int ClampLimit(int limit) { - // Fetch all sessions in the requested window across all tenants — drains - // every page so the platform metric covers the full date-bounded set - // (the previous maxResults:500 was a silent truncation on busy installs). - var allSessions = await _sessionRepo.GetAllSessionsAsync(tenantIdFilter: null, days: days); + if (limit < 1) return 1; + if (limit > MaxSessionLimit) return MaxSessionLimit; + return limit; + } + + // Single-session result of one storage round-trip. Holds the per-session + // metric (or null when no agent_metrics_snapshot was emitted) plus the + // raw inputs needed by the global delivery-latency and crash-rate + // aggregations — so we never re-fetch the same session's events. + private sealed record PerSessionData( + SessionAgentMetric? Metric, + List LatencyDeltasMs, + List> AgentStartedEvents); + + private async Task ComputePlatformMetricsInternalAsync(int days, int limit) + { + // Fetch only the newest `limit` sessions in the window. Each session below + // triggers its own GetSessionEventsAsync call; the per-session work is + // bounded by PerSessionFetchConcurrency so even 1000-session limits + // don't fan out 1000 simultaneous storage requests. + var sessionPage = await _sessionRepo.GetAllSessionsPageAsync( + tenantIdFilter: null, days: days, pageSize: limit, continuation: null); + var allSessions = sessionPage.Items; if (allSessions.Count == 0) { return new PlatformAgentMetricsResponse { Sessions = new List() }; } - // 2. For each session, fetch events and extract agent_metrics_snapshot data - var sessionMetrics = new List(); + // Mark the first 20 sessions as the delivery-latency sample so we + // only compute deltas for them (preserves the legacy "20 most recent + // for latency" semantics without a second event fetch). + var latencySampleSessionIds = allSessions + .Take(20) + .Select(s => s.SessionId) + .ToHashSet(StringComparer.Ordinal); - var tasks = allSessions.Select(async session => - { - try - { - var events = await _sessionRepo.GetSessionEventsAsync(session.TenantId, session.SessionId); - var snapshots = events - .Where(e => e.EventType == "agent_metrics_snapshot" && e.Data != null) - .Select(e => e.Data) - .ToList(); + var perSessionResults = await RunWithBoundedConcurrencyAsync( + allSessions, + PerSessionFetchConcurrency, + session => ProcessSessionAsync(session, latencySampleSessionIds)); - if (snapshots.Count == 0) return null; + var sessionMetrics = perSessionResults + .Where(r => r.Metric != null) + .Select(r => r.Metric!) + .ToList(); - var cpuValues = snapshots.Select(s => GetDouble(s, "agent_cpu_percent")).ToList(); - var wsValues = snapshots.Select(s => GetDouble(s, "agent_working_set_mb")).ToList(); - var pbValues = snapshots.Select(s => GetDouble(s, "agent_private_bytes_mb")).ToList(); - var latValues = snapshots.Select(s => GetDouble(s, "net_avg_latency_ms")).Where(v => v > 0).ToList(); - // V2 emits `spool_pending_item_count`; V1 emitted `spool_queue_depth`. - // Read V2 primary, fall back to V1 so legacy sessions still aggregate. - var spoolValues = snapshots.Select(s => GetDoubleFirst(s, "spool_pending_item_count", "spool_queue_depth")).ToList(); - - // V2-only spool fields. Each has a different aggregation semantic: - // peak_pending_item_count → monotonic per-process counter; Last() = best estimate of intra-tick peak - // file_size_bytes → instantaneous; Max() across snapshots - // total_enqueued_count → monotonic counter; Last() = total events emitted in session - var peakValues = snapshots.Select(s => GetDouble(s, "spool_peak_pending_item_count")).ToList(); - var fileSizeValues = snapshots.Select(s => GetDouble(s, "spool_file_size_bytes")).ToList(); - var totalEnqueuedValues = snapshots.Select(s => GetDouble(s, "spool_total_enqueued_count")).ToList(); - var spoolPressureDetected = events.Any(e => e.EventType == "spool_pressure_detected"); - - var lastSnapshot = snapshots.Last(); - - // Resolve agent version: prefer from snapshot, fallback to session - var agentVersion = snapshots - .Select(s => GetString(s, "agent_version")) - .FirstOrDefault(v => !string.IsNullOrEmpty(v)) - ?? session.AgentVersion - ?? "unknown"; - - return new SessionAgentMetric - { - SessionId = session.SessionId, - TenantId = session.TenantId, - DeviceName = session.DeviceName, - Manufacturer = session.Manufacturer, - Model = session.Model, - StartedAt = session.StartedAt.ToString("o"), - Status = session.Status.ToString(), - AgentVersion = agentVersion, - SnapshotCount = snapshots.Count, - TotalBytesUp = GetDouble(lastSnapshot, "net_total_bytes_up"), - TotalBytesDown = GetDouble(lastSnapshot, "net_total_bytes_down"), - TotalRequests = GetDouble(lastSnapshot, "net_total_requests"), - AvgCpu = cpuValues.Count > 0 ? cpuValues.Average() : 0, - MaxCpu = cpuValues.Count > 0 ? cpuValues.Max() : 0, - AvgWorkingSet = wsValues.Count > 0 ? wsValues.Average() : 0, - MaxWorkingSet = wsValues.Count > 0 ? wsValues.Max() : 0, - AvgPrivateBytes = pbValues.Count > 0 ? pbValues.Average() : 0, - AvgLatency = latValues.Count > 0 ? latValues.Average() : 0, - AvgSpoolDepth = spoolValues.Count > 0 ? spoolValues.Average() : 0, - MaxSpoolDepth = spoolValues.Count > 0 ? spoolValues.Max() : 0, - PeakSpoolDepth = peakValues.Count > 0 ? peakValues.Last() : 0, - MaxSpoolFileBytes = fileSizeValues.Count > 0 ? fileSizeValues.Max() : 0, - TotalEventsEmitted = totalEnqueuedValues.Count > 0 ? totalEnqueuedValues.Last() : 0, - SpoolPressureDetected = spoolPressureDetected - }; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to fetch events for session {SessionId}", session.SessionId); - return null; - } - }).ToList(); - - var results = await Task.WhenAll(tasks); - sessionMetrics = results.Where(r => r != null).ToList()!; - - // Compute delivery latency from all events across sessions - var deliveryLatency = ComputeDeliveryLatency(allSessions, _sessionRepo); - var crashRate = ComputeCrashRate(allSessions, _sessionRepo); - - // Run both in parallel - await Task.WhenAll(deliveryLatency, crashRate); + var deliveryLatency = AggregateDeliveryLatency(perSessionResults); + var crashRate = AggregateCrashRate(perSessionResults); return new PlatformAgentMetricsResponse { Sessions = sessionMetrics, - DeliveryLatency = deliveryLatency.Result, - CrashRate = crashRate.Result + DeliveryLatency = deliveryLatency, + CrashRate = crashRate }; } - private async Task ComputeDeliveryLatency( - List sessions, - ISessionRepository sessionRepo) + private async Task ProcessSessionAsync( + SessionSummary session, + HashSet latencySampleSessionIds) { try { - // Sample up to 20 recent sessions for latency data - var sampleSessions = sessions.Take(20).ToList(); - var allDeltas = new List(); + var events = await _sessionRepo.GetSessionEventsAsync(session.TenantId, session.SessionId); - var latencyTasks = sampleSessions.Select(async session => + // ── Latency deltas (only for sample-set sessions) ─────────── + List latencyDeltas = new(); + if (latencySampleSessionIds.Contains(session.SessionId)) { - try + foreach (var e in events) { - var events = await sessionRepo.GetSessionEventsAsync(session.TenantId, session.SessionId); - return events - .Where(e => e.ReceivedAt.HasValue && e.Timestamp != default) - .Select(e => (e.ReceivedAt!.Value - e.Timestamp).TotalMilliseconds) - .ToList(); - } - catch { return new List(); } - }).ToList(); - - var latencyResults = await Task.WhenAll(latencyTasks); - foreach (var deltas in latencyResults) - allDeltas.AddRange(deltas); - - if (allDeltas.Count == 0) - return new DeliveryLatencyMetrics(); - - var negativeCount = allDeltas.Count(d => d < 0); - var validDeltas = allDeltas.Where(d => d >= 0).OrderBy(d => d).ToList(); - - if (validDeltas.Count == 0) - return new DeliveryLatencyMetrics - { - SampleCount = allDeltas.Count, - ClockSkewPercent = 100.0 - }; - - return new DeliveryLatencyMetrics - { - P50Ms = Math.Round(Percentile(validDeltas, 0.50), 0), - P95Ms = Math.Round(Percentile(validDeltas, 0.95), 0), - P99Ms = Math.Round(Percentile(validDeltas, 0.99), 0), - AvgMs = Math.Round(validDeltas.Average(), 0), - SampleCount = allDeltas.Count, - ClockSkewPercent = Math.Round((double)negativeCount / allDeltas.Count * 100, 1) - }; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to compute delivery latency"); - return new DeliveryLatencyMetrics(); - } - } - - private async Task ComputeCrashRate( - List sessions, - ISessionRepository sessionRepo) - { - try - { - var metrics = new CrashRateMetrics(); - var exceptionCounts = new Dictionary(StringComparer.OrdinalIgnoreCase); - - var crashTasks = sessions.Select(async session => - { - try - { - var events = await sessionRepo.GetSessionEventsAsync(session.TenantId, session.SessionId); - return events - .Where(e => e.EventType == "agent_started" && e.Data != null) - .Select(e => e.Data) - .ToList(); - } - catch { return new List>(); } - }).ToList(); - - var crashResults = await Task.WhenAll(crashTasks); - foreach (var startEvents in crashResults) - { - foreach (var data in startEvents) - { - metrics.TotalStarts++; - var exitType = GetString(data, "previousExitType"); - switch (exitType) + if (e.ReceivedAt.HasValue && e.Timestamp != default) { - case "clean": - metrics.CleanExits++; - break; - case "exception_crash": - metrics.ExceptionCrashes++; - var exType = GetString(data, "previousCrashException"); - if (!string.IsNullOrEmpty(exType)) - { - exceptionCounts.TryGetValue(exType, out var count); - exceptionCounts[exType] = count + 1; - } - break; - case "hard_kill": - metrics.HardKills++; - break; - case "reboot_kill": - metrics.RebootKills++; - break; - default: - metrics.FirstRuns++; - break; + latencyDeltas.Add((e.ReceivedAt.Value - e.Timestamp).TotalMilliseconds); } } } - var nonFirstRuns = metrics.TotalStarts - metrics.FirstRuns; - metrics.CrashRatePercent = nonFirstRuns > 0 - ? Math.Round((double)metrics.ExceptionCrashes / nonFirstRuns * 100, 1) - : 0; - - metrics.TopExceptions = exceptionCounts - .OrderByDescending(kv => kv.Value) - .Take(5) - .Select(kv => new CrashExceptionSummary { ExceptionType = kv.Key, Count = kv.Value }) + // ── Agent-started events for crash rate ────────────────────── + var agentStartedEvents = events + .Where(e => e.EventType == "agent_started" && e.Data != null) + .Select(e => e.Data) .ToList(); - return metrics; + // ── Per-session metric (only when snapshots exist) ─────────── + var snapshots = events + .Where(e => e.EventType == "agent_metrics_snapshot" && e.Data != null) + .Select(e => e.Data) + .ToList(); + + if (snapshots.Count == 0) + { + return new PerSessionData(null, latencyDeltas, agentStartedEvents); + } + + var cpuValues = snapshots.Select(s => GetDouble(s, "agent_cpu_percent")).ToList(); + var wsValues = snapshots.Select(s => GetDouble(s, "agent_working_set_mb")).ToList(); + var pbValues = snapshots.Select(s => GetDouble(s, "agent_private_bytes_mb")).ToList(); + var latValues = snapshots.Select(s => GetDouble(s, "net_avg_latency_ms")).Where(v => v > 0).ToList(); + // V2 emits `spool_pending_item_count`; V1 emitted `spool_queue_depth`. + // Read V2 primary, fall back to V1 so legacy sessions still aggregate. + var spoolValues = snapshots.Select(s => GetDoubleFirst(s, "spool_pending_item_count", "spool_queue_depth")).ToList(); + + // V2-only spool fields. Each has a different aggregation semantic: + // peak_pending_item_count → monotonic per-process counter; Last() = best estimate of intra-tick peak + // file_size_bytes → instantaneous; Max() across snapshots + // total_enqueued_count → monotonic counter; Last() = total events emitted in session + var peakValues = snapshots.Select(s => GetDouble(s, "spool_peak_pending_item_count")).ToList(); + var fileSizeValues = snapshots.Select(s => GetDouble(s, "spool_file_size_bytes")).ToList(); + var totalEnqueuedValues = snapshots.Select(s => GetDouble(s, "spool_total_enqueued_count")).ToList(); + var spoolPressureDetected = events.Any(e => e.EventType == "spool_pressure_detected"); + + var lastSnapshot = snapshots.Last(); + + // Resolve agent version: prefer from snapshot, fallback to session + var agentVersion = snapshots + .Select(s => GetString(s, "agent_version")) + .FirstOrDefault(v => !string.IsNullOrEmpty(v)) + ?? session.AgentVersion + ?? "unknown"; + + var metric = new SessionAgentMetric + { + SessionId = session.SessionId, + TenantId = session.TenantId, + DeviceName = session.DeviceName, + Manufacturer = session.Manufacturer, + Model = session.Model, + StartedAt = session.StartedAt.ToString("o"), + Status = session.Status.ToString(), + AgentVersion = agentVersion, + SnapshotCount = snapshots.Count, + TotalBytesUp = GetDouble(lastSnapshot, "net_total_bytes_up"), + TotalBytesDown = GetDouble(lastSnapshot, "net_total_bytes_down"), + TotalRequests = GetDouble(lastSnapshot, "net_total_requests"), + AvgCpu = cpuValues.Count > 0 ? cpuValues.Average() : 0, + MaxCpu = cpuValues.Count > 0 ? cpuValues.Max() : 0, + AvgWorkingSet = wsValues.Count > 0 ? wsValues.Average() : 0, + MaxWorkingSet = wsValues.Count > 0 ? wsValues.Max() : 0, + AvgPrivateBytes = pbValues.Count > 0 ? pbValues.Average() : 0, + AvgLatency = latValues.Count > 0 ? latValues.Average() : 0, + AvgSpoolDepth = spoolValues.Count > 0 ? spoolValues.Average() : 0, + MaxSpoolDepth = spoolValues.Count > 0 ? spoolValues.Max() : 0, + PeakSpoolDepth = peakValues.Count > 0 ? peakValues.Last() : 0, + MaxSpoolFileBytes = fileSizeValues.Count > 0 ? fileSizeValues.Max() : 0, + TotalEventsEmitted = totalEnqueuedValues.Count > 0 ? totalEnqueuedValues.Last() : 0, + SpoolPressureDetected = spoolPressureDetected + }; + + return new PerSessionData(metric, latencyDeltas, agentStartedEvents); } catch (Exception ex) { - _logger.LogWarning(ex, "Failed to compute crash rate"); - return new CrashRateMetrics(); + _logger.LogWarning(ex, "Failed to fetch events for session {SessionId}", session.SessionId); + return new PerSessionData(null, new List(), new List>()); } } + // Bounded fan-out: equivalent to Task.WhenAll over `body(item)` but with + // at most `maxConcurrency` tasks in flight. Without this guard a 1000- + // session metric query would fire 1000 simultaneous storage requests + // and either throttle Azure Tables or run the worker out of file + // handles before responding. + private static async Task> RunWithBoundedConcurrencyAsync( + IReadOnlyList items, + int maxConcurrency, + Func> body) + { + using var sem = new SemaphoreSlim(maxConcurrency); + var tasks = items.Select(async item => + { + await sem.WaitAsync().ConfigureAwait(false); + try + { + return await body(item).ConfigureAwait(false); + } + finally + { + sem.Release(); + } + }); + var results = await Task.WhenAll(tasks).ConfigureAwait(false); + return results.ToList(); + } + + // Aggregate over the latency deltas already extracted by the per-session + // pass. No additional storage round-trips — the same events the metric + // pass walked have been pre-binned into PerSessionData.LatencyDeltasMs + // for the first-20-session sample. + private static DeliveryLatencyMetrics AggregateDeliveryLatency(IReadOnlyList perSession) + { + var allDeltas = new List(); + foreach (var r in perSession) allDeltas.AddRange(r.LatencyDeltasMs); + + if (allDeltas.Count == 0) return new DeliveryLatencyMetrics(); + + var negativeCount = allDeltas.Count(d => d < 0); + var validDeltas = allDeltas.Where(d => d >= 0).OrderBy(d => d).ToList(); + + if (validDeltas.Count == 0) + return new DeliveryLatencyMetrics + { + SampleCount = allDeltas.Count, + ClockSkewPercent = 100.0 + }; + + return new DeliveryLatencyMetrics + { + P50Ms = Math.Round(Percentile(validDeltas, 0.50), 0), + P95Ms = Math.Round(Percentile(validDeltas, 0.95), 0), + P99Ms = Math.Round(Percentile(validDeltas, 0.99), 0), + AvgMs = Math.Round(validDeltas.Average(), 0), + SampleCount = allDeltas.Count, + ClockSkewPercent = Math.Round((double)negativeCount / allDeltas.Count * 100, 1) + }; + } + + // Aggregate crash classifications over the agent_started events the + // per-session pass already collected — same idea as the latency + // aggregator: zero extra storage calls. + private static CrashRateMetrics AggregateCrashRate(IReadOnlyList perSession) + { + var metrics = new CrashRateMetrics(); + var exceptionCounts = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var r in perSession) + { + foreach (var data in r.AgentStartedEvents) + { + metrics.TotalStarts++; + var exitType = GetString(data, "previousExitType"); + switch (exitType) + { + case "clean": + metrics.CleanExits++; + break; + case "exception_crash": + metrics.ExceptionCrashes++; + var exType = GetString(data, "previousCrashException"); + if (!string.IsNullOrEmpty(exType)) + { + exceptionCounts.TryGetValue(exType, out var count); + exceptionCounts[exType] = count + 1; + } + break; + case "hard_kill": + metrics.HardKills++; + break; + case "reboot_kill": + metrics.RebootKills++; + break; + default: + metrics.FirstRuns++; + break; + } + } + } + + var nonFirstRuns = metrics.TotalStarts - metrics.FirstRuns; + metrics.CrashRatePercent = nonFirstRuns > 0 + ? Math.Round((double)metrics.ExceptionCrashes / nonFirstRuns * 100, 1) + : 0; + + metrics.TopExceptions = exceptionCounts + .OrderByDescending(kv => kv.Value) + .Take(5) + .Select(kv => new CrashExceptionSummary { ExceptionType = kv.Key, Count = kv.Value }) + .ToList(); + + return metrics; + } + private static double Percentile(List sortedValues, double percentile) { if (sortedValues.Count == 0) return 0; @@ -370,6 +423,7 @@ namespace AutopilotMonitor.Functions.Services public int ComputeDurationMs { get; set; } public bool FromCache { get; set; } public int WindowDays { get; set; } + public int SessionLimit { get; set; } } public class SessionAgentMetric diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsEventsSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsEventsSection.tsx index a18db3fc..25b50d5b 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsEventsSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsEventsSection.tsx @@ -44,7 +44,7 @@ const CATEGORY_STYLES: Record = { const ALL_CATEGORIES = ["Consent", "Maintenance", "Security", "Tenant", "Agent"]; -const PAGE_SIZE = 50; +const PAGE_SIZE = 20; function defaultIsoDateFrom(): string { const d = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); @@ -169,12 +169,18 @@ export function OpsEventsSection({ }, [categoryFilter, dateFromIso, dateToIso, getAccessToken, setError]); // Reset pagination + refetch whenever the filter window or category changes. + // fetchEvents is intentionally excluded from deps: getAccessToken's identity + // churns on every MSAL accounts-array refresh, which happens after each + // authenticatedFetch — leaving fetchEvents in deps causes the effect to + // re-fire after every successful page-N click, race a page-1 fetch against + // it, and snap the user back to page 1. useEffect(() => { setContinuation(null); setContinuationStack([]); setPageNumber(1); fetchEvents(null); - }, [categoryFilter, dateFromIso, dateToIso, fetchEvents]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [categoryFilter, dateFromIso, dateToIso]); const handleNextPage = () => { const nextCont = extractContinuation(nextLink); diff --git a/src/Web/autopilot-monitor-web/app/admin/components/SessionReportsSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/SessionReportsSection.tsx index 09cab751..c8f77269 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/SessionReportsSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/SessionReportsSection.tsx @@ -7,7 +7,7 @@ import { extractContinuation } from "@/lib/paginationLink"; import { isGuid } from "@/utils/inputValidation"; import { trackEvent } from "@/lib/appInsights"; -const PAGE_SIZE = 50; +const PAGE_SIZE = 20; interface SessionReport { reportId: string; @@ -174,12 +174,18 @@ export function SessionReportsSection({ }, [getAccessToken, setError]); // Initial load + reload whenever the applied tenant filter changes. + // fetchReports is intentionally excluded from deps: getAccessToken's identity + // churns on every MSAL accounts-array refresh, which happens after each + // authenticatedFetch — leaving fetchReports in deps causes the effect to + // re-fire after every successful page-N click, race a page-1 fetch against + // it, and snap the user back to page 1. useEffect(() => { setContinuation(null); setContinuationStack([]); setPageNumber(1); fetchReports(null, tenantFilterApplied); - }, [tenantFilterApplied, fetchReports]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [tenantFilterApplied]); const handleApplyTenantFilter = () => { const trimmed = tenantFilterInput.trim(); diff --git a/src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionAgentMetrics.tsx b/src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionAgentMetrics.tsx index 04f4b44b..a279a352 100644 --- a/src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionAgentMetrics.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/metrics/sections/SectionAgentMetrics.tsx @@ -144,21 +144,26 @@ export function SectionAgentMetrics() { const [loading, setLoading] = useState(true); const [sessionMetrics, setSessionMetrics] = useState([]); - const [sampleSize, setSampleSize] = useState(100); + // sampleSize drives the backend `limit` query, windowDays drives the backend + // `days` query. Each picker change forces a fresh fetch (previous version + // sliced client-side over an unbounded backend fetch and timed out on busy + // installs). Defaults stay snappy: 20 sessions over 30 days. + const [sampleSize, setSampleSize] = useState(20); + const [windowDays, setWindowDays] = useState(30); const [error, setError] = useState(null); const [versionFilter, setVersionFilter] = useState('all'); const [cacheInfo, setCacheInfo] = useState<{ fromCache: boolean; computeDurationMs: number; computedAt: string } | null>(null); const [deliveryLatency, setDeliveryLatency] = useState(null); const [crashRate, setCrashRate] = useState(null); - // ── Fetch pre-computed metrics from backend (with 5-min cache) ──────────── + // ── Fetch pre-computed metrics from backend (with 5-min per-(days,limit) cache) ──────────── - const fetchMetrics = useCallback(async () => { + const fetchMetrics = useCallback(async (limit: number, days: number) => { setLoading(true); setError(null); try { - const res = await authenticatedFetch(api.metrics.platform(), getAccessToken); + const res = await authenticatedFetch(api.metrics.platform({ limit, days }), getAccessToken); if (!res.ok) { if (res.status === 403) { @@ -216,9 +221,15 @@ export function SectionAgentMetrics() { } }, [getAccessToken, addNotification]); + // Refetch whenever the user picks a new sample size or window (both drive + // backend params, not a client-side slice). fetchMetrics is intentionally + // excluded from deps: getAccessToken's identity churns on every MSAL + // accounts refresh which would otherwise re-fire the effect after each + // request and ping-pong the loading spinner. useEffect(() => { - fetchMetrics(); - }, [fetchMetrics]); + fetchMetrics(sampleSize, windowDays); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [sampleSize, windowDays]); // ── Resolve agent version per session (pre-resolved by backend) ── @@ -236,16 +247,17 @@ export function SectionAgentMetrics() { return Array.from(versions).sort(); }, [sessionMetrics, resolveVersion]); - // ── Filtered metrics based on sample size and version filter ──────────────── + // ── Filtered metrics by version (sample-size cap is enforced server-side) ── const filteredMetrics = useMemo(() => { - // Sort by startedAt desc and take sampleSize most recent - const sorted = [...sessionMetrics] - .sort((a, b) => new Date(b.session.startedAt || 0).getTime() - new Date(a.session.startedAt || 0).getTime()) - .slice(0, sampleSize); + // Backend already returns the newest `limit` sessions in startedAt-desc order; + // re-sorting client-side is just defensive against repository-order drift. + const sorted = [...sessionMetrics].sort( + (a, b) => new Date(b.session.startedAt || 0).getTime() - new Date(a.session.startedAt || 0).getTime() + ); if (versionFilter === 'all') return sorted; return sorted.filter((sm) => resolveVersion(sm) === versionFilter); - }, [sessionMetrics, sampleSize, versionFilter, resolveVersion]); + }, [sessionMetrics, versionFilter, resolveVersion]); // ── Aggregated stats across filtered sessions ────────────────────────────── @@ -319,6 +331,19 @@ export function SectionAgentMetrics() { )}
+ + {availableVersions.length > 1 && ( <> @@ -347,7 +374,7 @@ export function SectionAgentMetrics() { )}