diff --git a/internal b/internal index 84f6aaf7..2f4d12b8 160000 --- a/internal +++ b/internal @@ -1 +1 @@ -Subproject commit 84f6aaf755c3d0dd62e437b72e0ed8c5df3226ee +Subproject commit 2f4d12b8df1192f1d5ba40b416086818c77f6e79 diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AppInstallIncompleteCloseTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AppInstallIncompleteCloseTests.cs new file mode 100644 index 00000000..fc9b0727 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/AppInstallIncompleteCloseTests.cs @@ -0,0 +1,63 @@ +using Azure.Data.Tables; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Observation-end closure of app rows: a row still InProgress when the session's observation +/// ended becomes "Incomplete" (terminal, non-failure, outcome unknown). Pins the pure per-row +/// predicate behind and +/// the disclosed bucket in the app-metrics payload. +/// +public class AppInstallIncompleteCloseTests +{ + private static TableEntity Row(string? status, DateTimeOffset? completedAt = null) + { + var row = new TableEntity("tenant", "session_App"); + if (status != null) row["Status"] = status; + if (completedAt.HasValue) row["CompletedAt"] = completedAt.Value; + return row; + } + + [Theory] + [InlineData("InProgress")] + [InlineData("")] + [InlineData(null)] + public void ShouldClose_OpenRow(string? status) + => Assert.True(TableStorageService.ShouldCloseAsIncomplete(Row(status))); + + [Theory] + [InlineData("Succeeded")] + [InlineData("Failed")] + [InlineData("Incomplete")] + public void ShouldClose_LeavesTerminalRowsAlone(string status) + => Assert.False(TableStorageService.ShouldCloseAsIncomplete(Row(status))); + + [Fact] + public void ShouldClose_NeverClosesARowWithAnEnd() + { + // Out-of-order arrival can leave a CompletedAt on a row whose status batch is late — + // an end is an end. + Assert.False(TableStorageService.ShouldCloseAsIncomplete(Row("InProgress", DateTimeOffset.UtcNow))); + } + + [Fact] + public void AppMetrics_DisclosesIncompleteOutsideTheRate() + { + var payload = MetricsMath.BuildAppMetricsPayload(new[] + { + new AppInstallSummary { AppName = "App", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 30 }, + new AppInstallSummary { AppName = "App", Status = "Failed", TerminalState = "Error", DurationSeconds = 30 }, + new AppInstallSummary { AppName = "App", Status = "Incomplete" }, + }); + + var group = Assert.Single(payload.TopFailingApps); + Assert.Equal(3, group.TotalInstalls); + Assert.Equal(1, group.Incomplete); + Assert.Equal(50.0, group.FailureRate); + Assert.Equal(1, payload.TotalIncomplete); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AppInstallTerminalStateAndReconcileTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AppInstallTerminalStateAndReconcileTests.cs index cee28816..eaf14e2c 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/AppInstallTerminalStateAndReconcileTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/AppInstallTerminalStateAndReconcileTests.cs @@ -155,6 +155,30 @@ public class AppInstallTerminalStateAndReconcileTests Assert.Equal("Succeeded", summary.Status); } + [Fact] + public void Reconcile_IncompleteIsSticky_AgainstLateProgressBatch() + { + // Closed by the observation-end step; a straggling started/progress batch must not reopen it. + var existing = new TableEntity("tenant", "session_App") { ["Status"] = "Incomplete" }; + var summary = new AppInstallSummary { AppName = "App", Status = "InProgress", StartedAt = T0 }; + + TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing); + + Assert.Equal("Incomplete", summary.Status); + } + + [Fact] + public void Reconcile_RealTerminalOverridesIncomplete() + { + // The completion arrived after the close step: real evidence beats the hedge. + var existing = new TableEntity("tenant", "session_App") { ["Status"] = "Incomplete" }; + var summary = new AppInstallSummary { AppName = "App", Status = "Succeeded", StartedAt = T0 }; + + TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing); + + Assert.Equal("Succeeded", summary.Status); + } + [Fact] public void Reconcile_Q4Guard_DoesNotFireWithoutStoredCompletedAt() { diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TypedContractCompletionWireParityTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TypedContractCompletionWireParityTests.cs index e1debc41..598e228a 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/TypedContractCompletionWireParityTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TypedContractCompletionWireParityTests.cs @@ -184,6 +184,7 @@ public class TypedContractCompletionWireParityTests totalInstalls = 4, totalSkipped = 1, totalUnmeasured = 1, + totalIncomplete = 0, totalCollisionExcluded = 0, slowestApps = new[] { @@ -195,6 +196,7 @@ public class TypedContractCompletionWireParityTests skipped = 1, unmeasured = 1, failed = 1, + incomplete = 0, failureRate = 33.3, avgDurationSeconds = 120.0, maxDurationSeconds = 240, @@ -225,6 +227,7 @@ public class TypedContractCompletionWireParityTests TotalInstalls = 4, TotalSkipped = 1, TotalUnmeasured = 1, + TotalIncomplete = 0, TotalCollisionExcluded = 0, SlowestApps = new[] { @@ -236,6 +239,7 @@ public class TypedContractCompletionWireParityTests Skipped = 1, Unmeasured = 1, Failed = 1, + Incomplete = 0, FailureRate = 33.3, AvgDurationSeconds = 120.0, MaxDurationSeconds = 240, @@ -684,6 +688,7 @@ public class TypedContractCompletionWireParityTests skipped = 0, unmeasured = 0, failed = 0, + incomplete = 0, failureRate = 0.0, avgDurationSeconds = 0.0, p95DurationSeconds = 0, @@ -715,6 +720,7 @@ public class TypedContractCompletionWireParityTests Skipped = 0, Unmeasured = 0, Failed = 0, + Incomplete = 0, FailureRate = 0.0, AvgDurationSeconds = 0.0, P95DurationSeconds = 0, diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMetricsRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMetricsRepository.cs index 057758d1..8a206c5d 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMetricsRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMetricsRepository.cs @@ -142,6 +142,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task ResolveEspBlockingForSessionAsync(string tenantId, string sessionId) => _storage.ResolveEspBlockingForSessionAsync(tenantId, sessionId); + public Task CloseOpenAppInstallsForSessionAsync(string tenantId, string sessionId) + => _storage.CloseOpenAppInstallsForSessionAsync(tenantId, sessionId); + public Task SaveTimeAttributionAggregateAsync(TimeAttributionDailyAggregate aggregate) => _storage.SaveTimeAttributionAggregateAsync(aggregate); diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs index a80cf4e3..77ac346d 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs @@ -101,6 +101,13 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage // complete. Positive evidence only (listed ⇒ EspBlocking=true, absent ⇒ stays // unknown). Idempotent + fail-soft like the counter reconcile above. await _storage.ResolveEspBlockingForSessionAsync(tenantId, sessionId); + // Observation end: app rows still InProgress on a terminal session never get a + // terminal event — close them as Incomplete (outcome unknown, not a failure). + // Early for the ingest path (verdict batch, the agent may still be watching), + // never wrong in the end state: a real terminal arriving later still wins via + // the sticky-status reconcile, and the shutdown batch re-runs the close for rows + // opened after the verdict. Idempotent + fail-soft. + await _storage.CloseOpenAppInstallsForSessionAsync(tenantId, sessionId); // F1 PR2: compute + persist the time-attribution breakdown once, now that the // terminal write stamped CompletedAt/DurationSeconds and the event stream is diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Apps/AppsAnalyticsHelper.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Apps/AppsAnalyticsHelper.cs index fe09c5d5..3405370a 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Apps/AppsAnalyticsHelper.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Apps/AppsAnalyticsHelper.cs @@ -291,6 +291,7 @@ namespace AutopilotMonitor.Functions.Functions.Apps var measured = installed.Where(MetricsMath.HasMeasuredDuration).ToList(); var succeeded = installed.Count; var failed = summaries.Count(s => s.Status == "Failed"); + var incomplete = summaries.Count(s => s.Status == "Incomplete"); var failureRate = MetricsMath.TerminalFailureRatePct(failed, succeeded); var avgDurationSeconds = measured.Count > 0 ? Math.Round(measured.Average(s => s.DurationSeconds), 0) : 0; var p95DurationSeconds = Percentile(measured.Select(s => s.DurationSeconds).ToList(), 0.95); @@ -423,6 +424,7 @@ namespace AutopilotMonitor.Functions.Functions.Apps Skipped = skipped, Unmeasured = installed.Count - measured.Count, Failed = failed, + Incomplete = incomplete, FailureRate = failureRate, AvgDurationSeconds = avgDurationSeconds, P95DurationSeconds = p95DurationSeconds, diff --git a/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs b/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs index 5ad590b3..5fdb274b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs +++ b/src/Backend/AutopilotMonitor.Functions/Helpers/MetricsMath.cs @@ -10,8 +10,8 @@ public static class MetricsMath /// /// App-install failure rate over finished installs only: Failed / (Failed + Succeeded), one /// decimal, 0 when nothing finished. The same outcome-quota convention as the enrollment - /// success rate — "InProgress" rows (still installing, or orphaned by a session that died - /// mid-install) never dilute the rate. Shared by the app-metrics payload and the Apps + /// success rate — "InProgress" rows (still installing) and "Incomplete" rows (observation + /// ended before any outcome) never dilute the rate. Shared by the app-metrics payload and the Apps /// dashboard aggregations so the definition can't drift between panes. /// public static double TerminalFailureRatePct(int failed, int succeeded) @@ -94,6 +94,8 @@ public static class MetricsMath var installed = succeededAll.Where(s => !IsSkipTerminalState(s)).ToList(); var measured = installed.Where(HasMeasuredDuration).ToList(); var failed = g.Where(s => s.Status == "Failed").ToList(); + // Observation ended before any outcome — disclosed, outside the rate and the durations. + var incomplete = g.Count(s => s.Status == "Incomplete"); var total = g.Count(); // DoAggregator is the single source for the DO rollup: it filters rows that actually @@ -109,6 +111,7 @@ public static class MetricsMath Skipped = skipped.Count, Unmeasured = installed.Count - measured.Count, Failed = failed.Count, + Incomplete = incomplete, // Skips leave the rate (they are not attempts); legacy rows stay in it. FailureRate = TerminalFailureRatePct(failed.Count, installed.Count), AvgDurationSeconds = measured.Count > 0 ? Math.Round(measured.Average(s => s.DurationSeconds), 0) : 0, @@ -151,6 +154,7 @@ public static class MetricsMath TotalInstalls = summaryList.Count, TotalSkipped = appGroups.Sum(a => a.Skipped), TotalUnmeasured = appGroups.Sum(a => a.Unmeasured), + TotalIncomplete = appGroups.Sum(a => a.Incomplete), TotalCollisionExcluded = totalCollisionExcluded, SlowestApps = slowestApps, TopFailingApps = topFailingApps, diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs index 87de5896..90bf6571 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs @@ -92,6 +92,9 @@ namespace AutopilotMonitor.Functions.Services classification.HasCompletionEvidenceCandidate = true; break; case "agent_shutting_down": + // Every shutdown ends this agent run's observation; the max-lifetime + // flavour additionally carries the verdict mapping below. + classification.AgentShutdownEvent = evt; if (IsMaxLifetimeAgentShutdown(evt)) classification.AgentMaxLifetimeShutdownEvent = evt; break; @@ -705,6 +708,14 @@ namespace AutopilotMonitor.Functions.Services public EnrollmentEvent? EspFailureEvent { get; set; } public EnrollmentEvent? SessionStalledEvent { get; set; } + /// + /// Any agent_shutting_down in the batch — the agent's last word for this run. + /// After the status write, a terminal session closes its still-open app rows on this + /// batch: apps that started after the verdict (the agent keeps watching after + /// enrollment_complete) got their row only after the terminal seam had already run. + /// + public EnrollmentEvent? AgentShutdownEvent { get; set; } + /// /// agent_shutting_down with Data.reason == "max_lifetime" — the V2 /// watchdog shutdown that deliberately carries no enrollment verdict. Mapped to a diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs index e3b6b0b8..684f8d0f 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.cs @@ -276,6 +276,21 @@ namespace AutopilotMonitor.Functions.Services updatedSession?.AgentVersion, skewScan); } + // Observation end: the batch carrying agent_shutting_down is the agent's last word. + // On a terminal session an app row still InProgress now will never get a terminal + // event — close it as Incomplete. The terminal seam already did this on the verdict + // batch; this re-run catches rows opened after the verdict (the IME keeps installing + // and the agent keeps watching after enrollment_complete). Non-terminal sessions + // (reboot shutdown, WhiteGlove part 1) keep their rows open. Idempotent + fail-soft. + if (classification.AgentShutdownEvent != null + && updatedSession != null + && (updatedSession.Status == SessionStatus.Succeeded + || updatedSession.Status == SessionStatus.Failed + || updatedSession.Status == SessionStatus.Incomplete)) + { + await _metricsRepo.CloseOpenAppInstallsForSessionAsync(request.TenantId, request.SessionId); + } + // Auto-analyze fan-out: enqueue a queue message instead of running fire-and-forget // Task.Run inside the function. The previous in-function approach could be killed // mid-flight by Functions scale-in (HTTP 200 returned → worker unloaded → rules diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs index 3f08a0bb..1eb23ea8 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs @@ -328,7 +328,12 @@ namespace AutopilotMonitor.Functions.Services // Idempotent, positive-evidence-only; also heals sessions that went // terminal before the resolution seam existed (hadNoRow backfill). if (eventCountMoved || hadNoRow) + { await _metricsRepo.ResolveEspBlockingForSessionAsync(session.TenantId, session.SessionId); + // Same late-batch reasoning: an app row opened after the agent's + // shutdown batch was never closed — the sweep is its self-heal. + await _metricsRepo.CloseOpenAppInstallsForSessionAsync(session.TenantId, session.SessionId); + } } if (breakdown == null) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Metrics.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Metrics.cs index 9db815c0..e25014c3 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Metrics.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Metrics.cs @@ -253,9 +253,11 @@ namespace AutopilotMonitor.Functions.Services // Status "InProgress" — a real (non-sentinel) value that Merge-mode would // happily write over a row the terminal batch already closed. Only another // terminal observation may change a terminal status (Failed→Succeeded on a - // successful retry stays legal). + // successful retry stays legal). Incomplete (observation-end close step) is + // terminal for the same purpose: a straggling progress-only batch must not + // reopen it, while a real terminal observation still wins. var existingStatus = existing.GetString("Status"); - if ((existingStatus == "Succeeded" || existingStatus == "Failed") && + if ((existingStatus == "Succeeded" || existingStatus == "Failed" || existingStatus == "Incomplete") && summary.Status != "Succeeded" && summary.Status != "Failed") { summary.Status = existingStatus; @@ -525,6 +527,66 @@ namespace AutopilotMonitor.Functions.Services return sets.Contains(appId); } + /// + /// Closes the session's app rows that never reached a terminal state: Status + /// "InProgress" (or missing) with no CompletedAt becomes "Incomplete" — terminal, + /// non-failure, outcome unknown. Runs at the terminal seam, again after the batch that + /// carries the agent's shutdown (rows opened after the verdict), and from the 30-day + /// sweep. Idempotent + fail-soft; a real terminal arriving later still wins through the + /// sticky-status reconcile. Returns the number of rows closed. + /// + public async Task CloseOpenAppInstallsForSessionAsync(string tenantId, string sessionId) + { + try + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); + + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.AppInstallSummaries); + var filter = $"PartitionKey eq '{tenantId}' and RowKey ge '{sessionId}_' and RowKey lt '{sessionId}`'"; + var query = tableClient.QueryAsync( + filter: filter, + select: new[] { "PartitionKey", "RowKey", "Status", "CompletedAt" }); + + var closed = 0; + await foreach (var row in query) + { + if (!ShouldCloseAsIncomplete(row)) continue; + + var update = new TableEntity(row.PartitionKey, row.RowKey) + { + ["Status"] = "Incomplete" + }; + await tableClient.UpsertEntityAsync(update, TableUpdateMode.Merge); + closed++; + } + + if (closed > 0) + _logger.LogInformation( + "Session {SessionId}: closed {Count} app rows as Incomplete (no terminal state when observation ended)", + sessionId, closed); + return closed; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Closing open app rows failed for session {SessionId} (fail-soft)", sessionId); + return 0; + } + } + + /// + /// Pure per-row predicate behind — + /// internal static so the contract is pinned by unit tests: only a row that is still + /// open (Status missing, empty or "InProgress", no CompletedAt) closes; every terminal + /// row, including an already-closed Incomplete one, is left alone (idempotency). + /// + internal static bool ShouldCloseAsIncomplete(TableEntity row) + { + if (row.GetDateTimeOffset("CompletedAt").HasValue) return false; + var status = row.GetString("Status"); + return string.IsNullOrEmpty(status) || status == "InProgress"; + } + /// /// Gets app install summaries for a tenant (fleet-level metrics). When /// is supplied, a server-side StartedAt ge filter is applied so a windowed view (e.g. the diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts index bc286d3e..1b53da86 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts @@ -254,6 +254,36 @@ describe('buildSessionCoverage — diagnostics package', () => { }); }); +describe('buildSessionCoverage — apps without a terminal state', () => { + // The shutdown emit of the termination handler: what was still in flight when the agent left. + const shutdownSummary = ev('app_tracking_summary', 900, { installingNames: ['Suite'], downloadingNames: [], pendingNames: ['Other'] }); + + it("names the apps still in flight in the agent's last summary on a terminal session", () => { + const c = buildSessionCoverage({ startedAt: at(0), status: 'Succeeded' }, [ + ev('agent_started', 5), + ev('app_tracking_summary', 300, { installingNames: ['Early'], downloadingNames: ['Later'] }), + shutdownSummary, + ]); + expect(c.apps).toEqual({ stillInstalling: ['Suite'], stillDownloading: [] }); + expect(c.gaps).toEqual([expect.stringContaining('1 app(s) still installing when the agent stopped observing (Suite)')]); + }); + + it('reports nothing while the session is still live, even with apps in flight', () => { + const c = buildSessionCoverage({ startedAt: at(0), status: 'InProgress' }, [ev('agent_started', 5), shutdownSummary]); + expect(c.apps).toEqual({ stillInstalling: [], stillDownloading: [] }); + expect(c.gaps).toEqual([]); + }); + + it("stays silent when the last summary has nothing in flight (apps that never started are the starved rule's finding)", () => { + const c = buildSessionCoverage({ startedAt: at(0), status: 'Failed' }, [ + ev('agent_started', 5), + ev('app_tracking_summary', 900, { installingNames: [], pendingNames: ['Other'] }), + ]); + expect(c.apps).toEqual({ stillInstalling: [], stillDownloading: [] }); + expect(c.gaps).toEqual([]); + }); +}); + describe('coverage wiring', () => { it('SUMMARY_EVENT_FIELDS carries every coverage slice, as data. entries only', () => { const fields = SUMMARY_EVENT_FIELDS.split(','); @@ -271,6 +301,7 @@ describe('coverage wiring', () => { 'agent_started', 'agent_late_start', 'historic_ime_replay_detected', 'ime_tracker_degraded', 'ime_pattern_hits', 'collector_degraded', 'spool_pressure_detected', 'telemetry_upload_poisoned', 'telemetry_upload_blocked', 'ingress_backpressure', 'disk_space_low', 'diagnostics_collecting', 'diagnostics_uploaded', 'diagnostics_upload_failed', + 'app_tracking_summary', ]; expect([...COVERAGE_EVENT_TYPES].sort()).toEqual([...consulted].sort()); }); diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index e8c084f7..68f53a65 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -470,6 +470,8 @@ export interface AppAnalyticsSummary { skipped: number; unmeasured: number; failed: number; + /** Still installing when the session's observation ended — outcome unknown, outside the rate. */ + incomplete: number; failureRate: number; avgDurationSeconds: number; p95DurationSeconds: number; @@ -558,6 +560,8 @@ export interface AppMetricsAppGroup { skipped: number; unmeasured: number; failed: number; + /** Still installing when the session's observation ended — outcome unknown, outside the rate. */ + incomplete: number; /** Failed / (failed + succeeded) as a percentage; skips never count as attempts. */ failureRate: number; /** Average measured FINAL-attempt duration (whole seconds); 0 with no measured installs. */ @@ -591,6 +595,8 @@ export interface AppMetricsResponse { totalInstalls: number; totalSkipped: number; totalUnmeasured: number; + /** Rows still installing when the session's observation ended — outcome unknown, outside the failure rate. */ + totalIncomplete: number; /** Rows excluded from per-app groups because their name-keyed row merged distinct appIds. */ totalCollisionExcluded: number; slowestApps: AppMetricsAppGroup[]; diff --git a/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts b/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts index 303e23de..0307d591 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts @@ -31,6 +31,7 @@ export const COVERAGE_EVENT_TYPES: ReadonlySet = new Set([ 'diagnostics_collecting', 'diagnostics_uploaded', 'diagnostics_upload_failed', + 'app_tracking_summary', ]); /** @@ -44,7 +45,8 @@ export const COVERAGE_EVENT_FIELDS = 'data.file,data.firstSkippedPattern,data.lineBudgetBreaks,data.regexTimeouts,data.oversizedLines,data.unanchoredPatterns,data.linesRead,' + 'data.collector,data.reason,data.errorType,' + 'data.pendingItemCount,data.pendingBytes,data.kind,data.itemCount,data.disk_free_gb,' + - 'data.truncated,data.includedFiles,data.includedBytes,data.skippedFiles,data.skippedByReason,data.problemsByKind'; + 'data.truncated,data.includedFiles,data.includedBytes,data.skippedFiles,data.skippedByReason,data.problemsByKind,' + + 'data.installingNames,data.downloadingNames'; export interface CoverageWindow { /** Timestamp of the first `agent_started` — the earliest moment anything was observed live. */ @@ -111,12 +113,20 @@ export interface DiagnosticsCoverage { problemsByKind?: Record; } +export interface AppsCoverage { + /** Apps the agent's last summary still listed as installing when observation ended (terminal session only). */ + stillInstalling: string[]; + /** Same for downloading. */ + stillDownloading: string[]; +} + export interface SessionCoverage { window: CoverageWindow; imeTracker: ImeTrackerCoverage; collectors: CollectorDegradation[]; upload: UploadCoverage; diagnostics: DiagnosticsCoverage; + apps: AppsCoverage; /** One calibrated line per observation gap; empty means nothing reported a gap. */ gaps: string[]; } @@ -165,12 +175,21 @@ function fmtCounts(m: Record): string { return Object.entries(m).map(([k, n]) => `${k}: ${n}`).join(', '); } +function names(v: unknown): string[] { + return Array.isArray(v) ? v.map(str).filter((s): s is string => s !== null) : []; +} + +/** Mirror of the backend SessionStatus terminal set (utils/sessionStatus.ts in the web app). */ +function isTerminalSessionStatus(status: unknown): boolean { + return status === 'Succeeded' || status === 'Failed' || status === 'Incomplete'; +} + /** * Folds the session's events (already stripped of historic-replay rows by the caller) into the * coverage block. Pure: no fetch, no clock. */ export function buildSessionCoverage( - session: { startedAt?: string | null }, + session: { startedAt?: string | null; status?: string | null }, events: readonly Ev[], ): SessionCoverage { const byType = new Map(); @@ -366,6 +385,25 @@ export function buildSessionCoverage( } } + // ── Apps without a terminal state ─────────────────────────────────────────── + // The agent's last app_tracking_summary (the shutdown emit) names what was still in flight. + // Only a terminal session turns that into a gap — while the session is live those apps are + // simply still installing. Apps that never started are the starved rule's finding, not a gap. + const apps: AppsCoverage = { stillInstalling: [], stillDownloading: [] }; + if (isTerminalSessionStatus(session.status)) { + const summary = last('app_tracking_summary'); + apps.stillInstalling = names(summary?.data?.installingNames); + apps.stillDownloading = names(summary?.data?.downloadingNames); + const inFlight = (verb: string, list: string[]): void => { + if (list.length === 0) return; + gaps.push( + `${list.length} app(s) still ${verb} when the agent stopped observing (${list.join(', ')}): outcome unknown — not observed to fail, not observed to finish.`, + ); + }; + inFlight('installing', apps.stillInstalling); + inFlight('downloading', apps.stillDownloading); + } + return { window: { observedFrom, @@ -380,6 +418,7 @@ export function buildSessionCoverage( collectors, upload, diagnostics, + apps, gaps, }; } diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMetricsRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMetricsRepository.cs index 0e6871c6..b16c884b 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMetricsRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMetricsRepository.cs @@ -137,6 +137,8 @@ namespace AutopilotMonitor.Shared.DataAccess Task GetSessionTimeBreakdownAsync(string tenantId, string sessionId); /// Re-joins the session's app rows against its latest esp_config_detected lists (positive evidence only). Idempotent + fail-soft — the terminal seam runs it once; the sweep re-runs it when late events changed the stream. Task ResolveEspBlockingForSessionAsync(string tenantId, string sessionId); + /// Closes the session's app rows that never reached a terminal state (InProgress → Incomplete, outcome unknown). Idempotent + fail-soft — the terminal seam, the agent-shutdown batch and the sweep all call it; a real terminal arriving later still wins. Returns the number of rows closed. + Task CloseOpenAppInstallsForSessionAsync(string tenantId, string sessionId); Task SaveTimeAttributionAggregateAsync(TimeAttributionDailyAggregate aggregate); /// Aggregate rows of one tenant partition ("global" allowed) for an inclusive date range. Task> GetTimeAttributionAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate); diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppInstallSummary.cs b/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppInstallSummary.cs index 456baf0f..471515ba 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppInstallSummary.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppInstallSummary.cs @@ -43,7 +43,12 @@ namespace AutopilotMonitor.Shared.Models public bool AppIdCollision { get; set; } /// - /// Lifecycle status: Succeeded, Failed, InProgress, or empty. + /// Lifecycle status: Succeeded, Failed, InProgress, Incomplete, or empty. + /// "Incomplete" is terminal and non-failure: the row was still InProgress when the + /// session's observation ended (terminal session, agent gone), so no outcome was ever + /// seen. Written only by the observation-end close step, never by an ingest fold; + /// CompletedAt and DurationSeconds stay unset (unknown, reported as unmeasured) and a + /// real terminal arriving later overrides it. /// Empty (default) is a sentinel meaning "no status-relevant event observed in the current /// aggregation batch". Aggregators only set a real value when they see started / completed / /// failed / skipped. The storage layer omits the column from the upsert when this is empty diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppMetricsApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppMetricsApiModels.cs index 15ae3132..c36322ea 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppMetricsApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppMetricsApiModels.cs @@ -16,6 +16,8 @@ namespace AutopilotMonitor.Shared.Models public int TotalInstalls { get; set; } public int TotalSkipped { get; set; } public int TotalUnmeasured { get; set; } + /// Rows still installing when the session's observation ended — outcome unknown, outside the failure rate. + public int TotalIncomplete { get; set; } /// Rows excluded from per-app groups because their name-keyed row merged distinct appIds. public int TotalCollisionExcluded { get; set; } public IReadOnlyList SlowestApps { get; set; } = default!; @@ -32,6 +34,8 @@ namespace AutopilotMonitor.Shared.Models public int Skipped { get; set; } public int Unmeasured { get; set; } public int Failed { get; set; } + /// Still installing when the session's observation ended — outcome unknown, outside the rate. + public int Incomplete { get; set; } /// Failed / (failed + succeeded) as a percentage; skips never count as attempts. public double FailureRate { get; set; } /// Average measured FINAL-attempt duration (whole seconds); 0 with no measured installs. diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppsDashboardApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppsDashboardApiModels.cs index 2873304f..2afa3770 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppsDashboardApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Metrics/AppsDashboardApiModels.cs @@ -82,6 +82,8 @@ namespace AutopilotMonitor.Shared.Models public int Skipped { get; set; } public int Unmeasured { get; set; } public int Failed { get; set; } + /// Still installing when the session's observation ended — outcome unknown, outside the rate. + public int Incomplete { get; set; } public double FailureRate { get; set; } public double AvgDurationSeconds { get; set; } public int P95DurationSeconds { get; set; } diff --git a/src/Web/autopilot-monitor-web/app/sessions/hooks/useSessionDerivedData.ts b/src/Web/autopilot-monitor-web/app/sessions/hooks/useSessionDerivedData.ts index c0f4dad2..307cd93d 100644 --- a/src/Web/autopilot-monitor-web/app/sessions/hooks/useSessionDerivedData.ts +++ b/src/Web/autopilot-monitor-web/app/sessions/hooks/useSessionDerivedData.ts @@ -26,6 +26,7 @@ export interface UseSessionDerivedDataReturn { gatherRulesSucceeded: boolean; displayStatus: string; enrollmentDurationFromEvents: string | null; + lastObservedAtMs: number | null; standbySeconds: number | null; phaseNamesMap: Record; phaseOrder: string[]; @@ -152,6 +153,24 @@ export function useSessionDerivedData( return `${Math.floor(durationSec / 3600)}h ${Math.floor((durationSec % 3600) / 60)}m`; }, [events]); + // Last moment the agent reported anything — the end of the observation window, in the + // device clock frame the event timestamps share. Deliberately NOT the enrollment duration + // above: that stops at enrollment_complete, while apps keep installing after the verdict + // for as long as the agent still watches. `session.lastEventAt` is the server-tracked + // maximum over every event; the loaded events are the fallback for rows predating it. + const sessionLastEventAt = session?.lastEventAt ?? null; + const lastObservedAtMs = useMemo(() => { + const fromSession = sessionLastEventAt ? new Date(sessionLastEventAt).getTime() : NaN; + if (Number.isFinite(fromSession)) return fromSession; + let max: number | null = null; + for (const e of events) { + if (e.source === "SystemTimelineWatcher") continue; + const t = new Date(e.timestamp).getTime(); + if (Number.isFinite(t) && (max === null || t > max)) max = t; + } + return max; + }, [events, sessionLastEventAt]); + // Total observed sleep/standby seconds (system_sleep_episode ground truth). The wall-clock // duration deliberately keeps the pause — this number tells the story next to it ("1h 20m, // 56m of it standby"). Dedup on enteredAt: the same episode can be observed twice (agent @@ -242,6 +261,7 @@ export function useSessionDerivedData( gatherRulesSucceeded: !!gatherRulesSucceeded, displayStatus, enrollmentDurationFromEvents, + lastObservedAtMs, standbySeconds, phaseNamesMap, phaseOrder, diff --git a/src/Web/autopilot-monitor-web/app/sessions/page.tsx b/src/Web/autopilot-monitor-web/app/sessions/page.tsx index fff2ea65..0c91de2e 100644 --- a/src/Web/autopilot-monitor-web/app/sessions/page.tsx +++ b/src/Web/autopilot-monitor-web/app/sessions/page.tsx @@ -17,6 +17,7 @@ import { useLatestVersions } from '@/lib/useLatestVersions'; import { useScriptDisplayNames } from '@/lib/scriptDisplayNames'; import { api } from "@/lib/api"; import { isGuid } from "@/utils/inputValidation"; +import { isTerminalStatus } from "@/utils/sessionStatus"; import { ApiError, fetchBlob, fetchJson, fetchOk, jsonBody } from "@/lib/apiClient"; import { useSessionAnalysis } from "./hooks/useSessionAnalysis"; @@ -197,6 +198,7 @@ function SessionDetailContent() { isGatherRulesSession, displayStatus, enrollmentDurationFromEvents, + lastObservedAtMs, standbySeconds, isSkipUserStatusPage, isWhiteGloveSession, @@ -209,6 +211,11 @@ function SessionDetailContent() { userEnrollGrouped, } = derived; + // Observation end for the progress panels: only once the session is terminal does a row + // without a terminal event stop being "still running" — it becomes incomplete, its timer + // bounded by the agent's last report. Live sessions pass null and keep ticking. + const observedUntilMs = isTerminalStatus(displayStatus) ? lastObservedAtMs : null; + // All phase keys currently present. For WhiteGlove sessions we use prefixed keys // (pre-X, user-X) to avoid collisions. const allPhaseKeys = useMemo( @@ -679,6 +686,7 @@ function SessionDetailContent() { e => e.eventType === "download_progress" || e.eventType === "app_download_started" || e.eventType === "app_install_skipped" )} summaryStats={appSummaryStats} + observedUntilMs={observedUntilMs} /> )} @@ -697,6 +705,7 @@ function SessionDetailContent() { || e.eventType === "realmjoin_package_started" || e.eventType === "realmjoin_package_completed" )} summaryStats={appSummaryStats} + observedUntilMs={observedUntilMs} /> )} diff --git a/src/Web/autopilot-monitor-web/components/DownloadProgress.tsx b/src/Web/autopilot-monitor-web/components/DownloadProgress.tsx index ed69b943..ba1c9678 100644 --- a/src/Web/autopilot-monitor-web/components/DownloadProgress.tsx +++ b/src/Web/autopilot-monitor-web/components/DownloadProgress.tsx @@ -61,6 +61,9 @@ interface SummaryStats { interface DownloadProgressProps { events: DownloadEvent[]; summaryStats?: SummaryStats | null; + // Epoch ms of the agent's last report, passed only once the session is terminal. A download + // that never finished by then is incomplete, not active. + observedUntilMs?: number | null; } interface DoStats { @@ -96,6 +99,8 @@ interface DownloadItem { // progress). Rows without it are phantom "completed" rows from the terminal // download_progress event and are dropped after folding. hasByteEvidence: boolean; + // Still downloading when the session's observation ended — outcome unknown, not active. + isIncomplete: boolean; firstSeenIndex: number; eventData?: Record; doStats?: DoStats | null; @@ -142,7 +147,7 @@ function effectiveDurationMs(dl: DownloadItem): number { return 0; } -export default function DownloadProgress({ events, summaryStats }: DownloadProgressProps) { +export default function DownloadProgress({ events, summaryStats, observedUntilMs = null }: DownloadProgressProps) { // Legacy-agent guard: drop download events replayed from a previous enrollment's IME log. // Silent (empty finals set) — the InstallProgress panel already reports the hidden count // for the same apps; a second note here would double-report them. @@ -250,18 +255,23 @@ export default function DownloadProgress({ events, summaryStats }: DownloadProgr isSkipped: isSkippedEvent || (existing?.isSkipped ?? false), isUninstall: (intent?.toLowerCase().includes("uninstall") ?? false) || (existing?.isUninstall ?? false), hasByteEvidence: hasByteActivity(filterInput) || (existing?.hasByteEvidence ?? false), + isIncomplete: false, firstSeenIndex: existing?.firstSeenIndex ?? insertionIndex++, eventData: d, doStats, }); } - return Array.from(downloadMap.values()).sort((a, b) => { + const folded = Array.from(downloadMap.values()).sort((a, b) => { // Keep stable visual order based on first appearance only. // This avoids active items jumping when completion status changes. return a.firstSeenIndex - b.firstSeenIndex; }); - }, [current]); + // Observation ended (terminal session): a download that never finished stays at its last + // observed bytes and is reported as incomplete rather than active. + if (observedUntilMs == null) return folded; + return folded.map(d => (!d.isComplete && !d.isSkipped && d.hasByteEvidence ? { ...d, isIncomplete: true } : d)); + }, [current, observedUntilMs]); const [expanded, setExpanded] = useState(true); const [showSkipped, setShowSkipped] = useState(false); @@ -327,7 +337,8 @@ export default function DownloadProgress({ events, summaryStats }: DownloadProgr // Pills stay install-scope (matching the "X of Y downloaded" header, whose summary // buckets uninstalls under skipped) — uninstall downloads get their own count, and // rows without any observed download activity sit behind the "no download" toggle. - const activeCount = downloads.filter(d => !d.isComplete && !d.isSkipped && !d.isUninstall && d.hasByteEvidence).length; + const activeCount = downloads.filter(d => !d.isComplete && !d.isSkipped && !d.isUninstall && d.hasByteEvidence && !d.isIncomplete).length; + const incompleteCount = downloads.filter(d => d.isIncomplete && !d.isUninstall).length; const completedCount = downloads.filter(d => d.isComplete && !d.isSkipped && !d.isUninstall && d.hasByteEvidence).length; const uninstallCount = downloads.filter(d => d.isUninstall && !d.isSkipped && d.hasByteEvidence).length; const noDownloadCount = downloads.filter(d => !d.isSkipped && !d.hasByteEvidence).length; @@ -389,6 +400,15 @@ export default function DownloadProgress({ events, summaryStats }: DownloadProgr {completedCount} completed )} + {incompleteCount > 0 && ( + // Slate like the session-level Incomplete badge: unknown outcome, not a failure. + + {incompleteCount} incomplete + + )} {uninstallCount > 0 && ( // Muted gray on purpose (dark-mode covered by the global gray overrides) — // it's context, not a state the user needs to act on. @@ -481,6 +501,11 @@ function DownloadItem({ download: dl, progressPercent }: { download: DownloadIte + ) : dl.isIncomplete ? ( + // Question mark in slate, no pulse — nothing is transferring any more. + + + ) : ( @@ -500,9 +525,17 @@ function DownloadItem({ download: dl, progressPercent }: { download: DownloadIte Uninstall )} + {dl.isIncomplete && ( + + Incomplete + + )}
- {!dl.isComplete && !dl.isSkipped && dl.downloadRateBps > 0 && ( + {!dl.isComplete && !dl.isSkipped && !dl.isIncomplete && dl.downloadRateBps > 0 && ( {formatThroughput(dl.downloadRateBps, "0 B/s")} )} {dl.eventData && Object.keys(dl.eventData).length > 0 && ( @@ -522,7 +555,7 @@ function DownloadItem({ download: dl, progressPercent }: { download: DownloadIte
diff --git a/src/Web/autopilot-monitor-web/components/InstallProgress.tsx b/src/Web/autopilot-monitor-web/components/InstallProgress.tsx index 67b8b4e5..49ed1dc6 100644 --- a/src/Web/autopilot-monitor-web/components/InstallProgress.tsx +++ b/src/Web/autopilot-monitor-web/components/InstallProgress.tsx @@ -3,7 +3,7 @@ import { useEffect, useMemo, useState } from "react"; import { getErrorCodeEntry, formatErrorCode, errorCodeTooltip } from "@/utils/errorCodeMap"; import { partitionHistoricReplayEvents } from "@/lib/historicReplay"; -import { buildInstallItems, isRebootOrRetryClass, type InstallEvent, type InstallItem, type InstallSource } from "@/lib/installProgress"; +import { applyObservationEnd, buildInstallItems, isRebootOrRetryClass, type InstallEvent, type InstallItem, type InstallSource } from "@/lib/installProgress"; import TruncatedLabel from "@/components/TruncatedLabel"; import PendingAppRow from "@/components/PendingAppRow"; @@ -21,6 +21,9 @@ interface SummaryStats { interface InstallProgressProps { events: InstallEvent[]; summaryStats?: SummaryStats | null; + // Epoch ms of the agent's last report, passed only once the session is terminal. A row still + // installing at that point renders as Incomplete instead of ticking against the wall clock. + observedUntilMs?: number | null; } // Finals counted by the historic-replay partition — one per hidden install, so the note @@ -56,7 +59,7 @@ function formatDuration(ms: number): string { return `${hours}h ${remainingMinutes}m`; } -export default function InstallProgress({ events, summaryStats }: InstallProgressProps) { +export default function InstallProgress({ events, summaryStats, observedUntilMs = null }: InstallProgressProps) { // Legacy-agent guard: split off app events replayed from a previous enrollment's IME log // (newer agents suppress them at the source) so week-old installs never render as current. // office_*/realmjoin_* events never carry rejectedSourceTimestamp and pass through untouched. @@ -65,7 +68,10 @@ export default function InstallProgress({ events, summaryStats }: InstallProgres [events] ); - const installs = useMemo(() => buildInstallItems(current), [current]); + const installs = useMemo( + () => applyObservationEnd(buildInstallItems(current), observedUntilMs), + [current, observedUntilMs] + ); const [expanded, setExpanded] = useState(true); const [showSkipped, setShowSkipped] = useState(false); @@ -101,7 +107,10 @@ export default function InstallProgress({ events, summaryStats }: InstallProgres if (installs.length === 0 && historicCount === 0) return null; - const activeCount = installs.filter(d => d.state === "Installing").length; + const activeCount = installs.filter(d => d.state === "Installing" && !d.isIncomplete).length; + // Still installing when observation ended — unknown outcome, counted apart from "active" + // so a finished session never claims an app is running. + const incompleteCount = installs.filter(d => d.isIncomplete).length; const completedCount = installs.filter(d => d.state === "Installed").length; // Enforced uninstall assignments — counted separately so the completed pill keeps // agreeing with the "X of Y installed" header (which buckets by intent agent-side). @@ -178,6 +187,15 @@ export default function InstallProgress({ events, summaryStats }: InstallProgres {likelyStuckCount} likely stuck )} + {incompleteCount > 0 && ( + // Slate like the session-level Incomplete badge: unknown outcome, not a failure. + + {incompleteCount} incomplete + + )} {postponedCount > 0 && ( {postponedCount} postponed @@ -237,17 +255,17 @@ function InstallItemRow({ item }: { item: InstallItem }) { const [liveElapsed, setLiveElapsed] = useState<{ startedAt: string; elapsedMs: number } | null>(null); useEffect(() => { - if (item.state !== "Installing" || !item.startedAt) return; + if (item.state !== "Installing" || item.isIncomplete || !item.startedAt) return; const startedAt = item.startedAt; const startTime = new Date(startedAt).getTime(); const tick = () => setLiveElapsed({ startedAt, elapsedMs: Date.now() - startTime }); tick(); const id = setInterval(tick, 1000); return () => clearInterval(id); - }, [item.state, item.startedAt]); + }, [item.state, item.isIncomplete, item.startedAt]); const elapsedMs = - item.state === "Installing" && item.startedAt && liveElapsed?.startedAt === item.startedAt + item.state === "Installing" && !item.isIncomplete && item.startedAt && liveElapsed?.startedAt === item.startedAt ? liveElapsed.elapsedMs : null; @@ -295,6 +313,12 @@ function InstallItemRow({ item }: { item: InstallItem }) { + ) : item.isIncomplete ? ( + // Same question-mark as "likely stuck" (we don't know), in slate: no pulse — nothing + // is running any more. + + + ) : ( @@ -364,6 +388,14 @@ function InstallItemRow({ item }: { item: InstallItem }) { {item.state === "Postponed" && ( Postponed )} + {item.isIncomplete && ( + + Incomplete + + )} {item.state === "Preinstalled" && (
+ {item.isIncomplete && item.observedMs != null && ( + // Lower bound: watched this long, then the agent's reports stopped. + + {`≥ ${formatDuration(item.observedMs)}`} + + )} {elapsedMs != null && elapsedMs > 0 && ( {formatDuration(elapsedMs)} )} diff --git a/src/Web/autopilot-monitor-web/lib/__tests__/installProgress.test.ts b/src/Web/autopilot-monitor-web/lib/__tests__/installProgress.test.ts index 8a4077bf..6b8c8436 100644 --- a/src/Web/autopilot-monitor-web/lib/__tests__/installProgress.test.ts +++ b/src/Web/autopilot-monitor-web/lib/__tests__/installProgress.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { buildInstallItems, isRebootOrRetryClass, type InstallEvent } from "@/lib/installProgress"; +import { applyObservationEnd, buildInstallItems, isRebootOrRetryClass, type InstallEvent } from "@/lib/installProgress"; function evt(eventType: string, timestamp: string, data: Record): InstallEvent { return { eventType, timestamp, data }; @@ -226,3 +226,41 @@ describe("isRebootOrRetryClass", () => { expect(isRebootOrRetryClass(undefined)).toBe(false); }); }); + +describe("applyObservationEnd", () => { + // Shape of session ac5660b8: the app started 143 s after enrollment_complete and was still + // installing when the agent sent its last event 30 s later — no terminal event ever came. + const stillInstalling = evt("app_install_started", "2026-09-09T06:45:18Z", { appName: "Suite", appId: "a1" }); + const finished = [ + evt("app_install_started", "2026-09-09T06:40:31Z", { appName: "Teams", appId: "t1" }), + evt("app_install_completed", "2026-09-09T06:41:05Z", { appName: "Teams", appId: "t1", state: "Installed" }), + ]; + const lastReport = Date.parse("2026-09-09T06:45:48Z"); + + it("leaves every row untouched while the session is still live (null observation end)", () => { + const items = buildInstallItems([stillInstalling, ...finished]); + expect(applyObservationEnd(items, null)).toBe(items); + expect(items.every(i => !i.isIncomplete)).toBe(true); + }); + + it("marks a row still installing at the observation end as incomplete with the watched span", () => { + const [suite] = applyObservationEnd(buildInstallItems([stillInstalling]), lastReport); + expect(suite.state).toBe("Installing"); + expect(suite.isIncomplete).toBe(true); + expect(suite.observedMs).toBe(30 * 1000); + }); + + it("never touches rows that reached a terminal state", () => { + const items = applyObservationEnd(buildInstallItems([...finished, stillInstalling]), lastReport); + const teams = items.find(i => i.appName === "Teams")!; + expect(teams.isIncomplete).toBe(false); + expect(teams.observedMs).toBeUndefined(); + expect(teams.durationMs).toBe(34 * 1000); + }); + + it("clamps the watched span to zero when the observation end precedes the start (clock skew)", () => { + const [suite] = applyObservationEnd(buildInstallItems([stillInstalling]), Date.parse("2026-09-09T06:45:00Z")); + expect(suite.isIncomplete).toBe(true); + expect(suite.observedMs).toBe(0); + }); +}); diff --git a/src/Web/autopilot-monitor-web/lib/installProgress.ts b/src/Web/autopilot-monitor-web/lib/installProgress.ts index e4c9445c..b8a1d0d4 100644 --- a/src/Web/autopilot-monitor-web/lib/installProgress.ts +++ b/src/Web/autopilot-monitor-web/lib/installProgress.ts @@ -94,6 +94,11 @@ export interface InstallItem { isLikelyStuck: boolean; isDetectionFailure: boolean; isInstallFailure: boolean; + // Set by applyObservationEnd, never by the fold: the row was still installing when the + // session's observation ended (agent gone, session terminal). Outcome unknown — neither a + // failure nor a success — and `observedMs` is the span actually watched, a lower bound. + isIncomplete: boolean; + observedMs?: number; firstSeenIndex: number; eventData?: Record; } @@ -185,7 +190,7 @@ export function buildInstallItems(events: InstallEvent[]): InstallItem[] { const isCompleted = (type === "app_install_completed" && !isSkippedCompletion) || type === "office_install_completed" || (type === "realmjoin_package_completed" && !rjFailed); const isFailed = type === "app_install_failed" || type === "office_install_failed" || rjFailed; - const base = { key, source, appName, appId, intent: intent ?? existing?.intent }; + const base = { key, source, appName, appId, intent: intent ?? existing?.intent, isIncomplete: false }; if (isStarted) { // Don't reset an app that already completed — later batch re-scans @@ -334,3 +339,19 @@ export function buildInstallItems(events: InstallEvent[]): InstallItem[] { return Array.from(installMap.values()).sort((a, b) => a.firstSeenIndex - b.firstSeenIndex); } + +/** + * Marks rows that were still installing when observation ended. `observedUntilMs` is the last + * moment the agent reported anything (session `lastEventAt`) and must be passed only once the + * session is terminal — `null` means "still live", and every row comes back untouched so the + * panel keeps its live timer. A start after the observation end (clock skew) clamps to 0. + */ +export function applyObservationEnd(items: InstallItem[], observedUntilMs: number | null): InstallItem[] { + if (observedUntilMs == null) return items; + return items.map(item => { + if (item.state !== "Installing" || !item.startedAt) return item; + const startedMs = new Date(item.startedAt).getTime(); + const observedMs = Number.isFinite(startedMs) ? Math.max(0, observedUntilMs - startedMs) : 0; + return { ...item, isIncomplete: true, observedMs }; + }); +} diff --git a/src/Web/autopilot-monitor-web/utils/shared-manifests.json b/src/Web/autopilot-monitor-web/utils/shared-manifests.json index 312c1f5d..c80bb83d 100644 --- a/src/Web/autopilot-monitor-web/utils/shared-manifests.json +++ b/src/Web/autopilot-monitor-web/utils/shared-manifests.json @@ -2845,6 +2845,14 @@ "name": "number" } }, + { + "name": "incomplete", + "type": { + "kind": "primitive", + "name": "number" + }, + "doc": "Still installing when the session's observation ended — outcome unknown, outside the rate." + }, { "name": "failureRate", "type": { @@ -3232,6 +3240,14 @@ "name": "number" } }, + { + "name": "incomplete", + "type": { + "kind": "primitive", + "name": "number" + }, + "doc": "Still installing when the session's observation ended — outcome unknown, outside the rate." + }, { "name": "failureRate", "type": { @@ -3398,6 +3414,14 @@ "name": "number" } }, + { + "name": "totalIncomplete", + "type": { + "kind": "primitive", + "name": "number" + }, + "doc": "Rows still installing when the session's observation ended — outcome unknown, outside the failure rate." + }, { "name": "totalCollisionExcluded", "type": { diff --git a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts index 5edfd9d9..25ec90ca 100644 --- a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts @@ -470,6 +470,8 @@ export interface AppAnalyticsSummary { skipped: number; unmeasured: number; failed: number; + /** Still installing when the session's observation ended — outcome unknown, outside the rate. */ + incomplete: number; failureRate: number; avgDurationSeconds: number; p95DurationSeconds: number; @@ -558,6 +560,8 @@ export interface AppMetricsAppGroup { skipped: number; unmeasured: number; failed: number; + /** Still installing when the session's observation ended — outcome unknown, outside the rate. */ + incomplete: number; /** Failed / (failed + succeeded) as a percentage; skips never count as attempts. */ failureRate: number; /** Average measured FINAL-attempt duration (whole seconds); 0 with no measured installs. */ @@ -591,6 +595,8 @@ export interface AppMetricsResponse { totalInstalls: number; totalSkipped: number; totalUnmeasured: number; + /** Rows still installing when the session's observation ended — outcome unknown, outside the failure rate. */ + totalIncomplete: number; /** Rows excluded from per-app groups because their name-keyed row merged distinct appIds. */ totalCollisionExcluded: number; slowestApps: AppMetricsAppGroup[];