feat(web,backend,mcp): app rows without a terminal state become Incomplete instead of running forever (D-237)

An app still installing when the session's observation ended has no terminal
event and never will. The session page ticked Date.now() - startedAt for it
("11h 39m" on a session that finished hours ago), and its AppInstallSummaries
row stayed InProgress forever - ranked behind failures in the app's session
list and counted in totalInstalls with no named bucket.

Web: applyObservationEnd marks such rows Incomplete once the session is
terminal (isTerminalStatus), bounded by session.lastEventAt; the panel shows a
slate Incomplete badge and the watched span as a lower bound, the "active"
pill excludes them, and the download panel carries the parallel flag. Covers
every historic session without a backfill.

Backend: CloseOpenAppInstallsForSessionAsync closes InProgress rows without a
CompletedAt as Incomplete - at the terminal seam, again on the ingest batch
that carries agent_shutting_down for a terminal session (rows opened after
the verdict), and from the 30-day sweep. The sticky-status reconcile keeps
Incomplete against a late progress-only batch and lets a real terminal win.
App metrics and app analytics disclose the bucket (incomplete,
totalIncomplete) outside the failure rate; manifest and wire types
regenerated.

MCP: get_session_summary coverage gains an apps block and a gaps line naming
the apps the agent's last app_tracking_summary still listed as installing or
downloading, terminal sessions only.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-09-10 00:02:24 +02:00
parent 5577f5e7e5
commit baa51102fd
27 changed files with 506 additions and 23 deletions
@@ -0,0 +1,63 @@
using Azure.Data.Tables;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared.Models;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Observation-end closure of app rows: a row still InProgress when the session's observation
/// ended becomes "Incomplete" (terminal, non-failure, outcome unknown). Pins the pure per-row
/// predicate behind <see cref="TableStorageService.CloseOpenAppInstallsForSessionAsync"/> and
/// the disclosed bucket in the app-metrics payload.
/// </summary>
public class AppInstallIncompleteCloseTests
{
private static TableEntity Row(string? status, DateTimeOffset? completedAt = null)
{
var row = new TableEntity("tenant", "session_App");
if (status != null) row["Status"] = status;
if (completedAt.HasValue) row["CompletedAt"] = completedAt.Value;
return row;
}
[Theory]
[InlineData("InProgress")]
[InlineData("")]
[InlineData(null)]
public void ShouldClose_OpenRow(string? status)
=> Assert.True(TableStorageService.ShouldCloseAsIncomplete(Row(status)));
[Theory]
[InlineData("Succeeded")]
[InlineData("Failed")]
[InlineData("Incomplete")]
public void ShouldClose_LeavesTerminalRowsAlone(string status)
=> Assert.False(TableStorageService.ShouldCloseAsIncomplete(Row(status)));
[Fact]
public void ShouldClose_NeverClosesARowWithAnEnd()
{
// Out-of-order arrival can leave a CompletedAt on a row whose status batch is late —
// an end is an end.
Assert.False(TableStorageService.ShouldCloseAsIncomplete(Row("InProgress", DateTimeOffset.UtcNow)));
}
[Fact]
public void AppMetrics_DisclosesIncompleteOutsideTheRate()
{
var payload = MetricsMath.BuildAppMetricsPayload(new[]
{
new AppInstallSummary { AppName = "App", Status = "Succeeded", TerminalState = "Installed", DurationSeconds = 30 },
new AppInstallSummary { AppName = "App", Status = "Failed", TerminalState = "Error", DurationSeconds = 30 },
new AppInstallSummary { AppName = "App", Status = "Incomplete" },
});
var group = Assert.Single(payload.TopFailingApps);
Assert.Equal(3, group.TotalInstalls);
Assert.Equal(1, group.Incomplete);
Assert.Equal(50.0, group.FailureRate);
Assert.Equal(1, payload.TotalIncomplete);
}
}
@@ -155,6 +155,30 @@ public class AppInstallTerminalStateAndReconcileTests
Assert.Equal("Succeeded", summary.Status);
}
[Fact]
public void Reconcile_IncompleteIsSticky_AgainstLateProgressBatch()
{
// Closed by the observation-end step; a straggling started/progress batch must not reopen it.
var existing = new TableEntity("tenant", "session_App") { ["Status"] = "Incomplete" };
var summary = new AppInstallSummary { AppName = "App", Status = "InProgress", StartedAt = T0 };
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
Assert.Equal("Incomplete", summary.Status);
}
[Fact]
public void Reconcile_RealTerminalOverridesIncomplete()
{
// The completion arrived after the close step: real evidence beats the hedge.
var existing = new TableEntity("tenant", "session_App") { ["Status"] = "Incomplete" };
var summary = new AppInstallSummary { AppName = "App", Status = "Succeeded", StartedAt = T0 };
TableStorageService.ReconcileAppInstallSummaryWithExisting(summary, existing);
Assert.Equal("Succeeded", summary.Status);
}
[Fact]
public void Reconcile_Q4Guard_DoesNotFireWithoutStoredCompletedAt()
{
@@ -184,6 +184,7 @@ public class TypedContractCompletionWireParityTests
totalInstalls = 4,
totalSkipped = 1,
totalUnmeasured = 1,
totalIncomplete = 0,
totalCollisionExcluded = 0,
slowestApps = new[]
{
@@ -195,6 +196,7 @@ public class TypedContractCompletionWireParityTests
skipped = 1,
unmeasured = 1,
failed = 1,
incomplete = 0,
failureRate = 33.3,
avgDurationSeconds = 120.0,
maxDurationSeconds = 240,
@@ -225,6 +227,7 @@ public class TypedContractCompletionWireParityTests
TotalInstalls = 4,
TotalSkipped = 1,
TotalUnmeasured = 1,
TotalIncomplete = 0,
TotalCollisionExcluded = 0,
SlowestApps = new[]
{
@@ -236,6 +239,7 @@ public class TypedContractCompletionWireParityTests
Skipped = 1,
Unmeasured = 1,
Failed = 1,
Incomplete = 0,
FailureRate = 33.3,
AvgDurationSeconds = 120.0,
MaxDurationSeconds = 240,
@@ -684,6 +688,7 @@ public class TypedContractCompletionWireParityTests
skipped = 0,
unmeasured = 0,
failed = 0,
incomplete = 0,
failureRate = 0.0,
avgDurationSeconds = 0.0,
p95DurationSeconds = 0,
@@ -715,6 +720,7 @@ public class TypedContractCompletionWireParityTests
Skipped = 0,
Unmeasured = 0,
Failed = 0,
Incomplete = 0,
FailureRate = 0.0,
AvgDurationSeconds = 0.0,
P95DurationSeconds = 0,
@@ -142,6 +142,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task ResolveEspBlockingForSessionAsync(string tenantId, string sessionId)
=> _storage.ResolveEspBlockingForSessionAsync(tenantId, sessionId);
public Task<int> CloseOpenAppInstallsForSessionAsync(string tenantId, string sessionId)
=> _storage.CloseOpenAppInstallsForSessionAsync(tenantId, sessionId);
public Task<bool> SaveTimeAttributionAggregateAsync(TimeAttributionDailyAggregate aggregate)
=> _storage.SaveTimeAttributionAggregateAsync(aggregate);
@@ -101,6 +101,13 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
// complete. Positive evidence only (listed ⇒ EspBlocking=true, absent ⇒ stays
// unknown). Idempotent + fail-soft like the counter reconcile above.
await _storage.ResolveEspBlockingForSessionAsync(tenantId, sessionId);
// Observation end: app rows still InProgress on a terminal session never get a
// terminal event — close them as Incomplete (outcome unknown, not a failure).
// Early for the ingest path (verdict batch, the agent may still be watching),
// never wrong in the end state: a real terminal arriving later still wins via
// the sticky-status reconcile, and the shutdown batch re-runs the close for rows
// opened after the verdict. Idempotent + fail-soft.
await _storage.CloseOpenAppInstallsForSessionAsync(tenantId, sessionId);
// F1 PR2: compute + persist the time-attribution breakdown once, now that the
// terminal write stamped CompletedAt/DurationSeconds and the event stream is
@@ -291,6 +291,7 @@ namespace AutopilotMonitor.Functions.Functions.Apps
var measured = installed.Where(MetricsMath.HasMeasuredDuration).ToList();
var succeeded = installed.Count;
var failed = summaries.Count(s => s.Status == "Failed");
var incomplete = summaries.Count(s => s.Status == "Incomplete");
var failureRate = MetricsMath.TerminalFailureRatePct(failed, succeeded);
var avgDurationSeconds = measured.Count > 0 ? Math.Round(measured.Average(s => s.DurationSeconds), 0) : 0;
var p95DurationSeconds = Percentile(measured.Select(s => s.DurationSeconds).ToList(), 0.95);
@@ -423,6 +424,7 @@ namespace AutopilotMonitor.Functions.Functions.Apps
Skipped = skipped,
Unmeasured = installed.Count - measured.Count,
Failed = failed,
Incomplete = incomplete,
FailureRate = failureRate,
AvgDurationSeconds = avgDurationSeconds,
P95DurationSeconds = p95DurationSeconds,
@@ -10,8 +10,8 @@ public static class MetricsMath
/// <summary>
/// App-install failure rate over finished installs only: Failed / (Failed + Succeeded), one
/// decimal, 0 when nothing finished. The same outcome-quota convention as the enrollment
/// success rate — "InProgress" rows (still installing, or orphaned by a session that died
/// mid-install) never dilute the rate. Shared by the app-metrics payload and the Apps
/// success rate — "InProgress" rows (still installing) and "Incomplete" rows (observation
/// ended before any outcome) never dilute the rate. Shared by the app-metrics payload and the Apps
/// dashboard aggregations so the definition can't drift between panes.
/// </summary>
public static double TerminalFailureRatePct(int failed, int succeeded)
@@ -94,6 +94,8 @@ public static class MetricsMath
var installed = succeededAll.Where(s => !IsSkipTerminalState(s)).ToList();
var measured = installed.Where(HasMeasuredDuration).ToList();
var failed = g.Where(s => s.Status == "Failed").ToList();
// Observation ended before any outcome — disclosed, outside the rate and the durations.
var incomplete = g.Count(s => s.Status == "Incomplete");
var total = g.Count();
// DoAggregator is the single source for the DO rollup: it filters rows that actually
@@ -109,6 +111,7 @@ public static class MetricsMath
Skipped = skipped.Count,
Unmeasured = installed.Count - measured.Count,
Failed = failed.Count,
Incomplete = incomplete,
// Skips leave the rate (they are not attempts); legacy rows stay in it.
FailureRate = TerminalFailureRatePct(failed.Count, installed.Count),
AvgDurationSeconds = measured.Count > 0 ? Math.Round(measured.Average(s => s.DurationSeconds), 0) : 0,
@@ -151,6 +154,7 @@ public static class MetricsMath
TotalInstalls = summaryList.Count,
TotalSkipped = appGroups.Sum(a => a.Skipped),
TotalUnmeasured = appGroups.Sum(a => a.Unmeasured),
TotalIncomplete = appGroups.Sum(a => a.Incomplete),
TotalCollisionExcluded = totalCollisionExcluded,
SlowestApps = slowestApps,
TopFailingApps = topFailingApps,
@@ -92,6 +92,9 @@ namespace AutopilotMonitor.Functions.Services
classification.HasCompletionEvidenceCandidate = true;
break;
case "agent_shutting_down":
// Every shutdown ends this agent run's observation; the max-lifetime
// flavour additionally carries the verdict mapping below.
classification.AgentShutdownEvent = evt;
if (IsMaxLifetimeAgentShutdown(evt))
classification.AgentMaxLifetimeShutdownEvent = evt;
break;
@@ -705,6 +708,14 @@ namespace AutopilotMonitor.Functions.Services
public EnrollmentEvent? EspFailureEvent { get; set; }
public EnrollmentEvent? SessionStalledEvent { get; set; }
/// <summary>
/// Any <c>agent_shutting_down</c> in the batch — the agent's last word for this run.
/// After the status write, a terminal session closes its still-open app rows on this
/// batch: apps that started after the verdict (the agent keeps watching after
/// enrollment_complete) got their row only after the terminal seam had already run.
/// </summary>
public EnrollmentEvent? AgentShutdownEvent { get; set; }
/// <summary>
/// <c>agent_shutting_down</c> with <c>Data.reason == "max_lifetime"</c> — the V2
/// watchdog shutdown that deliberately carries no enrollment verdict. Mapped to a
@@ -276,6 +276,21 @@ namespace AutopilotMonitor.Functions.Services
updatedSession?.AgentVersion, skewScan);
}
// Observation end: the batch carrying agent_shutting_down is the agent's last word.
// On a terminal session an app row still InProgress now will never get a terminal
// event — close it as Incomplete. The terminal seam already did this on the verdict
// batch; this re-run catches rows opened after the verdict (the IME keeps installing
// and the agent keeps watching after enrollment_complete). Non-terminal sessions
// (reboot shutdown, WhiteGlove part 1) keep their rows open. Idempotent + fail-soft.
if (classification.AgentShutdownEvent != null
&& updatedSession != null
&& (updatedSession.Status == SessionStatus.Succeeded
|| updatedSession.Status == SessionStatus.Failed
|| updatedSession.Status == SessionStatus.Incomplete))
{
await _metricsRepo.CloseOpenAppInstallsForSessionAsync(request.TenantId, request.SessionId);
}
// Auto-analyze fan-out: enqueue a queue message instead of running fire-and-forget
// Task.Run inside the function. The previous in-function approach could be killed
// mid-flight by Functions scale-in (HTTP 200 returned → worker unloaded → rules
@@ -328,7 +328,12 @@ namespace AutopilotMonitor.Functions.Services
// Idempotent, positive-evidence-only; also heals sessions that went
// terminal before the resolution seam existed (hadNoRow backfill).
if (eventCountMoved || hadNoRow)
{
await _metricsRepo.ResolveEspBlockingForSessionAsync(session.TenantId, session.SessionId);
// Same late-batch reasoning: an app row opened after the agent's
// shutdown batch was never closed — the sweep is its self-heal.
await _metricsRepo.CloseOpenAppInstallsForSessionAsync(session.TenantId, session.SessionId);
}
}
if (breakdown == null)
@@ -253,9 +253,11 @@ namespace AutopilotMonitor.Functions.Services
// Status "InProgress" — a real (non-sentinel) value that Merge-mode would
// happily write over a row the terminal batch already closed. Only another
// terminal observation may change a terminal status (Failed→Succeeded on a
// successful retry stays legal).
// successful retry stays legal). Incomplete (observation-end close step) is
// terminal for the same purpose: a straggling progress-only batch must not
// reopen it, while a real terminal observation still wins.
var existingStatus = existing.GetString("Status");
if ((existingStatus == "Succeeded" || existingStatus == "Failed") &&
if ((existingStatus == "Succeeded" || existingStatus == "Failed" || existingStatus == "Incomplete") &&
summary.Status != "Succeeded" && summary.Status != "Failed")
{
summary.Status = existingStatus;
@@ -525,6 +527,66 @@ namespace AutopilotMonitor.Functions.Services
return sets.Contains(appId);
}
/// <summary>
/// Closes the session's app rows that never reached a terminal state: Status
/// "InProgress" (or missing) with no CompletedAt becomes "Incomplete" — terminal,
/// non-failure, outcome unknown. Runs at the terminal seam, again after the batch that
/// carries the agent's shutdown (rows opened after the verdict), and from the 30-day
/// sweep. Idempotent + fail-soft; a real terminal arriving later still wins through the
/// sticky-status reconcile. Returns the number of rows closed.
/// </summary>
public async Task<int> CloseOpenAppInstallsForSessionAsync(string tenantId, string sessionId)
{
try
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.AppInstallSummaries);
var filter = $"PartitionKey eq '{tenantId}' and RowKey ge '{sessionId}_' and RowKey lt '{sessionId}`'";
var query = tableClient.QueryAsync<TableEntity>(
filter: filter,
select: new[] { "PartitionKey", "RowKey", "Status", "CompletedAt" });
var closed = 0;
await foreach (var row in query)
{
if (!ShouldCloseAsIncomplete(row)) continue;
var update = new TableEntity(row.PartitionKey, row.RowKey)
{
["Status"] = "Incomplete"
};
await tableClient.UpsertEntityAsync(update, TableUpdateMode.Merge);
closed++;
}
if (closed > 0)
_logger.LogInformation(
"Session {SessionId}: closed {Count} app rows as Incomplete (no terminal state when observation ended)",
sessionId, closed);
return closed;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Closing open app rows failed for session {SessionId} (fail-soft)", sessionId);
return 0;
}
}
/// <summary>
/// Pure per-row predicate behind <see cref="CloseOpenAppInstallsForSessionAsync"/> —
/// internal static so the contract is pinned by unit tests: only a row that is still
/// open (Status missing, empty or "InProgress", no CompletedAt) closes; every terminal
/// row, including an already-closed Incomplete one, is left alone (idempotency).
/// </summary>
internal static bool ShouldCloseAsIncomplete(TableEntity row)
{
if (row.GetDateTimeOffset("CompletedAt").HasValue) return false;
var status = row.GetString("Status");
return string.IsNullOrEmpty(status) || status == "InProgress";
}
/// <summary>
/// Gets app install summaries for a tenant (fleet-level metrics). When <paramref name="sinceUtc"/>
/// is supplied, a server-side <c>StartedAt ge</c> filter is applied so a windowed view (e.g. the
@@ -254,6 +254,36 @@ describe('buildSessionCoverage — diagnostics package', () => {
});
});
describe('buildSessionCoverage — apps without a terminal state', () => {
// The shutdown emit of the termination handler: what was still in flight when the agent left.
const shutdownSummary = ev('app_tracking_summary', 900, { installingNames: ['Suite'], downloadingNames: [], pendingNames: ['Other'] });
it("names the apps still in flight in the agent's last summary on a terminal session", () => {
const c = buildSessionCoverage({ startedAt: at(0), status: 'Succeeded' }, [
ev('agent_started', 5),
ev('app_tracking_summary', 300, { installingNames: ['Early'], downloadingNames: ['Later'] }),
shutdownSummary,
]);
expect(c.apps).toEqual({ stillInstalling: ['Suite'], stillDownloading: [] });
expect(c.gaps).toEqual([expect.stringContaining('1 app(s) still installing when the agent stopped observing (Suite)')]);
});
it('reports nothing while the session is still live, even with apps in flight', () => {
const c = buildSessionCoverage({ startedAt: at(0), status: 'InProgress' }, [ev('agent_started', 5), shutdownSummary]);
expect(c.apps).toEqual({ stillInstalling: [], stillDownloading: [] });
expect(c.gaps).toEqual([]);
});
it("stays silent when the last summary has nothing in flight (apps that never started are the starved rule's finding)", () => {
const c = buildSessionCoverage({ startedAt: at(0), status: 'Failed' }, [
ev('agent_started', 5),
ev('app_tracking_summary', 900, { installingNames: [], pendingNames: ['Other'] }),
]);
expect(c.apps).toEqual({ stillInstalling: [], stillDownloading: [] });
expect(c.gaps).toEqual([]);
});
});
describe('coverage wiring', () => {
it('SUMMARY_EVENT_FIELDS carries every coverage slice, as data.<key> entries only', () => {
const fields = SUMMARY_EVENT_FIELDS.split(',');
@@ -271,6 +301,7 @@ describe('coverage wiring', () => {
'agent_started', 'agent_late_start', 'historic_ime_replay_detected', 'ime_tracker_degraded', 'ime_pattern_hits',
'collector_degraded', 'spool_pressure_detected', 'telemetry_upload_poisoned', 'telemetry_upload_blocked',
'ingress_backpressure', 'disk_space_low', 'diagnostics_collecting', 'diagnostics_uploaded', 'diagnostics_upload_failed',
'app_tracking_summary',
];
expect([...COVERAGE_EVENT_TYPES].sort()).toEqual([...consulted].sort());
});
@@ -470,6 +470,8 @@ export interface AppAnalyticsSummary {
skipped: number;
unmeasured: number;
failed: number;
/** Still installing when the session's observation ended — outcome unknown, outside the rate. */
incomplete: number;
failureRate: number;
avgDurationSeconds: number;
p95DurationSeconds: number;
@@ -558,6 +560,8 @@ export interface AppMetricsAppGroup {
skipped: number;
unmeasured: number;
failed: number;
/** Still installing when the session's observation ended — outcome unknown, outside the rate. */
incomplete: number;
/** Failed / (failed + succeeded) as a percentage; skips never count as attempts. */
failureRate: number;
/** Average measured FINAL-attempt duration (whole seconds); 0 with no measured installs. */
@@ -591,6 +595,8 @@ export interface AppMetricsResponse {
totalInstalls: number;
totalSkipped: number;
totalUnmeasured: number;
/** Rows still installing when the session's observation ended — outcome unknown, outside the failure rate. */
totalIncomplete: number;
/** Rows excluded from per-app groups because their name-keyed row merged distinct appIds. */
totalCollisionExcluded: number;
slowestApps: AppMetricsAppGroup[];
@@ -31,6 +31,7 @@ export const COVERAGE_EVENT_TYPES: ReadonlySet<string> = new Set([
'diagnostics_collecting',
'diagnostics_uploaded',
'diagnostics_upload_failed',
'app_tracking_summary',
]);
/**
@@ -44,7 +45,8 @@ export const COVERAGE_EVENT_FIELDS =
'data.file,data.firstSkippedPattern,data.lineBudgetBreaks,data.regexTimeouts,data.oversizedLines,data.unanchoredPatterns,data.linesRead,' +
'data.collector,data.reason,data.errorType,' +
'data.pendingItemCount,data.pendingBytes,data.kind,data.itemCount,data.disk_free_gb,' +
'data.truncated,data.includedFiles,data.includedBytes,data.skippedFiles,data.skippedByReason,data.problemsByKind';
'data.truncated,data.includedFiles,data.includedBytes,data.skippedFiles,data.skippedByReason,data.problemsByKind,' +
'data.installingNames,data.downloadingNames';
export interface CoverageWindow {
/** Timestamp of the first `agent_started` — the earliest moment anything was observed live. */
@@ -111,12 +113,20 @@ export interface DiagnosticsCoverage {
problemsByKind?: Record<string, number>;
}
export interface AppsCoverage {
/** Apps the agent's last summary still listed as installing when observation ended (terminal session only). */
stillInstalling: string[];
/** Same for downloading. */
stillDownloading: string[];
}
export interface SessionCoverage {
window: CoverageWindow;
imeTracker: ImeTrackerCoverage;
collectors: CollectorDegradation[];
upload: UploadCoverage;
diagnostics: DiagnosticsCoverage;
apps: AppsCoverage;
/** One calibrated line per observation gap; empty means nothing reported a gap. */
gaps: string[];
}
@@ -165,12 +175,21 @@ function fmtCounts(m: Record<string, number>): string {
return Object.entries(m).map(([k, n]) => `${k}: ${n}`).join(', ');
}
function names(v: unknown): string[] {
return Array.isArray(v) ? v.map(str).filter((s): s is string => s !== null) : [];
}
/** Mirror of the backend SessionStatus terminal set (utils/sessionStatus.ts in the web app). */
function isTerminalSessionStatus(status: unknown): boolean {
return status === 'Succeeded' || status === 'Failed' || status === 'Incomplete';
}
/**
* Folds the session's events (already stripped of historic-replay rows by the caller) into the
* coverage block. Pure: no fetch, no clock.
*/
export function buildSessionCoverage(
session: { startedAt?: string | null },
session: { startedAt?: string | null; status?: string | null },
events: readonly Ev[],
): SessionCoverage {
const byType = new Map<string, Ev[]>();
@@ -366,6 +385,25 @@ export function buildSessionCoverage(
}
}
// ── Apps without a terminal state ───────────────────────────────────────────
// The agent's last app_tracking_summary (the shutdown emit) names what was still in flight.
// Only a terminal session turns that into a gap — while the session is live those apps are
// simply still installing. Apps that never started are the starved rule's finding, not a gap.
const apps: AppsCoverage = { stillInstalling: [], stillDownloading: [] };
if (isTerminalSessionStatus(session.status)) {
const summary = last('app_tracking_summary');
apps.stillInstalling = names(summary?.data?.installingNames);
apps.stillDownloading = names(summary?.data?.downloadingNames);
const inFlight = (verb: string, list: string[]): void => {
if (list.length === 0) return;
gaps.push(
`${list.length} app(s) still ${verb} when the agent stopped observing (${list.join(', ')}): outcome unknown — not observed to fail, not observed to finish.`,
);
};
inFlight('installing', apps.stillInstalling);
inFlight('downloading', apps.stillDownloading);
}
return {
window: {
observedFrom,
@@ -380,6 +418,7 @@ export function buildSessionCoverage(
collectors,
upload,
diagnostics,
apps,
gaps,
};
}
@@ -137,6 +137,8 @@ namespace AutopilotMonitor.Shared.DataAccess
Task<SessionTimeBreakdown?> GetSessionTimeBreakdownAsync(string tenantId, string sessionId);
/// <summary>Re-joins the session's app rows against its latest esp_config_detected lists (positive evidence only). Idempotent + fail-soft — the terminal seam runs it once; the sweep re-runs it when late events changed the stream.</summary>
Task ResolveEspBlockingForSessionAsync(string tenantId, string sessionId);
/// <summary>Closes the session's app rows that never reached a terminal state (InProgress → Incomplete, outcome unknown). Idempotent + fail-soft — the terminal seam, the agent-shutdown batch and the sweep all call it; a real terminal arriving later still wins. Returns the number of rows closed.</summary>
Task<int> CloseOpenAppInstallsForSessionAsync(string tenantId, string sessionId);
Task<bool> SaveTimeAttributionAggregateAsync(TimeAttributionDailyAggregate aggregate);
/// <summary>Aggregate rows of one tenant partition ("global" allowed) for an inclusive date range.</summary>
Task<List<TimeAttributionDailyAggregate>> GetTimeAttributionAggregatesAsync(string tenantId, DateTime startDate, DateTime endDate);
@@ -43,7 +43,12 @@ namespace AutopilotMonitor.Shared.Models
public bool AppIdCollision { get; set; }
/// <summary>
/// Lifecycle status: Succeeded, Failed, InProgress, or empty.
/// Lifecycle status: Succeeded, Failed, InProgress, Incomplete, or empty.
/// "Incomplete" is terminal and non-failure: the row was still InProgress when the
/// session's observation ended (terminal session, agent gone), so no outcome was ever
/// seen. Written only by the observation-end close step, never by an ingest fold;
/// CompletedAt and DurationSeconds stay unset (unknown, reported as unmeasured) and a
/// real terminal arriving later overrides it.
/// Empty (default) is a sentinel meaning "no status-relevant event observed in the current
/// aggregation batch". Aggregators only set a real value when they see started / completed /
/// failed / skipped. The storage layer omits the column from the upsert when this is empty
@@ -16,6 +16,8 @@ namespace AutopilotMonitor.Shared.Models
public int TotalInstalls { get; set; }
public int TotalSkipped { get; set; }
public int TotalUnmeasured { get; set; }
/// <summary>Rows still installing when the session's observation ended — outcome unknown, outside the failure rate.</summary>
public int TotalIncomplete { get; set; }
/// <summary>Rows excluded from per-app groups because their name-keyed row merged distinct appIds.</summary>
public int TotalCollisionExcluded { get; set; }
public IReadOnlyList<AppMetricsAppGroup> SlowestApps { get; set; } = default!;
@@ -32,6 +34,8 @@ namespace AutopilotMonitor.Shared.Models
public int Skipped { get; set; }
public int Unmeasured { get; set; }
public int Failed { get; set; }
/// <summary>Still installing when the session's observation ended — outcome unknown, outside the rate.</summary>
public int Incomplete { get; set; }
/// <summary>Failed / (failed + succeeded) as a percentage; skips never count as attempts.</summary>
public double FailureRate { get; set; }
/// <summary>Average measured FINAL-attempt duration (whole seconds); 0 with no measured installs.</summary>
@@ -82,6 +82,8 @@ namespace AutopilotMonitor.Shared.Models
public int Skipped { get; set; }
public int Unmeasured { get; set; }
public int Failed { get; set; }
/// <summary>Still installing when the session's observation ended — outcome unknown, outside the rate.</summary>
public int Incomplete { get; set; }
public double FailureRate { get; set; }
public double AvgDurationSeconds { get; set; }
public int P95DurationSeconds { get; set; }
@@ -26,6 +26,7 @@ export interface UseSessionDerivedDataReturn {
gatherRulesSucceeded: boolean;
displayStatus: string;
enrollmentDurationFromEvents: string | null;
lastObservedAtMs: number | null;
standbySeconds: number | null;
phaseNamesMap: Record<number, string>;
phaseOrder: string[];
@@ -152,6 +153,24 @@ export function useSessionDerivedData(
return `${Math.floor(durationSec / 3600)}h ${Math.floor((durationSec % 3600) / 60)}m`;
}, [events]);
// Last moment the agent reported anything — the end of the observation window, in the
// device clock frame the event timestamps share. Deliberately NOT the enrollment duration
// above: that stops at enrollment_complete, while apps keep installing after the verdict
// for as long as the agent still watches. `session.lastEventAt` is the server-tracked
// maximum over every event; the loaded events are the fallback for rows predating it.
const sessionLastEventAt = session?.lastEventAt ?? null;
const lastObservedAtMs = useMemo(() => {
const fromSession = sessionLastEventAt ? new Date(sessionLastEventAt).getTime() : NaN;
if (Number.isFinite(fromSession)) return fromSession;
let max: number | null = null;
for (const e of events) {
if (e.source === "SystemTimelineWatcher") continue;
const t = new Date(e.timestamp).getTime();
if (Number.isFinite(t) && (max === null || t > max)) max = t;
}
return max;
}, [events, sessionLastEventAt]);
// Total observed sleep/standby seconds (system_sleep_episode ground truth). The wall-clock
// duration deliberately keeps the pause — this number tells the story next to it ("1h 20m,
// 56m of it standby"). Dedup on enteredAt: the same episode can be observed twice (agent
@@ -242,6 +261,7 @@ export function useSessionDerivedData(
gatherRulesSucceeded: !!gatherRulesSucceeded,
displayStatus,
enrollmentDurationFromEvents,
lastObservedAtMs,
standbySeconds,
phaseNamesMap,
phaseOrder,
@@ -17,6 +17,7 @@ import { useLatestVersions } from '@/lib/useLatestVersions';
import { useScriptDisplayNames } from '@/lib/scriptDisplayNames';
import { api } from "@/lib/api";
import { isGuid } from "@/utils/inputValidation";
import { isTerminalStatus } from "@/utils/sessionStatus";
import { ApiError, fetchBlob, fetchJson, fetchOk, jsonBody } from "@/lib/apiClient";
import { useSessionAnalysis } from "./hooks/useSessionAnalysis";
@@ -197,6 +198,7 @@ function SessionDetailContent() {
isGatherRulesSession,
displayStatus,
enrollmentDurationFromEvents,
lastObservedAtMs,
standbySeconds,
isSkipUserStatusPage,
isWhiteGloveSession,
@@ -209,6 +211,11 @@ function SessionDetailContent() {
userEnrollGrouped,
} = derived;
// Observation end for the progress panels: only once the session is terminal does a row
// without a terminal event stop being "still running" — it becomes incomplete, its timer
// bounded by the agent's last report. Live sessions pass null and keep ticking.
const observedUntilMs = isTerminalStatus(displayStatus) ? lastObservedAtMs : null;
// All phase keys currently present. For WhiteGlove sessions we use prefixed keys
// (pre-X, user-X) to avoid collisions.
const allPhaseKeys = useMemo(
@@ -679,6 +686,7 @@ function SessionDetailContent() {
e => e.eventType === "download_progress" || e.eventType === "app_download_started" || e.eventType === "app_install_skipped"
)}
summaryStats={appSummaryStats}
observedUntilMs={observedUntilMs}
/>
</div>
)}
@@ -697,6 +705,7 @@ function SessionDetailContent() {
|| e.eventType === "realmjoin_package_started" || e.eventType === "realmjoin_package_completed"
)}
summaryStats={appSummaryStats}
observedUntilMs={observedUntilMs}
/>
</div>
)}
@@ -61,6 +61,9 @@ interface SummaryStats {
interface DownloadProgressProps {
events: DownloadEvent[];
summaryStats?: SummaryStats | null;
// Epoch ms of the agent's last report, passed only once the session is terminal. A download
// that never finished by then is incomplete, not active.
observedUntilMs?: number | null;
}
interface DoStats {
@@ -96,6 +99,8 @@ interface DownloadItem {
// progress). Rows without it are phantom "completed" rows from the terminal
// download_progress event and are dropped after folding.
hasByteEvidence: boolean;
// Still downloading when the session's observation ended — outcome unknown, not active.
isIncomplete: boolean;
firstSeenIndex: number;
eventData?: Record<string, unknown>;
doStats?: DoStats | null;
@@ -142,7 +147,7 @@ function effectiveDurationMs(dl: DownloadItem): number {
return 0;
}
export default function DownloadProgress({ events, summaryStats }: DownloadProgressProps) {
export default function DownloadProgress({ events, summaryStats, observedUntilMs = null }: DownloadProgressProps) {
// Legacy-agent guard: drop download events replayed from a previous enrollment's IME log.
// Silent (empty finals set) — the InstallProgress panel already reports the hidden count
// for the same apps; a second note here would double-report them.
@@ -250,18 +255,23 @@ export default function DownloadProgress({ events, summaryStats }: DownloadProgr
isSkipped: isSkippedEvent || (existing?.isSkipped ?? false),
isUninstall: (intent?.toLowerCase().includes("uninstall") ?? false) || (existing?.isUninstall ?? false),
hasByteEvidence: hasByteActivity(filterInput) || (existing?.hasByteEvidence ?? false),
isIncomplete: false,
firstSeenIndex: existing?.firstSeenIndex ?? insertionIndex++,
eventData: d,
doStats,
});
}
return Array.from(downloadMap.values()).sort((a, b) => {
const folded = Array.from(downloadMap.values()).sort((a, b) => {
// Keep stable visual order based on first appearance only.
// This avoids active items jumping when completion status changes.
return a.firstSeenIndex - b.firstSeenIndex;
});
}, [current]);
// Observation ended (terminal session): a download that never finished stays at its last
// observed bytes and is reported as incomplete rather than active.
if (observedUntilMs == null) return folded;
return folded.map(d => (!d.isComplete && !d.isSkipped && d.hasByteEvidence ? { ...d, isIncomplete: true } : d));
}, [current, observedUntilMs]);
const [expanded, setExpanded] = useState(true);
const [showSkipped, setShowSkipped] = useState(false);
@@ -327,7 +337,8 @@ export default function DownloadProgress({ events, summaryStats }: DownloadProgr
// Pills stay install-scope (matching the "X of Y downloaded" header, whose summary
// buckets uninstalls under skipped) — uninstall downloads get their own count, and
// rows without any observed download activity sit behind the "no download" toggle.
const activeCount = downloads.filter(d => !d.isComplete && !d.isSkipped && !d.isUninstall && d.hasByteEvidence).length;
const activeCount = downloads.filter(d => !d.isComplete && !d.isSkipped && !d.isUninstall && d.hasByteEvidence && !d.isIncomplete).length;
const incompleteCount = downloads.filter(d => d.isIncomplete && !d.isUninstall).length;
const completedCount = downloads.filter(d => d.isComplete && !d.isSkipped && !d.isUninstall && d.hasByteEvidence).length;
const uninstallCount = downloads.filter(d => d.isUninstall && !d.isSkipped && d.hasByteEvidence).length;
const noDownloadCount = downloads.filter(d => !d.isSkipped && !d.hasByteEvidence).length;
@@ -389,6 +400,15 @@ export default function DownloadProgress({ events, summaryStats }: DownloadProgr
{completedCount} completed
</span>
)}
{incompleteCount > 0 && (
// Slate like the session-level Incomplete badge: unknown outcome, not a failure.
<span
className="px-2 py-0.5 rounded-full bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300 font-medium"
title="Still downloading when the agent stopped observing — the outcome was never seen. Not a failure."
>
{incompleteCount} incomplete
</span>
)}
{uninstallCount > 0 && (
// Muted gray on purpose (dark-mode covered by the global gray overrides) —
// it's context, not a state the user needs to act on.
@@ -481,6 +501,11 @@ function DownloadItem({ download: dl, progressPercent }: { download: DownloadIte
<svg className="w-4 h-4 text-green-500 flex-shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M5 13l4 4L19 7" />
</svg>
) : dl.isIncomplete ? (
// Question mark in slate, no pulse — nothing is transferring any more.
<svg className="w-4 h-4 text-slate-400 flex-shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M8.228 9c.549-1.165 2.03-2 3.772-2 2.21 0 4 1.343 4 3 0 1.4-1.278 2.575-3.006 2.907-.542.104-.994.54-.994 1.093m0 3h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
) : (
<svg className="w-4 h-4 text-blue-500 flex-shrink-0 animate-pulse" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4" />
@@ -500,9 +525,17 @@ function DownloadItem({ download: dl, progressPercent }: { download: DownloadIte
Uninstall
</span>
)}
{dl.isIncomplete && (
<span
className="text-xs px-2 py-0.5 rounded-full bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300 font-medium"
title="Still downloading when the agent stopped observing. The outcome is unknown — the download may well have finished after the session ended."
>
Incomplete
</span>
)}
</div>
<div className="flex items-center space-x-3 text-xs text-gray-500 flex-shrink-0 ml-2">
{!dl.isComplete && !dl.isSkipped && dl.downloadRateBps > 0 && (
{!dl.isComplete && !dl.isSkipped && !dl.isIncomplete && dl.downloadRateBps > 0 && (
<span className="font-medium text-blue-600">{formatThroughput(dl.downloadRateBps, "0 B/s")}</span>
)}
{dl.eventData && Object.keys(dl.eventData).length > 0 && (
@@ -522,7 +555,7 @@ function DownloadItem({ download: dl, progressPercent }: { download: DownloadIte
<div className="w-full h-2 bg-gray-200 rounded-full overflow-hidden">
<div
className={`h-full rounded-full transition-all duration-500 ${
dl.isComplete ? "bg-green-500" : "bg-blue-500"
dl.isComplete ? "bg-green-500" : dl.isIncomplete ? "bg-slate-400" : "bg-blue-500"
}`}
style={{ width: `${progressPercent}%` }}
/>
@@ -3,7 +3,7 @@
import { useEffect, useMemo, useState } from "react";
import { getErrorCodeEntry, formatErrorCode, errorCodeTooltip } from "@/utils/errorCodeMap";
import { partitionHistoricReplayEvents } from "@/lib/historicReplay";
import { buildInstallItems, isRebootOrRetryClass, type InstallEvent, type InstallItem, type InstallSource } from "@/lib/installProgress";
import { applyObservationEnd, buildInstallItems, isRebootOrRetryClass, type InstallEvent, type InstallItem, type InstallSource } from "@/lib/installProgress";
import TruncatedLabel from "@/components/TruncatedLabel";
import PendingAppRow from "@/components/PendingAppRow";
@@ -21,6 +21,9 @@ interface SummaryStats {
interface InstallProgressProps {
events: InstallEvent[];
summaryStats?: SummaryStats | null;
// Epoch ms of the agent's last report, passed only once the session is terminal. A row still
// installing at that point renders as Incomplete instead of ticking against the wall clock.
observedUntilMs?: number | null;
}
// Finals counted by the historic-replay partition — one per hidden install, so the note
@@ -56,7 +59,7 @@ function formatDuration(ms: number): string {
return `${hours}h ${remainingMinutes}m`;
}
export default function InstallProgress({ events, summaryStats }: InstallProgressProps) {
export default function InstallProgress({ events, summaryStats, observedUntilMs = null }: InstallProgressProps) {
// Legacy-agent guard: split off app events replayed from a previous enrollment's IME log
// (newer agents suppress them at the source) so week-old installs never render as current.
// office_*/realmjoin_* events never carry rejectedSourceTimestamp and pass through untouched.
@@ -65,7 +68,10 @@ export default function InstallProgress({ events, summaryStats }: InstallProgres
[events]
);
const installs = useMemo(() => buildInstallItems(current), [current]);
const installs = useMemo(
() => applyObservationEnd(buildInstallItems(current), observedUntilMs),
[current, observedUntilMs]
);
const [expanded, setExpanded] = useState(true);
const [showSkipped, setShowSkipped] = useState(false);
@@ -101,7 +107,10 @@ export default function InstallProgress({ events, summaryStats }: InstallProgres
if (installs.length === 0 && historicCount === 0) return null;
const activeCount = installs.filter(d => d.state === "Installing").length;
const activeCount = installs.filter(d => d.state === "Installing" && !d.isIncomplete).length;
// Still installing when observation ended — unknown outcome, counted apart from "active"
// so a finished session never claims an app is running.
const incompleteCount = installs.filter(d => d.isIncomplete).length;
const completedCount = installs.filter(d => d.state === "Installed").length;
// Enforced uninstall assignments — counted separately so the completed pill keeps
// agreeing with the "X of Y installed" header (which buckets by intent agent-side).
@@ -178,6 +187,15 @@ export default function InstallProgress({ events, summaryStats }: InstallProgres
{likelyStuckCount} likely stuck
</span>
)}
{incompleteCount > 0 && (
// Slate like the session-level Incomplete badge: unknown outcome, not a failure.
<span
className="px-2 py-0.5 rounded-full bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300 font-medium"
title="Still installing when the agent stopped observing — the outcome was never seen. Not a failure."
>
{incompleteCount} incomplete
</span>
)}
{postponedCount > 0 && (
<span className="px-2 py-0.5 rounded-full bg-amber-100 text-amber-700 font-medium">
{postponedCount} postponed
@@ -237,17 +255,17 @@ function InstallItemRow({ item }: { item: InstallItem }) {
const [liveElapsed, setLiveElapsed] = useState<{ startedAt: string; elapsedMs: number } | null>(null);
useEffect(() => {
if (item.state !== "Installing" || !item.startedAt) return;
if (item.state !== "Installing" || item.isIncomplete || !item.startedAt) return;
const startedAt = item.startedAt;
const startTime = new Date(startedAt).getTime();
const tick = () => setLiveElapsed({ startedAt, elapsedMs: Date.now() - startTime });
tick();
const id = setInterval(tick, 1000);
return () => clearInterval(id);
}, [item.state, item.startedAt]);
}, [item.state, item.isIncomplete, item.startedAt]);
const elapsedMs =
item.state === "Installing" && item.startedAt && liveElapsed?.startedAt === item.startedAt
item.state === "Installing" && !item.isIncomplete && item.startedAt && liveElapsed?.startedAt === item.startedAt
? liveElapsed.elapsedMs
: null;
@@ -295,6 +313,12 @@ function InstallItemRow({ item }: { item: InstallItem }) {
<svg className="w-4 h-4 text-green-500 flex-shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M5 13l4 4L19 7" />
</svg>
) : item.isIncomplete ? (
// Same question-mark as "likely stuck" (we don't know), in slate: no pulse — nothing
// is running any more.
<svg className="w-4 h-4 text-slate-400 flex-shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M8.228 9c.549-1.165 2.03-2 3.772-2 2.21 0 4 1.343 4 3 0 1.4-1.278 2.575-3.006 2.907-.542.104-.994.54-.994 1.093m0 3h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" />
</svg>
) : (
<svg className="w-4 h-4 text-indigo-500 flex-shrink-0 animate-pulse" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 3v2m6-2v2M9 19v2m6-2v2M5 9H3m2 6H3m18-6h-2m2 6h-2M7 19h10a2 2 0 002-2V7a2 2 0 00-2-2H7a2 2 0 00-2 2v10a2 2 0 002 2zM9 9h6v6H9V9z" />
@@ -364,6 +388,14 @@ function InstallItemRow({ item }: { item: InstallItem }) {
{item.state === "Postponed" && (
<span className="text-xs px-2 py-0.5 rounded-full bg-amber-200 text-amber-700 font-medium">Postponed</span>
)}
{item.isIncomplete && (
<span
className="text-xs px-2 py-0.5 rounded-full bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300 font-medium"
title="Still installing when the agent stopped observing. The outcome is unknown — the install may well have finished after the session ended."
>
Incomplete
</span>
)}
{item.state === "Preinstalled" && (
<span
className="text-xs px-2 py-0.5 rounded-full bg-sky-200 text-sky-800 font-medium"
@@ -374,6 +406,15 @@ function InstallItemRow({ item }: { item: InstallItem }) {
)}
</div>
<div className="flex items-center space-x-3 text-xs text-gray-500 flex-shrink-0 ml-2">
{item.isIncomplete && item.observedMs != null && (
// Lower bound: watched this long, then the agent's reports stopped.
<span
className="font-medium text-gray-400 tabular-nums"
title={`Watched for ${formatDuration(item.observedMs)} until the agent's last report — the total install time is unknown.`}
>
{`≥ ${formatDuration(item.observedMs)}`}
</span>
)}
{elapsedMs != null && elapsedMs > 0 && (
<span className="font-medium text-indigo-600 tabular-nums">{formatDuration(elapsedMs)}</span>
)}
@@ -1,5 +1,5 @@
import { describe, it, expect } from "vitest";
import { buildInstallItems, isRebootOrRetryClass, type InstallEvent } from "@/lib/installProgress";
import { applyObservationEnd, buildInstallItems, isRebootOrRetryClass, type InstallEvent } from "@/lib/installProgress";
function evt(eventType: string, timestamp: string, data: Record<string, unknown>): InstallEvent {
return { eventType, timestamp, data };
@@ -226,3 +226,41 @@ describe("isRebootOrRetryClass", () => {
expect(isRebootOrRetryClass(undefined)).toBe(false);
});
});
describe("applyObservationEnd", () => {
// Shape of session ac5660b8: the app started 143 s after enrollment_complete and was still
// installing when the agent sent its last event 30 s later — no terminal event ever came.
const stillInstalling = evt("app_install_started", "2026-09-09T06:45:18Z", { appName: "Suite", appId: "a1" });
const finished = [
evt("app_install_started", "2026-09-09T06:40:31Z", { appName: "Teams", appId: "t1" }),
evt("app_install_completed", "2026-09-09T06:41:05Z", { appName: "Teams", appId: "t1", state: "Installed" }),
];
const lastReport = Date.parse("2026-09-09T06:45:48Z");
it("leaves every row untouched while the session is still live (null observation end)", () => {
const items = buildInstallItems([stillInstalling, ...finished]);
expect(applyObservationEnd(items, null)).toBe(items);
expect(items.every(i => !i.isIncomplete)).toBe(true);
});
it("marks a row still installing at the observation end as incomplete with the watched span", () => {
const [suite] = applyObservationEnd(buildInstallItems([stillInstalling]), lastReport);
expect(suite.state).toBe("Installing");
expect(suite.isIncomplete).toBe(true);
expect(suite.observedMs).toBe(30 * 1000);
});
it("never touches rows that reached a terminal state", () => {
const items = applyObservationEnd(buildInstallItems([...finished, stillInstalling]), lastReport);
const teams = items.find(i => i.appName === "Teams")!;
expect(teams.isIncomplete).toBe(false);
expect(teams.observedMs).toBeUndefined();
expect(teams.durationMs).toBe(34 * 1000);
});
it("clamps the watched span to zero when the observation end precedes the start (clock skew)", () => {
const [suite] = applyObservationEnd(buildInstallItems([stillInstalling]), Date.parse("2026-09-09T06:45:00Z"));
expect(suite.isIncomplete).toBe(true);
expect(suite.observedMs).toBe(0);
});
});
@@ -94,6 +94,11 @@ export interface InstallItem {
isLikelyStuck: boolean;
isDetectionFailure: boolean;
isInstallFailure: boolean;
// Set by applyObservationEnd, never by the fold: the row was still installing when the
// session's observation ended (agent gone, session terminal). Outcome unknown — neither a
// failure nor a success — and `observedMs` is the span actually watched, a lower bound.
isIncomplete: boolean;
observedMs?: number;
firstSeenIndex: number;
eventData?: Record<string, unknown>;
}
@@ -185,7 +190,7 @@ export function buildInstallItems(events: InstallEvent[]): InstallItem[] {
const isCompleted = (type === "app_install_completed" && !isSkippedCompletion) || type === "office_install_completed" || (type === "realmjoin_package_completed" && !rjFailed);
const isFailed = type === "app_install_failed" || type === "office_install_failed" || rjFailed;
const base = { key, source, appName, appId, intent: intent ?? existing?.intent };
const base = { key, source, appName, appId, intent: intent ?? existing?.intent, isIncomplete: false };
if (isStarted) {
// Don't reset an app that already completed — later batch re-scans
@@ -334,3 +339,19 @@ export function buildInstallItems(events: InstallEvent[]): InstallItem[] {
return Array.from(installMap.values()).sort((a, b) => a.firstSeenIndex - b.firstSeenIndex);
}
/**
* Marks rows that were still installing when observation ended. `observedUntilMs` is the last
* moment the agent reported anything (session `lastEventAt`) and must be passed only once the
* session is terminal — `null` means "still live", and every row comes back untouched so the
* panel keeps its live timer. A start after the observation end (clock skew) clamps to 0.
*/
export function applyObservationEnd(items: InstallItem[], observedUntilMs: number | null): InstallItem[] {
if (observedUntilMs == null) return items;
return items.map(item => {
if (item.state !== "Installing" || !item.startedAt) return item;
const startedMs = new Date(item.startedAt).getTime();
const observedMs = Number.isFinite(startedMs) ? Math.max(0, observedUntilMs - startedMs) : 0;
return { ...item, isIncomplete: true, observedMs };
});
}
@@ -2845,6 +2845,14 @@
"name": "number"
}
},
{
"name": "incomplete",
"type": {
"kind": "primitive",
"name": "number"
},
"doc": "Still installing when the session's observation ended — outcome unknown, outside the rate."
},
{
"name": "failureRate",
"type": {
@@ -3232,6 +3240,14 @@
"name": "number"
}
},
{
"name": "incomplete",
"type": {
"kind": "primitive",
"name": "number"
},
"doc": "Still installing when the session's observation ended — outcome unknown, outside the rate."
},
{
"name": "failureRate",
"type": {
@@ -3398,6 +3414,14 @@
"name": "number"
}
},
{
"name": "totalIncomplete",
"type": {
"kind": "primitive",
"name": "number"
},
"doc": "Rows still installing when the session's observation ended — outcome unknown, outside the failure rate."
},
{
"name": "totalCollisionExcluded",
"type": {
@@ -470,6 +470,8 @@ export interface AppAnalyticsSummary {
skipped: number;
unmeasured: number;
failed: number;
/** Still installing when the session's observation ended — outcome unknown, outside the rate. */
incomplete: number;
failureRate: number;
avgDurationSeconds: number;
p95DurationSeconds: number;
@@ -558,6 +560,8 @@ export interface AppMetricsAppGroup {
skipped: number;
unmeasured: number;
failed: number;
/** Still installing when the session's observation ended — outcome unknown, outside the rate. */
incomplete: number;
/** Failed / (failed + succeeded) as a percentage; skips never count as attempts. */
failureRate: number;
/** Average measured FINAL-attempt duration (whole seconds); 0 with no measured installs. */
@@ -591,6 +595,8 @@ export interface AppMetricsResponse {
totalInstalls: number;
totalSkipped: number;
totalUnmeasured: number;
/** Rows still installing when the session's observation ended — outcome unknown, outside the failure rate. */
totalIncomplete: number;
/** Rows excluded from per-app groups because their name-keyed row merged distinct appIds. */
totalCollisionExcluded: number;
slowestApps: AppMetricsAppGroup[];