From 049b6e5b4a2d8b62cdafd7cbd7cf4b1dea60b76b Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Fri, 10 Jul 2026 10:26:37 +0200 Subject: [PATCH] fix(backend): reconcile silent sessions to Succeeded when user provably finished setup (session 294ab5b4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The timeout classifier only read the ESP registry rollup, so a session where the user reached the desktop and completed Windows Hello — with only the RealmJoin completion gate still pending when the agent went silent — was labeled AwaitingUser (and would have graduated to Incomplete at grace expiry). Desktop arrival + a positive Hello terminal are the agent's own Classic completion AND-gate; the RJ gate self-releases after 60 min, so such a session's enrollment succeeded and only the completion report never left the device. - EnrollmentTimeoutClassifier: rollup now carries DesktopArrived, HelloResolved (hello_provisioning_completed/hello_skipped only) and RealmJoinDetected/Resolved (realmjoin_resolved/realmjoin_timeout); new rule 4 reconciles desktop+Hello sessions to Succeeded with an RJ-aware reason. Explicit failure and agent_emergency_break still win. - FailureSnapshotBuilder: schema v3 embeds the new evidence (helloResolved, realmJoinDetected, realmJoinResolved). - Web FailureSnapshotBlock: renders Hello completed + RealmJoin rows. - Design doc: decision table updated; desktop_arrived ALONE stays rejected as a completion signal. Co-Authored-By: Claude Fable 5 --- .../enrollment-status-reclassification.md | 23 +++- .../EnrollmentTimeoutClassifierTests.cs | 100 ++++++++++++++++++ .../FailureSnapshotBuilderTests.cs | 19 ++++ .../Services/EnrollmentTimeoutClassifier.cs | 59 ++++++++++- .../Services/FailureSnapshotBuilder.cs | 9 +- .../Services/MaintenanceService.cs | 2 + .../components/FailureSnapshotBlock.tsx | 18 ++++ 7 files changed, 221 insertions(+), 9 deletions(-) diff --git a/docs/design/enrollment-status-reclassification.md b/docs/design/enrollment-status-reclassification.md index 689f8a44..323c7afa 100644 --- a/docs/design/enrollment-status-reclassification.md +++ b/docs/design/enrollment-status-reclassification.md @@ -100,13 +100,28 @@ Given a non-terminal session past the inactivity window: 1. Explicit enrollment_failed / esp_failure present → Failed (real) 2. AccountSetup rollup = ALL subcategories succeeded OR enrollment_complete / whiteglove_complete present → Succeeded (reconcile) -3. DeviceSetup rollup = ALL subcategories succeeded (or 30s +3. agent_emergency_break present (agent provably gone) → Incomplete (skip grace) +4. desktop_arrived AND positive Hello terminal + (hello_provisioning_completed / hello_skipped) present → Succeeded (reconcile) +5. DeviceSetup rollup = ALL subcategories succeeded (or 30s fallback) AND no failure event: a. within grace (now - StartedAt < SessionGraceHours) → AwaitingUser (non-terminal) b. grace expired → Incomplete (terminal, non-failure) -4. else (silence before DeviceSetup complete, no failure) → Incomplete +6. else (silence before DeviceSetup complete, no failure) → Incomplete ``` +Step 4 (added 2026-07-10, trigger session `294ab5b4`): desktop arrival **plus** a positive +Hello terminal is exactly the agent's own Classic completion AND‑gate — once both are in, +the only thing that can still block the agent's `enrollment_complete` is the RealmJoin +completion gate, which self‑releases via a 60‑min hard timeout. A session silent long +enough to reach the sweep is past every agent‑side wait: the enrollment succeeded and only +the completion report never left the device (shutdown / process kill / egress cut). +Labeling it `AwaitingUser` is factually wrong — the user was provably there. The reason +string distinguishes "RealmJoin deployment never reported completion" (RJ detected but +unresolved) from the generic "agent went silent before reporting completion". The failure +snapshot (schema v3) carries the evidence: `helloResolved`, `realmJoinDetected`, +`realmJoinResolved`. + Rationale for each bucket sizing is in the Evidence table above. ⚠️ Gate step 2/3 on the **subcategory rollup**, not `categorySucceeded`. @@ -197,7 +212,9 @@ Platform‑wide the same reclassification de‑noises every tenant's failure rat - **Agent heartbeat** — rejected: continuous client load + zombie risk; the completion truth is already persisted (registry/ESP tracking survives reboots) and the wait belongs in the backend. -- **`desktop_arrived` as completion** — rejected: fires while the user phase is still running. +- **`desktop_arrived` ALONE as completion** — rejected: fires while the user phase is still + running. (`desktop_arrived` **+ a positive Hello terminal** is accepted since 2026-07-10 — + that pair is the agent's own Classic completion AND‑gate, see classification step 4.) ## Acceptance criteria diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs index cdbaeee2..147c8c0e 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs @@ -88,6 +88,28 @@ public class EnrollmentTimeoutClassifierTests Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("agent_emergency_break") }).HasAgentEmergencyBreak); } + [Fact] + public void ExtractRollup_detects_desktop_hello_and_realmjoin_evidence() + { + var r = EnrollmentTimeoutClassifier.ExtractRollup(new[] + { + Evt("desktop_arrived"), Evt("hello_provisioning_completed"), Evt("realmjoin_detected"), + }); + Assert.True(r.DesktopArrived); + Assert.True(r.HelloResolved); + Assert.True(r.RealmJoinDetected); + Assert.False(r.RealmJoinResolved); + + // hello_skipped is the other positive Hello terminal (agent raises HelloResolved for it). + Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("hello_skipped") }).HelloResolved); + // Negative Hello terminals leave the agent waiting — must NOT count as resolved. + Assert.False(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("hello_provisioning_failed") }).HelloResolved); + Assert.False(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("hello_completion_timeout") }).HelloResolved); + // Both RealmJoin gate terminals count as resolved (phase 110 or 60-min hard timeout). + Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("realmjoin_resolved") }).RealmJoinResolved); + Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("realmjoin_timeout") }).RealmJoinResolved); + } + [Fact] public void Classify_emergency_break_skips_grace_and_is_Incomplete() { @@ -138,6 +160,84 @@ public class EnrollmentTimeoutClassifierTests Assert.Equal(SessionStatus.Succeeded, status); } + // -------- "user completed setup" reconcile (session 294ab5b4) -------- + + [Fact] + public void Classify_desktop_plus_hello_with_unresolved_realmjoin_reconciles_to_Succeeded() + { + // Session 294ab5b4 replay: DeviceSetup 4/4, AccountSetup frozen at 1/5 after the user + // hit the desktop, Hello provisioned, RealmJoin detected but never resolved — agent + // went silent mid-deployment. The user was provably there; "AwaitingUser" is wrong. + var (status, reason) = Classify(new[] + { + Esp(DeviceSetup44), Esp(AccountSetup15), + Evt("desktop_arrived"), Evt("hello_provisioning_completed"), Evt("realmjoin_detected"), + }, hoursSinceStart: 6); + Assert.Equal(SessionStatus.Succeeded, status); + Assert.Contains("RealmJoin", reason); + } + + [Fact] + public void Classify_desktop_plus_hello_without_realmjoin_reconciles_to_Succeeded() + { + // Both Classic completion prerequisites in and no gate pending: the agent died in the + // narrow window before it could report enrollment_complete. + var (status, reason) = Classify(new[] + { + Esp(DeviceSetup44), Esp(AccountSetup15), + Evt("desktop_arrived"), Evt("hello_provisioning_completed"), + }, hoursSinceStart: 6); + Assert.Equal(SessionStatus.Succeeded, status); + Assert.Contains("silent before reporting completion", reason); + } + + [Fact] + public void Classify_desktop_plus_hello_skipped_also_reconciles_to_Succeeded() + { + var (status, _) = Classify(new[] + { + Esp(DeviceSetup44), Evt("desktop_arrived"), Evt("hello_skipped"), + }, hoursSinceStart: 6); + Assert.Equal(SessionStatus.Succeeded, status); + } + + [Fact] + public void Classify_desktop_without_hello_terminal_stays_AwaitingUser() + { + // desktop_arrived alone is explicitly NOT a completion signal (design doc) — the user + // may still be mid Hello wizard / user phase. Falls through to the AwaitingUser rule. + var (status, _) = Classify(new[] + { + Esp(DeviceSetup44), Esp(AccountSetup15), Evt("desktop_arrived"), + }, hoursSinceStart: 6); + Assert.Equal(SessionStatus.AwaitingUser, status); + } + + [Fact] + public void Classify_explicit_failure_beats_desktop_plus_hello() + { + var (status, _) = Classify(new[] + { + Esp(DeviceSetup44), Evt("desktop_arrived"), Evt("hello_provisioning_completed"), + Evt("enrollment_failed"), + }, hoursSinceStart: 6); + Assert.Equal(SessionStatus.Failed, status); + } + + [Fact] + public void Classify_emergency_break_beats_desktop_plus_hello() + { + // The break means the agent stayed alive to the 48h absolute cap WITHOUT completing — + // despite both prerequisites being in, something blocked completion for two days. + // That is not a success; the honest verdict stays Incomplete. + var (status, _) = Classify(new[] + { + Esp(DeviceSetup44), Evt("desktop_arrived"), Evt("hello_provisioning_completed"), + Evt("agent_emergency_break"), + }, hoursSinceStart: 50); + Assert.Equal(SessionStatus.Incomplete, status); + } + [Fact] public void Classify_device_provisioned_user_phase_pending_within_grace_is_AwaitingUser() { diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/FailureSnapshotBuilderTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/FailureSnapshotBuilderTests.cs index 978bcace..e44a01f4 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/FailureSnapshotBuilderTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/FailureSnapshotBuilderTests.cs @@ -311,6 +311,25 @@ public class FailureSnapshotBuilderTests Assert.DoesNotContain("hello_terminal", missing); } + [Fact] + public void Build_records_user_completion_evidence_for_reconcile_rule() + { + // Schema v3 (session 294ab5b4): helloResolved + realmJoin* are the evidence behind the + // "user completed setup" reconcile verdict — the snapshot must show operators the same + // facts the classifier used. + var json = FailureSnapshotBuilder.Build(new[] + { + Event("desktop_arrived", Now.AddHours(-5)), + Event("hello_provisioning_completed", Now.AddHours(-5).AddMinutes(6)), + Event("realmjoin_detected", Now.AddHours(-5).AddMinutes(1)), + }, Now); + + var obj = JObject.Parse(json!); + Assert.True((bool)obj["helloResolved"]!); + Assert.True((bool)obj["realmJoinDetected"]!); + Assert.False((bool)obj["realmJoinResolved"]!); + } + // ============================================================================ // Timing / stale silence // ============================================================================ diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs b/src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs index 074ff36e..95b011c5 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs @@ -78,7 +78,11 @@ namespace AutopilotMonitor.Functions.Services /// /// Distilled ESP lifecycle facts extracted from a session's event stream — the inputs /// to . Also embedded into the failure snapshot so - /// operators see the same evidence the classifier used. + /// operators see the same evidence the classifier used. Besides the ESP registry rollup + /// this carries the user-presence evidence (desktop arrival + positive Hello terminal — + /// the agent's Classic completion prerequisites) and the RealmJoin gate state, so the + /// classifier can tell "user was provably there" apart from "awaiting user" (session + /// 294ab5b4). /// public readonly record struct EspProvisioningRollup( bool DeviceSetupAllSucceeded, @@ -87,7 +91,11 @@ namespace AutopilotMonitor.Functions.Services bool AccountSetupAllSucceeded, bool HasExplicitFailure, bool HasTerminalComplete, - bool HasAgentEmergencyBreak); + bool HasAgentEmergencyBreak, + bool DesktopArrived, + bool HelloResolved, + bool RealmJoinDetected, + bool RealmJoinResolved); /// /// Walk a session's events and distill the ESP rollup. Tolerant of missing/empty input @@ -103,6 +111,8 @@ namespace AutopilotMonitor.Functions.Services int acctBestN = 0, acctBestM = 0; bool acctFallbackAll = false; bool hasFailure = false, hasComplete = false, hasEmergencyBreak = false; + bool desktopArrived = false, helloResolved = false; + bool realmJoinDetected = false, realmJoinResolved = false; foreach (var evt in events) { @@ -114,6 +124,19 @@ namespace AutopilotMonitor.Functions.Services hasComplete = true; else if (Eq(type, "agent_emergency_break")) hasEmergencyBreak = true; + else if (Eq(type, "desktop_arrived")) + desktopArrived = true; + // Positive Hello terminals only (provisioned / skipped) — these are what raises + // the agent's HelloResolved completion fact. _failed / _blocked / _timeout leave + // the agent still waiting and must not count as "user finished setup". + else if (Eq(type, "hello_provisioning_completed") || Eq(type, "hello_skipped")) + helloResolved = true; + else if (Eq(type, "realmjoin_detected")) + realmJoinDetected = true; + // Either terminal opens the agent-side RealmJoin gate: phase 110 or the 60-min + // hard timeout (both dual-emitted to the timeline by the agent/engine). + else if (Eq(type, "realmjoin_resolved") || Eq(type, "realmjoin_timeout")) + realmJoinResolved = true; var msg = evt.Message; if (string.IsNullOrEmpty(msg)) continue; @@ -152,7 +175,11 @@ namespace AutopilotMonitor.Functions.Services AccountSetupAllSucceeded: acctAll, HasExplicitFailure: hasFailure, HasTerminalComplete: hasComplete, - HasAgentEmergencyBreak: hasEmergencyBreak); + HasAgentEmergencyBreak: hasEmergencyBreak, + DesktopArrived: desktopArrived, + HelloResolved: helloResolved, + RealmJoinDetected: realmJoinDetected, + RealmJoinResolved: realmJoinResolved); } /// @@ -184,7 +211,29 @@ namespace AutopilotMonitor.Functions.Services return (SessionStatus.Incomplete, "Agent emergency break fired (absolute session-age cap) — agent gone without completion"); - // 4. Device Setup fully provisioned (device is AADJ + MDM-enrolled), user phase pending. + // 4. The user demonstrably finished setup: a real-user desktop was observed AND + // Windows Hello reached a positive terminal (provisioned or skipped). Those are + // exactly the agent's Classic completion prerequisites — the only thing that can + // still block enrollment_complete on the device is the RealmJoin completion gate, + // and that self-releases via a 60-min hard timeout. A session still silent by the + // time this sweep runs is therefore past every agent-side wait: the enrollment + // itself succeeded and only the final completion report never left the device + // (shutdown / process kill / egress cut — session 294ab5b4). Labeling this + // "AwaitingUser" would be factually wrong (the user was provably there), so + // reconcile to Succeeded with the honest reason. Note desktop arrival ALONE + // remains explicitly rejected as a completion signal (design doc) — it fires + // while the user phase is still running; the Hello terminal is what proves the + // user finished. + if (rollup.DesktopArrived && rollup.HelloResolved) + { + var detail = rollup.RealmJoinDetected && !rollup.RealmJoinResolved + ? "RealmJoin deployment never reported completion before the agent went silent" + : "agent went silent before reporting completion"; + return (SessionStatus.Succeeded, + $"Reconciled at timeout: user completed setup (desktop + Windows Hello) — {detail}"); + } + + // 5. Device Setup fully provisioned (device is AADJ + MDM-enrolled), user phase pending. if (rollup.DeviceSetupAllSucceeded) { var elapsedHours = (nowUtc - startedAtUtc).TotalHours; @@ -202,7 +251,7 @@ namespace AutopilotMonitor.Functions.Services $"(last Account Setup {rollup.AccountSetupSucceededCount}/{rollup.AccountSetupTotal})"); } - // 5. Silent before Device Setup completed, with no explicit failure → unknown, not a failure. + // 6. Silent before Device Setup completed, with no explicit failure → unknown, not a failure. return (SessionStatus.Incomplete, "No Device Setup completion or explicit failure signal observed before timeout"); } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/FailureSnapshotBuilder.cs b/src/Backend/AutopilotMonitor.Functions/Services/FailureSnapshotBuilder.cs index 9d37f681..76fc38e0 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/FailureSnapshotBuilder.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/FailureSnapshotBuilder.cs @@ -27,7 +27,11 @@ namespace AutopilotMonitor.Functions.Services // v2 (2026-07-08): added the ESP DeviceSetup/AccountSetup subcategory rollup fields // (deviceSetupAllSucceeded / accountSetup*) that the timeout reclassification uses — // see docs/design/enrollment-status-reclassification.md. - public const int CurrentSchemaVersion = 2; + // v3 (2026-07-10): added helloResolved + realmJoinDetected/realmJoinResolved — the + // evidence behind the "user completed setup" reconcile rule (session 294ab5b4: user + // reached desktop + finished Hello, only the RealmJoin gate was pending when the + // agent went silent; previously mislabeled AwaitingUser). + public const int CurrentSchemaVersion = 3; // Canonical signals we expect to have seen by the time a healthy session completes. // Missing entries are surfaced in the snapshot under "missingSignals". Some entries @@ -236,6 +240,9 @@ namespace AutopilotMonitor.Functions.Services accountSetupSucceededCount = espRollup.AccountSetupSucceededCount, accountSetupTotal = espRollup.AccountSetupTotal, accountSetupAllSucceeded = espRollup.AccountSetupAllSucceeded, + helloResolved = espRollup.HelloResolved, + realmJoinDetected = espRollup.RealmJoinDetected, + realmJoinResolved = espRollup.RealmJoinResolved, missingSignals = missing, }; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index d6844a76..f00acae7 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -369,6 +369,8 @@ namespace AutopilotMonitor.Functions.Services // emits instead of blindly failing every silent session // (docs/design/enrollment-status-reclassification.md): explicit failure → Failed; // Account Setup all-succeeded / enrollment_complete → Succeeded (reconcile); + // desktop + positive Hello terminal observed → Succeeded (user provably finished + // setup; only the completion report never left the device — session 294ab5b4); // Device Setup done + within grace → AwaitingUser, else Incomplete; silent before // Device Setup → Incomplete. Never Failed without an explicit failure signal. var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents); diff --git a/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/FailureSnapshotBlock.tsx b/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/FailureSnapshotBlock.tsx index 24f17acb..53caa9c2 100644 --- a/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/FailureSnapshotBlock.tsx +++ b/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/FailureSnapshotBlock.tsx @@ -27,6 +27,10 @@ interface FailureSnapshot { isHybridJoin?: boolean | null; enrollmentType?: string | null; lastNetworkState?: string | null; + // Schema v3 (2026-07-10): evidence behind the "user completed setup" reconcile rule. + helloResolved?: boolean; + realmJoinDetected?: boolean; + realmJoinResolved?: boolean; missingSignals?: string[]; } @@ -80,6 +84,12 @@ export default function FailureSnapshotBlock({ + {snapshot.helloResolved !== undefined && ( + + )} + {snapshot.realmJoinDetected !== undefined && ( + + )} @@ -135,3 +145,11 @@ function fmtHelloPolicy(s: FailureSnapshot): string { if (s.helloPolicyEnabled === false) return "detected (disabled)"; return "detected"; } + +function fmtRealmJoin(s: FailureSnapshot): string { + if (!s.realmJoinDetected) return "not detected"; + // "unresolved" = RealmJoin never reported CompletedFirstDeployment (phase 110) and the + // 60-min gate timeout wasn't observed either — the deployment was still in flight when + // the agent went silent. + return s.realmJoinResolved ? "detected (resolved)" : "detected (unresolved)"; +}