From e7ea431ccfeebf4b3fa480b361ff786579eb4b2e Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 23 Jul 2026 17:03:44 +0200 Subject: [PATCH] feat(esp): recovery story for terminally reported ESP failures (session 4910a5a5) DeviceSetup/Apps failed, the user pressed "Try again" ~23 min later, apps recovered to 10/10 and DeviceSetup resolved - yet the advisory_completion window un-defanged the stale failure and failed the live enrollment, because the advisory variant was exempt from all re-arm/rebase guards. Engine fix (DecisionCore): - New facts EspAdvisoryFailureResolvedUtc + EspAdvisoryFailureCategory (state-borne; deadline payload would be lost on re-arm/rebase). - TryResolveAdvisoryOnCategoryRecovery in both ProvisioningComplete handlers: category match sets the fact + emits esp_failure_advisory_resolved. - Guards now (!hasAdvisoryAnchor || advisoryResolved): enforcement-progress re-arm, hello-never-observed promote and the reboot rebase apply again once the failure demonstrably un-happened; unresolved advisories keep the legacy un-defang exactly. - Truthful terminal reason esp_recovered_without_completion_evidence (+ failureRecovered=true, LastFailureTrigger=DeadlineFired so the likely-stuck app promotion stays off). Story events (agent tracker): - esp_failure_retry_detected: failed subcategory leaves "failed" after the terminal fire - consistent with the user pressing "Try again". - esp_failure_recovered: previously failed category reaches success. - EspFailureDetectedEventArgs.LikelyCulpritApps (never-started first, cap 3) rides the EspTerminalFailure payload so esp_failure_advisory / enrollment_failed name the culprit app directly. Surfacing: MCP resource-catalog, web timeline badges (Recovered / User retry pills, culprit chip on advisory), gather-rule autocomplete, ANALYZE-ESP-006 "ESP Failure Recovered After User Retry". Tests: AdvisoryRecoveryTests (13, incl. the 527/611/635/671 field-ordinal replay that no longer fails the session), EspFailureRecoveryStoryTests (5), snapshot-builder guards 18 -> 20. All suites green (DecisionCore 493, V2.Core 2018, Backend 3201, web vitest 589, MCP 449 incl. drift). Co-Authored-By: Claude Fable 5 --- rules/analyze/ANALYZE-ESP-006.json | 90 +++++ rules/dist/analyze-rules.json | 104 +++++ .../EspFailureRecoveryStoryTests.cs | 256 ++++++++++++ .../EspFailureDetectedEventArgs.cs | 17 +- .../ProvisioningStatusTracker.cs | 190 ++++++++- .../EspAndHelloTrackerAdapter.cs | 9 + .../AdvisoryRecoveryTests.cs | 382 ++++++++++++++++++ .../DecisionStateSnapshotBuilderTests.cs | 15 +- .../src/resource-catalog.ts | 3 + .../Engine/DecisionEngine.Edge.cs | 127 +++++- .../Engine/DecisionEngine.SelfDeploying.cs | 38 +- .../Engine/DecisionEngine.Shared.cs | 28 +- .../Engine/DecisionStateSignalCensus.cs | 11 + .../Engine/DecisionStateSnapshotBuilder.cs | 2 + .../State/DecisionState.cs | 36 +- .../State/DecisionStateBuilder.cs | 26 +- .../AutopilotMonitor.Shared/Constants.cs | 11 + .../app/gather-rules/eventTypes.ts | 6 + .../[sessionId]/components/EventTimeline.tsx | 40 ++ 19 files changed, 1360 insertions(+), 31 deletions(-) create mode 100644 rules/analyze/ANALYZE-ESP-006.json create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/EspFailureRecoveryStoryTests.cs create mode 100644 src/Agent/AutopilotMonitor.DecisionCore.Tests/AdvisoryRecoveryTests.cs diff --git a/rules/analyze/ANALYZE-ESP-006.json b/rules/analyze/ANALYZE-ESP-006.json new file mode 100644 index 00000000..029383c8 --- /dev/null +++ b/rules/analyze/ANALYZE-ESP-006.json @@ -0,0 +1,90 @@ +{ + "$schema": "../schema/analyze-rule.schema.json", + "ruleId": "ANALYZE-ESP-006", + "title": "ESP Failure Recovered After User Retry", + "description": "Detects the recovery story: the ESP reported a terminal failure (typically DeviceSetup/Apps), the user most likely pressed 'Try again' on the failure page, the failed step re-ran and completed successfully. The earlier failure did not stick — the enrollment continued past it.", + "severity": "info", + "category": "esp", + "version": "1.0.0", + "enabled": true, + "baseConfidence": 85, + "conditions": [ + { + "signal": "advisory", + "source": "event_type", + "eventType": "esp_failure_advisory", + "operator": "exists", + "value": "", + "required": true + }, + { + "signal": "recovered", + "source": "event_type", + "eventType": "esp_failure_recovered", + "operator": "exists", + "value": "", + "required": true + }, + { + "signal": "retry", + "source": "event_type", + "eventType": "esp_failure_retry_detected", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "category", + "source": "event_type", + "eventType": "esp_failure_recovered", + "dataField": "category", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "failedSubcategory", + "source": "event_type", + "eventType": "esp_failure_recovered", + "dataField": "failedSubcategory", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "likelyCulpritApps", + "source": "event_type", + "eventType": "esp_failure_advisory", + "dataField": "likelyCulpritApps", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "minutesSinceFailure", + "source": "event_type", + "eventType": "esp_failure_recovered", + "dataField": "minutesSinceFailure", + "operator": "exists", + "value": "", + "required": false + } + ], + "confidenceFactors": [ + { "signal": "retry", "condition": "exists", "weight": 10 }, + { "signal": "likelyCulpritApps", "condition": "exists", "weight": 5 } + ], + "explanation": "The Enrollment Status Page reported a **failure** in the `{{category}}` phase (`{{failedSubcategory}}` subcategory, most likely app(s): `{{likelyCulpritApps}}`) — and then **recovered**: the failed step left its failed state and completed successfully about `{{minutesSinceFailure}}` min later.\n\nThat pattern is consistent with the user pressing **'Try again'** on the ESP failure page (the profile allows failure-page actions; the agent cannot observe the click itself). The failure the user saw was real, but it did not stick — the ESP re-ran the step, it succeeded, and the enrollment continued.\n\nRead the session timeline as one arc: `esp_failure_advisory` (the failure, with the likely culprit app) → `esp_failure_retry_detected` (the retry) → `esp_failure_recovered` (the recovery). If this pattern repeats across devices for the same app, that app's install is flaky on first attempt — fix the app rather than the devices. Any literal `{{…}}` placeholder means that detail was not recorded for this session.", + "remediation": [ + { + "title": "Make the first install attempt stick", + "steps": [ + "Check the likely culprit app(s) (`{{likelyCulpritApps}}`) in Intune: an app that fails on the first ESP pass but succeeds on retry usually has a dependency/ordering or detection-timing problem.", + "If the app shows 'never started installing' in esp_failure_settle_started, verify its assignment and applicability rules — ESP counted it as required, but IME never processed it.", + "If the pattern repeats across the tenant, consider removing the app from the ESP blocking list until the first-attempt install is reliable — users are seeing an avoidable failure screen.", + "No device-side action is needed for this session: the failure recovered and the enrollment continued." + ] + } + ], + "tags": ["esp", "recovery", "try-again", "retry", "continue-anyway", "enrollment"] +} diff --git a/rules/dist/analyze-rules.json b/rules/dist/analyze-rules.json index 10dde5a2..fe751a62 100644 --- a/rules/dist/analyze-rules.json +++ b/rules/dist/analyze-rules.json @@ -1884,6 +1884,110 @@ "user-experience" ] }, + { + "ruleId": "ANALYZE-ESP-006", + "title": "ESP Failure Recovered After User Retry", + "description": "Detects the recovery story: the ESP reported a terminal failure (typically DeviceSetup/Apps), the user most likely pressed 'Try again' on the failure page, the failed step re-ran and completed successfully. The earlier failure did not stick — the enrollment continued past it.", + "severity": "info", + "category": "esp", + "version": "1.0.0", + "enabled": true, + "baseConfidence": 85, + "conditions": [ + { + "signal": "advisory", + "source": "event_type", + "eventType": "esp_failure_advisory", + "operator": "exists", + "value": "", + "required": true + }, + { + "signal": "recovered", + "source": "event_type", + "eventType": "esp_failure_recovered", + "operator": "exists", + "value": "", + "required": true + }, + { + "signal": "retry", + "source": "event_type", + "eventType": "esp_failure_retry_detected", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "category", + "source": "event_type", + "eventType": "esp_failure_recovered", + "dataField": "category", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "failedSubcategory", + "source": "event_type", + "eventType": "esp_failure_recovered", + "dataField": "failedSubcategory", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "likelyCulpritApps", + "source": "event_type", + "eventType": "esp_failure_advisory", + "dataField": "likelyCulpritApps", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "minutesSinceFailure", + "source": "event_type", + "eventType": "esp_failure_recovered", + "dataField": "minutesSinceFailure", + "operator": "exists", + "value": "", + "required": false + } + ], + "confidenceFactors": [ + { + "signal": "retry", + "condition": "exists", + "weight": 10 + }, + { + "signal": "likelyCulpritApps", + "condition": "exists", + "weight": 5 + } + ], + "explanation": "The Enrollment Status Page reported a **failure** in the `{{category}}` phase (`{{failedSubcategory}}` subcategory, most likely app(s): `{{likelyCulpritApps}}`) — and then **recovered**: the failed step left its failed state and completed successfully about `{{minutesSinceFailure}}` min later.\n\nThat pattern is consistent with the user pressing **'Try again'** on the ESP failure page (the profile allows failure-page actions; the agent cannot observe the click itself). The failure the user saw was real, but it did not stick — the ESP re-ran the step, it succeeded, and the enrollment continued.\n\nRead the session timeline as one arc: `esp_failure_advisory` (the failure, with the likely culprit app) → `esp_failure_retry_detected` (the retry) → `esp_failure_recovered` (the recovery). If this pattern repeats across devices for the same app, that app's install is flaky on first attempt — fix the app rather than the devices. Any literal `{{…}}` placeholder means that detail was not recorded for this session.", + "remediation": [ + { + "title": "Make the first install attempt stick", + "steps": [ + "Check the likely culprit app(s) (`{{likelyCulpritApps}}`) in Intune: an app that fails on the first ESP pass but succeeds on retry usually has a dependency/ordering or detection-timing problem.", + "If the app shows 'never started installing' in esp_failure_settle_started, verify its assignment and applicability rules — ESP counted it as required, but IME never processed it.", + "If the pattern repeats across the tenant, consider removing the app from the ESP blocking list until the first-attempt install is reliable — users are seeing an avoidable failure screen.", + "No device-side action is needed for this session: the failure recovered and the enrollment continued." + ] + } + ], + "tags": [ + "esp", + "recovery", + "try-again", + "retry", + "continue-anyway", + "enrollment" + ] + }, { "ruleId": "ANALYZE-ID-001", "title": "Expected Machine Certificate Not Found", diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/EspFailureRecoveryStoryTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/EspFailureRecoveryStoryTests.cs new file mode 100644 index 00000000..722c3fb5 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/EspFailureRecoveryStoryTests.cs @@ -0,0 +1,256 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals +{ + /// + /// Session 4910a5a5 (2026-07-23) — post-terminal recovery story. Field sequence: + /// DeviceSetup/Apps failed (culprit app never started installing) → settle window expired + /// → terminal EspFailureDetected → the user pressed "Try again" ~23 min later → Apps left + /// the failed state → 10/10 installed → DeviceSetup resolved. Pre-fix, that recovery only + /// appeared as a generic subcategory_state_change. Coverage: + /// + /// esp_failure_retry_detected fires once when the failed subcategory leaves + /// the failed state AFTER the terminal fire. + /// esp_failure_recovered fires once when the category then resolves to + /// success. + /// An in-settle-window recovery keeps emitting only + /// esp_failure_settle_recovered — the post-terminal pair stays silent. + /// Apps failures carry LikelyCulpritApps on the EspFailureDetected args, + /// never-started apps ranked first. + /// + /// + public sealed class EspFailureRecoveryStoryTests + { + private static readonly DateTime Fixed = new DateTime(2026, 7, 23, 4, 30, 0, DateTimeKind.Utc); + + private sealed class Fixture : IDisposable + { + public TempDirectory Tmp { get; } = new TempDirectory(); + public FakeSignalIngressSink Sink { get; } = new FakeSignalIngressSink(); + public List EspFailures { get; } = new List(); + public ProvisioningStatusTracker Tracker { get; } + + public Fixture(Func>? packageStatesProbe = null) + { + var clock = new VirtualClock(Fixed); + var post = new InformationalEventPost(Sink, clock); + var logger = new AgentLogger(Tmp.Path, AgentLogLevel.Info); + Tracker = new ProvisioningStatusTracker( + sessionId: "S1", + tenantId: "T1", + post: post, + logger: logger, + appxScanner: new FakeAppxDeploymentFailureScanner(), + backgroundDispatcher: action => action(), + packageStatesProbe: packageStatesProbe); + Tracker.EspFailureDetected += (_, args) => EspFailures.Add(args); + } + + public IReadOnlyList<(string EventType, IReadOnlyDictionary Data)> CapturedEvents() + { + return Sink.Posted + .Where(p => p.Kind == DecisionSignalKind.InformationalEvent && p.Payload != null) + .Select(p => + { + var eventType = p.Payload!.TryGetValue(SignalPayloadKeys.EventType, out var et) ? et : ""; + var data = p.TypedPayload as IReadOnlyDictionary + ?? (IReadOnlyDictionary)new Dictionary(); + return (eventType, data); + }) + .ToList(); + } + + /// Drive the session-4910a5a5 shape up to the terminal fire. + public void DriveToTerminalDeviceSetupAppsFailure() + { + Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (9 of 10 installed)""} + }"); + Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (Error)""} + }"); + Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status"); + } + + public void Dispose() + { + Tracker.Dispose(); + Tmp.Dispose(); + } + } + + [Fact] + public void PostTerminalRetry_EmitsRetryDetected_Once() + { + using var f = new Fixture(); + f.DriveToTerminalDeviceSetupAppsFailure(); + Assert.Single(f.EspFailures); + + // "Try again": Apps leaves the failed state well after the settle window. + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""} + }"); + + var retry = Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_retry_detected"); + Assert.Equal("DeviceSetup", (string)retry.Data["category"]); + Assert.Equal("Apps", (string)retry.Data["subcategory"]); + Assert.Equal("failed", (string)retry.Data["previousState"]); + Assert.Equal("inProgress", (string)retry.Data["newState"]); + Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", (string)retry.Data["failureType"]); + Assert.True(retry.Data.ContainsKey("minutesSinceFailure")); + + // A later unrelated transition does not re-emit. + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed)""} + }"); + Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_retry_detected"); + } + + [Fact] + public void PostTerminalRecovery_EmitsRecovered_OnCategorySuccess() + { + using var f = new Fixture(); + f.DriveToTerminalDeviceSetupAppsFailure(); + + // Retry → all apps installed → Windows resolves the category boolean. + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""} + }"); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": true, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed)""} + }"); + + var recovered = Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_recovered"); + Assert.Equal("DeviceSetup", (string)recovered.Data["category"]); + Assert.Equal("Apps", (string)recovered.Data["failedSubcategory"]); + Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", (string)recovered.Data["failureType"]); + Assert.True(recovered.Data.ContainsKey("minutesSinceFailure")); + + // Duplicate category-success writes do not re-emit (fire-once). + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": true, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed, take 2)""} + }"); + Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_recovered"); + } + + [Fact] + public void InSettleWindowRecovery_KeepsPostTerminalPairSilent() + { + using var f = new Fixture(); + + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (0 of 1 installed)""} + }"); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (Error)""} + }"); + // Recovery INSIDE the settle window — terminal fire is suppressed. + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""} + }"); + f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status"); + Assert.Empty(f.EspFailures); + + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": true, + ""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (1 of 1 installed)""} + }"); + + var events = f.CapturedEvents(); + Assert.Contains(events, e => e.EventType == "esp_failure_settle_recovered"); + Assert.DoesNotContain(events, e => e.EventType == "esp_failure_retry_detected"); + Assert.DoesNotContain(events, e => e.EventType == "esp_failure_recovered"); + } + + [Fact] + public void AppsFailure_TerminalArgs_CarryLikelyCulprits_NeverStartedFirst() + { + var neverStarted = BuildApp("app-bg", "Teams Backgrounds v2", + AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown, + downloadingSeen: false); + var startedButPending = BuildApp("app-slow", "Slow Installer", + AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown, + downloadingSeen: true); + var installed = BuildApp("app-ok", "Falcon Sensor", + AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Installed, + downloadingSeen: true); + using var f = new Fixture(() => new[] { startedButPending, neverStarted, installed }); + + f.DriveToTerminalDeviceSetupAppsFailure(); + + var fired = Assert.Single(f.EspFailures); + Assert.Equal(2, fired.LikelyCulpritApps.Count); + // Never-started app ranked first — it starved the ESP gate without any IME event. + Assert.Equal("Teams Backgrounds v2", fired.LikelyCulpritApps[0]); + Assert.Equal("Slow Installer", fired.LikelyCulpritApps[1]); + } + + [Fact] + public void NonAppsFailure_TerminalArgs_CarryNoCulprits() + { + var neverStarted = BuildApp("app-bg", "Teams Backgrounds v2", + AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown, + downloadingSeen: false); + using var f = new Fixture(() => new[] { neverStarted }); + + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Working""} + }"); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Failed""} + }"); + f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status"); + + var fired = Assert.Single(f.EspFailures); + Assert.Empty(fired.LikelyCulpritApps); + } + + private static AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState BuildApp( + string id, string name, + AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState state, + bool downloadingSeen) + { + var pkg = new AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState(id, 0); + pkg.UpdateName(name); + pkg.UpdateIntent(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppIntent.Install); + pkg.UpdateTargeted(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppTargeted.Device); + // DownloadingOrInstallingSeen is derived inside UpdateState (Downloading..Installing + // range) — drive it through a Downloading transition instead of setting it directly. + if (downloadingSeen) + pkg.UpdateState(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Downloading); + if (state != AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown + && state != AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Downloading) + pkg.UpdateState(state); + return pkg; + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspFailureDetectedEventArgs.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspFailureDetectedEventArgs.cs index 186ad407..85e3769d 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspFailureDetectedEventArgs.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspFailureDetectedEventArgs.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals { @@ -38,16 +39,30 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals /// public string Category { get; } + /// + /// Session 4910a5a5: names of the tracked apps most likely behind an Apps-subcategory + /// failure (capped, never-started apps ranked first — those starve ESP's app gate + /// without ever producing an IME install event). Snapshot from + /// ProvisioningStatusTracker.SnapshotTrackedAppsNotCompleted at failure-arming + /// time; empty for non-Apps failures and event-log-derived failures. Propagated onto + /// the EspTerminalFailure signal payload so the DecisionEngine's + /// esp_failure_advisory / enrollment_failed events can name the culprit + /// directly. + /// + public IReadOnlyList LikelyCulpritApps { get; } + public EspFailureDetectedEventArgs( string failureType, string errorCode = null, string failedSubcategory = null, - string category = null) + string category = null, + IReadOnlyList likelyCulpritApps = null) { FailureType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType; ErrorCode = string.IsNullOrEmpty(errorCode) ? null : errorCode; FailedSubcategory = string.IsNullOrEmpty(failedSubcategory) ? null : failedSubcategory; Category = string.IsNullOrEmpty(category) ? null : category; + LikelyCulpritApps = likelyCulpritApps ?? Array.Empty(); } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs index c2454234..fbc2d996 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs @@ -116,6 +116,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals // failure details that fire after the settle window expires. private Dictionary _provisioningFailureSettleTimers; private Dictionary _provisioningFailureSettleArgs; + // Session 4910a5a5 — post-terminal recovery story. Populated when the settle window + // expires and EspFailureDetected actually fires (terminal path); consumed by the + // esp_failure_retry_detected / esp_failure_recovered emitters so a late "Try again" + // recovery (23 min after the failure in the field session) is visible on the timeline + // instead of hiding inside a generic subcategory_state_change. Keyed by category-name. + private Dictionary _provisioningFailureTerminalArgs; + private Dictionary _provisioningFailureTerminalUtc; + private HashSet _postFailureRetryEmitted; + private HashSet _postFailureRecoveredEmitted; // Session 2bc884b6: AppX enrichment scan seams + fire-once guard. The scanner and the // dispatcher are injectable so tests never touch the real event log or the threadpool. private readonly IAppxDeploymentFailureScanner _appxScanner; @@ -285,6 +294,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _lastSubcategoryStates = new Dictionary>(StringComparer.OrdinalIgnoreCase); _provisioningFailureSettleTimers = new Dictionary(StringComparer.OrdinalIgnoreCase); _provisioningFailureSettleArgs = new Dictionary(StringComparer.OrdinalIgnoreCase); + _provisioningFailureTerminalArgs = new Dictionary(StringComparer.OrdinalIgnoreCase); + _provisioningFailureTerminalUtc = new Dictionary(StringComparer.OrdinalIgnoreCase); + _postFailureRetryEmitted = new HashSet(StringComparer.OrdinalIgnoreCase); + _postFailureRecoveredEmitted = new HashSet(StringComparer.OrdinalIgnoreCase); _deviceSetupProvisioningCompleteFired = false; _deviceSetupFallbackTimer = null; _accountSetupProvisioningCompleteFired = false; @@ -665,6 +678,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals // null-ing here is correctness-neutral on the start-condition side. ResetAccountSetupFallbackTimer(); _accountSetupProvisioningCompleteFired = true; + MaybeEmitPostFailureRecovered("AccountSetupCategory.Status", "AccountSetup"); try { AccountSetupProvisioningComplete?.Invoke(this, EventArgs.Empty); } catch (Exception ex) { _logger.Error("AccountSetupProvisioningComplete handler failed (fallback path)", ex); } } @@ -833,6 +847,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals // null-ing here is correctness-neutral on the start-condition side. ResetDeviceSetupFallbackTimer(); _deviceSetupProvisioningCompleteFired = true; + MaybeEmitPostFailureRecovered("DeviceSetupCategory.Status", "DeviceSetup"); try { DeviceSetupProvisioningComplete?.Invoke(this, EventArgs.Empty); } catch (Exception ex) { _logger.Error("DeviceSetupProvisioningComplete handler failed (fallback path)", ex); } } @@ -946,12 +961,25 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals var severity = categorySucceeded == false ? EventSeverity.Warning : EventSeverity.Info; EmitProvisioningEvent(categoryLabel, categorySucceeded, statusText, subcategories, severity); EmitRawRegistryDump(categoryLabel, jsonValue, $"category_resolved_{(categorySucceeded == true ? "success" : "failed")}"); + + // Session 4910a5a5: category resolving to success after a terminally + // fired failure = the failure recovered (fallback-path successes call + // this from their own timer handlers). + if (categorySucceeded == true) + { + MaybeEmitPostFailureRecovered(categoryName, categoryLabel); + } } else { var transitions = DetectSubcategoryTransitions(categoryName, subcategories); StoreSubcategoryStates(categoryName, subcategories); + // Session 4910a5a5: after a terminally fired failure, the failed + // subcategory leaving "failed" is the ESP re-running the step — + // consistent with the user pressing "Try again" on the failure page. + MaybeEmitPostFailureRetry(categoryName, categoryLabel, transitions); + var failureTransitions = transitions.Where(t => t.IsFailure).ToList(); if (transitions.Count > 0) { @@ -1025,6 +1053,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals $"escalating as {failureTypeName} (errorCode={errorCode ?? "n/a"})"); _logger.Warning($"Provisioning failure detected: {failureTypeName}"); + // Session 4910a5a5: name the likely culprit app(s) directly on the failure args so + // the DecisionEngine's advisory/terminal events carry them — never-started apps + // first (they starve ESP's app gate without ever producing an IME install event), + // capped at 3 names. + IReadOnlyList likelyCulpritApps = null; + if (string.Equals(failedSubcategoryName, "Apps", StringComparison.OrdinalIgnoreCase)) + { + likelyCulpritApps = SnapshotTrackedAppsNotCompleted() + .OrderBy(a => a["everStartedInstalling"] is bool started && started ? 1 : 0) + .Select(a => a["appName"]?.ToString()) + .Where(n => !string.IsNullOrEmpty(n)) + .Take(3) + .ToList(); + } + // Caller already holds _stateLock. ArmFailureSettleTimer( categoryName: categoryName, @@ -1032,7 +1075,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals failureType: failureTypeName, errorCode: errorCode, failedSubcategory: failedSubcategoryName, - category: categoryLabel)); + category: categoryLabel, + likelyCulpritApps: likelyCulpritApps)); // Settle-window owns the actual EspFailureDetected fire. The legacy "fire-after-emit" // path in CheckProvisioningStatus is now a no-op for this code-path; it stays in place @@ -1292,6 +1336,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _provisioningCategoriesResolved.Remove(categoryName); } } + else + { + // Session 4910a5a5: remember the terminally fired failure so a LATE + // recovery (outside the settle window — e.g. user pressing "Try again" + // minutes later) can be told as esp_failure_retry_detected / + // esp_failure_recovered instead of hiding in a generic state-change. + _provisioningFailureTerminalArgs[categoryName] = args; + _provisioningFailureTerminalUtc[categoryName] = DateTime.UtcNow; + } } if (recovered) @@ -1382,6 +1435,133 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals }); } + /// + /// Session 4910a5a5: after a terminally fired failure (settle window expired, + /// EspFailureDetected raised), the failed subcategory leaving the failed state + /// means the ESP re-ran the step — consistent with the user pressing "Try again" on + /// the ESP failure page (the agent cannot observe the click itself). Emits + /// esp_failure_retry_detected once per terminally fired category. Caller holds + /// . + /// + private void MaybeEmitPostFailureRetry(string categoryName, string categoryLabel, List transitions) + { + if (transitions == null || transitions.Count == 0) return; + if (_postFailureRetryEmitted.Contains(categoryName)) return; + if (_provisioningFailureTerminalArgs == null + || !_provisioningFailureTerminalArgs.TryGetValue(categoryName, out var terminalArgs)) + { + return; // No terminal fire on record — in-window recovery is settle_recovered's job. + } + + SubcategoryTransition? retryTransition = null; + foreach (var t in transitions) + { + if (string.Equals(t.OldState, "failed", StringComparison.OrdinalIgnoreCase) + && !string.Equals(t.NewState, "failed", StringComparison.OrdinalIgnoreCase)) + { + retryTransition = t; + break; + } + } + if (retryTransition == null) return; + var retry = retryTransition.Value; + + _postFailureRetryEmitted.Add(categoryName); + + var minutesSinceFailure = MinutesSinceTerminalFailure(categoryName); + var data = new Dictionary + { + { "category", categoryLabel }, + { "subcategory", retry.SubcategoryName }, + { "previousState", retry.OldState }, + { "newState", retry.NewState }, + { "failureType", terminalArgs.FailureType }, + { "minutesSinceFailure", minutesSinceFailure }, + }; + if (!string.IsNullOrEmpty(terminalArgs.ErrorCode)) + data["errorCode"] = terminalArgs.ErrorCode; + + _post.Emit(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.EspFailureRetryDetected, + Severity = EventSeverity.Info, + Source = "EspAndHelloTracker", + Phase = EnrollmentPhase.Unknown, + Message = $"ESP retry detected — {categoryLabel}/{retry.SubcategoryName} left the failed state " + + $"({retry.OldState} → {retry.NewState}) {minutesSinceFailure} min after the failure — " + + "consistent with the user pressing 'Try again' on the ESP failure page.", + Data = data, + ImmediateUpload = true + }); + } + + /// + /// Session 4910a5a5: the category behind a terminally fired failure reached success + /// (categorySucceeded=true or the all-subcategories-succeeded fallback) — the failure + /// demonstrably recovered. Emits esp_failure_recovered once per terminally + /// fired category so the timeline tells "failed → retried → recovered" instead of + /// leaving the stale failure as the last word. Caller holds . + /// + private void MaybeEmitPostFailureRecovered(string categoryName, string categoryLabel) + { + if (_postFailureRecoveredEmitted == null || _postFailureRecoveredEmitted.Contains(categoryName)) return; + if (_provisioningFailureTerminalArgs == null + || !_provisioningFailureTerminalArgs.TryGetValue(categoryName, out var terminalArgs)) + { + return; + } + + _postFailureRecoveredEmitted.Add(categoryName); + + var minutesSinceFailure = MinutesSinceTerminalFailure(categoryName); + string subcategoryStatus = null; + if (terminalArgs.FailedSubcategory != null + && _lastSubcategoryStates != null + && _lastSubcategoryStates.TryGetValue(categoryName, out var subs) + && subs.TryGetValue(terminalArgs.FailedSubcategory, out var currentState)) + { + subcategoryStatus = currentState; + } + + var data = new Dictionary + { + { "category", categoryLabel }, + { "failedSubcategory", terminalArgs.FailedSubcategory ?? "category-level" }, + { "failureType", terminalArgs.FailureType }, + { "minutesSinceFailure", minutesSinceFailure }, + }; + if (!string.IsNullOrEmpty(terminalArgs.ErrorCode)) + data["errorCode"] = terminalArgs.ErrorCode; + if (subcategoryStatus != null) + data["subcategoryState"] = subcategoryStatus; + + _post.Emit(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.EspFailureRecovered, + Severity = EventSeverity.Info, + Source = "EspAndHelloTracker", + Phase = EnrollmentPhase.Unknown, + Message = $"ESP failure recovered — {categoryLabel}/{terminalArgs.FailedSubcategory ?? "category-level"} " + + $"completed successfully {minutesSinceFailure} min after the failure; the earlier ESP failure no longer applies.", + Data = data, + ImmediateUpload = true + }); + } + + private int MinutesSinceTerminalFailure(string categoryName) + { + if (_provisioningFailureTerminalUtc != null + && _provisioningFailureTerminalUtc.TryGetValue(categoryName, out var firedAt)) + { + return Math.Max(0, (int)Math.Round((DateTime.UtcNow - firedAt).TotalMinutes)); + } + return 0; + } + private void EmitFailureSettleStarted(EspFailureDetectedEventArgs args) { var data = new Dictionary @@ -1863,6 +2043,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _provisioningFailureSettleTimers = new Dictionary(StringComparer.OrdinalIgnoreCase); if (_provisioningFailureSettleArgs == null) _provisioningFailureSettleArgs = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (_provisioningFailureTerminalArgs == null) + _provisioningFailureTerminalArgs = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (_provisioningFailureTerminalUtc == null) + _provisioningFailureTerminalUtc = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (_postFailureRetryEmitted == null) + _postFailureRetryEmitted = new HashSet(StringComparer.OrdinalIgnoreCase); + if (_postFailureRecoveredEmitted == null) + _postFailureRecoveredEmitted = new HashSet(StringComparer.OrdinalIgnoreCase); } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/EspAndHelloTrackerAdapter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/EspAndHelloTrackerAdapter.cs index bbc48f0f..92223bcb 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/EspAndHelloTrackerAdapter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/EspAndHelloTrackerAdapter.cs @@ -1,6 +1,7 @@ #nullable enable using System; using System.Collections.Generic; +using System.Globalization; using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; using AutopilotMonitor.Agent.V2.Core.Orchestration; using AutopilotMonitor.DecisionCore.Engine; @@ -267,6 +268,14 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters payload["failedSubcategory"] = failedSubcategory!; if (!string.IsNullOrEmpty(category)) payload["category"] = category!; + // Session 4910a5a5: culprit-app names ride the signal so the DecisionEngine's + // esp_failure_advisory / enrollment_failed events can name them directly. + var culprits = args?.LikelyCulpritApps; + if (culprits != null && culprits.Count > 0) + { + payload["likelyCulpritApps"] = string.Join(", ", culprits); + payload["likelyCulpritAppCount"] = culprits.Count.ToString(CultureInfo.InvariantCulture); + } _ingress.Post( kind: DecisionSignalKind.EspTerminalFailure, diff --git a/src/Agent/AutopilotMonitor.DecisionCore.Tests/AdvisoryRecoveryTests.cs b/src/Agent/AutopilotMonitor.DecisionCore.Tests/AdvisoryRecoveryTests.cs new file mode 100644 index 00000000..41ad4a87 --- /dev/null +++ b/src/Agent/AutopilotMonitor.DecisionCore.Tests/AdvisoryRecoveryTests.cs @@ -0,0 +1,382 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Serialization; +using AutopilotMonitor.DecisionCore.Signals; +using AutopilotMonitor.DecisionCore.State; +using Xunit; + +namespace AutopilotMonitor.DecisionCore.Tests +{ + /// + /// Session 4910a5a5 (2026-07-23) — advisory recovery. Field failure: DeviceSetup/Apps + /// failed (culprit app never started installing), the advisory defang armed the + /// advisory_completion window, the user pressed "Try again" ~23 min later, the apps + /// re-ran to 10/10 and DeviceSetupProvisioningComplete arrived — yet the window + /// fired 30 min after the advisory and un-defanged the stale failure + /// (enrollment_failed: esp_terminal_failure) on a live enrollment mid-AccountSetup, + /// because the advisory variant was exempt from ALL re-arm/rebase guards. Coverage: + /// + /// The advisory records the failed category + culprit apps. + /// A ProvisioningComplete of the SAME category sets the resolved fact + emits + /// esp_failure_advisory_resolved; a different category does not. + /// Deadline fire after recovery + enforcement progress → re-arm, not Failed. + /// Deadline fire after recovery without progress → Failed with the truthful + /// esp_recovered_without_completion_evidence reason and + /// LastFailureTrigger=DeadlineFired (likely-stuck promotion off). + /// Unrecovered advisory keeps the legacy un-defang exactly (regression guard). + /// Reboot re-bases a recovered-advisory window (and still does NOT touch an + /// unrecovered one). + /// Snapshot round-trip + census surfacing of the new facts. + /// + /// + public sealed class AdvisoryRecoveryTests + { + private static readonly DateTime T0 = new DateTime(2026, 7, 23, 3, 30, 0, DateTimeKind.Utc); + + private static DecisionSignal MakeSignal( + long ordinal, + DecisionSignalKind kind, + DateTime occurredAtUtc, + IReadOnlyDictionary? payload = null) + { + return new DecisionSignal( + sessionSignalOrdinal: ordinal, + sessionTraceOrdinal: ordinal, + kind: kind, + kindSchemaVersion: 1, + occurredAtUtc: occurredAtUtc, + sourceOrigin: "test", + evidence: new Evidence(EvidenceKind.Synthetic, $"{kind}-{ordinal}", "test"), + payload: payload); + } + + private static DecisionSignal DeadlineFired(long ordinal, DateTime occurredAtUtc) => + MakeSignal(ordinal, DecisionSignalKind.DeadlineFired, occurredAtUtc, + new Dictionary { [SignalPayloadKeys.Deadline] = DeadlineNames.AdvisoryCompletion }); + + private static DecisionEffect? FindTimelineEffect(DecisionStep step, string eventType) => + step.Effects.FirstOrDefault(e => + e.Kind == DecisionEffectKind.EmitEventTimelineEntry + && e.Parameters != null + && e.Parameters.TryGetValue("eventType", out var et) + && et == eventType); + + private static ActiveDeadline? FindDeadline(DecisionState state, string name) => + state.Deadlines.FirstOrDefault(d => d.Name == name); + + /// + /// Replays the session-4910a5a5 shape up to (but excluding) the ESP terminal failure: + /// Classic UserDriven, ContinueAnyway profile, Hello disabled, DeviceSetup entered, + /// AccountSetup entered (04:18, BEFORE the 04:34 DeviceSetup/Apps failure — the IME + /// phase line landed while device apps were still at 9/10). + /// + private static DecisionState SetupSession(DecisionEngine engine) + { + var state = DecisionState.CreateInitial("sess-4910a5a5", "tenant-e46bc88e", T0); + state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState; + state = engine.Reduce(state, MakeSignal( + 5, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1), + new Dictionary + { + [SignalPayloadKeys.SkipUserEsp] = "false", + [SignalPayloadKeys.SkipDeviceEsp] = "false", + [SignalPayloadKeys.EspSyncFailureTimeoutMinutes] = "60", + [SignalPayloadKeys.EspAllowContinueAnyway] = "true", + })).NewState; + state = engine.Reduce(state, MakeSignal( + 8, DecisionSignalKind.HelloPolicyDetected, T0.AddMinutes(2), + new Dictionary { [SignalPayloadKeys.HelloEnabled] = "false" })).NewState; + state = engine.Reduce(state, MakeSignal( + 10, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(6), + new Dictionary { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState; + state = engine.Reduce(state, MakeSignal( + 20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(48), + new Dictionary { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState; + return state; + } + + private static DecisionStep ApplyDeviceSetupAppsFailure(DecisionEngine engine, DecisionState state, long ordinal = 50) => + engine.Reduce(state, MakeSignal( + ordinal, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(64), + new Dictionary + { + ["failureType"] = "Provisioning_DeviceSetup_Apps_Failed", + ["failedSubcategory"] = "Apps", + ["category"] = "DeviceSetup", + ["likelyCulpritApps"] = "Oriflame Cosmectics - Teams Backgroundsv v2", + ["likelyCulpritAppCount"] = "1", + })); + + private static DecisionStep ApplyDeviceSetupResolved(DecisionEngine engine, DecisionState state, long ordinal = 60) => + engine.Reduce(state, MakeSignal( + ordinal, DecisionSignalKind.DeviceSetupProvisioningComplete, T0.AddMinutes(87))); + + // ================================================== advisory arming context ==== + + [Fact] + public void Advisory_RecordsFailedCategory_AndCulpritApps() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + + var step = ApplyDeviceSetupAppsFailure(engine, state); + + Assert.NotNull(step.NewState.EspAdvisoryFailureRecordedUtc); + Assert.Equal("DeviceSetup", step.NewState.EspAdvisoryFailureCategory?.Value); + Assert.Null(step.NewState.EspAdvisoryFailureResolvedUtc); + + var advisory = FindTimelineEffect(step, "esp_failure_advisory"); + Assert.NotNull(advisory); + Assert.Equal("Oriflame Cosmectics - Teams Backgroundsv v2", advisory!.Parameters!["likelyCulpritApps"]); + } + + // ===================================================== recovery hook ==== + + [Fact] + public void DeviceSetupComplete_SameCategory_ResolvesAdvisory_AndEmitsStoryEvent() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + + var step = ApplyDeviceSetupResolved(engine, state); + + Assert.NotNull(step.NewState.EspAdvisoryFailureResolvedUtc); + Assert.Equal(60, step.NewState.EspAdvisoryFailureResolvedUtc!.SourceSignalOrdinal); + // The anchor itself stays — the advisory happened; it just no longer blocks. + Assert.NotNull(step.NewState.EspAdvisoryFailureRecordedUtc); + + var resolved = FindTimelineEffect(step, "esp_failure_advisory_resolved"); + Assert.NotNull(resolved); + Assert.Equal("DeviceSetup", resolved!.Parameters!["category"]); + Assert.Equal("23", resolved.Parameters["minutesSinceAdvisory"]); + } + + [Fact] + public void DeviceSetupComplete_DifferentCategory_DoesNotResolve() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + // Advisory for AccountSetup — DeviceSetup resolving must not defuse it. + state = engine.Reduce(state, MakeSignal( + 50, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(64), + new Dictionary + { + ["failureType"] = "Provisioning_AccountSetup_Apps_Failed", + ["failedSubcategory"] = "Apps", + ["category"] = "AccountSetup", + })).NewState; + + var step = ApplyDeviceSetupResolved(engine, state); + + Assert.Null(step.NewState.EspAdvisoryFailureResolvedUtc); + Assert.Null(FindTimelineEffect(step, "esp_failure_advisory_resolved")); + } + + [Fact] + public void DeviceSetupComplete_Duplicate_ResolvesOnlyOnce() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + state = ApplyDeviceSetupResolved(engine, state).NewState; + + // Post-reboot duplicate (AnchorAlreadySet passthrough branch). + var duplicate = engine.Reduce(state, MakeSignal( + 65, DecisionSignalKind.DeviceSetupProvisioningComplete, T0.AddMinutes(92))); + + Assert.Equal(60, duplicate.NewState.EspAdvisoryFailureResolvedUtc!.SourceSignalOrdinal); + Assert.Null(FindTimelineEffect(duplicate, "esp_failure_advisory_resolved")); + } + + // ================================================ deadline fire semantics ==== + + [Fact] + public void DeadlineFire_AfterRecovery_WithEnforcementProgress_ReArmsInsteadOfFailing() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + state = ApplyDeviceSetupResolved(engine, state).NewState; + // Post-reboot IME re-assert of AccountSetup (session shape: 05:02:59, ordinal 654) + // — user-phase enforcement demonstrably progressed since the arming at ordinal 50. + state = engine.Reduce(state, MakeSignal( + 70, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(92), + new Dictionary { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState; + + var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94))); + + // Not failed — the window re-arms with fresh baselines instead. + Assert.Null(step.NewState.Outcome); + Assert.NotEqual(SessionStage.Failed, step.NewState.Stage); + var rearmed = FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion); + Assert.NotNull(rearmed); + Assert.Equal(T0.AddMinutes(94).AddMinutes(30), rearmed!.DueAtUtc); + Assert.Contains(step.Effects, e => e.Kind == DecisionEffectKind.ScheduleDeadline); + } + + [Fact] + public void DeadlineFire_AfterRecovery_NoProgress_FailsWithRecoveredReason() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + state = ApplyDeviceSetupResolved(engine, state).NewState; + + var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94))); + + Assert.Equal(SessionStage.Failed, step.NewState.Stage); + Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome); + + var failed = FindTimelineEffect(step, "enrollment_failed"); + Assert.NotNull(failed); + // Truthful reason: the original failure recovered — do not blame it. + Assert.Equal("esp_recovered_without_completion_evidence", failed!.Parameters!["reason"]); + Assert.Equal("true", failed.Parameters["failureRecovered"]); + Assert.Equal("advisory_recovered_window_expired_without_completion_evidence", failed.Parameters["advisoryReason"]); + // Likely-stuck app promotion stays off: apps are not stuck, they succeeded. + Assert.Equal(nameof(DecisionSignalKind.DeadlineFired), step.NewState.LastFailureTrigger?.Value); + } + + [Fact] + public void DeadlineFire_WithoutRecovery_KeepsLegacyUndefang() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + // No recovery — even with enforcement progress the advisory variant must + // un-defang exactly as before (its anchor failure never un-happened). + state = engine.Reduce(state, MakeSignal( + 70, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(92), + new Dictionary { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState; + + var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94))); + + Assert.Equal(SessionStage.Failed, step.NewState.Stage); + var failed = FindTimelineEffect(step, "enrollment_failed"); + Assert.NotNull(failed); + Assert.Equal("esp_terminal_failure", failed!.Parameters!["reason"]); + Assert.Equal(nameof(DecisionSignalKind.EspTerminalFailure), step.NewState.LastFailureTrigger?.Value); + } + + // ======================================================= reboot rebase ==== + + [Fact] + public void Reboot_AfterRecovery_RebasesAdvisoryWindow() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + state = ApplyDeviceSetupResolved(engine, state).NewState; + + var step = engine.Reduce(state, MakeSignal( + 70, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92))); + + Assert.Contains(step.Effects, e => + e.Kind == DecisionEffectKind.CancelDeadline + && e.CancelDeadlineName == DeadlineNames.AdvisoryCompletion); + var rebased = FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion); + Assert.NotNull(rebased); + Assert.Equal(T0.AddMinutes(92).AddMinutes(30), rebased!.DueAtUtc); + } + + [Fact] + public void Reboot_WithoutRecovery_DoesNotTouchAdvisoryWindow() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + var originalDue = FindDeadline(state, DeadlineNames.AdvisoryCompletion)!.DueAtUtc; + + var step = engine.Reduce(state, MakeSignal( + 70, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92))); + + Assert.DoesNotContain(step.Effects, e => e.Kind == DecisionEffectKind.CancelDeadline); + Assert.Equal(originalDue, FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion)!.DueAtUtc); + } + + // ============================================== full session-4910a5a5 replay ==== + + [Fact] + public void Session4910a5a5_FailureRecoveryRebootFire_DoesNotFailTheLiveEnrollment() + { + // Field ordinals: 527 EspTerminalFailure → 611 DeviceSetupProvisioningComplete + // (post "Try again", apps 10/10) → 635 SystemRebootObserved → 654 AccountSetup + // re-assert → 671 DeadlineFired. Pre-fix outcome: enrollment_failed at the fire. + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state, ordinal: 527).NewState; + state = ApplyDeviceSetupResolved(engine, state, ordinal: 611).NewState; + state = engine.Reduce(state, MakeSignal( + 635, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92))).NewState; + state = engine.Reduce(state, MakeSignal( + 654, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(93), + new Dictionary { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState; + + var step = engine.Reduce(state, DeadlineFired(671, T0.AddMinutes(94))); + + Assert.Null(step.NewState.Outcome); + Assert.NotEqual(SessionStage.Failed, step.NewState.Stage); + Assert.NotNull(FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion)); + } + + // =============================================== serialization + census ==== + + [Fact] + public void SnapshotRoundTrip_PreservesRecoveryFacts() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + state = ApplyDeviceSetupResolved(engine, state).NewState; + + var roundtripped = StateSerializer.Deserialize(StateSerializer.Serialize(state)); + + Assert.Equal("DeviceSetup", roundtripped.EspAdvisoryFailureCategory?.Value); + Assert.Equal(state.EspAdvisoryFailureResolvedUtc!.Value, roundtripped.EspAdvisoryFailureResolvedUtc?.Value); + Assert.Equal(state.EspAdvisoryFailureResolvedUtc.SourceSignalOrdinal, roundtripped.EspAdvisoryFailureResolvedUtc?.SourceSignalOrdinal); + } + + [Fact] + public void LegacySnapshot_WithoutRecoveryFacts_DeserializesToNull() + { + // Simulate a snapshot written by a pre-fix agent: strip the new properties from + // the JSON entirely — deserialization must default them to null (additive compat). + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + state = ApplyDeviceSetupResolved(engine, state).NewState; + + var legacy = Newtonsoft.Json.Linq.JObject.Parse(StateSerializer.Serialize(state)); + foreach (var prop in legacy.Properties() + .Where(p => p.Name.IndexOf("EspAdvisoryFailureResolved", StringComparison.OrdinalIgnoreCase) >= 0 + || p.Name.IndexOf("EspAdvisoryFailureCategory", StringComparison.OrdinalIgnoreCase) >= 0) + .ToList()) + { + prop.Remove(); + } + + var roundtripped = StateSerializer.Deserialize(legacy.ToString()); + Assert.Null(roundtripped.EspAdvisoryFailureResolvedUtc); + Assert.Null(roundtripped.EspAdvisoryFailureCategory); + // The pre-existing advisory anchor still round-trips. + Assert.NotNull(roundtripped.EspAdvisoryFailureRecordedUtc); + } + + [Fact] + public void Census_SurfacesAdvisoryResolved() + { + var engine = new DecisionEngine(); + var state = SetupSession(engine); + state = ApplyDeviceSetupAppsFailure(engine, state).NewState; + state = ApplyDeviceSetupResolved(engine, state).NewState; + + var census = DecisionStateSignalCensus.Build(state); + + Assert.Contains("esp_advisory_failure_resolved", census.SignalsSeen); + Assert.True(census.SignalTimestamps.ContainsKey("espAdvisoryFailureResolved")); + } + } +} diff --git a/src/Agent/AutopilotMonitor.DecisionCore.Tests/DecisionStateSnapshotBuilderTests.cs b/src/Agent/AutopilotMonitor.DecisionCore.Tests/DecisionStateSnapshotBuilderTests.cs index 57c13aa0..c8b5f7b0 100644 --- a/src/Agent/AutopilotMonitor.DecisionCore.Tests/DecisionStateSnapshotBuilderTests.cs +++ b/src/Agent/AutopilotMonitor.DecisionCore.Tests/DecisionStateSnapshotBuilderTests.cs @@ -114,7 +114,10 @@ namespace AutopilotMonitor.DecisionCore.Tests // state-change-only dedupe anchor for the completion_waiting event → 17. // Session 772fe502 (2026-07-13) added `helloWizardStartedUtc` — genuine // Shell-Core 62404 wizard launch; vetoes the policy-disabled Hello skip → 18. - Assert.Equal(18, facts.Count); + // Session 4910a5a5 (2026-07-23) added `espAdvisoryFailureResolvedUtc` (the failed + // category recovered — lifts the advisory's re-arm/rebase guard exemption) and + // `espAdvisoryFailureCategory` (which category the advisory was for) → 20. + Assert.Equal(20, facts.Count); Assert.All(facts.Values, v => Assert.Null(v)); } @@ -236,10 +239,12 @@ namespace AutopilotMonitor.DecisionCore.Tests // `completionWaitingFingerprint` as the state-change-only dedupe anchor for // the completion_waiting event → 17. Session 772fe502 (2026-07-13) added // `helloWizardStartedUtc` — the genuine Shell-Core 62404 wizard-launch fact - // that vetoes/retracts the policy-disabled Hello skip → 18. If this number - // ever changes, both the count expectation AND the actual snapshot output - // need to evolve in lockstep. - Assert.Equal(18, expectedFactKeys.Count); + // that vetoes/retracts the policy-disabled Hello skip → 18. Session 4910a5a5 + // (2026-07-23) added `espAdvisoryFailureResolvedUtc` + `espAdvisoryFailureCategory` + // — advisory-recovery facts that lift the AdvisoryCompletion guard exemption → 20. + // If this number ever changes, both the count expectation AND the actual snapshot + // output need to evolve in lockstep. + Assert.Equal(20, expectedFactKeys.Count); var snapshot = DecisionStateSnapshotBuilder.Build(DecisionState.CreateInitial("s", "t")); var facts = (Dictionary)snapshot["facts"]!; diff --git a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts index 16b72483..2bfc1038 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts @@ -23,6 +23,9 @@ export const EVENT_TYPES_CATALOG = { 'esp_failure_advisory', 'esp_failure_settle_started', 'esp_failure_settle_recovered', + 'esp_failure_retry_detected', + 'esp_failure_recovered', + 'esp_failure_advisory_resolved', 'esp_exiting', 'mdm_policy_reboot_required', 'esp_config_detected', diff --git a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs index 0a219cf3..59bce8af 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs @@ -73,6 +73,11 @@ namespace AutopilotMonitor.DecisionCore.Engine /// The advisory variant ( /// set) is deliberately NOT touched: its anchor is a real ESP terminal failure and /// a reboot does not un-happen that failure — the window must still un-defang it. + /// EXCEPTION (session 4910a5a5, 2026-07-23): when + /// is set the failed + /// category demonstrably recovered to success — the failure DID un-happen, and a + /// reboot mid-recovery (ESP continuing into AccountSetup) must re-base the window + /// exactly like the esp-exit variant instead of letting it fail a live enrollment. /// /// Returns the effects (cancel + schedule + a fingerprint-bypassing /// completion_waiting carrying the re-based due-time, mirroring the @@ -84,7 +89,9 @@ namespace AutopilotMonitor.DecisionCore.Engine DecisionStateBuilder builder, DecisionSignal signal) { - if (state.EspAdvisoryFailureRecordedUtc != null || !HasAdvisoryCompletionDeadline(state)) + var unresolvedAdvisoryAnchored = state.EspAdvisoryFailureRecordedUtc != null + && state.EspAdvisoryFailureResolvedUtc == null; + if (unresolvedAdvisoryAnchored || !HasAdvisoryCompletionDeadline(state)) { return Array.Empty(); } @@ -139,6 +146,10 @@ namespace AutopilotMonitor.DecisionCore.Engine "mayHaveContinuedAnyway", "continueAnywayHint", "advisoryReason", + // Session 4910a5a5 — recovery story context. + "failureRecovered", + "likelyCulpritApps", + "likelyCulpritAppCount", }; /// @@ -408,10 +419,17 @@ namespace AutopilotMonitor.DecisionCore.Engine string reason, Dictionary parameters) { + // Session 4910a5a5: remember WHICH category failed so the recovery hook + // (TryResolveAdvisoryOnCategoryRecovery) can later match a ProvisioningComplete + // signal of the same category against this advisory. Absent on event-log-derived + // failures — the hook then never matches. + string? failedCategory = null; + signal.Payload?.TryGetValue("category", out failedCategory); + var builder = state.ToBuilder() .WithStepIndex(nextStep) .WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal) - .WithEspAdvisoryFailureRecorded(signal.OccurredAtUtc, signal.SessionSignalOrdinal); + .WithEspAdvisoryFailureRecorded(signal.OccurredAtUtc, signal.SessionSignalOrdinal, failedCategory); // Stage / Outcome deliberately untouched — monitoring continues. The session's // normal completion paths (Hello/Desktop, IME pattern, AccountSetup provisioning @@ -454,6 +472,63 @@ namespace AutopilotMonitor.DecisionCore.Engine return new DecisionStep(newState, transition, effects); } + /// + /// Session 4910a5a5 (2026-07-23) — recovery hook for a defanged ESP failure. When a + /// DeviceSetupProvisioningComplete / AccountSetupProvisioningComplete + /// signal reports the SAME category the advisory failure was recorded for, the failure + /// demonstrably un-happened (DeviceSetup/Apps failed → user pressed "Try again" ~23 min + /// later → apps re-ran to 10/10 → category resolved). Sets the set-once + /// fact on + /// and returns an esp_failure_advisory_resolved + /// timeline effect telling the recovery story; returns null (builder untouched) when + /// there is no advisory, it is already resolved, or the category does not match. + /// + /// The fact lifts the advisory variant's exemption from the AdvisoryCompletion + /// re-arm/rebase guards (see and + /// ) — without it the window + /// un-defangs the stale failure and fails a live enrollment mid-AccountSetup. + /// + /// + private static DecisionEffect? TryResolveAdvisoryOnCategoryRecovery( + DecisionState state, + DecisionStateBuilder builder, + DecisionSignal signal, + string resolvedCategory) + { + if (state.EspAdvisoryFailureRecordedUtc == null) return null; + if (state.EspAdvisoryFailureResolvedUtc != null) return null; + + var advisoryCategory = state.EspAdvisoryFailureCategory?.Value; + if (!string.Equals(advisoryCategory, resolvedCategory, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + builder.WithEspAdvisoryFailureResolved(signal.OccurredAtUtc, signal.SessionSignalOrdinal); + + var minutesSinceAdvisory = Math.Max(0, + (int)Math.Round((signal.OccurredAtUtc - state.EspAdvisoryFailureRecordedUtc.Value).TotalMinutes)); + var continueAnywayAllowed = state.ScenarioObservations?.EspAllowContinueAnyway?.Value == true; + + var parameters = new Dictionary + { + ["eventType"] = SharedConstants.EventTypes.EspFailureAdvisoryResolved, + ["source"] = "DecisionEngine", + ["severity"] = "Info", + ["immediateUpload"] = "false", + ["message"] = $"ESP failure advisory resolved: {resolvedCategory} completed successfully " + + $"{minutesSinceAdvisory} min after the advisory — the ESP re-ran the failed step " + + "(consistent with a user-initiated retry on the ESP failure page" + + (continueAnywayAllowed ? "; the profile allows Try again / Continue anyway" : "") + + "). The earlier failure no longer blocks completion.", + ["category"] = resolvedCategory, + ["minutesSinceAdvisory"] = minutesSinceAdvisory.ToString(CultureInfo.InvariantCulture), + ["espAllowContinueAnyway"] = continueAnywayAllowed ? "true" : "false", + }; + + return new DecisionEffect(DecisionEffectKind.EmitEventTimelineEntry, parameters: parameters); + } + /// /// fired — the resolution window after a /// completion dead-end expired without one of the normal completion paths terminating @@ -511,6 +586,12 @@ namespace AutopilotMonitor.DecisionCore.Engine var deadlineStillArmed = armedDeadline != null; var hasAdvisoryAnchor = state.EspAdvisoryFailureRecordedUtc != null; + // Session 4910a5a5: once the failed category demonstrably recovered + // (TryResolveAdvisoryOnCategoryRecovery set the fact), the advisory variant's + // "a real ESP terminal failure does not un-happen" premise is disproven — the + // guards below that protect live enrollments (enforcement-progress re-arm, + // hello-never-observed promote) apply to it again. + var advisoryResolved = state.EspAdvisoryFailureResolvedUtc != null; // Presence-only anchor (L5, delta review 2026-07-02). The esp-exit arming site only // runs when AccountSetupEnteredUtc is already set, so both facts being present is // the anchor; `deadlineStillArmed` below is what rejects stray timer fires. The old @@ -635,8 +716,13 @@ namespace AutopilotMonitor.DecisionCore.Engine // When user-phase enforcement demonstrably progressed since arming, the dead-end // shape is disproven: re-arm a fresh window with updated baselines instead of // failing a live enrollment mid-install. The advisory variant is exempt — its - // anchor is a real ESP terminal failure that a busy IME does not un-happen. - if (!hasAdvisoryAnchor && HasEnforcementProgressSinceArming(state, armedDeadline!)) + // anchor is a real ESP terminal failure that a busy IME does not un-happen — + // UNLESS the failed category later resolved to success (session 4910a5a5: + // DeviceSetup/Apps failed, user retried, apps reached 10/10, DeviceSetup resolved + // — the failure DID un-happen, so a live AccountSetup must not be failed on the + // stale advisory). Convergent either way: each re-arm renews the baselines, so a + // session that stalls after the recovery still fails one window later. + if ((!hasAdvisoryAnchor || advisoryResolved) && HasEnforcementProgressSinceArming(state, armedDeadline!)) { var rearmedDeadline = BuildAdvisoryCompletionDeadline(state, signal); builder.AddDeadline(rearmedDeadline); @@ -692,7 +778,7 @@ namespace AutopilotMonitor.DecisionCore.Engine // through Finalizing. Deliberately does NOT require the IME user-session gate — // the FP sessions never saw an IME user-session completion (that gate guards // completion evidence, not user presence; presence is the DAD-validated desktop). - if (!hasAdvisoryAnchor + if ((!hasAdvisoryAnchor || advisoryResolved) && desktopArrived && state.HelloResolvedUtc == null && state.HelloPolicyEnabled == null @@ -741,7 +827,30 @@ namespace AutopilotMonitor.DecisionCore.Engine // stays the deadline, not EspTerminalFailure). string reason; Dictionary parameters; - if (hasAdvisoryAnchor) + if (hasAdvisoryAnchor && advisoryResolved) + { + // Session 4910a5a5: the original ESP failure demonstrably recovered (failed + // category resolved to success after the advisory), so claiming + // `esp_terminal_failure` would blame a failure that un-happened. The device + // recovered its device phase but then produced no completion evidence for a + // whole (re-armed, progress-gated) window. LastFailureTrigger stays the + // deadline so the EnrollmentTerminationHandler's likely-stuck app promotion + // stays OFF — the apps are not stuck, they succeeded. + reason = "esp_recovered_without_completion_evidence"; + parameters = BuildEspFailureParameters( + signal: signal, + reason: reason, + advisoryPath: false, + observations: state.ScenarioObservations); + parameters["advisoryReason"] = "advisory_recovered_window_expired_without_completion_evidence"; + parameters["failureRecovered"] = "true"; + if (state.EspAdvisoryFailureCategory != null) + { + parameters["category"] = state.EspAdvisoryFailureCategory.Value; + } + builder.WithLastFailureTrigger(nameof(DecisionSignalKind.DeadlineFired), signal.SessionSignalOrdinal); + } + else if (hasAdvisoryAnchor) { // Registry-derived failure context (failureType/errorCode/...) lived on the // original signal and is already on the wire via the esp_failure_advisory @@ -885,6 +994,12 @@ namespace AutopilotMonitor.DecisionCore.Engine parameters["failedSubcategory"] = failedSubcategory; if (signal.Payload.TryGetValue("category", out var category) && !string.IsNullOrEmpty(category)) parameters["category"] = category; + // Session 4910a5a5 — tracker-snapshotted culprit-app names for Apps failures, + // so the advisory/terminal event tells WHICH app ESP most likely failed on. + if (signal.Payload.TryGetValue("likelyCulpritApps", out var culpritApps) && !string.IsNullOrEmpty(culpritApps)) + parameters["likelyCulpritApps"] = culpritApps; + if (signal.Payload.TryGetValue("likelyCulpritAppCount", out var culpritCount) && !string.IsNullOrEmpty(culpritCount)) + parameters["likelyCulpritAppCount"] = culpritCount; } // FirstSync-derived ESP failure-handling settings — observable hint that diff --git a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.SelfDeploying.cs b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.SelfDeploying.cs index b9d65d45..e904e4af 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.SelfDeploying.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.SelfDeploying.cs @@ -98,17 +98,28 @@ namespace AutopilotMonitor.DecisionCore.Engine // was already armed (or already cancelled by AccountSetup). Pass-through. if (state.DeviceSetupResolvedUtc != null) { - var passthroughState = state.ToBuilder() + var passthroughBuilder = state.ToBuilder() .WithStepIndex(nextStep) - .WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal) - .Build(); + .WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal); + // Session 4910a5a5 recovery hook: a duplicate arrival (e.g. the post-reboot + // registry re-read) can still be the first evidence that the advisory's failed + // category recovered — set-once + category-match inside the helper keep this + // a no-op everywhere else. + var passthroughResolveEffect = TryResolveAdvisoryOnCategoryRecovery( + state, passthroughBuilder, signal, resolvedCategory: "DeviceSetup"); + var passthroughState = passthroughBuilder.Build(); var passthroughTransition = BuildTakenTransition( before: state, signal: signal, toStage: state.Stage, nextStepIndex: nextStep, trigger: nameof(DecisionSignalKind.DeviceSetupProvisioningComplete) + ":AnchorAlreadySet"); - return new DecisionStep(passthroughState, passthroughTransition, Array.Empty()); + return new DecisionStep( + passthroughState, + passthroughTransition, + passthroughResolveEffect != null + ? new[] { passthroughResolveEffect } + : Array.Empty()); } // Step 2 — Set the DeviceSetupResolvedUtc anchor UNCONDITIONALLY. Even when Classic @@ -119,6 +130,13 @@ namespace AutopilotMonitor.DecisionCore.Engine .WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal); builder.DeviceSetupResolvedUtc = new SignalFact(signal.OccurredAtUtc, signal.SessionSignalOrdinal); + // Session 4910a5a5 recovery hook: DeviceSetup resolving to success while a + // DeviceSetup advisory failure is on record means the failure un-happened (user + // "Try again" retry). Sets the set-once resolved fact + emits the + // esp_failure_advisory_resolved story event; no-op otherwise. + var advisoryResolveEffect = TryResolveAdvisoryOnCategoryRecovery( + state, builder, signal, resolvedCategory: "DeviceSetup"); + // Step 3 — Classic-path short-circuit: AccountSetup already started. The DeviceOnly // deadline is moot (already cancelled by HandleEspPhaseChangedV1's AccountSetup branch, // or never armed under the new code). No deadline arm here. @@ -136,7 +154,12 @@ namespace AutopilotMonitor.DecisionCore.Engine toStage: state.Stage, nextStepIndex: nextStep, trigger: nameof(DecisionSignalKind.DeviceSetupProvisioningComplete) + ":AccountSetupAlreadyEntered"); - return new DecisionStep(classicState, classicTransition, Array.Empty()); + return new DecisionStep( + classicState, + classicTransition, + advisoryResolveEffect != null + ? new[] { advisoryResolveEffect } + : Array.Empty()); } // Step 4 — Arm the deadline. Cancel any existing DeviceOnlyEspDetection first @@ -156,13 +179,14 @@ namespace AutopilotMonitor.DecisionCore.Engine nextStepIndex: nextStep, trigger: nameof(DecisionSignalKind.DeviceSetupProvisioningComplete) + ":DeadlineArmed"); - var effects = new[] + var effects = new List(3) { new DecisionEffect(DecisionEffectKind.CancelDeadline, cancelDeadlineName: DeadlineNames.DeviceOnlyEspDetection), new DecisionEffect(DecisionEffectKind.ScheduleDeadline, deadline: deadline), }; + if (advisoryResolveEffect != null) effects.Add(advisoryResolveEffect); - return new DecisionStep(newState, transition, effects); + return new DecisionStep(newState, transition, effects.ToArray()); } /// diff --git a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Shared.cs b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Shared.cs index 651d6e81..9036b8e4 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Shared.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Shared.cs @@ -836,6 +836,13 @@ namespace AutopilotMonitor.DecisionCore.Engine .WithStepIndex(nextStep) .WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal); + // Session 4910a5a5 recovery hook (AccountSetup counterpart): the user-phase + // category resolving to success while an AccountSetup advisory failure is on + // record means that failure un-happened. Sets the set-once resolved fact + emits + // the esp_failure_advisory_resolved story event; no-op otherwise. + var advisoryResolveEffect = TryResolveAdvisoryOnCategoryRecovery( + state, builder, signal, resolvedCategory: "AccountSetup"); + var alreadyRecorded = state.AccountSetupProvisioningSucceededUtc != null; if (!alreadyRecorded) { @@ -876,10 +883,10 @@ namespace AutopilotMonitor.DecisionCore.Engine toStage: state.Stage, nextStepIndex: nextStep, trigger: nameof(DecisionSignalKind.AccountSetupProvisioningComplete)); - return new DecisionStep( - newState, - transition, - waitingEffect != null ? new[] { waitingEffect } : Array.Empty()); + var noPromoteEffects = new List(2); + if (advisoryResolveEffect != null) noPromoteEffects.Add(advisoryResolveEffect); + if (waitingEffect != null) noPromoteEffects.Add(waitingEffect); + return new DecisionStep(newState, transition, noPromoteEffects.ToArray()); } // Deferred-completion parity (session caa6cf50 fix, 2026-06-11): when the strong gate @@ -910,14 +917,18 @@ namespace AutopilotMonitor.DecisionCore.Engine builder.CancelDeadline(DeadlineNames.HelloSafety); } + var deferredLeadingEffects = new List(2); + if (advisoryResolveEffect != null) deferredLeadingEffects.Add(advisoryResolveEffect); + if (helloSafetyCancelEffect != null) deferredLeadingEffects.Add(helloSafetyCancelEffect); + return CompleteThroughFinalizingOrDefer( state: state, signal: signal, preparedBuilder: builder, nextStepIndex: nextStep, trigger: nameof(DecisionSignalKind.AccountSetupProvisioningComplete) + ":DeferredCompletion", - leadingEffects: helloSafetyCancelEffect != null - ? new[] { helloSafetyCancelEffect } + leadingEffects: deferredLeadingEffects.Count > 0 + ? deferredLeadingEffects.ToArray() : null); } @@ -946,12 +957,13 @@ namespace AutopilotMonitor.DecisionCore.Engine nextStepIndex: nextStep, trigger: nameof(DecisionSignalKind.AccountSetupProvisioningComplete) + ":DeferredPromote"); - var promotedEffects = new[] + var promotedEffects = new List(2) { new DecisionEffect(DecisionEffectKind.ScheduleDeadline, deadline: helloSafety), }; + if (advisoryResolveEffect != null) promotedEffects.Add(advisoryResolveEffect); - return new DecisionStep(promotedState, promotedTransition, promotedEffects); + return new DecisionStep(promotedState, promotedTransition, promotedEffects.ToArray()); } /// diff --git a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionStateSignalCensus.cs b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionStateSignalCensus.cs index 2e1300ea..b367ff90 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionStateSignalCensus.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionStateSignalCensus.cs @@ -110,6 +110,17 @@ namespace AutopilotMonitor.DecisionCore.Engine timestamps["imeUserSessionCompleted"] = FormatUtc(state.ImeUserSessionCompletedUtc.Value); evidence["imeUserSessionCompleted"] = TimestampedEvidence(state.ImeUserSessionCompletedUtc); } + // Session 4910a5a5 — the ESP category behind a defanged advisory failure later + // resolved to success (user "Try again" recovery). Surfaced so terminal audit + // trails show that the original failure demonstrably un-happened — the + // discriminant between "failed and stayed failed" and "failed, recovered, but + // produced no completion evidence". + if (state.EspAdvisoryFailureResolvedUtc != null) + { + seen.Add("esp_advisory_failure_resolved"); + timestamps["espAdvisoryFailureResolved"] = FormatUtc(state.EspAdvisoryFailureResolvedUtc.Value); + evidence["espAdvisoryFailureResolved"] = TimestampedEvidence(state.EspAdvisoryFailureResolvedUtc); + } // Session 772fe502 — genuine Hello-wizard launch (Shell-Core 62404, CXID AADHello/ // NGC). Surfaced in the census so terminal audit trails show whether a wizard was // observed — the discriminant between "Hello skipped by policy" and "Hello skipped diff --git a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionStateSnapshotBuilder.cs b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionStateSnapshotBuilder.cs index f65b0131..801ac3b3 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionStateSnapshotBuilder.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionStateSnapshotBuilder.cs @@ -63,6 +63,8 @@ namespace AutopilotMonitor.DecisionCore.Engine ["helloPolicyEnabled"] = SerializeFact(state.HelloPolicyEnabled, v => (object)v), ["lastFailureTrigger"] = SerializeFact(state.LastFailureTrigger, v => v), ["espAdvisoryFailureRecordedUtc"] = SerializeFact(state.EspAdvisoryFailureRecordedUtc, FormatUtc), + ["espAdvisoryFailureResolvedUtc"] = SerializeFact(state.EspAdvisoryFailureResolvedUtc, FormatUtc), + ["espAdvisoryFailureCategory"] = SerializeFact(state.EspAdvisoryFailureCategory, v => v), ["imeUserSessionCompletedUtc"] = SerializeFact(state.ImeUserSessionCompletedUtc, FormatUtc), ["completionWaitingFingerprint"] = SerializeFact(state.CompletionWaitingFingerprint, v => v), ["helloWizardStartedUtc"] = SerializeFact(state.HelloWizardStartedUtc, FormatUtc), diff --git a/src/Shared/AutopilotMonitor.DecisionCore/State/DecisionState.cs b/src/Shared/AutopilotMonitor.DecisionCore/State/DecisionState.cs index 7d981b81..108ad2b8 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/State/DecisionState.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/State/DecisionState.cs @@ -64,7 +64,9 @@ namespace AutopilotMonitor.DecisionCore.State SignalFact? espAdvisoryFailureRecordedUtc = null, SignalFact? imeUserSessionCompletedUtc = null, SignalFact? completionWaitingFingerprint = null, - SignalFact? helloWizardStartedUtc = null) + SignalFact? helloWizardStartedUtc = null, + SignalFact? espAdvisoryFailureResolvedUtc = null, + SignalFact? espAdvisoryFailureCategory = null) { if (string.IsNullOrEmpty(sessionId)) { @@ -108,6 +110,8 @@ namespace AutopilotMonitor.DecisionCore.State ImeUserSessionCompletedUtc = imeUserSessionCompletedUtc; CompletionWaitingFingerprint = completionWaitingFingerprint; HelloWizardStartedUtc = helloWizardStartedUtc; + EspAdvisoryFailureResolvedUtc = espAdvisoryFailureResolvedUtc; + EspAdvisoryFailureCategory = espAdvisoryFailureCategory; } public string SessionId { get; } @@ -320,6 +324,36 @@ namespace AutopilotMonitor.DecisionCore.State /// public SignalFact? HelloWizardStartedUtc { get; } + /// + /// Set when the ESP category behind a recorded advisory failure + /// () later resolves to success — i.e. the + /// failure demonstrably un-happened. Session 4910a5a5 (2026-07-23): DeviceSetup/Apps + /// failed, the user pressed "Try again" ~23 min later, the apps re-ran to 10/10 and + /// DeviceSetupProvisioningComplete arrived — yet the AdvisoryCompletion + /// window still un-defanged the stale failure and failed the live enrollment. + /// + /// Set-once by the shared recovery hook in the ProvisioningComplete handlers when the + /// resolved category matches the armFailedCategory carried on the armed + /// AdvisoryCompletion deadline. Once set, the advisory variant loses its + /// exemption from the enforcement-progress re-arm, the reboot rebase and the Hello + /// promotes — those guards exist to protect live enrollments, and with this fact the + /// "a real ESP terminal failure does not un-happen" premise is disproven for this + /// session. Additive-nullable, no snapshot-schema bump. + /// + /// + public SignalFact? EspAdvisoryFailureResolvedUtc { get; } + + /// + /// The ESP category (DeviceSetup / AccountSetup) named by the advisory + /// failure's signal payload, recorded alongside . + /// Lives on state — not on the AdvisoryCompletion deadline payload — because + /// re-arm and reboot-rebase sites rebuild that deadline and would silently drop a + /// payload-carried category. Null when the arming signal carried no category (e.g. + /// event-log-derived failures); the recovery hook then never matches and behavior is + /// unchanged. Additive-nullable, no snapshot-schema bump. + /// + public SignalFact? EspAdvisoryFailureCategory { get; } + public string SchemaVersion { get; } /// diff --git a/src/Shared/AutopilotMonitor.DecisionCore/State/DecisionStateBuilder.cs b/src/Shared/AutopilotMonitor.DecisionCore/State/DecisionStateBuilder.cs index 9e339df7..bb683162 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/State/DecisionStateBuilder.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/State/DecisionStateBuilder.cs @@ -51,6 +51,8 @@ namespace AutopilotMonitor.DecisionCore.State ImeUserSessionCompletedUtc = source.ImeUserSessionCompletedUtc; CompletionWaitingFingerprint = source.CompletionWaitingFingerprint; HelloWizardStartedUtc = source.HelloWizardStartedUtc; + EspAdvisoryFailureResolvedUtc = source.EspAdvisoryFailureResolvedUtc; + EspAdvisoryFailureCategory = source.EspAdvisoryFailureCategory; } public string SessionId { get; set; } @@ -84,6 +86,8 @@ namespace AutopilotMonitor.DecisionCore.State public SignalFact? ImeUserSessionCompletedUtc { get; set; } public SignalFact? CompletionWaitingFingerprint { get; set; } public SignalFact? HelloWizardStartedUtc { get; set; } + public SignalFact? EspAdvisoryFailureResolvedUtc { get; set; } + public SignalFact? EspAdvisoryFailureCategory { get; set; } public string SchemaVersion { get; set; } // ---------- fluent helpers for the most common reducer operations ---------- @@ -197,9 +201,25 @@ namespace AutopilotMonitor.DecisionCore.State /// Fire-once gate: subsequent EspTerminalFailure signals see this fact set and /// dead-end without further effect. /// - public DecisionStateBuilder WithEspAdvisoryFailureRecorded(DateTime utc, long sourceSignalOrdinal) + public DecisionStateBuilder WithEspAdvisoryFailureRecorded( + DateTime utc, long sourceSignalOrdinal, string? failedCategory = null) { EspAdvisoryFailureRecordedUtc = new SignalFact(utc, sourceSignalOrdinal); + if (!string.IsNullOrEmpty(failedCategory)) + { + EspAdvisoryFailureCategory = new SignalFact(failedCategory!, sourceSignalOrdinal); + } + return this; + } + + /// + /// Record that the ESP category behind the recorded advisory failure later resolved to + /// success (session 4910a5a5 — "Try again" recovery). Set-once; lifts the advisory + /// variant's exemption from the AdvisoryCompletion re-arm/rebase guards. + /// + public DecisionStateBuilder WithEspAdvisoryFailureResolved(DateTime utc, long sourceSignalOrdinal) + { + EspAdvisoryFailureResolvedUtc = new SignalFact(utc, sourceSignalOrdinal); return this; } @@ -236,6 +256,8 @@ namespace AutopilotMonitor.DecisionCore.State espAdvisoryFailureRecordedUtc: EspAdvisoryFailureRecordedUtc, imeUserSessionCompletedUtc: ImeUserSessionCompletedUtc, completionWaitingFingerprint: CompletionWaitingFingerprint, - helloWizardStartedUtc: HelloWizardStartedUtc); + helloWizardStartedUtc: HelloWizardStartedUtc, + espAdvisoryFailureResolvedUtc: EspAdvisoryFailureResolvedUtc, + espAdvisoryFailureCategory: EspAdvisoryFailureCategory); } } diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 48d47949..4f0f0f76 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -539,6 +539,17 @@ namespace AutopilotMonitor.Shared public const string EspFailureSettleRecovered = "esp_failure_settle_recovered"; // The failed subcategory left the "failed" state during the settle window (e.g. ESP "Try again" retry, session c071e92b) — the terminal EspFailureDetected fire is suppressed and monitoring continues; a subsequent failure re-arms a fresh settle window. public const string EspAppxFailureAnalysis = "esp_appx_failure_analysis"; // One-shot AppXDeploymentServer/Operational scan during the ESP failure settle window: candidate MSIX/Store packages behind an "Apps (0x…)" subcategory failure invisible to ImeLogTracker (session 2bc884b6). Assessment, not a confirmed root cause. public const string EspProvisioningRaw = "esp_provisioning_raw"; + // Recovery story AFTER a terminally fired ESP failure (session 4910a5a5, 2026-07-23 — + // DeviceSetup/Apps failed, user pressed "Try again" ~23 min later, apps re-ran to + // 10/10). Distinct from EspFailureSettleRecovered, which covers recovery INSIDE the + // 30s settle window (terminal fire suppressed); these fire when the failure already + // went terminal (advisory/enrollment_failed) and the registry later disproves it. + public const string EspFailureRetryDetected = "esp_failure_retry_detected"; // failed subcategory left "failed" post-terminal — consistent with the user pressing "Try again" + public const string EspFailureRecovered = "esp_failure_recovered"; // previously failed category reached all-subcategories-succeeded post-terminal + // DecisionEngine counterpart (survives reboots via snapshot, unlike the in-run tracker + // pair above): the category behind a defanged esp_failure_advisory later resolved to + // success — the advisory no longer blocks; re-arm/rebase guards apply again. + public const string EspFailureAdvisoryResolved = "esp_failure_advisory_resolved"; public const string EspFailure = "esp_failure"; public const string EspExiting = "esp_exiting"; diff --git a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts index 5f20b209..7414e1b0 100644 --- a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts +++ b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts @@ -98,6 +98,12 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [ description: "ESP reported a subcategory failure but the device had already progressed to AccountSetup with ContinueAnyway enabled — non-terminal advisory, the agent continues monitoring." }, { value: "esp_appx_failure_analysis", label: "esp_appx_failure_analysis", category: "esp", description: "AppX deployment log scan during the ESP failure settle window: suspected MSIX/Store package candidates behind an Apps-subcategory failure (assessment, not a confirmed root cause)." }, + { value: "esp_failure_retry_detected", label: "esp_failure_retry_detected", category: "esp", + description: "After a terminally reported ESP failure, the failed subcategory left the failed state — consistent with the user pressing 'Try again' on the ESP failure page." }, + { value: "esp_failure_recovered", label: "esp_failure_recovered", category: "esp", + description: "The ESP category behind a terminally reported failure later completed successfully — the earlier failure recovered (e.g. after a user retry)." }, + { value: "esp_failure_advisory_resolved", label: "esp_failure_advisory_resolved", category: "esp", + description: "DecisionEngine: the category behind a defanged esp_failure_advisory resolved to success — the advisory no longer blocks completion." }, // -------- App installs -------- { value: "app_install_started", label: "app_install_started", category: "app", diff --git a/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/EventTimeline.tsx b/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/EventTimeline.tsx index 59b340aa..f117b56b 100644 --- a/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/EventTimeline.tsx +++ b/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/EventTimeline.tsx @@ -521,6 +521,46 @@ function EventRow({ event, showScriptOutput }: { event: EnrollmentEvent; showScr subcategory: {String(sub)} )} + {event.data?.likelyCulpritApps && ( + + Likely app: {String(event.data.likelyCulpritApps)} + + )} + + ); + })()} + {/* Recovery-story badges (session 4910a5a5): a terminally reported ESP failure that + later un-happened — the user retried ("Try again"), the failed step re-ran and + recovered. Green/teal palette so the timeline visually closes the earlier red/amber + failure arc. */} + {(event.eventType === "esp_failure_retry_detected" || event.eventType === "esp_failure_recovered" || event.eventType === "esp_failure_advisory_resolved") && (() => { + const isRetry = event.eventType === "esp_failure_retry_detected"; + const label = isRetry ? "User retry" : "Recovered"; + const badgeCls = isRetry ? "bg-sky-100 text-sky-800" : "bg-emerald-100 text-emerald-800"; + const cat = event.data?.category; + const sub = event.data?.subcategory ?? event.data?.failedSubcategory; + const mins = event.data?.minutesSinceFailure ?? event.data?.minutesSinceAdvisory; + return ( +
+ + {label} + + {cat && ( + + {String(cat)}{sub ? <> / {String(sub)} : null} + + )} + {mins !== undefined && mins !== null && ( + {String(mins)} min after the failure + )}
); })()}