From e8e062d24a4f92ccf8ef2bd1d5f858f48c2a569e Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 21 May 2026 16:37:28 +0200 Subject: [PATCH] feat(agent/v2,decision): surface ESP failure HRESULT + 30s settle window Session 9d052230 dropped enrollment_failed with reason esp_terminal_failure but the underlying HRESULT 0x87d1041c (Application not detected after install) sat unparsed in the ESP subcategory statusText, and the agent terminated before IME could emit the matching app_install_failed. Two-part fix: - ProvisioningStatusTracker arms a 30s settle window between ESP-subcategory failure detection and EspFailureDetected fire. Gives ImeLogTracker a chance to surface the app-level failure on the timeline. Emits esp_failure_settle_started for observability. - HRESULT extracted via language-invariant regex from the failed subcategory statusText, surfaced top-level on esp_provisioning_status (failedSubcategoryErrorCode) and propagated through new EspFailureDetectedEventArgs to the EspTerminalFailure signal payload. DecisionEngine.Edge writes failureType/errorCode/failedSubcategory/ category onto the enrollment_failed event parameters. UI renders an HRESULT badge with description from errorCodeMap.ts (analogous to app_install_failed). V2-only; V1 (Agent.Core) untouched. ConfigVersion unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ProvisioningFailureSettleWindowTests.cs | 179 ++++++++++++++ .../EspFailureDetectedHResultTests.cs | 143 ++++++++++++ .../SystemSignals/EspAndHelloTracker.cs | 55 ++++- .../EspFailureDetectedEventArgs.cs | 53 +++++ .../ProvisioningStatusTracker.cs | 218 ++++++++++++++++-- .../EspAndHelloTrackerAdapter.cs | 38 ++- .../ProvisioningStatusTrackerAdapter.cs | 49 ++-- .../buildcounter.txt | 2 +- .../DecisionAuditTrailTests.cs | 60 +++++ .../Engine/DecisionEngine.Edge.cs | 28 ++- .../[sessionId]/components/EventTimeline.tsx | 28 +++ 11 files changed, 793 insertions(+), 60 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningFailureSettleWindowTests.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/SignalAdapters/EspFailureDetectedHResultTests.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspFailureDetectedEventArgs.cs diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningFailureSettleWindowTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningFailureSettleWindowTests.cs new file mode 100644 index 00000000..f005affb --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningFailureSettleWindowTests.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals +{ + /// + /// Session 9d052230 (2026-05-21) regression coverage. When ESP DeviceSetup + /// Apps transitions to failed with a Windows HRESULT in its + /// statusText (e.g. "Apps (0x87d1041c)"), the tracker must: + /// + /// NOT fire immediately — + /// a 30 s settle window gives ImeLogTracker time to emit the underlying + /// app_install_failed event so the timeline carries the app-level failure. + /// Emit esp_failure_settle_started (Info) for observability. + /// Emit esp_provisioning_status with a top-level + /// failedSubcategoryErrorCode field so the UI / RuleEngine can match on it + /// without parsing nested statusText. + /// On settle-window expiry, fire EspFailureDetected with enriched + /// . + /// + /// + public sealed class ProvisioningFailureSettleWindowTests + { + private static readonly DateTime Fixed = new DateTime(2026, 5, 21, 12, 25, 0, DateTimeKind.Utc); + + private sealed class Fixture : IDisposable + { + public TempDirectory Tmp { get; } = new TempDirectory(); + public FakeSignalIngressSink Sink { get; } = new FakeSignalIngressSink(); + public List EspFailures { get; } = new List(); + public ProvisioningStatusTracker Tracker { get; } + + public Fixture() + { + var clock = new VirtualClock(Fixed); + var post = new InformationalEventPost(Sink, clock); + var logger = new AgentLogger(Tmp.Path, AgentLogLevel.Info); + Tracker = new ProvisioningStatusTracker( + sessionId: "S1", + tenantId: "T1", + post: post, + logger: logger); + Tracker.EspFailureDetected += (_, args) => EspFailures.Add(args); + } + + // Posted EnrollmentEvent equivalents: filter the sink for InformationalEvent signals, + // pull the eventType from payload and the data from typedPayload. + public IReadOnlyList<(string EventType, IReadOnlyDictionary Data)> CapturedEvents() + { + return Sink.Posted + .Where(p => p.Kind == DecisionSignalKind.InformationalEvent && p.Payload != null) + .Select(p => + { + var eventType = p.Payload!.TryGetValue(SignalPayloadKeys.EventType, out var et) ? et : ""; + var data = p.TypedPayload as IReadOnlyDictionary + ?? (IReadOnlyDictionary)new Dictionary(); + return (eventType, data); + }) + .ToList(); + } + + public void Dispose() + { + Tracker.Dispose(); + Tmp.Dispose(); + } + } + + [Fact] + public void FailedAppsSubcategoryWithHResult_armsSettleWindow_DoesNotFireImmediately() + { + using var f = new Fixture(); + + // First seen — Apps in_progress. + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (4 of 4 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Identifying)""} + }"); + + // Apps transitions to failed with HRESULT in statusText. + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (4 of 4 applied)""}, + ""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (0x87d1041c)""} + }"); + + // EspFailureDetected is delayed by the settle window — no immediate fire. + Assert.Empty(f.EspFailures); + + var captured = f.CapturedEvents(); + + // esp_failure_settle_started observability event is emitted. + var settleStarted = captured.SingleOrDefault(e => e.EventType == "esp_failure_settle_started"); + Assert.NotEqual(default, settleStarted); + Assert.Equal("DeviceSetup", (string)settleStarted.Data["category"]); + Assert.Equal("Apps", (string)settleStarted.Data["failedSubcategory"]); + Assert.Equal("0x87d1041c", (string)settleStarted.Data["errorCode"]); + Assert.Equal(ProvisioningStatusTracker.ProvisioningFailureSettleWindowSeconds, (int)settleStarted.Data["settleSeconds"]); + + // esp_provisioning_status (subcategory_state_change) carries failedSubcategoryErrorCode top-level. + var statusEvents = captured.Where(e => e.EventType == AutopilotMonitor.Shared.Constants.EventTypes.EspProvisioningStatus).ToList(); + var transitionEvent = statusEvents.Last(e => + e.Data.TryGetValue("changeType", out var ct) && (string)ct == "subcategory_state_change"); + Assert.Equal("0x87d1041c", (string)transitionEvent.Data["failedSubcategoryErrorCode"]); + Assert.Equal("Apps", (string)transitionEvent.Data["failedSubcategories"]); + + // TriggerSettleTimerForTest synchronously runs the timer callback. + f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status"); + + var fired = Assert.Single(f.EspFailures); + Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", fired.FailureType); + Assert.Equal("0x87d1041c", fired.ErrorCode); + Assert.Equal("Apps", fired.FailedSubcategory); + Assert.Equal("DeviceSetup", fired.Category); + } + + [Fact] + public void FailedSubcategoryWithoutHResult_stillArmsSettleWindow_ErrorCodeIsNull() + { + using var f = new Fixture(); + + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Working""} + }"); + + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": false, + ""SecurityPoliciesSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Failed""} + }"); + + Assert.Empty(f.EspFailures); + var captured = f.CapturedEvents(); + var settleStarted = captured.SingleOrDefault(e => e.EventType == "esp_failure_settle_started"); + Assert.NotEqual(default, settleStarted); + Assert.False(settleStarted.Data.ContainsKey("errorCode")); + + f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status"); + + var fired = Assert.Single(f.EspFailures); + Assert.Null(fired.ErrorCode); + Assert.Equal("SecurityPolicies", fired.FailedSubcategory); + } + + [Fact] + public void RepeatedFailedProcessing_armsSettleTimer_OnlyOnce() + { + using var f = new Fixture(); + + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Identifying)""} + }"); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": null, + ""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (0x87d1041c)""} + }"); + // Re-process with the same failed-state JSON — fire-once guard must prevent a + // second settle-window arm and a duplicate esp_failure_settle_started event. + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{ + ""categorySucceeded"": false, + ""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (0x87d1041c)""} + }"); + + var captured = f.CapturedEvents(); + Assert.Single(captured.Where(e => e.EventType == "esp_failure_settle_started")); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/SignalAdapters/EspFailureDetectedHResultTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/SignalAdapters/EspFailureDetectedHResultTests.cs new file mode 100644 index 00000000..b9226098 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/SignalAdapters/EspFailureDetectedHResultTests.cs @@ -0,0 +1,143 @@ +using System; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.SignalAdapters; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.DecisionCore.Signals; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters +{ + /// + /// Session 9d052230 (2026-05-21) regression coverage: ESP-Apps failure with a Windows HRESULT + /// embedded in the subcategory statusText must surface + /// errorCode/failedSubcategory/category on the + /// signal payload so the DecisionEngine + /// can propagate it to the terminal enrollment_failed event. + /// + public sealed class EspFailureDetectedHResultTests + { + private static readonly DateTime Fixed = new DateTime(2026, 5, 21, 12, 25, 41, DateTimeKind.Utc); + + // ------------------------------------------------------------------ Regex extraction + + [Theory] + [InlineData("Apps (0x87d1041c)", "0x87d1041c")] + [InlineData("Security policies (0x80070005)", "0x80070005")] + [InlineData("Certificates (No setup needed)", null)] + [InlineData("Apps (Identifying)", null)] + [InlineData("Apps (0X87D1041C)", "0x87d1041c")] + [InlineData("Network connections (1 of 1 added)", null)] + [InlineData("", null)] + [InlineData(null, null)] + public void TryExtractErrorCode_extracts_parenthesised_hex_tail(string statusText, string expected) + { + var actual = ProvisioningStatusTracker.TryExtractErrorCode(statusText); + Assert.Equal(expected, actual); + } + + // ------------------------------------------------------------------ EspAndHelloTrackerAdapter + + [Fact] + public void Coordinator_adapter_forwards_errorCode_failedSubcategory_category_to_signal_payload() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + var clock = new VirtualClock(Fixed); + var ingress = new FakeSignalIngressSink(); + var coordinator = new EspAndHelloTracker( + sessionId: "S1", + tenantId: "T1", + post: new InformationalEventPost(new FakeSignalIngressSink(), clock), + logger: logger); + + try + { + using var adapter = new EspAndHelloTrackerAdapter(coordinator, ingress, clock); + + adapter.TriggerEspFailureFromTest(new EspFailureDetectedEventArgs( + failureType: "Provisioning_DeviceSetup_Apps_Failed", + errorCode: "0x87d1041c", + failedSubcategory: "Apps", + category: "DeviceSetup")); + + var posted = Assert.Single(ingress.Posted); + Assert.Equal(DecisionSignalKind.EspTerminalFailure, posted.Kind); + Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", posted.Payload!["failureType"]); + Assert.Equal("0x87d1041c", posted.Payload!["errorCode"]); + Assert.Equal("Apps", posted.Payload!["failedSubcategory"]); + Assert.Equal("DeviceSetup", posted.Payload!["category"]); + Assert.Equal("0x87d1041c", posted.Evidence.DerivationInputs!["errorCode"]); + } + finally { coordinator.Dispose(); } + } + + [Fact] + public void Coordinator_adapter_omits_optional_keys_when_only_failureType_is_set() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + var clock = new VirtualClock(Fixed); + var ingress = new FakeSignalIngressSink(); + var coordinator = new EspAndHelloTracker( + sessionId: "S1", + tenantId: "T1", + post: new InformationalEventPost(new FakeSignalIngressSink(), clock), + logger: logger); + + try + { + using var adapter = new EspAndHelloTrackerAdapter(coordinator, ingress, clock); + + // Backwards-compat: ShellCore-derived failures use the string overload — + // no HRESULT, no subcategory, no category surface. + adapter.TriggerEspFailureFromTest("ESPProgress_Timeout"); + + var posted = Assert.Single(ingress.Posted); + Assert.Equal("ESPProgress_Timeout", posted.Payload!["failureType"]); + Assert.False(posted.Payload!.ContainsKey("errorCode")); + Assert.False(posted.Payload!.ContainsKey("failedSubcategory")); + Assert.False(posted.Payload!.ContainsKey("category")); + } + finally { coordinator.Dispose(); } + } + + // ------------------------------------------------------------------ ProvisioningStatusTrackerAdapter + + [Fact] + public void Provisioning_adapter_forwards_full_args_to_signal_payload() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + var clock = new VirtualClock(Fixed); + var ingress = new FakeSignalIngressSink(); + var tracker = new ProvisioningStatusTracker( + sessionId: "S1", + tenantId: "T1", + post: new InformationalEventPost(new FakeSignalIngressSink(), clock), + logger: logger); + + try + { + using var adapter = new ProvisioningStatusTrackerAdapter(tracker, ingress, clock); + + adapter.TriggerEspFailureFromTest(new EspFailureDetectedEventArgs( + failureType: "Provisioning_DeviceSetup_Apps_Failed", + errorCode: "0x87d1041c", + failedSubcategory: "Apps", + category: "DeviceSetup")); + + var posted = Assert.Single(ingress.Posted); + Assert.Equal(DecisionSignalKind.EspTerminalFailure, posted.Kind); + Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", posted.Payload!["failureType"]); + Assert.Equal("0x87d1041c", posted.Payload!["errorCode"]); + Assert.Equal("Apps", posted.Payload!["failedSubcategory"]); + Assert.Equal("DeviceSetup", posted.Payload!["category"]); + } + finally { tracker.Dispose(); } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs index 13ef7b11..3daa0468 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs @@ -69,9 +69,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals /// /// Fired when an ESP failure is detected (ESPProgress_Failure, _Timeout, _Abort, - /// WhiteGlove_Failed, Provisioning_*_Failed, etc.). The string is the structured failure type. + /// WhiteGlove_Failed, Provisioning_*_Failed, etc.). + /// + /// carries the structured failure + /// identifier. Registry-derived failures from + /// additionally enrich , + /// , and + /// ; event-log-derived failures from + /// only set . + /// /// - public event EventHandler EspFailureDetected; + public event EventHandler EspFailureDetected; /// /// Fired when DeviceSetup provisioning status shows categorySucceeded=true (or fallback confirmed). @@ -248,7 +256,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _helloTracker); _shellCoreTracker.FinalizingSetupPhaseTriggered += OnFinalizingSetupPhaseTriggered; _shellCoreTracker.WhiteGloveCompleted += OnWhiteGloveCompleted; - _shellCoreTracker.EspFailureDetected += OnEspFailureDetected; + _shellCoreTracker.EspFailureDetected += OnShellCoreEspFailureDetected; _shellCoreTracker.EspExited += OnEspExited; _shellCoreTracker.Start(); @@ -257,7 +265,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _tenantId, _post, _logger); - _provisioningTracker.EspFailureDetected += OnEspFailureDetected; + _provisioningTracker.EspFailureDetected += OnProvisioningEspFailureDetected; _provisioningTracker.DeviceSetupProvisioningComplete += OnDeviceSetupProvisioningComplete; _provisioningTracker.AccountSetupProvisioningComplete += OnAccountSetupProvisioningComplete; _provisioningTracker.Start(); @@ -310,7 +318,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals { _shellCoreTracker.FinalizingSetupPhaseTriggered -= OnFinalizingSetupPhaseTriggered; _shellCoreTracker.WhiteGloveCompleted -= OnWhiteGloveCompleted; - _shellCoreTracker.EspFailureDetected -= OnEspFailureDetected; + _shellCoreTracker.EspFailureDetected -= OnShellCoreEspFailureDetected; _shellCoreTracker.EspExited -= OnEspExited; _shellCoreTracker.Stop(); } @@ -323,7 +331,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals if (_provisioningTracker == null) return; try { - _provisioningTracker.EspFailureDetected -= OnEspFailureDetected; + _provisioningTracker.EspFailureDetected -= OnProvisioningEspFailureDetected; _provisioningTracker.DeviceSetupProvisioningComplete -= OnDeviceSetupProvisioningComplete; _provisioningTracker.AccountSetupProvisioningComplete -= OnAccountSetupProvisioningComplete; _provisioningTracker.Stop("tracker_stopped"); @@ -463,17 +471,40 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals finally { LastEventOccurredAtUtc = null; } } - private void OnEspFailureDetected(object sender, string failureType) + // Forwarder for ShellCoreTracker (event-log-derived ESP failures, e.g. Shell-Core 62407 + // failure descriptions). Source-event timestamp is mirrored so the adapter can stamp the + // DecisionSignal with the historical instant on backfill. ShellCoreTracker has no HRESULT + // surface, so only FailureType is set on the args. + private void OnShellCoreEspFailureDetected(object sender, string failureType) { - // EspFailureDetected fires from ShellCoreTracker (carries timestamp) AND - // ProvisioningStatusTracker (no timestamp surfaced today — adapter falls back - // to clock). Cast attempt below picks up the timestamp on the ShellCore path. LastEventOccurredAtUtc = (sender as ShellCoreTracker)?.LastEventOccurredAtUtc; - try { EspFailureDetected?.Invoke(this, failureType); } - catch (Exception ex) { _logger.Error($"Error forwarding EspFailureDetected for '{failureType}'", ex); } + try + { + EspFailureDetected?.Invoke(this, new EspFailureDetectedEventArgs(failureType)); + } + catch (Exception ex) { _logger.Error($"Error forwarding ShellCore EspFailureDetected for '{failureType}'", ex); } finally { LastEventOccurredAtUtc = null; } } + // Forwarder for ProvisioningStatusTracker (registry-derived ESP failures). Args carry the + // full failure detail (FailureType, ErrorCode, FailedSubcategory, Category) extracted + // from the failed subcategory's statusText. Provisioning has no source-event timestamp + // surfaced today, so the adapter falls back to clock. + private void OnProvisioningEspFailureDetected(object sender, EspFailureDetectedEventArgs args) + { + LastEventOccurredAtUtc = null; + try + { + EspFailureDetected?.Invoke(this, args); + } + catch (Exception ex) + { + _logger.Error( + $"Error forwarding Provisioning EspFailureDetected for '{args?.FailureType ?? "n/a"}'", + ex); + } + } + private void OnDeviceSetupProvisioningComplete(object sender, EventArgs e) { // Provisioning tracker doesn't surface a timestamp; adapter falls back to clock. diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspFailureDetectedEventArgs.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspFailureDetectedEventArgs.cs new file mode 100644 index 00000000..186ad407 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspFailureDetectedEventArgs.cs @@ -0,0 +1,53 @@ +using System; + +namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals +{ + /// + /// Event payload for and the + /// coordinator-forwarded . + /// + /// is the structured ID consumed by the DecisionEngine and the + /// audit-trail (e.g. Provisioning_DeviceSetup_Apps_Failed). The remaining fields are + /// populated from registry-derived ESP failures (); + /// event-log-derived failures () only carry + /// and leave the rest null. + /// + /// + public sealed class EspFailureDetectedEventArgs : EventArgs + { + /// Structured failure-type identifier (always set; never null/empty). + public string FailureType { get; } + + /// + /// Windows HRESULT extracted from the failed subcategory's statusText + /// (e.g. 0x87d1041c). Null when no HRESULT pattern matched or when the failure + /// source is event-log-derived. Lower-cased hex with 0x prefix. + /// + public string ErrorCode { get; } + + /// + /// Name of the failed subcategory (e.g. Apps, SecurityPolicies, + /// Certificates). Null when no subcategory could be identified + /// (category-level failure). + /// + public string FailedSubcategory { get; } + + /// + /// ESP category label (DevicePreparation, DeviceSetup, + /// AccountSetup). Null when not applicable. + /// + public string Category { get; } + + public EspFailureDetectedEventArgs( + string failureType, + string errorCode = null, + string failedSubcategory = null, + string category = null) + { + FailureType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType; + ErrorCode = string.IsNullOrEmpty(errorCode) ? null : errorCode; + FailedSubcategory = string.IsNullOrEmpty(failedSubcategory) ? null : failedSubcategory; + Category = string.IsNullOrEmpty(category) ? null : category; + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs index e437f87e..168463d9 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading; using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop; @@ -48,6 +49,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals internal const int AccountSetupFallbackDelaySeconds = 30; internal const int ProvisioningDebounceMilliseconds = 1000; + // Session 9d052230 fix: synthetic delay between "ESP subcategory failed detected" and the + // EspFailureDetected event fire. Gives ImeLogTracker a window to surface a matching + // app_install_failed event (with hresult capture) so the session timeline carries the + // app-level failure that ESP aggregated into the registry. Without this window the + // DecisionEngine transitions to Failed on the first registry observation, the agent + // terminates, and any post-ESP IME log entries are lost (race between ESP registry + // write and IME workload log). + internal const int ProvisioningFailureSettleWindowSeconds = 30; + + // Regex for HRESULT extraction from subcategory statusText (e.g. "Apps (0x87d1041c)"). + // Language-invariant: the parenthesised hex tail is consistent across Windows locales. + private static readonly Regex HResultPattern = new Regex( + @"\((0[xX][0-9a-fA-F]{8})\)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly string[] ProvisioningCategoryNames = { "DevicePreparationCategory.Status", @@ -86,9 +102,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals private bool _saveWhiteGloveSuccessResultSeen; // Track SaveWhiteGloveSuccessResult state transitions for observability (null → notStarted → succeeded) private string _lastSaveWhiteGloveState; + // Settle-window timers + pending args per category (session 9d052230 fix). Keyed by + // category-name (e.g. "DeviceSetupCategory.Status"). Pending args carry the enriched + // failure details that fire after the settle window expires. + private Dictionary _provisioningFailureSettleTimers; + private Dictionary _provisioningFailureSettleArgs; private readonly object _stateLock = new object(); - public event EventHandler EspFailureDetected; + public event EventHandler EspFailureDetected; public event EventHandler DeviceSetupProvisioningComplete; /// /// Fires once when AccountSetupCategory.Status resolves to @@ -235,6 +256,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _provisioningFailureFired = new HashSet(StringComparer.OrdinalIgnoreCase); _provisioningCategoriesResolved = new HashSet(StringComparer.OrdinalIgnoreCase); _lastSubcategoryStates = new Dictionary>(StringComparer.OrdinalIgnoreCase); + _provisioningFailureSettleTimers = new Dictionary(StringComparer.OrdinalIgnoreCase); + _provisioningFailureSettleArgs = new Dictionary(StringComparer.OrdinalIgnoreCase); _deviceSetupProvisioningCompleteFired = false; _deviceSetupFallbackTimer = null; _accountSetupProvisioningCompleteFired = false; @@ -275,6 +298,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _accountSetupFallbackTimer?.Dispose(); _accountSetupFallbackTimer = null; + if (_provisioningFailureSettleTimers != null) + { + foreach (var t in _provisioningFailureSettleTimers.Values) + { + try { t?.Dispose(); } catch { /* swallow */ } + } + _provisioningFailureSettleTimers.Clear(); + } + if (_provisioningWatcher != null) { _provisioningWatcher.Dispose(); @@ -354,8 +386,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals return; } - bool failureDetected = false; - string failureType = null; int changedCount = 0, unchangedCount = 0, missingCount = 0; foreach (var categoryName in ProvisioningCategoryNames) @@ -380,13 +410,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _logger.Trace($"CheckProvisioningStatus: {categoryName} — raw JSON: {jsonValue}"); _lastProvisioningJson[categoryName] = jsonValue; - var result = ProcessCategoryStatus(categoryName, jsonValue); - - if (result.IsFailed) - { - failureDetected = true; - failureType = result.FailureType; - } + // TryFireProvisioningFailure (inside ProcessCategoryStatus) arms the + // 30 s settle timer when a failure is detected — the actual + // EspFailureDetected fire happens in OnProvisioningFailureSettleExpired. + ProcessCategoryStatus(categoryName, jsonValue); } _logger.Trace($"CheckProvisioningStatus: summary — changed={changedCount}, unchanged={unchangedCount}, missing={missingCount}, " + @@ -403,13 +430,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _provisioningWatcher?.RequestStop(); } - // Fire EspFailureDetected AFTER event emission (event in spool before agent reacts) - if (failureDetected && failureType != null) - { - try { EspFailureDetected?.Invoke(this, failureType); } - catch (Exception ex) { _logger.Error($"EspFailureDetected handler failed for '{failureType}'", ex); } - } - // Fire DeviceSetupProvisioningComplete when DeviceSetup resolves with success. if (!_deviceSetupProvisioningCompleteFired && _lastCategorySucceeded.TryGetValue("DeviceSetupCategory.Status", out var dsSucceeded) @@ -957,16 +977,150 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _provisioningFailureFired.Add(categoryName); - var failedSubcategory = FindFailedSubcategory(subcategories); - var failureTypeName = failedSubcategory != null - ? $"Provisioning_{categoryLabel}_{failedSubcategory}_Failed" + var failedSubcategoryInfo = FindFailedSubcategoryInfo(subcategories); + var failedSubcategoryName = failedSubcategoryInfo?.Name; + var failureTypeName = failedSubcategoryName != null + ? $"Provisioning_{categoryLabel}_{failedSubcategoryName}_Failed" : $"Provisioning_{categoryLabel}_Failed"; - _logger.Info($"Provisioning subcategory failure: {categoryLabel}/{failedSubcategory ?? "category-level"} — escalating as {failureTypeName}"); + var errorCode = TryExtractErrorCode(failedSubcategoryInfo?.StatusText); + + _logger.Info( + $"Provisioning subcategory failure: {categoryLabel}/{failedSubcategoryName ?? "category-level"} — " + + $"escalating as {failureTypeName} (errorCode={errorCode ?? "n/a"})"); _logger.Warning($"Provisioning failure detected: {failureTypeName}"); - return ProvisioningResult.Failure(failureTypeName); + + // Caller already holds _stateLock. + ArmFailureSettleTimer( + categoryName: categoryName, + args: new EspFailureDetectedEventArgs( + failureType: failureTypeName, + errorCode: errorCode, + failedSubcategory: failedSubcategoryName, + category: categoryLabel)); + + // Settle-window owns the actual EspFailureDetected fire. The legacy "fire-after-emit" + // path in CheckProvisioningStatus is now a no-op for this code-path; it stays in place + // only for direct callers that may still expect a synchronous failure-type string. + return ProvisioningResult.NoAction; } + /// + /// Session 9d052230 fix: arms a 30 s settle timer per category. While the timer is + /// pending, ImeLogTracker can still surface app_install_failed events with hresult + /// capture so the session timeline carries the underlying app-level failure that ESP + /// aggregated into a single "Apps: failed (0x…)" registry write. On expiry the + /// EspFailureDetected event is fired with the enriched args. + /// + /// Caller holds . Pending args are stored under the same lock + /// so the timer callback re-acquires it before invocation. A single per-session emit + /// of esp_failure_settle_started is sufficient — fire-once is gated by + /// which TryFireProvisioningFailure checks. + /// + /// + private void ArmFailureSettleTimer(string categoryName, EspFailureDetectedEventArgs args) + { + _provisioningFailureSettleArgs[categoryName] = args; + + EmitFailureSettleStarted(args); + + var timer = new Timer( + _ => OnProvisioningFailureSettleExpired(categoryName), + state: null, + dueTime: TimeSpan.FromSeconds(ProvisioningFailureSettleWindowSeconds), + period: Timeout.InfiniteTimeSpan); + + _provisioningFailureSettleTimers[categoryName] = timer; + + _logger.Info( + $"Provisioning failure settle window armed: category={args.Category}, " + + $"failedSubcategory={args.FailedSubcategory ?? "n/a"}, errorCode={args.ErrorCode ?? "n/a"}, " + + $"settleSeconds={ProvisioningFailureSettleWindowSeconds}"); + } + + private void OnProvisioningFailureSettleExpired(string categoryName) + { + EspFailureDetectedEventArgs args = null; + try + { + lock (_stateLock) + { + if (_provisioningFailureSettleArgs == null + || !_provisioningFailureSettleArgs.TryGetValue(categoryName, out args)) + { + _logger.Debug($"Failure settle timer expired but no pending args for {categoryName} — ignoring"); + return; + } + _provisioningFailureSettleArgs.Remove(categoryName); + + if (_provisioningFailureSettleTimers != null + && _provisioningFailureSettleTimers.TryGetValue(categoryName, out var timer)) + { + try { timer?.Dispose(); } catch { /* swallow */ } + _provisioningFailureSettleTimers.Remove(categoryName); + } + } + + _logger.Info( + $"Provisioning failure settle window elapsed — firing EspFailureDetected: " + + $"category={args.Category}, failedSubcategory={args.FailedSubcategory ?? "n/a"}, " + + $"errorCode={args.ErrorCode ?? "n/a"}, failureType={args.FailureType}"); + + try { EspFailureDetected?.Invoke(this, args); } + catch (Exception ex) { _logger.Error($"EspFailureDetected handler failed for '{args.FailureType}'", ex); } + } + catch (Exception ex) + { + _logger.Error("OnProvisioningFailureSettleExpired callback failed", ex); + } + } + + private void EmitFailureSettleStarted(EspFailureDetectedEventArgs args) + { + var data = new Dictionary + { + { "category", args.Category ?? "unknown" }, + { "failedSubcategory", args.FailedSubcategory ?? "category-level" }, + { "failureType", args.FailureType }, + { "settleSeconds", ProvisioningFailureSettleWindowSeconds }, + { "reason", "wait_for_late_ime_signals" } + }; + if (!string.IsNullOrEmpty(args.ErrorCode)) + data["errorCode"] = args.ErrorCode; + + _post.Emit(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = "esp_failure_settle_started", + Severity = EventSeverity.Info, + Source = "EspAndHelloTracker", + Phase = EnrollmentPhase.Unknown, + Message = $"ESP failure settle window started ({ProvisioningFailureSettleWindowSeconds}s) — " + + $"{args.Category}/{args.FailedSubcategory ?? "category-level"}" + + (string.IsNullOrEmpty(args.ErrorCode) ? "" : $" (errorCode={args.ErrorCode})"), + Data = data + }); + } + + /// + /// Extracts a Windows HRESULT (e.g. 0x87d1041c) from a subcategory statusText. + /// Returns the normalised lower-case hex form with 0x prefix, or null when no + /// HRESULT pattern is present. Pattern is language-invariant (parenthesised hex tail). + /// + internal static string TryExtractErrorCode(string statusText) + { + if (string.IsNullOrEmpty(statusText)) return null; + var match = HResultPattern.Match(statusText); + if (!match.Success) return null; + var raw = match.Groups[1].Value; + return "0x" + raw.Substring(2).ToLowerInvariant(); + } + + /// Test seam — drive the settle-timer callback synchronously. + internal void TriggerSettleTimerForTest(string categoryName) + => OnProvisioningFailureSettleExpired(categoryName); + private bool HasCategorySucceededChanged(string categoryName, bool? newValue) { if (!_lastCategorySucceeded.TryGetValue(categoryName, out var oldValue)) @@ -1069,6 +1223,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals if (failedNames.Count > 0) { eventData["failedSubcategories"] = string.Join(",", failedNames); + + // Session 9d052230: surface HRESULT from the first failed subcategory's + // statusText at top-level so the UI / RuleEngine can match it without + // parsing nested statusText strings. Pattern is language-invariant. + var firstFailedSub = subcategories + .FirstOrDefault(s => string.Equals(s.State, "failed", StringComparison.OrdinalIgnoreCase)); + var firstFailedCode = TryExtractErrorCode(firstFailedSub?.StatusText); + if (!string.IsNullOrEmpty(firstFailedCode)) + { + eventData["failedSubcategoryErrorCode"] = firstFailedCode; + } } } @@ -1204,11 +1369,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals } internal static string FindFailedSubcategory(List subcategories) + => FindFailedSubcategoryInfo(subcategories)?.Name; + + internal static SubcategoryInfo FindFailedSubcategoryInfo(List subcategories) { foreach (var sub in subcategories) { if (string.Equals(sub.State, "failed", StringComparison.OrdinalIgnoreCase)) - return sub.Name; + return sub; } return null; } @@ -1324,6 +1492,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _provisioningCategoriesResolved = new HashSet(StringComparer.OrdinalIgnoreCase); if (_lastSubcategoryStates == null) _lastSubcategoryStates = new Dictionary>(StringComparer.OrdinalIgnoreCase); + if (_provisioningFailureSettleTimers == null) + _provisioningFailureSettleTimers = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (_provisioningFailureSettleArgs == null) + _provisioningFailureSettleArgs = new Dictionary(StringComparer.OrdinalIgnoreCase); } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/EspAndHelloTrackerAdapter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/EspAndHelloTrackerAdapter.cs index bca9bea5..72366891 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/EspAndHelloTrackerAdapter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/EspAndHelloTrackerAdapter.cs @@ -89,7 +89,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters private void OnHelloCompleted(object sender, EventArgs e) => EmitHello(ReadHelloOutcome()); private void OnFinalizing(object sender, string reason) => EmitFinalizing(reason); private void OnWhiteGloveCompleted(object sender, EventArgs e) => EmitWhiteGlove(); - private void OnEspFailure(object sender, string failureType) => EmitEspFailure(failureType); + private void OnEspFailure(object sender, EspFailureDetectedEventArgs args) => EmitEspFailure(args); private void OnDeviceSetupComplete(object sender, EventArgs e) => EmitDeviceSetupComplete(); private void OnAccountSetupComplete(object sender, EventArgs e) => EmitAccountSetupComplete(); private void OnHelloPolicyDetected(bool helloEnabled, string source) => EmitHelloPolicy(helloEnabled, source); @@ -98,7 +98,9 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters internal void TriggerHelloFromTest(string? helloOutcome) => EmitHello(helloOutcome); internal void TriggerFinalizingFromTest(string reason) => EmitFinalizing(reason); internal void TriggerWhiteGloveFromTest() => EmitWhiteGlove(); - internal void TriggerEspFailureFromTest(string failureType) => EmitEspFailure(failureType); + internal void TriggerEspFailureFromTest(string failureType) + => EmitEspFailure(new EspFailureDetectedEventArgs(failureType)); + internal void TriggerEspFailureFromTest(EspFailureDetectedEventArgs args) => EmitEspFailure(args); internal void TriggerDeviceSetupCompleteFromTest() => EmitDeviceSetupComplete(); internal void TriggerAccountSetupCompleteFromTest() => EmitAccountSetupComplete(); internal void TriggerHelloPolicyDetectedFromTest(bool helloEnabled, string source) => EmitHelloPolicy(helloEnabled, source); @@ -226,20 +228,41 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters derivationInputs: derivationInputs)); } - private void EmitEspFailure(string failureType) + private void EmitEspFailure(EspFailureDetectedEventArgs args) { if (_espFailurePosted) return; _espFailurePosted = true; - var safeType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType!; + var safeType = args?.FailureType ?? "unknown"; + var errorCode = args?.ErrorCode; + var failedSubcategory = args?.FailedSubcategory; + var category = args?.Category; + var now = ResolveOccurredAt(out var derivedFromClock); var derivationInputs = new Dictionary(StringComparer.Ordinal) { ["subSource"] = "ShellCoreTracker+ProvisioningStatusTracker (merged)", ["failureType"] = safeType, }; + if (!string.IsNullOrEmpty(errorCode)) + derivationInputs["errorCode"] = errorCode!; + if (!string.IsNullOrEmpty(failedSubcategory)) + derivationInputs["failedSubcategory"] = failedSubcategory!; + if (!string.IsNullOrEmpty(category)) + derivationInputs["category"] = category!; TagDerivedTimestamp(derivationInputs, derivedFromClock); + var payload = new Dictionary(StringComparer.Ordinal) + { + ["failureType"] = safeType, + }; + if (!string.IsNullOrEmpty(errorCode)) + payload["errorCode"] = errorCode!; + if (!string.IsNullOrEmpty(failedSubcategory)) + payload["failedSubcategory"] = failedSubcategory!; + if (!string.IsNullOrEmpty(category)) + payload["category"] = category!; + _ingress.Post( kind: DecisionSignalKind.EspTerminalFailure, occurredAtUtc: now, @@ -247,12 +270,9 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters evidence: new Evidence( kind: EvidenceKind.Derived, identifier: DetectorId, - summary: $"ESP terminal failure (coordinator-merged, type={safeType})", + summary: $"ESP terminal failure (coordinator-merged, type={safeType}, errorCode={errorCode ?? "n/a"})", derivationInputs: derivationInputs), - payload: new Dictionary(StringComparer.Ordinal) - { - ["failureType"] = safeType, - }); + payload: payload); } // PR4 (882fef64 debrief) — once per session: post HelloPolicyDetected so the engine diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ProvisioningStatusTrackerAdapter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ProvisioningStatusTrackerAdapter.cs index 59192e41..afe6d79f 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ProvisioningStatusTrackerAdapter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ProvisioningStatusTrackerAdapter.cs @@ -59,11 +59,13 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters private void OnDeviceSetupComplete(object sender, EventArgs e) => EmitDeviceSetupComplete(); private void OnAccountSetupComplete(object sender, EventArgs e) => EmitAccountSetupComplete(); - private void OnEspFailure(object sender, string failureType) => EmitEspFailure(failureType); + private void OnEspFailure(object sender, EspFailureDetectedEventArgs args) => EmitEspFailure(args); internal void TriggerDeviceSetupCompleteFromTest() => EmitDeviceSetupComplete(); internal void TriggerAccountSetupCompleteFromTest() => EmitAccountSetupComplete(); - internal void TriggerEspFailureFromTest(string failureType) => EmitEspFailure(failureType); + internal void TriggerEspFailureFromTest(string failureType) + => EmitEspFailure(new EspFailureDetectedEventArgs(failureType)); + internal void TriggerEspFailureFromTest(EspFailureDetectedEventArgs args) => EmitEspFailure(args); private void EmitDeviceSetupComplete() { @@ -123,12 +125,38 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters }); } - private void EmitEspFailure(string failureType) + private void EmitEspFailure(EspFailureDetectedEventArgs args) { if (_espFailurePosted) return; _espFailurePosted = true; - var safeFailureType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType!; + var safeFailureType = args?.FailureType ?? "unknown"; + var errorCode = args?.ErrorCode; + var failedSubcategory = args?.FailedSubcategory; + var category = args?.Category; + + var derivationInputs = new Dictionary(StringComparer.Ordinal) + { + ["registrySource"] = @"HKLM\SOFTWARE\Microsoft\Provisioning\AutopilotSettings\*.Status", + ["failureType"] = safeFailureType, + }; + if (!string.IsNullOrEmpty(errorCode)) + derivationInputs["errorCode"] = errorCode!; + if (!string.IsNullOrEmpty(failedSubcategory)) + derivationInputs["failedSubcategory"] = failedSubcategory!; + if (!string.IsNullOrEmpty(category)) + derivationInputs["category"] = category!; + + var payload = new Dictionary(StringComparer.Ordinal) + { + ["failureType"] = safeFailureType, + }; + if (!string.IsNullOrEmpty(errorCode)) + payload["errorCode"] = errorCode!; + if (!string.IsNullOrEmpty(failedSubcategory)) + payload["failedSubcategory"] = failedSubcategory!; + if (!string.IsNullOrEmpty(category)) + payload["category"] = category!; _ingress.Post( kind: DecisionSignalKind.EspTerminalFailure, @@ -137,16 +165,9 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters evidence: new Evidence( kind: EvidenceKind.Derived, identifier: "provisioning-status-tracker-v1", - summary: $"ESP terminal failure from provisioning registry (type={safeFailureType})", - derivationInputs: new Dictionary(StringComparer.Ordinal) - { - ["registrySource"] = @"HKLM\SOFTWARE\Microsoft\Provisioning\AutopilotSettings\*.Status", - ["failureType"] = safeFailureType, - }), - payload: new Dictionary(StringComparer.Ordinal) - { - ["failureType"] = safeFailureType, - }); + summary: $"ESP terminal failure from provisioning registry (type={safeFailureType}, errorCode={errorCode ?? "n/a"})", + derivationInputs: derivationInputs), + payload: payload); } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt index 1cc5be4f..dca34d14 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt @@ -1 +1 @@ -838 +842 diff --git a/src/Agent/AutopilotMonitor.DecisionCore.Tests/DecisionAuditTrailTests.cs b/src/Agent/AutopilotMonitor.DecisionCore.Tests/DecisionAuditTrailTests.cs index 54451d8b..afd08c96 100644 --- a/src/Agent/AutopilotMonitor.DecisionCore.Tests/DecisionAuditTrailTests.cs +++ b/src/Agent/AutopilotMonitor.DecisionCore.Tests/DecisionAuditTrailTests.cs @@ -332,6 +332,66 @@ namespace AutopilotMonitor.DecisionCore.Tests Assert.Equal("policy_apply_timeout", payload["reason"]); } + /// + /// Session 9d052230 regression — when the adapter posts a registry-derived ESP failure + /// with HRESULT enrichment, the terminal enrollment_failed event MUST carry + /// errorCode, failureType, failedSubcategory, and category + /// on the effect parameters so the UI can render the HRESULT badge + description. + /// + [Fact] + public void EspTerminalFailure_surfaces_errorCode_and_failedSubcategory_on_terminal_event() + { + var engine = new DecisionEngine(); + var state = DecisionState.CreateInitial("sess-esp-hresult", "tenant-esp-hresult"); + state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState; + + var step = engine.Reduce(state, MakeSignal( + 40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(15), + new Dictionary + { + ["failureType"] = "Provisioning_DeviceSetup_Apps_Failed", + ["errorCode"] = "0x87d1041c", + ["failedSubcategory"] = "Apps", + ["category"] = "DeviceSetup", + })); + + Assert.Equal(SessionStage.Failed, step.NewState.Stage); + + var effect = SingleTimelineEffect(step, "enrollment_failed"); + Assert.NotNull(effect.Parameters); + + // reason defaults to the stable enum literal because the signal did not override it. + Assert.Equal("esp_terminal_failure", effect.Parameters!["reason"]); + Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", effect.Parameters!["failureType"]); + Assert.Equal("0x87d1041c", effect.Parameters!["errorCode"]); + Assert.Equal("Apps", effect.Parameters!["failedSubcategory"]); + Assert.Equal("DeviceSetup", effect.Parameters!["category"]); + } + + /// + /// Backwards-compat — a signal without HRESULT enrichment (e.g. ShellCore-derived) must + /// still produce a valid enrollment_failed event with just the default reason. + /// + [Fact] + public void EspTerminalFailure_works_without_errorCode_payload() + { + var engine = new DecisionEngine(); + var state = DecisionState.CreateInitial("sess-esp-bare", "tenant-esp-bare"); + state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState; + + var step = engine.Reduce(state, MakeSignal( + 40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(15), + new Dictionary { ["failureType"] = "ESPProgress_Timeout" })); + + var effect = SingleTimelineEffect(step, "enrollment_failed"); + Assert.NotNull(effect.Parameters); + + Assert.Equal("esp_terminal_failure", effect.Parameters!["reason"]); + Assert.Equal("ESPProgress_Timeout", effect.Parameters!["failureType"]); + Assert.False(effect.Parameters!.ContainsKey("errorCode")); + Assert.False(effect.Parameters!.ContainsKey("failedSubcategory")); + } + [Fact] public void EspTerminalFailure_event_enriched_with_continueAnyway_and_timeout_facts() { diff --git a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs index 75e38f31..a7e28f98 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs @@ -46,12 +46,24 @@ namespace AutopilotMonitor.DecisionCore.Engine /// to with , /// clears deadlines, and emits enrollment_failed. No classifier re-run — /// ESP terminal failure is definitive. + /// + /// The terminal-event reason is always the stable enum literal esp_terminal_failure. + /// Session 9d052230: registry-derived failures additionally carry failureType, + /// errorCode (HRESULT), failedSubcategory, and category on the signal + /// payload — those propagate verbatim to the timeline-event parameters so the UI can + /// render the HRESULT badge + description without re-parsing the ESP statusText. + /// /// private DecisionStep HandleEspTerminalFailureV1(DecisionState state, DecisionSignal signal) { var nextStep = state.StepIndex + 1; - var reason = signal.Payload != null && signal.Payload.TryGetValue("reason", out var r) + // Default to the stable enum literal so the timeline-event "reason" is predictable + // across all sources of EspTerminalFailure. Direct test signals (and any future + // adapter that wants to override) can still inject a specific reason via payload. + var reason = signal.Payload != null + && signal.Payload.TryGetValue("reason", out var r) + && !string.IsNullOrEmpty(r) ? r : "esp_terminal_failure"; @@ -78,6 +90,20 @@ namespace AutopilotMonitor.DecisionCore.Engine ["reason"] = reason, }; + // Session 9d052230 — surface registry-derived ESP failure detail (HRESULT etc) on the + // terminal event so the UI can render it without parsing nested ESP statusText. + if (signal.Payload != null) + { + if (signal.Payload.TryGetValue("failureType", out var failureType) && !string.IsNullOrEmpty(failureType)) + parameters["failureType"] = failureType; + if (signal.Payload.TryGetValue("errorCode", out var errorCode) && !string.IsNullOrEmpty(errorCode)) + parameters["errorCode"] = errorCode; + if (signal.Payload.TryGetValue("failedSubcategory", out var failedSubcategory) && !string.IsNullOrEmpty(failedSubcategory)) + parameters["failedSubcategory"] = failedSubcategory; + if (signal.Payload.TryGetValue("category", out var category) && !string.IsNullOrEmpty(category)) + parameters["category"] = category; + } + // Enrich the audit payload with the FirstSync-derived ESP failure-handling settings, // so the timeline carries the observable fact that Microsoft's ESP profile may have // offered the user a "Continue anyway" button — the agent's terminal-failure verdict diff --git a/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/EventTimeline.tsx b/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/EventTimeline.tsx index a1a56728..b1d24f6d 100644 --- a/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/EventTimeline.tsx +++ b/src/Web/autopilot-monitor-web/app/sessions/[sessionId]/components/EventTimeline.tsx @@ -478,6 +478,34 @@ function EventRow({ event, showScriptOutput }: { event: EnrollmentEvent; showScr ); })()} + {/* HRESULT badge for ESP failures (enrollment_failed via esp_terminal_failure, + esp_provisioning_status failed-subcategory). The HRESULT is extracted from the ESP + registry statusText by the agent (e.g. "Apps (0x87d1041c)") and surfaced as + top-level event data so the UI can render it without parsing nested text. */} + {(event.eventType === "enrollment_failed" || event.eventType === "esp_provisioning_status") && (() => { + const code = event.data?.errorCode ?? event.data?.failedSubcategoryErrorCode; + if (!code) return null; + const codeStr = String(code); + const entry = getErrorCodeEntry(codeStr); + const sub = event.data?.failedSubcategory ?? event.data?.failedSubcategories; + return ( +
+ + HRESULT: {formatErrorCode(codeStr)} + + {entry && ( + + {entry.description} + + )} + {sub && ( + + subcategory: {String(sub)} + + )} +
+ ); + })()}
Source: {event.source} Seq: {event.sequence}