feat(agent/v2,decision): surface ESP failure HRESULT + 30s settle window

Session 9d052230 dropped enrollment_failed with reason esp_terminal_failure
but the underlying HRESULT 0x87d1041c (Application not detected after install)
sat unparsed in the ESP subcategory statusText, and the agent terminated before
IME could emit the matching app_install_failed. Two-part fix:

- ProvisioningStatusTracker arms a 30s settle window between ESP-subcategory
  failure detection and EspFailureDetected fire. Gives ImeLogTracker a chance
  to surface the app-level failure on the timeline. Emits esp_failure_settle_started
  for observability.
- HRESULT extracted via language-invariant regex from the failed subcategory
  statusText, surfaced top-level on esp_provisioning_status (failedSubcategoryErrorCode)
  and propagated through new EspFailureDetectedEventArgs to the EspTerminalFailure
  signal payload. DecisionEngine.Edge writes failureType/errorCode/failedSubcategory/
  category onto the enrollment_failed event parameters. UI renders an HRESULT badge
  with description from errorCodeMap.ts (analogous to app_install_failed).

V2-only; V1 (Agent.Core) untouched. ConfigVersion unchanged.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-05-21 16:37:28 +02:00
parent a36bf8bf0c
commit e8e062d24a
11 changed files with 793 additions and 60 deletions
@@ -0,0 +1,179 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
using AutopilotMonitor.DecisionCore.Engine;
using AutopilotMonitor.DecisionCore.Signals;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
{
/// <summary>
/// Session 9d052230 (2026-05-21) regression coverage. When ESP DeviceSetup
/// <c>Apps</c> transitions to <c>failed</c> with a Windows HRESULT in its
/// <c>statusText</c> (e.g. <c>"Apps (0x87d1041c)"</c>), the tracker must:
/// <list type="bullet">
/// <item>NOT fire <see cref="ProvisioningStatusTracker.EspFailureDetected"/> immediately —
/// a 30 s settle window gives ImeLogTracker time to emit the underlying
/// <c>app_install_failed</c> event so the timeline carries the app-level failure.</item>
/// <item>Emit <c>esp_failure_settle_started</c> (Info) for observability.</item>
/// <item>Emit <c>esp_provisioning_status</c> with a top-level
/// <c>failedSubcategoryErrorCode</c> field so the UI / RuleEngine can match on it
/// without parsing nested statusText.</item>
/// <item>On settle-window expiry, fire <c>EspFailureDetected</c> with enriched
/// <see cref="EspFailureDetectedEventArgs"/>.</item>
/// </list>
/// </summary>
public sealed class ProvisioningFailureSettleWindowTests
{
private static readonly DateTime Fixed = new DateTime(2026, 5, 21, 12, 25, 0, DateTimeKind.Utc);
private sealed class Fixture : IDisposable
{
public TempDirectory Tmp { get; } = new TempDirectory();
public FakeSignalIngressSink Sink { get; } = new FakeSignalIngressSink();
public List<EspFailureDetectedEventArgs> EspFailures { get; } = new List<EspFailureDetectedEventArgs>();
public ProvisioningStatusTracker Tracker { get; }
public Fixture()
{
var clock = new VirtualClock(Fixed);
var post = new InformationalEventPost(Sink, clock);
var logger = new AgentLogger(Tmp.Path, AgentLogLevel.Info);
Tracker = new ProvisioningStatusTracker(
sessionId: "S1",
tenantId: "T1",
post: post,
logger: logger);
Tracker.EspFailureDetected += (_, args) => EspFailures.Add(args);
}
// Posted EnrollmentEvent equivalents: filter the sink for InformationalEvent signals,
// pull the eventType from payload and the data from typedPayload.
public IReadOnlyList<(string EventType, IReadOnlyDictionary<string, object> Data)> CapturedEvents()
{
return Sink.Posted
.Where(p => p.Kind == DecisionSignalKind.InformationalEvent && p.Payload != null)
.Select(p =>
{
var eventType = p.Payload!.TryGetValue(SignalPayloadKeys.EventType, out var et) ? et : "";
var data = p.TypedPayload as IReadOnlyDictionary<string, object>
?? (IReadOnlyDictionary<string, object>)new Dictionary<string, object>();
return (eventType, data);
})
.ToList();
}
public void Dispose()
{
Tracker.Dispose();
Tmp.Dispose();
}
}
[Fact]
public void FailedAppsSubcategoryWithHResult_armsSettleWindow_DoesNotFireImmediately()
{
using var f = new Fixture();
// First seen — Apps in_progress.
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
""categorySucceeded"": null,
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (4 of 4 applied)""},
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Identifying)""}
}");
// Apps transitions to failed with HRESULT in statusText.
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
""categorySucceeded"": null,
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (4 of 4 applied)""},
""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (0x87d1041c)""}
}");
// EspFailureDetected is delayed by the settle window — no immediate fire.
Assert.Empty(f.EspFailures);
var captured = f.CapturedEvents();
// esp_failure_settle_started observability event is emitted.
var settleStarted = captured.SingleOrDefault(e => e.EventType == "esp_failure_settle_started");
Assert.NotEqual(default, settleStarted);
Assert.Equal("DeviceSetup", (string)settleStarted.Data["category"]);
Assert.Equal("Apps", (string)settleStarted.Data["failedSubcategory"]);
Assert.Equal("0x87d1041c", (string)settleStarted.Data["errorCode"]);
Assert.Equal(ProvisioningStatusTracker.ProvisioningFailureSettleWindowSeconds, (int)settleStarted.Data["settleSeconds"]);
// esp_provisioning_status (subcategory_state_change) carries failedSubcategoryErrorCode top-level.
var statusEvents = captured.Where(e => e.EventType == AutopilotMonitor.Shared.Constants.EventTypes.EspProvisioningStatus).ToList();
var transitionEvent = statusEvents.Last(e =>
e.Data.TryGetValue("changeType", out var ct) && (string)ct == "subcategory_state_change");
Assert.Equal("0x87d1041c", (string)transitionEvent.Data["failedSubcategoryErrorCode"]);
Assert.Equal("Apps", (string)transitionEvent.Data["failedSubcategories"]);
// TriggerSettleTimerForTest synchronously runs the timer callback.
f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status");
var fired = Assert.Single(f.EspFailures);
Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", fired.FailureType);
Assert.Equal("0x87d1041c", fired.ErrorCode);
Assert.Equal("Apps", fired.FailedSubcategory);
Assert.Equal("DeviceSetup", fired.Category);
}
[Fact]
public void FailedSubcategoryWithoutHResult_stillArmsSettleWindow_ErrorCodeIsNull()
{
using var f = new Fixture();
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
""categorySucceeded"": null,
""SecurityPoliciesSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Working""}
}");
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
""categorySucceeded"": false,
""SecurityPoliciesSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Failed""}
}");
Assert.Empty(f.EspFailures);
var captured = f.CapturedEvents();
var settleStarted = captured.SingleOrDefault(e => e.EventType == "esp_failure_settle_started");
Assert.NotEqual(default, settleStarted);
Assert.False(settleStarted.Data.ContainsKey("errorCode"));
f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status");
var fired = Assert.Single(f.EspFailures);
Assert.Null(fired.ErrorCode);
Assert.Equal("SecurityPolicies", fired.FailedSubcategory);
}
[Fact]
public void RepeatedFailedProcessing_armsSettleTimer_OnlyOnce()
{
using var f = new Fixture();
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
""categorySucceeded"": null,
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Identifying)""}
}");
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
""categorySucceeded"": null,
""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (0x87d1041c)""}
}");
// Re-process with the same failed-state JSON — fire-once guard must prevent a
// second settle-window arm and a duplicate esp_failure_settle_started event.
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
""categorySucceeded"": false,
""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (0x87d1041c)""}
}");
var captured = f.CapturedEvents();
Assert.Single(captured.Where(e => e.EventType == "esp_failure_settle_started"));
}
}
}
@@ -0,0 +1,143 @@
using System;
using System.Linq;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
using AutopilotMonitor.DecisionCore.Signals;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
{
/// <summary>
/// Session 9d052230 (2026-05-21) regression coverage: ESP-Apps failure with a Windows HRESULT
/// embedded in the subcategory statusText must surface
/// <c>errorCode</c>/<c>failedSubcategory</c>/<c>category</c> on the
/// <see cref="DecisionSignalKind.EspTerminalFailure"/> signal payload so the DecisionEngine
/// can propagate it to the terminal <c>enrollment_failed</c> event.
/// </summary>
public sealed class EspFailureDetectedHResultTests
{
private static readonly DateTime Fixed = new DateTime(2026, 5, 21, 12, 25, 41, DateTimeKind.Utc);
// ------------------------------------------------------------------ Regex extraction
[Theory]
[InlineData("Apps (0x87d1041c)", "0x87d1041c")]
[InlineData("Security policies (0x80070005)", "0x80070005")]
[InlineData("Certificates (No setup needed)", null)]
[InlineData("Apps (Identifying)", null)]
[InlineData("Apps (0X87D1041C)", "0x87d1041c")]
[InlineData("Network connections (1 of 1 added)", null)]
[InlineData("", null)]
[InlineData(null, null)]
public void TryExtractErrorCode_extracts_parenthesised_hex_tail(string statusText, string expected)
{
var actual = ProvisioningStatusTracker.TryExtractErrorCode(statusText);
Assert.Equal(expected, actual);
}
// ------------------------------------------------------------------ EspAndHelloTrackerAdapter
[Fact]
public void Coordinator_adapter_forwards_errorCode_failedSubcategory_category_to_signal_payload()
{
using var tmp = new TempDirectory();
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
var clock = new VirtualClock(Fixed);
var ingress = new FakeSignalIngressSink();
var coordinator = new EspAndHelloTracker(
sessionId: "S1",
tenantId: "T1",
post: new InformationalEventPost(new FakeSignalIngressSink(), clock),
logger: logger);
try
{
using var adapter = new EspAndHelloTrackerAdapter(coordinator, ingress, clock);
adapter.TriggerEspFailureFromTest(new EspFailureDetectedEventArgs(
failureType: "Provisioning_DeviceSetup_Apps_Failed",
errorCode: "0x87d1041c",
failedSubcategory: "Apps",
category: "DeviceSetup"));
var posted = Assert.Single(ingress.Posted);
Assert.Equal(DecisionSignalKind.EspTerminalFailure, posted.Kind);
Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", posted.Payload!["failureType"]);
Assert.Equal("0x87d1041c", posted.Payload!["errorCode"]);
Assert.Equal("Apps", posted.Payload!["failedSubcategory"]);
Assert.Equal("DeviceSetup", posted.Payload!["category"]);
Assert.Equal("0x87d1041c", posted.Evidence.DerivationInputs!["errorCode"]);
}
finally { coordinator.Dispose(); }
}
[Fact]
public void Coordinator_adapter_omits_optional_keys_when_only_failureType_is_set()
{
using var tmp = new TempDirectory();
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
var clock = new VirtualClock(Fixed);
var ingress = new FakeSignalIngressSink();
var coordinator = new EspAndHelloTracker(
sessionId: "S1",
tenantId: "T1",
post: new InformationalEventPost(new FakeSignalIngressSink(), clock),
logger: logger);
try
{
using var adapter = new EspAndHelloTrackerAdapter(coordinator, ingress, clock);
// Backwards-compat: ShellCore-derived failures use the string overload —
// no HRESULT, no subcategory, no category surface.
adapter.TriggerEspFailureFromTest("ESPProgress_Timeout");
var posted = Assert.Single(ingress.Posted);
Assert.Equal("ESPProgress_Timeout", posted.Payload!["failureType"]);
Assert.False(posted.Payload!.ContainsKey("errorCode"));
Assert.False(posted.Payload!.ContainsKey("failedSubcategory"));
Assert.False(posted.Payload!.ContainsKey("category"));
}
finally { coordinator.Dispose(); }
}
// ------------------------------------------------------------------ ProvisioningStatusTrackerAdapter
[Fact]
public void Provisioning_adapter_forwards_full_args_to_signal_payload()
{
using var tmp = new TempDirectory();
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
var clock = new VirtualClock(Fixed);
var ingress = new FakeSignalIngressSink();
var tracker = new ProvisioningStatusTracker(
sessionId: "S1",
tenantId: "T1",
post: new InformationalEventPost(new FakeSignalIngressSink(), clock),
logger: logger);
try
{
using var adapter = new ProvisioningStatusTrackerAdapter(tracker, ingress, clock);
adapter.TriggerEspFailureFromTest(new EspFailureDetectedEventArgs(
failureType: "Provisioning_DeviceSetup_Apps_Failed",
errorCode: "0x87d1041c",
failedSubcategory: "Apps",
category: "DeviceSetup"));
var posted = Assert.Single(ingress.Posted);
Assert.Equal(DecisionSignalKind.EspTerminalFailure, posted.Kind);
Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", posted.Payload!["failureType"]);
Assert.Equal("0x87d1041c", posted.Payload!["errorCode"]);
Assert.Equal("Apps", posted.Payload!["failedSubcategory"]);
Assert.Equal("DeviceSetup", posted.Payload!["category"]);
}
finally { tracker.Dispose(); }
}
}
}
@@ -69,9 +69,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
/// <summary>
/// Fired when an ESP failure is detected (ESPProgress_Failure, _Timeout, _Abort,
/// WhiteGlove_Failed, Provisioning_*_Failed, etc.). The string is the structured failure type.
/// WhiteGlove_Failed, Provisioning_*_Failed, etc.).
/// <para>
/// <see cref="EspFailureDetectedEventArgs.FailureType"/> carries the structured failure
/// identifier. Registry-derived failures from <see cref="ProvisioningStatusTracker"/>
/// additionally enrich <see cref="EspFailureDetectedEventArgs.ErrorCode"/>,
/// <see cref="EspFailureDetectedEventArgs.FailedSubcategory"/>, and
/// <see cref="EspFailureDetectedEventArgs.Category"/>; event-log-derived failures from
/// <see cref="ShellCoreTracker"/> only set <see cref="EspFailureDetectedEventArgs.FailureType"/>.
/// </para>
/// </summary>
public event EventHandler<string> EspFailureDetected;
public event EventHandler<EspFailureDetectedEventArgs> EspFailureDetected;
/// <summary>
/// Fired when DeviceSetup provisioning status shows categorySucceeded=true (or fallback confirmed).
@@ -248,7 +256,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_helloTracker);
_shellCoreTracker.FinalizingSetupPhaseTriggered += OnFinalizingSetupPhaseTriggered;
_shellCoreTracker.WhiteGloveCompleted += OnWhiteGloveCompleted;
_shellCoreTracker.EspFailureDetected += OnEspFailureDetected;
_shellCoreTracker.EspFailureDetected += OnShellCoreEspFailureDetected;
_shellCoreTracker.EspExited += OnEspExited;
_shellCoreTracker.Start();
@@ -257,7 +265,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_tenantId,
_post,
_logger);
_provisioningTracker.EspFailureDetected += OnEspFailureDetected;
_provisioningTracker.EspFailureDetected += OnProvisioningEspFailureDetected;
_provisioningTracker.DeviceSetupProvisioningComplete += OnDeviceSetupProvisioningComplete;
_provisioningTracker.AccountSetupProvisioningComplete += OnAccountSetupProvisioningComplete;
_provisioningTracker.Start();
@@ -310,7 +318,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
{
_shellCoreTracker.FinalizingSetupPhaseTriggered -= OnFinalizingSetupPhaseTriggered;
_shellCoreTracker.WhiteGloveCompleted -= OnWhiteGloveCompleted;
_shellCoreTracker.EspFailureDetected -= OnEspFailureDetected;
_shellCoreTracker.EspFailureDetected -= OnShellCoreEspFailureDetected;
_shellCoreTracker.EspExited -= OnEspExited;
_shellCoreTracker.Stop();
}
@@ -323,7 +331,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
if (_provisioningTracker == null) return;
try
{
_provisioningTracker.EspFailureDetected -= OnEspFailureDetected;
_provisioningTracker.EspFailureDetected -= OnProvisioningEspFailureDetected;
_provisioningTracker.DeviceSetupProvisioningComplete -= OnDeviceSetupProvisioningComplete;
_provisioningTracker.AccountSetupProvisioningComplete -= OnAccountSetupProvisioningComplete;
_provisioningTracker.Stop("tracker_stopped");
@@ -463,17 +471,40 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
finally { LastEventOccurredAtUtc = null; }
}
private void OnEspFailureDetected(object sender, string failureType)
// Forwarder for ShellCoreTracker (event-log-derived ESP failures, e.g. Shell-Core 62407
// failure descriptions). Source-event timestamp is mirrored so the adapter can stamp the
// DecisionSignal with the historical instant on backfill. ShellCoreTracker has no HRESULT
// surface, so only FailureType is set on the args.
private void OnShellCoreEspFailureDetected(object sender, string failureType)
{
// EspFailureDetected fires from ShellCoreTracker (carries timestamp) AND
// ProvisioningStatusTracker (no timestamp surfaced today — adapter falls back
// to clock). Cast attempt below picks up the timestamp on the ShellCore path.
LastEventOccurredAtUtc = (sender as ShellCoreTracker)?.LastEventOccurredAtUtc;
try { EspFailureDetected?.Invoke(this, failureType); }
catch (Exception ex) { _logger.Error($"Error forwarding EspFailureDetected for '{failureType}'", ex); }
try
{
EspFailureDetected?.Invoke(this, new EspFailureDetectedEventArgs(failureType));
}
catch (Exception ex) { _logger.Error($"Error forwarding ShellCore EspFailureDetected for '{failureType}'", ex); }
finally { LastEventOccurredAtUtc = null; }
}
// Forwarder for ProvisioningStatusTracker (registry-derived ESP failures). Args carry the
// full failure detail (FailureType, ErrorCode, FailedSubcategory, Category) extracted
// from the failed subcategory's statusText. Provisioning has no source-event timestamp
// surfaced today, so the adapter falls back to clock.
private void OnProvisioningEspFailureDetected(object sender, EspFailureDetectedEventArgs args)
{
LastEventOccurredAtUtc = null;
try
{
EspFailureDetected?.Invoke(this, args);
}
catch (Exception ex)
{
_logger.Error(
$"Error forwarding Provisioning EspFailureDetected for '{args?.FailureType ?? "n/a"}'",
ex);
}
}
private void OnDeviceSetupProvisioningComplete(object sender, EventArgs e)
{
// Provisioning tracker doesn't surface a timestamp; adapter falls back to clock.
@@ -0,0 +1,53 @@
using System;
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
{
/// <summary>
/// Event payload for <see cref="ProvisioningStatusTracker.EspFailureDetected"/> and the
/// coordinator-forwarded <see cref="EspAndHelloTracker.EspFailureDetected"/>.
/// <para>
/// <see cref="FailureType"/> is the structured ID consumed by the DecisionEngine and the
/// audit-trail (e.g. <c>Provisioning_DeviceSetup_Apps_Failed</c>). The remaining fields are
/// populated from registry-derived ESP failures (<see cref="ProvisioningStatusTracker"/>);
/// event-log-derived failures (<see cref="ShellCoreTracker"/>) only carry
/// <see cref="FailureType"/> and leave the rest null.
/// </para>
/// </summary>
public sealed class EspFailureDetectedEventArgs : EventArgs
{
/// <summary>Structured failure-type identifier (always set; never null/empty).</summary>
public string FailureType { get; }
/// <summary>
/// Windows HRESULT extracted from the failed subcategory's <c>statusText</c>
/// (e.g. <c>0x87d1041c</c>). Null when no HRESULT pattern matched or when the failure
/// source is event-log-derived. Lower-cased hex with <c>0x</c> prefix.
/// </summary>
public string ErrorCode { get; }
/// <summary>
/// Name of the failed subcategory (e.g. <c>Apps</c>, <c>SecurityPolicies</c>,
/// <c>Certificates</c>). Null when no subcategory could be identified
/// (category-level failure).
/// </summary>
public string FailedSubcategory { get; }
/// <summary>
/// ESP category label (<c>DevicePreparation</c>, <c>DeviceSetup</c>,
/// <c>AccountSetup</c>). Null when not applicable.
/// </summary>
public string Category { get; }
public EspFailureDetectedEventArgs(
string failureType,
string errorCode = null,
string failedSubcategory = null,
string category = null)
{
FailureType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType;
ErrorCode = string.IsNullOrEmpty(errorCode) ? null : errorCode;
FailedSubcategory = string.IsNullOrEmpty(failedSubcategory) ? null : failedSubcategory;
Category = string.IsNullOrEmpty(category) ? null : category;
}
}
}
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
@@ -48,6 +49,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
internal const int AccountSetupFallbackDelaySeconds = 30;
internal const int ProvisioningDebounceMilliseconds = 1000;
// Session 9d052230 fix: synthetic delay between "ESP subcategory failed detected" and the
// EspFailureDetected event fire. Gives ImeLogTracker a window to surface a matching
// app_install_failed event (with hresult capture) so the session timeline carries the
// app-level failure that ESP aggregated into the registry. Without this window the
// DecisionEngine transitions to Failed on the first registry observation, the agent
// terminates, and any post-ESP IME log entries are lost (race between ESP registry
// write and IME workload log).
internal const int ProvisioningFailureSettleWindowSeconds = 30;
// Regex for HRESULT extraction from subcategory statusText (e.g. "Apps (0x87d1041c)").
// Language-invariant: the parenthesised hex tail is consistent across Windows locales.
private static readonly Regex HResultPattern = new Regex(
@"\((0[xX][0-9a-fA-F]{8})\)",
RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly string[] ProvisioningCategoryNames =
{
"DevicePreparationCategory.Status",
@@ -86,9 +102,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
private bool _saveWhiteGloveSuccessResultSeen;
// Track SaveWhiteGloveSuccessResult state transitions for observability (null → notStarted → succeeded)
private string _lastSaveWhiteGloveState;
// Settle-window timers + pending args per category (session 9d052230 fix). Keyed by
// category-name (e.g. "DeviceSetupCategory.Status"). Pending args carry the enriched
// failure details that fire after the settle window expires.
private Dictionary<string, Timer> _provisioningFailureSettleTimers;
private Dictionary<string, EspFailureDetectedEventArgs> _provisioningFailureSettleArgs;
private readonly object _stateLock = new object();
public event EventHandler<string> EspFailureDetected;
public event EventHandler<EspFailureDetectedEventArgs> EspFailureDetected;
public event EventHandler DeviceSetupProvisioningComplete;
/// <summary>
/// Fires once when <c>AccountSetupCategory.Status</c> resolves to
@@ -235,6 +256,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_provisioningFailureFired = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
_provisioningCategoriesResolved = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
_lastSubcategoryStates = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
_provisioningFailureSettleTimers = new Dictionary<string, Timer>(StringComparer.OrdinalIgnoreCase);
_provisioningFailureSettleArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
_deviceSetupProvisioningCompleteFired = false;
_deviceSetupFallbackTimer = null;
_accountSetupProvisioningCompleteFired = false;
@@ -275,6 +298,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_accountSetupFallbackTimer?.Dispose();
_accountSetupFallbackTimer = null;
if (_provisioningFailureSettleTimers != null)
{
foreach (var t in _provisioningFailureSettleTimers.Values)
{
try { t?.Dispose(); } catch { /* swallow */ }
}
_provisioningFailureSettleTimers.Clear();
}
if (_provisioningWatcher != null)
{
_provisioningWatcher.Dispose();
@@ -354,8 +386,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
return;
}
bool failureDetected = false;
string failureType = null;
int changedCount = 0, unchangedCount = 0, missingCount = 0;
foreach (var categoryName in ProvisioningCategoryNames)
@@ -380,13 +410,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_logger.Trace($"CheckProvisioningStatus: {categoryName} — raw JSON: {jsonValue}");
_lastProvisioningJson[categoryName] = jsonValue;
var result = ProcessCategoryStatus(categoryName, jsonValue);
if (result.IsFailed)
{
failureDetected = true;
failureType = result.FailureType;
}
// TryFireProvisioningFailure (inside ProcessCategoryStatus) arms the
// 30 s settle timer when a failure is detected — the actual
// EspFailureDetected fire happens in OnProvisioningFailureSettleExpired.
ProcessCategoryStatus(categoryName, jsonValue);
}
_logger.Trace($"CheckProvisioningStatus: summary — changed={changedCount}, unchanged={unchangedCount}, missing={missingCount}, " +
@@ -403,13 +430,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_provisioningWatcher?.RequestStop();
}
// Fire EspFailureDetected AFTER event emission (event in spool before agent reacts)
if (failureDetected && failureType != null)
{
try { EspFailureDetected?.Invoke(this, failureType); }
catch (Exception ex) { _logger.Error($"EspFailureDetected handler failed for '{failureType}'", ex); }
}
// Fire DeviceSetupProvisioningComplete when DeviceSetup resolves with success.
if (!_deviceSetupProvisioningCompleteFired
&& _lastCategorySucceeded.TryGetValue("DeviceSetupCategory.Status", out var dsSucceeded)
@@ -957,16 +977,150 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_provisioningFailureFired.Add(categoryName);
var failedSubcategory = FindFailedSubcategory(subcategories);
var failureTypeName = failedSubcategory != null
? $"Provisioning_{categoryLabel}_{failedSubcategory}_Failed"
var failedSubcategoryInfo = FindFailedSubcategoryInfo(subcategories);
var failedSubcategoryName = failedSubcategoryInfo?.Name;
var failureTypeName = failedSubcategoryName != null
? $"Provisioning_{categoryLabel}_{failedSubcategoryName}_Failed"
: $"Provisioning_{categoryLabel}_Failed";
_logger.Info($"Provisioning subcategory failure: {categoryLabel}/{failedSubcategory ?? "category-level"} — escalating as {failureTypeName}");
var errorCode = TryExtractErrorCode(failedSubcategoryInfo?.StatusText);
_logger.Info(
$"Provisioning subcategory failure: {categoryLabel}/{failedSubcategoryName ?? "category-level"} — " +
$"escalating as {failureTypeName} (errorCode={errorCode ?? "n/a"})");
_logger.Warning($"Provisioning failure detected: {failureTypeName}");
return ProvisioningResult.Failure(failureTypeName);
// Caller already holds _stateLock.
ArmFailureSettleTimer(
categoryName: categoryName,
args: new EspFailureDetectedEventArgs(
failureType: failureTypeName,
errorCode: errorCode,
failedSubcategory: failedSubcategoryName,
category: categoryLabel));
// Settle-window owns the actual EspFailureDetected fire. The legacy "fire-after-emit"
// path in CheckProvisioningStatus is now a no-op for this code-path; it stays in place
// only for direct callers that may still expect a synchronous failure-type string.
return ProvisioningResult.NoAction;
}
/// <summary>
/// Session 9d052230 fix: arms a 30 s settle timer per category. While the timer is
/// pending, ImeLogTracker can still surface app_install_failed events with hresult
/// capture so the session timeline carries the underlying app-level failure that ESP
/// aggregated into a single "Apps: failed (0x…)" registry write. On expiry the
/// EspFailureDetected event is fired with the enriched args.
/// <para>
/// Caller holds <see cref="_stateLock"/>. Pending args are stored under the same lock
/// so the timer callback re-acquires it before invocation. A single per-session emit
/// of <c>esp_failure_settle_started</c> is sufficient — fire-once is gated by
/// <see cref="_provisioningFailureFired"/> which TryFireProvisioningFailure checks.
/// </para>
/// </summary>
private void ArmFailureSettleTimer(string categoryName, EspFailureDetectedEventArgs args)
{
_provisioningFailureSettleArgs[categoryName] = args;
EmitFailureSettleStarted(args);
var timer = new Timer(
_ => OnProvisioningFailureSettleExpired(categoryName),
state: null,
dueTime: TimeSpan.FromSeconds(ProvisioningFailureSettleWindowSeconds),
period: Timeout.InfiniteTimeSpan);
_provisioningFailureSettleTimers[categoryName] = timer;
_logger.Info(
$"Provisioning failure settle window armed: category={args.Category}, " +
$"failedSubcategory={args.FailedSubcategory ?? "n/a"}, errorCode={args.ErrorCode ?? "n/a"}, " +
$"settleSeconds={ProvisioningFailureSettleWindowSeconds}");
}
private void OnProvisioningFailureSettleExpired(string categoryName)
{
EspFailureDetectedEventArgs args = null;
try
{
lock (_stateLock)
{
if (_provisioningFailureSettleArgs == null
|| !_provisioningFailureSettleArgs.TryGetValue(categoryName, out args))
{
_logger.Debug($"Failure settle timer expired but no pending args for {categoryName} — ignoring");
return;
}
_provisioningFailureSettleArgs.Remove(categoryName);
if (_provisioningFailureSettleTimers != null
&& _provisioningFailureSettleTimers.TryGetValue(categoryName, out var timer))
{
try { timer?.Dispose(); } catch { /* swallow */ }
_provisioningFailureSettleTimers.Remove(categoryName);
}
}
_logger.Info(
$"Provisioning failure settle window elapsed — firing EspFailureDetected: " +
$"category={args.Category}, failedSubcategory={args.FailedSubcategory ?? "n/a"}, " +
$"errorCode={args.ErrorCode ?? "n/a"}, failureType={args.FailureType}");
try { EspFailureDetected?.Invoke(this, args); }
catch (Exception ex) { _logger.Error($"EspFailureDetected handler failed for '{args.FailureType}'", ex); }
}
catch (Exception ex)
{
_logger.Error("OnProvisioningFailureSettleExpired callback failed", ex);
}
}
private void EmitFailureSettleStarted(EspFailureDetectedEventArgs args)
{
var data = new Dictionary<string, object>
{
{ "category", args.Category ?? "unknown" },
{ "failedSubcategory", args.FailedSubcategory ?? "category-level" },
{ "failureType", args.FailureType },
{ "settleSeconds", ProvisioningFailureSettleWindowSeconds },
{ "reason", "wait_for_late_ime_signals" }
};
if (!string.IsNullOrEmpty(args.ErrorCode))
data["errorCode"] = args.ErrorCode;
_post.Emit(new EnrollmentEvent
{
SessionId = _sessionId,
TenantId = _tenantId,
EventType = "esp_failure_settle_started",
Severity = EventSeverity.Info,
Source = "EspAndHelloTracker",
Phase = EnrollmentPhase.Unknown,
Message = $"ESP failure settle window started ({ProvisioningFailureSettleWindowSeconds}s) — " +
$"{args.Category}/{args.FailedSubcategory ?? "category-level"}" +
(string.IsNullOrEmpty(args.ErrorCode) ? "" : $" (errorCode={args.ErrorCode})"),
Data = data
});
}
/// <summary>
/// Extracts a Windows HRESULT (e.g. <c>0x87d1041c</c>) from a subcategory statusText.
/// Returns the normalised lower-case hex form with <c>0x</c> prefix, or null when no
/// HRESULT pattern is present. Pattern is language-invariant (parenthesised hex tail).
/// </summary>
internal static string TryExtractErrorCode(string statusText)
{
if (string.IsNullOrEmpty(statusText)) return null;
var match = HResultPattern.Match(statusText);
if (!match.Success) return null;
var raw = match.Groups[1].Value;
return "0x" + raw.Substring(2).ToLowerInvariant();
}
/// <summary>Test seam — drive the settle-timer callback synchronously.</summary>
internal void TriggerSettleTimerForTest(string categoryName)
=> OnProvisioningFailureSettleExpired(categoryName);
private bool HasCategorySucceededChanged(string categoryName, bool? newValue)
{
if (!_lastCategorySucceeded.TryGetValue(categoryName, out var oldValue))
@@ -1069,6 +1223,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
if (failedNames.Count > 0)
{
eventData["failedSubcategories"] = string.Join(",", failedNames);
// Session 9d052230: surface HRESULT from the first failed subcategory's
// statusText at top-level so the UI / RuleEngine can match it without
// parsing nested statusText strings. Pattern is language-invariant.
var firstFailedSub = subcategories
.FirstOrDefault(s => string.Equals(s.State, "failed", StringComparison.OrdinalIgnoreCase));
var firstFailedCode = TryExtractErrorCode(firstFailedSub?.StatusText);
if (!string.IsNullOrEmpty(firstFailedCode))
{
eventData["failedSubcategoryErrorCode"] = firstFailedCode;
}
}
}
@@ -1204,11 +1369,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
}
internal static string FindFailedSubcategory(List<SubcategoryInfo> subcategories)
=> FindFailedSubcategoryInfo(subcategories)?.Name;
internal static SubcategoryInfo FindFailedSubcategoryInfo(List<SubcategoryInfo> subcategories)
{
foreach (var sub in subcategories)
{
if (string.Equals(sub.State, "failed", StringComparison.OrdinalIgnoreCase))
return sub.Name;
return sub;
}
return null;
}
@@ -1324,6 +1492,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_provisioningCategoriesResolved = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (_lastSubcategoryStates == null)
_lastSubcategoryStates = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
if (_provisioningFailureSettleTimers == null)
_provisioningFailureSettleTimers = new Dictionary<string, Timer>(StringComparer.OrdinalIgnoreCase);
if (_provisioningFailureSettleArgs == null)
_provisioningFailureSettleArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
}
}
@@ -89,7 +89,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
private void OnHelloCompleted(object sender, EventArgs e) => EmitHello(ReadHelloOutcome());
private void OnFinalizing(object sender, string reason) => EmitFinalizing(reason);
private void OnWhiteGloveCompleted(object sender, EventArgs e) => EmitWhiteGlove();
private void OnEspFailure(object sender, string failureType) => EmitEspFailure(failureType);
private void OnEspFailure(object sender, EspFailureDetectedEventArgs args) => EmitEspFailure(args);
private void OnDeviceSetupComplete(object sender, EventArgs e) => EmitDeviceSetupComplete();
private void OnAccountSetupComplete(object sender, EventArgs e) => EmitAccountSetupComplete();
private void OnHelloPolicyDetected(bool helloEnabled, string source) => EmitHelloPolicy(helloEnabled, source);
@@ -98,7 +98,9 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
internal void TriggerHelloFromTest(string? helloOutcome) => EmitHello(helloOutcome);
internal void TriggerFinalizingFromTest(string reason) => EmitFinalizing(reason);
internal void TriggerWhiteGloveFromTest() => EmitWhiteGlove();
internal void TriggerEspFailureFromTest(string failureType) => EmitEspFailure(failureType);
internal void TriggerEspFailureFromTest(string failureType)
=> EmitEspFailure(new EspFailureDetectedEventArgs(failureType));
internal void TriggerEspFailureFromTest(EspFailureDetectedEventArgs args) => EmitEspFailure(args);
internal void TriggerDeviceSetupCompleteFromTest() => EmitDeviceSetupComplete();
internal void TriggerAccountSetupCompleteFromTest() => EmitAccountSetupComplete();
internal void TriggerHelloPolicyDetectedFromTest(bool helloEnabled, string source) => EmitHelloPolicy(helloEnabled, source);
@@ -226,20 +228,41 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
derivationInputs: derivationInputs));
}
private void EmitEspFailure(string failureType)
private void EmitEspFailure(EspFailureDetectedEventArgs args)
{
if (_espFailurePosted) return;
_espFailurePosted = true;
var safeType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType!;
var safeType = args?.FailureType ?? "unknown";
var errorCode = args?.ErrorCode;
var failedSubcategory = args?.FailedSubcategory;
var category = args?.Category;
var now = ResolveOccurredAt(out var derivedFromClock);
var derivationInputs = new Dictionary<string, string>(StringComparer.Ordinal)
{
["subSource"] = "ShellCoreTracker+ProvisioningStatusTracker (merged)",
["failureType"] = safeType,
};
if (!string.IsNullOrEmpty(errorCode))
derivationInputs["errorCode"] = errorCode!;
if (!string.IsNullOrEmpty(failedSubcategory))
derivationInputs["failedSubcategory"] = failedSubcategory!;
if (!string.IsNullOrEmpty(category))
derivationInputs["category"] = category!;
TagDerivedTimestamp(derivationInputs, derivedFromClock);
var payload = new Dictionary<string, string>(StringComparer.Ordinal)
{
["failureType"] = safeType,
};
if (!string.IsNullOrEmpty(errorCode))
payload["errorCode"] = errorCode!;
if (!string.IsNullOrEmpty(failedSubcategory))
payload["failedSubcategory"] = failedSubcategory!;
if (!string.IsNullOrEmpty(category))
payload["category"] = category!;
_ingress.Post(
kind: DecisionSignalKind.EspTerminalFailure,
occurredAtUtc: now,
@@ -247,12 +270,9 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
evidence: new Evidence(
kind: EvidenceKind.Derived,
identifier: DetectorId,
summary: $"ESP terminal failure (coordinator-merged, type={safeType})",
summary: $"ESP terminal failure (coordinator-merged, type={safeType}, errorCode={errorCode ?? "n/a"})",
derivationInputs: derivationInputs),
payload: new Dictionary<string, string>(StringComparer.Ordinal)
{
["failureType"] = safeType,
});
payload: payload);
}
// PR4 (882fef64 debrief) — once per session: post HelloPolicyDetected so the engine
@@ -59,11 +59,13 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
private void OnDeviceSetupComplete(object sender, EventArgs e) => EmitDeviceSetupComplete();
private void OnAccountSetupComplete(object sender, EventArgs e) => EmitAccountSetupComplete();
private void OnEspFailure(object sender, string failureType) => EmitEspFailure(failureType);
private void OnEspFailure(object sender, EspFailureDetectedEventArgs args) => EmitEspFailure(args);
internal void TriggerDeviceSetupCompleteFromTest() => EmitDeviceSetupComplete();
internal void TriggerAccountSetupCompleteFromTest() => EmitAccountSetupComplete();
internal void TriggerEspFailureFromTest(string failureType) => EmitEspFailure(failureType);
internal void TriggerEspFailureFromTest(string failureType)
=> EmitEspFailure(new EspFailureDetectedEventArgs(failureType));
internal void TriggerEspFailureFromTest(EspFailureDetectedEventArgs args) => EmitEspFailure(args);
private void EmitDeviceSetupComplete()
{
@@ -123,12 +125,38 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
});
}
private void EmitEspFailure(string failureType)
private void EmitEspFailure(EspFailureDetectedEventArgs args)
{
if (_espFailurePosted) return;
_espFailurePosted = true;
var safeFailureType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType!;
var safeFailureType = args?.FailureType ?? "unknown";
var errorCode = args?.ErrorCode;
var failedSubcategory = args?.FailedSubcategory;
var category = args?.Category;
var derivationInputs = new Dictionary<string, string>(StringComparer.Ordinal)
{
["registrySource"] = @"HKLM\SOFTWARE\Microsoft\Provisioning\AutopilotSettings\*.Status",
["failureType"] = safeFailureType,
};
if (!string.IsNullOrEmpty(errorCode))
derivationInputs["errorCode"] = errorCode!;
if (!string.IsNullOrEmpty(failedSubcategory))
derivationInputs["failedSubcategory"] = failedSubcategory!;
if (!string.IsNullOrEmpty(category))
derivationInputs["category"] = category!;
var payload = new Dictionary<string, string>(StringComparer.Ordinal)
{
["failureType"] = safeFailureType,
};
if (!string.IsNullOrEmpty(errorCode))
payload["errorCode"] = errorCode!;
if (!string.IsNullOrEmpty(failedSubcategory))
payload["failedSubcategory"] = failedSubcategory!;
if (!string.IsNullOrEmpty(category))
payload["category"] = category!;
_ingress.Post(
kind: DecisionSignalKind.EspTerminalFailure,
@@ -137,16 +165,9 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
evidence: new Evidence(
kind: EvidenceKind.Derived,
identifier: "provisioning-status-tracker-v1",
summary: $"ESP terminal failure from provisioning registry (type={safeFailureType})",
derivationInputs: new Dictionary<string, string>(StringComparer.Ordinal)
{
["registrySource"] = @"HKLM\SOFTWARE\Microsoft\Provisioning\AutopilotSettings\*.Status",
["failureType"] = safeFailureType,
}),
payload: new Dictionary<string, string>(StringComparer.Ordinal)
{
["failureType"] = safeFailureType,
});
summary: $"ESP terminal failure from provisioning registry (type={safeFailureType}, errorCode={errorCode ?? "n/a"})",
derivationInputs: derivationInputs),
payload: payload);
}
}
}
@@ -1 +1 @@
838
842
@@ -332,6 +332,66 @@ namespace AutopilotMonitor.DecisionCore.Tests
Assert.Equal("policy_apply_timeout", payload["reason"]);
}
/// <summary>
/// Session 9d052230 regression — when the adapter posts a registry-derived ESP failure
/// with HRESULT enrichment, the terminal <c>enrollment_failed</c> event MUST carry
/// <c>errorCode</c>, <c>failureType</c>, <c>failedSubcategory</c>, and <c>category</c>
/// on the effect parameters so the UI can render the HRESULT badge + description.
/// </summary>
[Fact]
public void EspTerminalFailure_surfaces_errorCode_and_failedSubcategory_on_terminal_event()
{
var engine = new DecisionEngine();
var state = DecisionState.CreateInitial("sess-esp-hresult", "tenant-esp-hresult");
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
var step = engine.Reduce(state, MakeSignal(
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(15),
new Dictionary<string, string>
{
["failureType"] = "Provisioning_DeviceSetup_Apps_Failed",
["errorCode"] = "0x87d1041c",
["failedSubcategory"] = "Apps",
["category"] = "DeviceSetup",
}));
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
var effect = SingleTimelineEffect(step, "enrollment_failed");
Assert.NotNull(effect.Parameters);
// reason defaults to the stable enum literal because the signal did not override it.
Assert.Equal("esp_terminal_failure", effect.Parameters!["reason"]);
Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", effect.Parameters!["failureType"]);
Assert.Equal("0x87d1041c", effect.Parameters!["errorCode"]);
Assert.Equal("Apps", effect.Parameters!["failedSubcategory"]);
Assert.Equal("DeviceSetup", effect.Parameters!["category"]);
}
/// <summary>
/// Backwards-compat — a signal without HRESULT enrichment (e.g. ShellCore-derived) must
/// still produce a valid <c>enrollment_failed</c> event with just the default reason.
/// </summary>
[Fact]
public void EspTerminalFailure_works_without_errorCode_payload()
{
var engine = new DecisionEngine();
var state = DecisionState.CreateInitial("sess-esp-bare", "tenant-esp-bare");
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
var step = engine.Reduce(state, MakeSignal(
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(15),
new Dictionary<string, string> { ["failureType"] = "ESPProgress_Timeout" }));
var effect = SingleTimelineEffect(step, "enrollment_failed");
Assert.NotNull(effect.Parameters);
Assert.Equal("esp_terminal_failure", effect.Parameters!["reason"]);
Assert.Equal("ESPProgress_Timeout", effect.Parameters!["failureType"]);
Assert.False(effect.Parameters!.ContainsKey("errorCode"));
Assert.False(effect.Parameters!.ContainsKey("failedSubcategory"));
}
[Fact]
public void EspTerminalFailure_event_enriched_with_continueAnyway_and_timeout_facts()
{
@@ -46,12 +46,24 @@ namespace AutopilotMonitor.DecisionCore.Engine
/// to <see cref="SessionStage.Failed"/> with <see cref="SessionOutcome.EnrollmentFailed"/>,
/// clears deadlines, and emits <c>enrollment_failed</c>. No classifier re-run —
/// ESP terminal failure is definitive.
/// <para>
/// The terminal-event reason is always the stable enum literal <c>esp_terminal_failure</c>.
/// Session 9d052230: registry-derived failures additionally carry <c>failureType</c>,
/// <c>errorCode</c> (HRESULT), <c>failedSubcategory</c>, and <c>category</c> on the signal
/// payload — those propagate verbatim to the timeline-event parameters so the UI can
/// render the HRESULT badge + description without re-parsing the ESP statusText.
/// </para>
/// </summary>
private DecisionStep HandleEspTerminalFailureV1(DecisionState state, DecisionSignal signal)
{
var nextStep = state.StepIndex + 1;
var reason = signal.Payload != null && signal.Payload.TryGetValue("reason", out var r)
// Default to the stable enum literal so the timeline-event "reason" is predictable
// across all sources of EspTerminalFailure. Direct test signals (and any future
// adapter that wants to override) can still inject a specific reason via payload.
var reason = signal.Payload != null
&& signal.Payload.TryGetValue("reason", out var r)
&& !string.IsNullOrEmpty(r)
? r
: "esp_terminal_failure";
@@ -78,6 +90,20 @@ namespace AutopilotMonitor.DecisionCore.Engine
["reason"] = reason,
};
// Session 9d052230 — surface registry-derived ESP failure detail (HRESULT etc) on the
// terminal event so the UI can render it without parsing nested ESP statusText.
if (signal.Payload != null)
{
if (signal.Payload.TryGetValue("failureType", out var failureType) && !string.IsNullOrEmpty(failureType))
parameters["failureType"] = failureType;
if (signal.Payload.TryGetValue("errorCode", out var errorCode) && !string.IsNullOrEmpty(errorCode))
parameters["errorCode"] = errorCode;
if (signal.Payload.TryGetValue("failedSubcategory", out var failedSubcategory) && !string.IsNullOrEmpty(failedSubcategory))
parameters["failedSubcategory"] = failedSubcategory;
if (signal.Payload.TryGetValue("category", out var category) && !string.IsNullOrEmpty(category))
parameters["category"] = category;
}
// Enrich the audit payload with the FirstSync-derived ESP failure-handling settings,
// so the timeline carries the observable fact that Microsoft's ESP profile may have
// offered the user a "Continue anyway" button — the agent's terminal-failure verdict
@@ -478,6 +478,34 @@ function EventRow({ event, showScriptOutput }: { event: EnrollmentEvent; showScr
</div>
);
})()}
{/* HRESULT badge for ESP failures (enrollment_failed via esp_terminal_failure,
esp_provisioning_status failed-subcategory). The HRESULT is extracted from the ESP
registry statusText by the agent (e.g. "Apps (0x87d1041c)") and surfaced as
top-level event data so the UI can render it without parsing nested text. */}
{(event.eventType === "enrollment_failed" || event.eventType === "esp_provisioning_status") && (() => {
const code = event.data?.errorCode ?? event.data?.failedSubcategoryErrorCode;
if (!code) return null;
const codeStr = String(code);
const entry = getErrorCodeEntry(codeStr);
const sub = event.data?.failedSubcategory ?? event.data?.failedSubcategories;
return (
<div className="mt-1 flex flex-wrap items-center gap-2 text-xs">
<span className="px-1.5 py-0.5 rounded bg-red-100 text-red-800 font-mono font-medium">
HRESULT: {formatErrorCode(codeStr)}
</span>
{entry && (
<span className="text-red-600" title={`${entry.source} (${entry.confidence} confidence)`}>
{entry.description}
</span>
)}
{sub && (
<span className="text-gray-500">
subcategory: <span className="font-mono">{String(sub)}</span>
</span>
)}
</div>
);
})()}
<div className="mt-1 flex items-center gap-3 text-xs text-gray-500">
<span>Source: {event.source}</span>
<span>Seq: {event.sequence}</span>