mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
fix(decisioncore,agent/v2): defang ESP terminal failure when ContinueAnyway + AccountSetup entered
Session 4fa5a2d4 in tenant c9787ba2 hit 199/202 false-positive enrollment_failed events: ESP
profile sets BlockInStatusPage=6 (ContinueAnyway=true), Windows transitioned to AccountSetup,
then wrote DeviceSetupCategory=failed (Certificates) 13 min later — the reducer terminated
even though the device demonstrably kept progressing. PR1 Tier-1 defang downgrades these
EspTerminalFailure signals to a non-terminal esp_failure_advisory event while keeping
monitoring armed (Hello/Desktop/IME completion paths continue to drive the session).
DecisionEngine:
- New SignalFact<DateTime>? EspAdvisoryFailureRecordedUtc on DecisionState as the fire-once
gate (additive ctor param, optional at end — preserves Newtonsoft positional-binding for
pre-PR1 v4 snapshots).
- HandleEspTerminalFailureV1: when ScenarioObservations.EspAllowContinueAnyway==true AND
AccountSetupEnteredUtc!=null, emit esp_failure_advisory (severity=Warning,
advisoryReason=esp_failure_defanged_continueanyway_with_accountsetup) instead of
transitioning to Failed. Stage / deadlines / outcome stay untouched.
- Duplicate handling: a second EspTerminalFailure dead-ends UNLESS the duplicate carries
registry-specific keys (errorCode/failedSubcategory/category) — those only appear in the
ProvisioningStatusTracker path and indicate enrichment of an earlier ShellCoreTracker
advisory. failureType alone is NOT a registry marker (both adapters emit it).
- New BuildEspFailureAuditTrail helper merges a whitelisted set of failure-context keys into
the audit-trail typedPayload so HRESULT/subcategory survive the wire. Closes a pre-existing
gap on the enrollment_failed path too.
Bootstrap:
- EnrollmentOrchestrator.PostEspConfigDetectedBootstrap switched from EspSkipConfigurationProbe.
Read() (tuple SkipUser/SkipDevice) to ReadFull() so EspAllowContinueAnyway +
EspSyncFailureTimeoutMinutes are synchronously available before any EspTerminalFailure
signal — defeats the fire-and-forget DeviceInfoCollector.CollectAll race that would have
left Tier 1 with ScenarioObservations.EspAllowContinueAnyway==null.
Tests:
- EspFailureAdvisoryDefangTests (9 new): advisory path, fire-once idempotency, enrichment
on registry-detail second-fire, dead-end on ShellCore-only duplicate, three negative paths
(ContinueAnyway false / unknown / no AccountSetup), TypedPayload whitelist merge on both
advisory + Failed paths.
- SerializationTests: backcompat test for old v4 snapshots without EspAdvisoryFailureRecordedUtc.
- DecisionStateSnapshotBuilderTests: fact count bumped from 14 to 15.
- EdgeScenarioTests + esp-failure-advisory-continueanyway-v1.jsonl: end-to-end replay against
the canonical Session 4fa5a2d4 signal stream including a duplicate dead-end.
- EnrollmentOrchestratorEspConfigBootstrapTests: 4 existing tests migrated to ScopedFullOverride,
4 new tests cover ContinueAnyway + SyncFailureTimeout propagation, class gated behind
[Collection("SerialThreading")] since the override targets a static probe slot.
UI:
- EventTimeline renders esp_failure_advisory with an amber "Advisory (ContinueAnyway)" badge
alongside the HRESULT/subcategory chips so analysts can distinguish defanged from real
failures at a glance.
- gather-rules/eventTypes.ts mirrors the new constant for autocomplete.
Build: dotnet build AutopilotMonitor.sln green; xUnit Backend(1937) + V2.Core(1139) +
Agent.Core(538) + DecisionCore(357) + SummaryDialog(44) = 4015 passed, 0 failed.
Web tsc clean, vitest 384/384 passed.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+129
-8
@@ -24,7 +24,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
/// <c>ShouldTransitionToAwaitingHello</c> guard would block the legitimate promotion —
|
||||
/// leaving the session stuck, because <see cref="AutopilotMonitor.Agent.V2.Core.SignalAdapters.EspAndHelloTrackerAdapter"/>
|
||||
/// forwards Finalizing exactly once.
|
||||
/// <para>
|
||||
/// PR1 (Session 4fa5a2d4, 2026-05-22) — bootstrap migrated from <c>EspSkipConfigurationProbe.Read()</c>
|
||||
/// (Tuple SkipUser/SkipDevice only) to <c>ReadFull()</c> so EspAllowContinueAnyway +
|
||||
/// EspSyncFailureTimeoutMinutes are synchronously available before any EspTerminalFailure
|
||||
/// signal can arrive. Tests migrated from <see cref="EspSkipConfigurationProbe.ScopedOverride"/>
|
||||
/// to <see cref="EspSkipConfigurationProbe.ScopedFullOverride"/>. Both overrides set a
|
||||
/// <b>static</b> slot on the probe (<c>TestOverride</c>/<c>FullTestOverride</c>) — the
|
||||
/// class is gated behind <c>[Collection("SerialThreading")]</c> so parallel orchestrator
|
||||
/// tests in other classes do not race on the static slot.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Collection("SerialThreading")]
|
||||
public sealed class EnrollmentOrchestratorEspConfigBootstrapTests
|
||||
{
|
||||
private static DateTime At => new DateTime(2026, 4, 24, 10, 0, 0, DateTimeKind.Utc);
|
||||
@@ -46,8 +57,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
public void Start_posts_EspConfigDetected_before_collectors_when_FirstSync_available()
|
||||
{
|
||||
using var rig = new EnrollmentOrchestratorRig(At);
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedOverride(
|
||||
_log => ((bool?)true, (bool?)false));
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedFullOverride(
|
||||
_log => new EspFirstSyncSnapshot(
|
||||
skipUser: true,
|
||||
skipDevice: false,
|
||||
blockInStatusPage: null,
|
||||
syncFailureTimeoutMinutes: null));
|
||||
using var orchestrator = rig.Build();
|
||||
|
||||
orchestrator.Start();
|
||||
@@ -67,8 +82,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
// very early boot) the bootstrap no-ops so the SignalLog does not carry a
|
||||
// meaningless signal. The collector's later CollectAll is the backup path.
|
||||
using var rig = new EnrollmentOrchestratorRig(At);
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedOverride(
|
||||
_log => ((bool?)null, (bool?)null));
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedFullOverride(_log => EspFirstSyncSnapshot.Empty);
|
||||
using var orchestrator = rig.Build();
|
||||
|
||||
orchestrator.Start();
|
||||
@@ -85,8 +99,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
// later from a subsequent post (DeviceInfoCollector.CollectEspConfiguration or
|
||||
// CollectAtEnrollmentStart).
|
||||
using var rig = new EnrollmentOrchestratorRig(At);
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedOverride(
|
||||
_log => ((bool?)null, (bool?)false));
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedFullOverride(
|
||||
_log => new EspFirstSyncSnapshot(
|
||||
skipUser: null,
|
||||
skipDevice: false,
|
||||
blockInStatusPage: null,
|
||||
syncFailureTimeoutMinutes: null));
|
||||
using var orchestrator = rig.Build();
|
||||
|
||||
orchestrator.Start();
|
||||
@@ -107,8 +125,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
// kind could drive Classic stage promotion. SessionStarted itself is always first
|
||||
// (ordinal 0); EspConfigDetected should come right after.
|
||||
using var rig = new EnrollmentOrchestratorRig(At);
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedOverride(
|
||||
_log => ((bool?)true, (bool?)false));
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedFullOverride(
|
||||
_log => new EspFirstSyncSnapshot(
|
||||
skipUser: true,
|
||||
skipDevice: false,
|
||||
blockInStatusPage: null,
|
||||
syncFailureTimeoutMinutes: null));
|
||||
using var orchestrator = rig.Build();
|
||||
|
||||
orchestrator.Start();
|
||||
@@ -122,5 +144,104 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
signals.TakeWhile(s => s.SessionSignalOrdinal < espConfig.SessionSignalOrdinal),
|
||||
s => s.Kind == DecisionSignalKind.EspPhaseChanged || s.Kind == DecisionSignalKind.EspExiting);
|
||||
}
|
||||
|
||||
// ============================================================ PR1 (4fa5a2d4) =====
|
||||
|
||||
[Fact]
|
||||
public void Start_posts_EspConfigDetected_with_ContinueAnywayAndSyncFailureTimeout_FromReadFull()
|
||||
{
|
||||
// PR1 (Session 4fa5a2d4, 2026-05-22) — ReadFull-Switch carries EspAllowContinueAnyway +
|
||||
// EspSyncFailureTimeoutMinutes synchronously, so Tier 1 advisory-defang in
|
||||
// HandleEspTerminalFailureV1 has the ContinueAnyway fact before any EspTerminalFailure
|
||||
// signal can arrive (otherwise the fire-and-forget DeviceInfoCollector.CollectAll
|
||||
// would race and the reducer would see ScenarioObservations.EspAllowContinueAnyway==null).
|
||||
//
|
||||
// BlockInStatusPage=6 → AllowContinueAnyway=true, AllowTryAgain=true, AllowReset=false.
|
||||
using var rig = new EnrollmentOrchestratorRig(At);
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedFullOverride(
|
||||
_log => new EspFirstSyncSnapshot(
|
||||
skipUser: false,
|
||||
skipDevice: false,
|
||||
blockInStatusPage: 6,
|
||||
syncFailureTimeoutMinutes: 30));
|
||||
using var orchestrator = rig.Build();
|
||||
|
||||
orchestrator.Start();
|
||||
orchestrator.Stop();
|
||||
|
||||
var signals = ReadSignalLog(rig.StateDir);
|
||||
var espConfig = Assert.Single(signals, s => s.Kind == DecisionSignalKind.EspConfigDetected);
|
||||
Assert.Equal("EnrollmentOrchestrator", espConfig.SourceOrigin);
|
||||
Assert.Equal("false", espConfig.Payload![SignalPayloadKeys.SkipUserEsp]);
|
||||
Assert.Equal("false", espConfig.Payload![SignalPayloadKeys.SkipDeviceEsp]);
|
||||
Assert.Equal("true", espConfig.Payload![SignalPayloadKeys.EspAllowContinueAnyway]);
|
||||
Assert.Equal("30", espConfig.Payload![SignalPayloadKeys.EspSyncFailureTimeoutMinutes]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Start_posts_EspConfigDetected_with_ContinueAnywayFalse_when_BlockInStatusPage_zero()
|
||||
{
|
||||
// Strict ESP profile: BlockInStatusPage=0 disables all three user buttons (Reset,
|
||||
// TryAgain, ContinueAnyway). The bootstrap surfaces "false" verbatim so the reducer
|
||||
// can distinguish "ContinueAnyway explicitly disabled" from "FirstSync unread".
|
||||
using var rig = new EnrollmentOrchestratorRig(At);
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedFullOverride(
|
||||
_log => new EspFirstSyncSnapshot(
|
||||
skipUser: false,
|
||||
skipDevice: false,
|
||||
blockInStatusPage: 0,
|
||||
syncFailureTimeoutMinutes: 30));
|
||||
using var orchestrator = rig.Build();
|
||||
|
||||
orchestrator.Start();
|
||||
orchestrator.Stop();
|
||||
|
||||
var signals = ReadSignalLog(rig.StateDir);
|
||||
var espConfig = Assert.Single(signals, s => s.Kind == DecisionSignalKind.EspConfigDetected);
|
||||
Assert.Equal("false", espConfig.Payload![SignalPayloadKeys.EspAllowContinueAnyway]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Start_skips_EspConfigDetected_post_when_all_four_fields_null()
|
||||
{
|
||||
// No-op guard updated for ReadFull(): if SkipUser, SkipDevice, BlockInStatusPage AND
|
||||
// SyncFailureTimeout are all null, the bootstrap stays silent — the collector's later
|
||||
// CollectAll is the backup path.
|
||||
using var rig = new EnrollmentOrchestratorRig(At);
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedFullOverride(_log => EspFirstSyncSnapshot.Empty);
|
||||
using var orchestrator = rig.Build();
|
||||
|
||||
orchestrator.Start();
|
||||
orchestrator.Stop();
|
||||
|
||||
var signals = ReadSignalLog(rig.StateDir);
|
||||
Assert.DoesNotContain(signals, s => s.Kind == DecisionSignalKind.EspConfigDetected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Start_posts_EspConfigDetected_when_only_ContinueAnyway_field_present()
|
||||
{
|
||||
// Edge case: FirstSync registry has only BlockInStatusPage set (e.g. partial write
|
||||
// from Windows or trimmed registry). The bootstrap still posts because
|
||||
// ContinueAnyway is decoded from BlockInStatusPage, even though SkipUser/SkipDevice
|
||||
// remain null. Confirms the no-op guard checks all four fields.
|
||||
using var rig = new EnrollmentOrchestratorRig(At);
|
||||
using var _ = new EspSkipConfigurationProbe.ScopedFullOverride(
|
||||
_log => new EspFirstSyncSnapshot(
|
||||
skipUser: null,
|
||||
skipDevice: null,
|
||||
blockInStatusPage: 6,
|
||||
syncFailureTimeoutMinutes: null));
|
||||
using var orchestrator = rig.Build();
|
||||
|
||||
orchestrator.Start();
|
||||
orchestrator.Stop();
|
||||
|
||||
var signals = ReadSignalLog(rig.StateDir);
|
||||
var espConfig = Assert.Single(signals, s => s.Kind == DecisionSignalKind.EspConfigDetected);
|
||||
Assert.False(espConfig.Payload!.ContainsKey(SignalPayloadKeys.SkipUserEsp));
|
||||
Assert.False(espConfig.Payload!.ContainsKey(SignalPayloadKeys.SkipDeviceEsp));
|
||||
Assert.Equal("true", espConfig.Payload![SignalPayloadKeys.EspAllowContinueAnyway]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -859,8 +859,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
try
|
||||
{
|
||||
var (skipUser, skipDevice) = Monitoring.Enrollment.SystemSignals.EspSkipConfigurationProbe.Read(_logger);
|
||||
if (skipUser == null && skipDevice == null)
|
||||
// PR1 (Session 4fa5a2d4, 2026-05-22): switched from Read() (Tuple skipUser/skipDevice
|
||||
// only) to ReadFull() so EspAllowContinueAnyway + EspSyncFailureTimeoutMinutes are
|
||||
// synchronously available before any EspTerminalFailure signal can arrive. Without
|
||||
// this, the Tier 1 advisory-defang in HandleEspTerminalFailureV1 would race the
|
||||
// fire-and-forget DeviceInfoCollector.CollectAll and see EspAllowContinueAnyway==null.
|
||||
var snapshot = Monitoring.Enrollment.SystemSignals.EspSkipConfigurationProbe.ReadFull(_logger);
|
||||
if (snapshot.SkipUser == null
|
||||
&& snapshot.SkipDevice == null
|
||||
&& snapshot.SyncFailureTimeoutMinutes == null
|
||||
&& snapshot.AllowContinueAnyway == null)
|
||||
{
|
||||
_logger.Debug(
|
||||
"EnrollmentOrchestrator: EspConfigDetected bootstrap skipped — FirstSync not yet populated; DeviceInfoCollector will post when CollectAll runs.");
|
||||
@@ -868,10 +876,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
}
|
||||
|
||||
var payload = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
if (skipUser.HasValue)
|
||||
payload[SignalPayloadKeys.SkipUserEsp] = skipUser.Value ? "true" : "false";
|
||||
if (skipDevice.HasValue)
|
||||
payload[SignalPayloadKeys.SkipDeviceEsp] = skipDevice.Value ? "true" : "false";
|
||||
if (snapshot.SkipUser.HasValue)
|
||||
payload[SignalPayloadKeys.SkipUserEsp] = snapshot.SkipUser.Value ? "true" : "false";
|
||||
if (snapshot.SkipDevice.HasValue)
|
||||
payload[SignalPayloadKeys.SkipDeviceEsp] = snapshot.SkipDevice.Value ? "true" : "false";
|
||||
if (snapshot.SyncFailureTimeoutMinutes.HasValue && snapshot.SyncFailureTimeoutMinutes.Value > 0)
|
||||
payload[SignalPayloadKeys.EspSyncFailureTimeoutMinutes] = snapshot.SyncFailureTimeoutMinutes.Value
|
||||
.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
if (snapshot.AllowContinueAnyway.HasValue)
|
||||
payload[SignalPayloadKeys.EspAllowContinueAnyway] = snapshot.AllowContinueAnyway.Value ? "true" : "false";
|
||||
|
||||
_ingress!.Post(
|
||||
kind: DecisionSignalKind.EspConfigDetected,
|
||||
@@ -880,7 +893,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
evidence: new Evidence(
|
||||
kind: EvidenceKind.Raw,
|
||||
identifier: "esp_config_detected_bootstrap",
|
||||
summary: $"SkipUser={skipUser?.ToString() ?? "unknown"}, SkipDevice={skipDevice?.ToString() ?? "unknown"}",
|
||||
summary: $"SkipUser={snapshot.SkipUser?.ToString() ?? "unknown"}, SkipDevice={snapshot.SkipDevice?.ToString() ?? "unknown"}, AllowContinueAnyway={snapshot.AllowContinueAnyway?.ToString() ?? "unknown"}",
|
||||
derivationInputs: new Dictionary<string, string>(payload, StringComparer.Ordinal)
|
||||
{
|
||||
["source"] = "registry_firstsync_bootstrap",
|
||||
@@ -888,7 +901,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
payload: payload);
|
||||
|
||||
_logger.Info(
|
||||
$"EnrollmentOrchestrator: EspConfigDetected bootstrap posted (SkipUser={skipUser?.ToString() ?? "unknown"}, SkipDevice={skipDevice?.ToString() ?? "unknown"}).");
|
||||
$"EnrollmentOrchestrator: EspConfigDetected bootstrap posted (SkipUser={snapshot.SkipUser?.ToString() ?? "unknown"}, SkipDevice={snapshot.SkipDevice?.ToString() ?? "unknown"}, AllowContinueAnyway={snapshot.AllowContinueAnyway?.ToString() ?? "unknown"}, SyncFailureTimeoutMinutes={snapshot.SyncFailureTimeoutMinutes?.ToString() ?? "unknown"}).");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -1 +1 @@
|
||||
861
|
||||
868
|
||||
|
||||
@@ -1 +1 @@
|
||||
1397
|
||||
1398
|
||||
|
||||
@@ -105,7 +105,10 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
// as the strong post-AccountSetup gate → 13 slots.
|
||||
// 88a53223 SelfDeploying defang (Plan v9) added `deviceSetupResolvedUtc` so the
|
||||
// new DeviceOnlyEspDetection deadline arms from DeviceSetup-END not -START → 14.
|
||||
Assert.Equal(14, facts.Count);
|
||||
// PR1 ContinueAnyway-Defang (Session 4fa5a2d4, 2026-05-22) added
|
||||
// `espAdvisoryFailureRecordedUtc` as the fire-once gate for downgraded ESP
|
||||
// terminal failures → 15.
|
||||
Assert.Equal(15, facts.Count);
|
||||
Assert.All(facts.Values, v => Assert.Null(v));
|
||||
}
|
||||
|
||||
@@ -219,9 +222,12 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
// `accountSetupProvisioningSucceededUtc` as the strong post-AccountSetup
|
||||
// gate → 13. 88a53223 SelfDeploying defang (Plan v9, 2026-05-21) added
|
||||
// `deviceSetupResolvedUtc` as the DeviceSetup-END anchor for the new
|
||||
// deadline arm-point → 14. If this number ever changes, both the count
|
||||
// expectation AND the actual snapshot output need to evolve in lockstep.
|
||||
Assert.Equal(14, expectedFactKeys.Count);
|
||||
// deadline arm-point → 14. PR1 ContinueAnyway-Defang (Session 4fa5a2d4,
|
||||
// 2026-05-22) added `espAdvisoryFailureRecordedUtc` as the fire-once gate
|
||||
// for downgraded ESP terminal failures → 15. If this number ever changes,
|
||||
// both the count expectation AND the actual snapshot output need to evolve
|
||||
// in lockstep.
|
||||
Assert.Equal(15, expectedFactKeys.Count);
|
||||
|
||||
var snapshot = DecisionStateSnapshotBuilder.Build(DecisionState.CreateInitial("s", "t"));
|
||||
var facts = (Dictionary<string, object?>)snapshot["facts"]!;
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.DecisionCore.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// PR1 — ContinueAnyway-aware ESP terminal-failure defang (Session 4fa5a2d4, 2026-05-22).
|
||||
/// Coverage for the advisory path added to <c>HandleEspTerminalFailureV1</c>:
|
||||
/// <list type="bullet">
|
||||
/// <item>Advisory-Demote when ESP profile permits ContinueAnyway AND AccountSetup has been entered.</item>
|
||||
/// <item>Fire-once idempotency via <c>EspAdvisoryFailureRecordedUtc</c>.</item>
|
||||
/// <item>Negative paths (ContinueAnyway false / unknown / AccountSetup not entered) still terminate.</item>
|
||||
/// <item>Whitelist-merge of failure-context keys into the audit-trail typed payload (both paths).</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class EspFailureAdvisoryDefangTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 5, 22, 8, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static DecisionSignal MakeSignal(
|
||||
long ordinal,
|
||||
DecisionSignalKind kind,
|
||||
DateTime occurredAtUtc,
|
||||
IReadOnlyDictionary<string, string>? payload = null,
|
||||
string sourceOrigin = "test")
|
||||
{
|
||||
return new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: kind,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: occurredAtUtc,
|
||||
sourceOrigin: sourceOrigin,
|
||||
evidence: new Evidence(EvidenceKind.Synthetic, $"{kind}-{ordinal}", "test"),
|
||||
payload: payload);
|
||||
}
|
||||
|
||||
private static DecisionEffect SingleTimelineEffect(DecisionStep step, string eventType) =>
|
||||
step.Effects.Single(e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry
|
||||
&& e.Parameters != null
|
||||
&& e.Parameters.TryGetValue("eventType", out var et)
|
||||
&& et == eventType);
|
||||
|
||||
/// <summary>
|
||||
/// Builds a state that has ESP-ContinueAnyway observed and AccountSetup entered — the
|
||||
/// canonical "advisory eligible" preconditions. Mirrors the production wire: bootstrap
|
||||
/// fires EspConfigDetected, ESP transitions to DeviceSetup then AccountSetup before the
|
||||
/// late DeviceSetup/Certificates failure registry-write arrives.
|
||||
/// </summary>
|
||||
private static DecisionState SetupContinueAnywayWithAccountSetupEntered(DecisionEngine engine, bool continueAnyway = true)
|
||||
{
|
||||
var state = DecisionState.CreateInitial("sess-advisory", "tenant-advisory");
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
10, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[SignalPayloadKeys.SkipUserEsp] = "false",
|
||||
[SignalPayloadKeys.SkipDeviceEsp] = "false",
|
||||
[SignalPayloadKeys.EspSyncFailureTimeoutMinutes] = "30",
|
||||
[SignalPayloadKeys.EspAllowContinueAnyway] = continueAnyway ? "true" : "false",
|
||||
})).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
30, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(15),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
return state;
|
||||
}
|
||||
|
||||
// =============================================== advisory-defang positive path ====
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_WhenContinueAnywayAndAccountSetupEntered_DemotesToAdvisory()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupContinueAnywayWithAccountSetupEntered(engine);
|
||||
|
||||
var preStage = state.Stage;
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(30),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Certificates_Failed",
|
||||
["failedSubcategory"] = "Certificates",
|
||||
["category"] = "DeviceSetup",
|
||||
}));
|
||||
|
||||
// No transition to Failed — stage unchanged, no terminal outcome.
|
||||
Assert.Equal(preStage, step.NewState.Stage);
|
||||
Assert.Null(step.NewState.Outcome);
|
||||
|
||||
// The advisory fact is recorded — the fire-once gate for duplicates.
|
||||
Assert.NotNull(step.NewState.EspAdvisoryFailureRecordedUtc);
|
||||
Assert.Equal(T0.AddMinutes(30), step.NewState.EspAdvisoryFailureRecordedUtc!.Value);
|
||||
Assert.Equal(40, step.NewState.EspAdvisoryFailureRecordedUtc!.SourceSignalOrdinal);
|
||||
|
||||
// LastFailureTrigger is NOT set on the advisory path — only the Failed-path
|
||||
// EnrollmentTerminationHandler relies on that fact.
|
||||
Assert.Null(step.NewState.LastFailureTrigger);
|
||||
|
||||
// Deadlines stay armed — monitoring continues.
|
||||
// (We don't assert on a specific deadline name because the test setup doesn't arm one,
|
||||
// but the fact that the reducer didn't ClearDeadlines() means the existing list survives.)
|
||||
|
||||
// Single esp_failure_advisory effect, no enrollment_failed.
|
||||
var advisory = SingleTimelineEffect(step, "esp_failure_advisory");
|
||||
Assert.Equal("esp_terminal_failure", advisory.Parameters!["reason"]);
|
||||
Assert.Equal("esp_failure_defanged_continueanyway_with_accountsetup", advisory.Parameters!["advisoryReason"]);
|
||||
// Severity is explicit (not derived from "_failed"-suffix DeriveSeverity default,
|
||||
// which would give Info for advisory). The UI renders this as amber/warning.
|
||||
Assert.Equal("Warning", advisory.Parameters!["severity"]);
|
||||
Assert.Equal("Certificates", advisory.Parameters!["failedSubcategory"]);
|
||||
Assert.Equal("DeviceSetup", advisory.Parameters!["category"]);
|
||||
Assert.Equal("true", advisory.Parameters!["espAllowContinueAnyway"]);
|
||||
Assert.Equal("true", advisory.Parameters!["mayHaveContinuedAnyway"]);
|
||||
Assert.Contains("AccountSetup", advisory.Parameters!["continueAnywayHint"]);
|
||||
|
||||
Assert.DoesNotContain(step.Effects, e =>
|
||||
e.Parameters != null
|
||||
&& e.Parameters.TryGetValue("eventType", out var et)
|
||||
&& et == "enrollment_failed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_AdvisoryEffect_TypedPayloadCarriesAuditTrailAndFailureContext()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupContinueAnywayWithAccountSetupEntered(engine);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(30),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Certificates_Failed",
|
||||
["errorCode"] = "0x87d1041c",
|
||||
["failedSubcategory"] = "Certificates",
|
||||
["category"] = "DeviceSetup",
|
||||
}));
|
||||
|
||||
var advisory = SingleTimelineEffect(step, "esp_failure_advisory");
|
||||
var payload = Assert.IsType<Dictionary<string, object>>(advisory.TypedPayload);
|
||||
|
||||
// Audit-trail anchors present.
|
||||
Assert.Equal("DecisionEngine", payload["decisionSource"]);
|
||||
Assert.Equal(nameof(DecisionSignalKind.EspTerminalFailure), payload["trigger"]);
|
||||
Assert.True(payload.ContainsKey("signalsSeen"));
|
||||
Assert.True(payload.ContainsKey("scenario"));
|
||||
|
||||
// Failure-context whitelist merged in (Codex review #18).
|
||||
Assert.Equal("Provisioning_DeviceSetup_Certificates_Failed", payload["failureType"]);
|
||||
Assert.Equal("0x87d1041c", payload["errorCode"]);
|
||||
Assert.Equal("Certificates", payload["failedSubcategory"]);
|
||||
Assert.Equal("DeviceSetup", payload["category"]);
|
||||
Assert.Equal("true", payload["espAllowContinueAnyway"]);
|
||||
Assert.Equal("true", payload["mayHaveContinuedAnyway"]);
|
||||
Assert.Equal("esp_failure_defanged_continueanyway_with_accountsetup", payload["advisoryReason"]);
|
||||
|
||||
// Emitter-metadata keys must NOT be in the typed payload (Codex review #18).
|
||||
Assert.False(payload.ContainsKey("eventType"));
|
||||
}
|
||||
|
||||
// =================================================== idempotency / duplicate ====
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_AfterAdvisory_PureDuplicateDeadEnds()
|
||||
{
|
||||
// Pure duplicate: ShellCoreTracker fires after ProvisioningStatusTracker already did
|
||||
// the rich emission. The ShellCoreTrackerAdapter (line 141) always includes
|
||||
// failureType in its payload but never the registry-specific keys (errorCode /
|
||||
// failedSubcategory / category) — those come exclusively from the registry path.
|
||||
// This test mirrors that production reality so the dead-end gate is exercised
|
||||
// against the actual adapter shape, not a strawman "reason-only" payload.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupContinueAnywayWithAccountSetupEntered(engine);
|
||||
|
||||
// First firing — full registry-derived payload.
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(30),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Certificates_Failed",
|
||||
["failedSubcategory"] = "Certificates",
|
||||
["category"] = "DeviceSetup",
|
||||
})).NewState;
|
||||
Assert.NotNull(state.EspAdvisoryFailureRecordedUtc);
|
||||
var firstAdvisoryUtc = state.EspAdvisoryFailureRecordedUtc!.Value;
|
||||
var preStage = state.Stage;
|
||||
var preStepIndex = state.StepIndex;
|
||||
|
||||
// Second firing — ShellCoreTracker shape: failureType only, no registry detail.
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
50, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(31),
|
||||
new Dictionary<string, string> { ["failureType"] = "ESPProgress_Timeout" }));
|
||||
|
||||
// Dead-end transition, no effects. failureType alone must NOT count as registry
|
||||
// enrichment — otherwise ShellCoreTracker would always trigger a redundant
|
||||
// follow-up advisory after a rich Provisioning advisory.
|
||||
Assert.Equal(preStage, step.NewState.Stage);
|
||||
Assert.Null(step.NewState.Outcome);
|
||||
Assert.Empty(step.Effects);
|
||||
Assert.False(step.Transition.Taken);
|
||||
Assert.Equal("esp_terminal_failure_advisory_already_recorded", step.Transition.DeadEndReason);
|
||||
|
||||
// Step bookkeeping advanced (signal counted as processed).
|
||||
Assert.Equal(preStepIndex + 1, step.NewState.StepIndex);
|
||||
Assert.Equal(50, step.NewState.LastAppliedSignalOrdinal);
|
||||
|
||||
// Original advisory fact unchanged — first record wins.
|
||||
Assert.Equal(firstAdvisoryUtc, step.NewState.EspAdvisoryFailureRecordedUtc!.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_AfterAdvisory_ReasonOnlySignalAlsoDeadEnds()
|
||||
{
|
||||
// Defensive coverage for the rarer case where a duplicate carries only a `reason`
|
||||
// string with no failure-context keys at all (e.g. a synthetic test signal or a
|
||||
// hypothetical future adapter that doesn't propagate failureType). The dead-end
|
||||
// path must still fire.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupContinueAnywayWithAccountSetupEntered(engine);
|
||||
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(30),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Certificates_Failed",
|
||||
["failedSubcategory"] = "Certificates",
|
||||
["category"] = "DeviceSetup",
|
||||
})).NewState;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
50, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(31),
|
||||
new Dictionary<string, string> { ["reason"] = "esp_terminal_failure_account_setup" }));
|
||||
|
||||
Assert.Empty(step.Effects);
|
||||
Assert.False(step.Transition.Taken);
|
||||
Assert.Equal("esp_terminal_failure_advisory_already_recorded", step.Transition.DeadEndReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_AfterAdvisory_EnrichmentSignalEmitsFollowupAdvisory()
|
||||
{
|
||||
// Real-world race: ShellCoreTracker fires first with sparse payload (just `reason`),
|
||||
// ProvisioningStatusTracker fires later with full registry-derived detail
|
||||
// (failureType/errorCode/failedSubcategory/category). The duplicate-guard must NOT
|
||||
// blindly drop the second signal — that would lose the HRESULT and subcategory the
|
||||
// UI and KQL filters rely on.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupContinueAnywayWithAccountSetupEntered(engine);
|
||||
|
||||
// First firing — sparse ShellCoreTracker payload.
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(30),
|
||||
new Dictionary<string, string> { ["reason"] = "esp_terminal_failure_account_setup" })).NewState;
|
||||
Assert.NotNull(state.EspAdvisoryFailureRecordedUtc);
|
||||
var firstAdvisoryUtc = state.EspAdvisoryFailureRecordedUtc!.Value;
|
||||
var preStage = state.Stage;
|
||||
|
||||
// Second firing — registry-derived enrichment.
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
50, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(31),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Certificates_Failed",
|
||||
["errorCode"] = "0x87d1041c",
|
||||
["failedSubcategory"] = "Certificates",
|
||||
["category"] = "DeviceSetup",
|
||||
}));
|
||||
|
||||
// Taken transition (NOT a dead-end), stage unchanged, advisory anchor untouched.
|
||||
Assert.Equal(preStage, step.NewState.Stage);
|
||||
Assert.Null(step.NewState.Outcome);
|
||||
Assert.True(step.Transition.Taken);
|
||||
Assert.Equal(firstAdvisoryUtc, step.NewState.EspAdvisoryFailureRecordedUtc!.Value);
|
||||
|
||||
// Follow-up advisory event carries the enrichment payload + distinct advisoryReason.
|
||||
var advisory = SingleTimelineEffect(step, "esp_failure_advisory");
|
||||
Assert.Equal("esp_failure_advisory_enriched_from_duplicate", advisory.Parameters!["advisoryReason"]);
|
||||
Assert.Equal("Warning", advisory.Parameters!["severity"]);
|
||||
Assert.Equal("Provisioning_DeviceSetup_Certificates_Failed", advisory.Parameters!["failureType"]);
|
||||
Assert.Equal("0x87d1041c", advisory.Parameters!["errorCode"]);
|
||||
Assert.Equal("Certificates", advisory.Parameters!["failedSubcategory"]);
|
||||
Assert.Equal("DeviceSetup", advisory.Parameters!["category"]);
|
||||
|
||||
// TypedPayload also carries the merged failure context (whitelist via Build helper).
|
||||
var payload = Assert.IsType<Dictionary<string, object>>(advisory.TypedPayload);
|
||||
Assert.Equal("0x87d1041c", payload["errorCode"]);
|
||||
Assert.Equal("esp_failure_advisory_enriched_from_duplicate", payload["advisoryReason"]);
|
||||
}
|
||||
|
||||
// ======================================================== negative pathways =====
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_WhenContinueAnywayTrue_ButNoAccountSetupEntered_StillFails()
|
||||
{
|
||||
// Subset of the SetupContinueAnyway… helper: only stage to DeviceSetup, never AccountSetup.
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("sess-no-as", "tenant-no-as");
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
10, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspAllowContinueAnyway] = "true" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState;
|
||||
|
||||
// Sanity: AccountSetup never entered.
|
||||
Assert.Null(state.AccountSetupEnteredUtc);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(15),
|
||||
new Dictionary<string, string> { ["failedSubcategory"] = "Certificates", ["category"] = "DeviceSetup" }));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome);
|
||||
Assert.Null(step.NewState.EspAdvisoryFailureRecordedUtc);
|
||||
|
||||
// Failed-path event emitted, NOT advisory.
|
||||
SingleTimelineEffect(step, "enrollment_failed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_WhenContinueAnywayFalse_AndAccountSetupEntered_StillFails()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupContinueAnywayWithAccountSetupEntered(engine, continueAnyway: false);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(30),
|
||||
new Dictionary<string, string> { ["failedSubcategory"] = "Certificates", ["category"] = "DeviceSetup" }));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome);
|
||||
Assert.Null(step.NewState.EspAdvisoryFailureRecordedUtc);
|
||||
|
||||
SingleTimelineEffect(step, "enrollment_failed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_WhenContinueAnywayUnknown_DefaultsToFailedPath()
|
||||
{
|
||||
// Race-condition coverage: bootstrap probe + DeviceInfoCollector both missed the
|
||||
// FirstSync read (very early boot, agent crashed and replayed signals from log).
|
||||
// EspAllowContinueAnyway is null → reducer defensively treats as "not advisory".
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("sess-unknown", "tenant-unknown");
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
30, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(15),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
// EspConfigDetected was never posted → EspAllowContinueAnyway stays null.
|
||||
Assert.Null(state.ScenarioObservations.EspAllowContinueAnyway);
|
||||
Assert.NotNull(state.AccountSetupEnteredUtc);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(30),
|
||||
new Dictionary<string, string> { ["failedSubcategory"] = "Certificates" }));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.Null(step.NewState.EspAdvisoryFailureRecordedUtc);
|
||||
SingleTimelineEffect(step, "enrollment_failed");
|
||||
}
|
||||
|
||||
// ============================================ typed-payload merge (failed path) ==
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_FailedPath_WithContinueAnywayHints_TypedPayloadContainsAllKeys()
|
||||
{
|
||||
// ContinueAnyway=true + AccountSetupEnteredUtc=null → Advisory-Guard does NOT trigger,
|
||||
// Failed path runs. But the ContinueAnyway hint keys ARE present on the parameter bag
|
||||
// and must be merged into the typed payload (Codex review #2 / #18 — the existing
|
||||
// enrollment_failed event lost these keys in wire-Data before this fix).
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("sess-merge", "tenant-merge");
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
10, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[SignalPayloadKeys.EspAllowContinueAnyway] = "true",
|
||||
[SignalPayloadKeys.EspSyncFailureTimeoutMinutes] = "30",
|
||||
})).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState;
|
||||
|
||||
Assert.Null(state.AccountSetupEnteredUtc);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(15),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Certificates_Failed",
|
||||
["errorCode"] = "0xdeadbeef",
|
||||
["failedSubcategory"] = "Certificates",
|
||||
["category"] = "DeviceSetup",
|
||||
}));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
var effect = SingleTimelineEffect(step, "enrollment_failed");
|
||||
var payload = Assert.IsType<Dictionary<string, object>>(effect.TypedPayload);
|
||||
|
||||
// Audit-trail anchors (DecisionAuditTrailBuilder output).
|
||||
Assert.Equal("DecisionEngine", payload["decisionSource"]);
|
||||
Assert.Equal(nameof(DecisionSignalKind.EspTerminalFailure), payload["trigger"]);
|
||||
|
||||
// Failure-context whitelist all present in typedPayload.
|
||||
Assert.Equal("Provisioning_DeviceSetup_Certificates_Failed", payload["failureType"]);
|
||||
Assert.Equal("0xdeadbeef", payload["errorCode"]);
|
||||
Assert.Equal("Certificates", payload["failedSubcategory"]);
|
||||
Assert.Equal("DeviceSetup", payload["category"]);
|
||||
Assert.Equal("30", payload["espSyncFailureTimeoutMinutes"]);
|
||||
Assert.Equal("true", payload["espAllowContinueAnyway"]);
|
||||
Assert.Equal("true", payload["mayHaveContinuedAnyway"]);
|
||||
Assert.True(payload.ContainsKey("continueAnywayHint"));
|
||||
|
||||
// No advisoryReason in Failed-path output.
|
||||
Assert.False(payload.ContainsKey("advisoryReason"));
|
||||
// Emitter-metadata not duplicated.
|
||||
Assert.False(payload.ContainsKey("eventType"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspTerminalFailure_FailedPath_WithoutContinueAnyway_TypedPayloadOmitsContinueAnywayKeys()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("sess-merge-neg", "tenant-merge-neg");
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
10, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[SignalPayloadKeys.EspAllowContinueAnyway] = "false",
|
||||
})).NewState;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(15),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Certificates_Failed",
|
||||
["errorCode"] = "0xdeadbeef",
|
||||
}));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
var effect = SingleTimelineEffect(step, "enrollment_failed");
|
||||
var payload = Assert.IsType<Dictionary<string, object>>(effect.TypedPayload);
|
||||
|
||||
// Failure-context present.
|
||||
Assert.Equal("Provisioning_DeviceSetup_Certificates_Failed", payload["failureType"]);
|
||||
Assert.Equal("0xdeadbeef", payload["errorCode"]);
|
||||
|
||||
// espAllowContinueAnyway=false is recorded but no hint/mayHave is set.
|
||||
Assert.Equal("false", payload["espAllowContinueAnyway"]);
|
||||
Assert.False(payload.ContainsKey("mayHaveContinuedAnyway"));
|
||||
Assert.False(payload.ContainsKey("continueAnywayHint"));
|
||||
Assert.False(payload.ContainsKey("advisoryReason"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -65,6 +65,46 @@ namespace AutopilotMonitor.DecisionCore.Tests.Scenarios
|
||||
Assert.Equal("EspTerminalFailure", result.FinalState.LastFailureTrigger!.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspFailureAdvisoryContinueAnyway_replays_RealSession4fa5a2d4Pattern_endsInEspAccountSetup_notFailed()
|
||||
{
|
||||
// PR1 (Session 4fa5a2d4, 2026-05-22) — ContinueAnyway-aware ESP terminal-failure
|
||||
// defang. The fixture replays the canonical race that produced 199/202 false
|
||||
// positives in tenant c9787ba2: ContinueAnyway=true profile, AccountSetup entered,
|
||||
// then DeviceSetup/Certificates registry-write flips to failed. Without PR1 this
|
||||
// would land in Failed; PR1 records EspAdvisoryFailureRecordedUtc and keeps the
|
||||
// stage at EspAccountSetup so the agent stays in monitoring.
|
||||
var signals = LoadFixture("esp-failure-advisory-continueanyway-v1.jsonl");
|
||||
var result = NewHarness().Replay("session-anon-advisory", "tenant-anon-advisory", signals);
|
||||
|
||||
// Critical: stage is NOT Failed — the advisory path preserved EspAccountSetup.
|
||||
Assert.NotEqual(SessionStage.Failed, result.FinalState.Stage);
|
||||
Assert.Equal(SessionStage.EspAccountSetup, result.FinalState.Stage);
|
||||
Assert.Null(result.FinalState.Outcome);
|
||||
|
||||
// The fire-once gate is set so the duplicate EspTerminalFailure from
|
||||
// ShellCoreTracker (signal #5 in the fixture) dead-ends without effect.
|
||||
Assert.NotNull(result.FinalState.EspAdvisoryFailureRecordedUtc);
|
||||
|
||||
// The LastFailureTrigger fact is NOT set on the advisory path — that fact is the
|
||||
// EnrollmentTerminationHandler's gate for promote-installing-apps-to-stuck and only
|
||||
// fires when the session genuinely terminates.
|
||||
Assert.Null(result.FinalState.LastFailureTrigger);
|
||||
|
||||
// Two EspTerminalFailure signals in the fixture, two transitions: first one
|
||||
// is the advisory (Taken=true, stage preserved); second one is the duplicate
|
||||
// dead-end (Taken=false, DeadEndReason set).
|
||||
var failureTransitions = new List<DecisionTransition>();
|
||||
foreach (var t in result.Transitions)
|
||||
{
|
||||
if (t.Trigger == "EspTerminalFailure") failureTransitions.Add(t);
|
||||
}
|
||||
Assert.Equal(2, failureTransitions.Count);
|
||||
Assert.True(failureTransitions[0].Taken);
|
||||
Assert.False(failureTransitions[1].Taken);
|
||||
Assert.Equal("esp_terminal_failure_advisory_already_recorded", failureTransitions[1].DeadEndReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RestartRecovery_replayTwice_producesIdenticalFinalStepHash()
|
||||
{
|
||||
|
||||
@@ -382,6 +382,61 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
Assert.Equal("v3", deserialized.SchemaVersion);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Deserialize_v4Snapshot_withoutEspAdvisoryFailureRecordedUtc_field_yieldsNullAnchor()
|
||||
{
|
||||
// Rollout-safety regression (PR1 ContinueAnyway-Defang, Session 4fa5a2d4, 2026-05-22):
|
||||
// EspAdvisoryFailureRecordedUtc is an additive nullable field appended to the end of
|
||||
// the DecisionState ctor (Codex review #9). Old persisted snapshots from v4 versions
|
||||
// before 2026-05-22 DO NOT contain this property. Loading such a snapshot under the
|
||||
// PR1+ code must succeed and produce EspAdvisoryFailureRecordedUtc == null —
|
||||
// otherwise sessions in flight at agent-upgrade time fail to rehydrate.
|
||||
const string priorV4Json = @"{
|
||||
""SessionId"": ""sess-rehydrate"",
|
||||
""TenantId"": ""tenant-rehydrate"",
|
||||
""Stage"": ""EspAccountSetup"",
|
||||
""Outcome"": null,
|
||||
""CurrentEnrollmentPhase"": null,
|
||||
""DeviceSetupEnteredUtc"": null,
|
||||
""AccountSetupEnteredUtc"": null,
|
||||
""FinalizingEnteredUtc"": null,
|
||||
""AccountSetupProvisioningSucceededUtc"": null,
|
||||
""EspFinalExitUtc"": null,
|
||||
""DesktopArrivedUtc"": null,
|
||||
""HelloResolvedUtc"": null,
|
||||
""SystemRebootUtc"": null,
|
||||
""HelloOutcome"": null,
|
||||
""ImeMatchedPatternId"": null,
|
||||
""Deadlines"": [],
|
||||
""LastAppliedSignalOrdinal"": 42,
|
||||
""StepIndex"": 100,
|
||||
""AppInstallFacts"": null,
|
||||
""ScenarioProfile"": null,
|
||||
""ScenarioObservations"": null,
|
||||
""ClassifierOutcomes"": null,
|
||||
""HelloPolicyEnabled"": null,
|
||||
""AgentBootUtc"": ""2026-05-22T08:00:00Z"",
|
||||
""LastFailureTrigger"": null,
|
||||
""RealmJoinFacts"": null,
|
||||
""DeviceSetupResolvedUtc"": null,
|
||||
""SchemaVersion"": ""v4""
|
||||
}";
|
||||
|
||||
var deserialized = StateSerializer.Deserialize(priorV4Json);
|
||||
|
||||
// Critical: the missing EspAdvisoryFailureRecordedUtc field deserializes to null
|
||||
// (the optional ctor parameter default). No ArgumentException, no JsonSerialization-
|
||||
// Exception. The next EspTerminalFailure signal can then write the fact normally.
|
||||
Assert.Null(deserialized.EspAdvisoryFailureRecordedUtc);
|
||||
|
||||
// Other state survived.
|
||||
Assert.Equal("sess-rehydrate", deserialized.SessionId);
|
||||
Assert.Equal(SessionStage.EspAccountSetup, deserialized.Stage);
|
||||
Assert.Equal(42L, deserialized.LastAppliedSignalOrdinal);
|
||||
Assert.Equal(100, deserialized.StepIndex);
|
||||
Assert.Equal("v4", deserialized.SchemaVersion);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScenarioProfile_stateRoundtripsThroughStateSerializer_enumsAsStrings()
|
||||
{
|
||||
|
||||
@@ -42,10 +42,49 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handle <see cref="DecisionSignalKind.EspTerminalFailure"/>. Directly transitions
|
||||
/// to <see cref="SessionStage.Failed"/> with <see cref="SessionOutcome.EnrollmentFailed"/>,
|
||||
/// clears deadlines, and emits <c>enrollment_failed</c>. No classifier re-run —
|
||||
/// ESP terminal failure is definitive.
|
||||
/// Whitelist of <c>parameters</c>-keys that are merged into the audit-trail typed payload
|
||||
/// for ESP-failure-related timeline events (PR1, Session 4fa5a2d4). Emitter-metadata
|
||||
/// keys like <c>eventType</c>, <c>severity</c>, <c>source</c>, <c>phase</c> are
|
||||
/// deliberately NOT in the whitelist — they belong on the event header, not duplicated
|
||||
/// into wire-data. Codex review finding #18.
|
||||
/// </summary>
|
||||
private static readonly string[] FailureContextKeysWhitelist = new[]
|
||||
{
|
||||
"failureType",
|
||||
"errorCode",
|
||||
"failedSubcategory",
|
||||
"category",
|
||||
"espSyncFailureTimeoutMinutes",
|
||||
"espAllowContinueAnyway",
|
||||
"mayHaveContinuedAnyway",
|
||||
"continueAnywayHint",
|
||||
"advisoryReason",
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Handle <see cref="DecisionSignalKind.EspTerminalFailure"/>. Two paths:
|
||||
/// <list type="bullet">
|
||||
/// <item>
|
||||
/// <b>Advisory defang</b> — when the ESP profile permits "Continue anyway"
|
||||
/// (<c>ScenarioObservations.EspAllowContinueAnyway == true</c>) AND
|
||||
/// <see cref="DecisionState.AccountSetupEnteredUtc"/> is already set, the device has
|
||||
/// demonstrably progressed past DeviceSetup. The signal is downgraded to an
|
||||
/// <c>esp_failure_advisory</c> timeline entry, the stage stays unchanged, deadlines
|
||||
/// remain armed, and normal completion paths (Hello/Desktop, IME pattern,
|
||||
/// AccountSetupProvisioningComplete) continue to drive the session. Session 4fa5a2d4
|
||||
/// (2026-05-22) — defangs the 199/202 false-positive failures observed in tenant
|
||||
/// c9787ba2 where ContinueAnyway was on and the device kept progressing.
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// <b>Terminal failure</b> — default path. Transitions to <see cref="SessionStage.Failed"/>
|
||||
/// with <see cref="SessionOutcome.EnrollmentFailed"/>, clears deadlines, and emits
|
||||
/// <c>enrollment_failed</c>. No classifier re-run — ESP terminal failure is definitive.
|
||||
/// </item>
|
||||
/// </list>
|
||||
/// Duplicates: once <c>EspAdvisoryFailureRecordedUtc</c> is set, subsequent
|
||||
/// <c>EspTerminalFailure</c> signals (e.g. from <c>ShellCoreTracker</c> in addition to
|
||||
/// <c>ProvisioningStatusTracker</c>) dead-end without effects so the timeline does not
|
||||
/// accumulate redundant advisories.
|
||||
/// <para>
|
||||
/// The terminal-event reason is always the stable enum literal <c>esp_terminal_failure</c>.
|
||||
/// Session 9d052230: registry-derived failures additionally carry <c>failureType</c>,
|
||||
@@ -56,6 +95,42 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
/// </summary>
|
||||
private DecisionStep HandleEspTerminalFailureV1(DecisionState state, DecisionSignal signal)
|
||||
{
|
||||
// After an advisory was recorded: a duplicate EspTerminalFailure from another tracker
|
||||
// source may either be a true duplicate (same context, no new info) OR an enrichment
|
||||
// (e.g. ShellCoreTracker fired first with sparse payload, then ProvisioningStatusTracker
|
||||
// fires later with the registry-derived errorCode/failedSubcategory/category). The two
|
||||
// sources can fire in either order (eventlog and registry race). We don't want to
|
||||
// drop the richer payload if it arrives second — that would hide the HRESULT and
|
||||
// subcategory the UI/KQL need.
|
||||
//
|
||||
// Gate the enrichment on registry-specific keys ONLY (errorCode/failedSubcategory/
|
||||
// category). failureType is posted by BOTH ShellCoreTracker and ProvisioningStatusTracker
|
||||
// (their adapters always include it), so a ShellCoreTracker signal arriving AFTER a
|
||||
// rich Provisioning advisory would otherwise be misclassified as enrichment and emit
|
||||
// a redundant follow-up event.
|
||||
if (state.EspAdvisoryFailureRecordedUtc != null)
|
||||
{
|
||||
var carriesRegistryDetail = signal.Payload != null && (
|
||||
signal.Payload.ContainsKey("errorCode")
|
||||
|| signal.Payload.ContainsKey("failedSubcategory")
|
||||
|| signal.Payload.ContainsKey("category"));
|
||||
|
||||
if (carriesRegistryDetail)
|
||||
{
|
||||
return BuildAdvisoryEnrichmentStep(state, signal);
|
||||
}
|
||||
|
||||
var dupNextStep = state.StepIndex + 1;
|
||||
var dupState = BumpStepBookkeeping(state, signal);
|
||||
var dupTransition = BuildDeadEndTransition(
|
||||
state: state,
|
||||
signal: signal,
|
||||
nextStepIndex: dupNextStep,
|
||||
trigger: nameof(DecisionSignalKind.EspTerminalFailure),
|
||||
deadEndReason: "esp_terminal_failure_advisory_already_recorded");
|
||||
return new DecisionStep(dupState, dupTransition, Array.Empty<DecisionEffect>());
|
||||
}
|
||||
|
||||
var nextStep = state.StepIndex + 1;
|
||||
|
||||
// Default to the stable enum literal so the timeline-event "reason" is predictable
|
||||
@@ -67,6 +142,126 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
? r
|
||||
: "esp_terminal_failure";
|
||||
|
||||
// Build parameter bag once — both paths consume it via BuildEspFailureAuditTrail.
|
||||
var observations = state.ScenarioObservations;
|
||||
var continueAnywayEnabled = observations?.EspAllowContinueAnyway?.Value == true;
|
||||
var advisoryEligible = continueAnywayEnabled && state.AccountSetupEnteredUtc != null;
|
||||
var parameters = BuildEspFailureParameters(
|
||||
signal: signal,
|
||||
reason: reason,
|
||||
advisoryPath: advisoryEligible,
|
||||
observations: observations);
|
||||
|
||||
if (advisoryEligible)
|
||||
{
|
||||
return BuildAdvisoryStep(state, signal, nextStep, reason, parameters);
|
||||
}
|
||||
|
||||
return BuildFailedStep(state, signal, nextStep, reason, parameters);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Build the advisory-enrichment step that runs when an EspTerminalFailure duplicate
|
||||
/// arrives AFTER an advisory was already recorded AND the duplicate carries new
|
||||
/// failure-context (failureType/errorCode/failedSubcategory/category). Stage and the
|
||||
/// EspAdvisoryFailureRecordedUtc-fact stay unchanged — the first signal's anchor wins.
|
||||
/// A second <c>esp_failure_advisory</c> event is emitted carrying the enriched payload
|
||||
/// so the timeline / UI gain the registry-derived HRESULT and subcategory detail.
|
||||
/// </summary>
|
||||
private DecisionStep BuildAdvisoryEnrichmentStep(DecisionState state, DecisionSignal signal)
|
||||
{
|
||||
var nextStep = state.StepIndex + 1;
|
||||
var reason = signal.Payload != null
|
||||
&& signal.Payload.TryGetValue("reason", out var r)
|
||||
&& !string.IsNullOrEmpty(r)
|
||||
? r
|
||||
: "esp_terminal_failure";
|
||||
|
||||
var parameters = BuildEspFailureParameters(
|
||||
signal: signal,
|
||||
reason: reason,
|
||||
advisoryPath: true,
|
||||
observations: state.ScenarioObservations);
|
||||
|
||||
// Override advisoryReason so the timeline can distinguish "first advisory at gate-set"
|
||||
// from "later enrichment of the same advisory". Both events share eventType so the
|
||||
// UI render path is identical.
|
||||
parameters["advisoryReason"] = "esp_failure_advisory_enriched_from_duplicate";
|
||||
|
||||
// Bookkeeping-only state update — the advisory-anchor (EspAdvisoryFailureRecordedUtc)
|
||||
// intentionally stays at the first signal's ordinal/UTC.
|
||||
var newState = BumpStepBookkeeping(state, signal);
|
||||
|
||||
var transition = BuildTakenTransition(
|
||||
before: state,
|
||||
signal: signal,
|
||||
toStage: state.Stage,
|
||||
nextStepIndex: nextStep,
|
||||
trigger: nameof(DecisionSignalKind.EspTerminalFailure));
|
||||
|
||||
var effects = new[]
|
||||
{
|
||||
new DecisionEffect(
|
||||
DecisionEffectKind.EmitEventTimelineEntry,
|
||||
parameters: parameters,
|
||||
typedPayload: BuildEspFailureAuditTrail(
|
||||
postState: newState,
|
||||
decidedStage: state.Stage,
|
||||
trigger: nameof(DecisionSignalKind.EspTerminalFailure),
|
||||
failureReason: reason,
|
||||
parameters: parameters)),
|
||||
};
|
||||
|
||||
return new DecisionStep(newState, transition, effects);
|
||||
}
|
||||
|
||||
private DecisionStep BuildAdvisoryStep(
|
||||
DecisionState state,
|
||||
DecisionSignal signal,
|
||||
int nextStep,
|
||||
string reason,
|
||||
Dictionary<string, string> parameters)
|
||||
{
|
||||
var builder = state.ToBuilder()
|
||||
.WithStepIndex(nextStep)
|
||||
.WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal)
|
||||
.WithEspAdvisoryFailureRecorded(signal.OccurredAtUtc, signal.SessionSignalOrdinal);
|
||||
|
||||
// Stage / Outcome / Deadlines deliberately untouched — monitoring continues. The
|
||||
// session's normal completion paths (Hello/Desktop, IME pattern, AccountSetup
|
||||
// provisioning complete) still drive it.
|
||||
var newState = builder.Build();
|
||||
|
||||
var transition = BuildTakenTransition(
|
||||
before: state,
|
||||
signal: signal,
|
||||
toStage: state.Stage,
|
||||
nextStepIndex: nextStep,
|
||||
trigger: nameof(DecisionSignalKind.EspTerminalFailure));
|
||||
|
||||
var effects = new[]
|
||||
{
|
||||
new DecisionEffect(
|
||||
DecisionEffectKind.EmitEventTimelineEntry,
|
||||
parameters: parameters,
|
||||
typedPayload: BuildEspFailureAuditTrail(
|
||||
postState: newState,
|
||||
decidedStage: state.Stage,
|
||||
trigger: nameof(DecisionSignalKind.EspTerminalFailure),
|
||||
failureReason: reason,
|
||||
parameters: parameters)),
|
||||
};
|
||||
|
||||
return new DecisionStep(newState, transition, effects);
|
||||
}
|
||||
|
||||
private DecisionStep BuildFailedStep(
|
||||
DecisionState state,
|
||||
DecisionSignal signal,
|
||||
int nextStep,
|
||||
string reason,
|
||||
Dictionary<string, string> parameters)
|
||||
{
|
||||
var builder = state.ToBuilder()
|
||||
.WithStage(SessionStage.Failed)
|
||||
.WithOutcome(SessionOutcome.EnrollmentFailed)
|
||||
@@ -84,14 +279,58 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
nextStepIndex: nextStep,
|
||||
trigger: nameof(DecisionSignalKind.EspTerminalFailure));
|
||||
|
||||
var effects = new[]
|
||||
{
|
||||
new DecisionEffect(
|
||||
DecisionEffectKind.EmitEventTimelineEntry,
|
||||
parameters: parameters,
|
||||
typedPayload: BuildEspFailureAuditTrail(
|
||||
postState: newState,
|
||||
decidedStage: SessionStage.Failed,
|
||||
trigger: nameof(DecisionSignalKind.EspTerminalFailure),
|
||||
failureReason: reason,
|
||||
parameters: parameters)),
|
||||
};
|
||||
|
||||
return new DecisionStep(newState, transition, effects);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Build the timeline-event parameter bag for the advisory and failed paths. The
|
||||
/// <c>eventType</c> distinguishes the two; failure-context keys
|
||||
/// (failureType/errorCode/failedSubcategory/category) carry the registry-derived detail
|
||||
/// from the originating ESP signal so the UI / KQL can render the HRESULT badge without
|
||||
/// re-parsing the ESP statusText. ContinueAnyway hint keys are only added when the
|
||||
/// ScenarioObservations actually carry an <c>EspAllowContinueAnyway</c> fact — absence
|
||||
/// of those keys distinguishes "ContinueAnyway unknown" from "ContinueAnyway explicitly
|
||||
/// false" in the wire payload.
|
||||
/// </summary>
|
||||
private static Dictionary<string, string> BuildEspFailureParameters(
|
||||
DecisionSignal signal,
|
||||
string reason,
|
||||
bool advisoryPath,
|
||||
State.EnrollmentScenarioObservations? observations)
|
||||
{
|
||||
var parameters = new Dictionary<string, string>
|
||||
{
|
||||
["eventType"] = "enrollment_failed",
|
||||
["eventType"] = advisoryPath
|
||||
? SharedConstants.EventTypes.EspFailureAdvisory
|
||||
: "enrollment_failed",
|
||||
["reason"] = reason,
|
||||
};
|
||||
|
||||
// Session 9d052230 — surface registry-derived ESP failure detail (HRESULT etc) on the
|
||||
// terminal event so the UI can render it without parsing nested ESP statusText.
|
||||
if (advisoryPath)
|
||||
{
|
||||
// Advisory is a non-terminal hint: ESP reported a failure but the device kept
|
||||
// progressing past it. Warning matches the UI's amber badge and the semantic
|
||||
// weight ("something worth knowing but not session-fatal"). The Failed path
|
||||
// relies on the emitter's DeriveSeverity("_failed" → Error) default.
|
||||
parameters["severity"] = "Warning";
|
||||
parameters["advisoryReason"] = "esp_failure_defanged_continueanyway_with_accountsetup";
|
||||
}
|
||||
|
||||
// Session 9d052230 — surface registry-derived ESP failure detail (HRESULT etc) on
|
||||
// the terminal event so the UI can render it without parsing nested ESP statusText.
|
||||
if (signal.Payload != null)
|
||||
{
|
||||
if (signal.Payload.TryGetValue("failureType", out var failureType) && !string.IsNullOrEmpty(failureType))
|
||||
@@ -104,12 +343,8 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
parameters["category"] = category;
|
||||
}
|
||||
|
||||
// Enrich the audit payload with the FirstSync-derived ESP failure-handling settings,
|
||||
// so the timeline carries the observable fact that Microsoft's ESP profile may have
|
||||
// offered the user a "Continue anyway" button — the agent's terminal-failure verdict
|
||||
// does not preclude the user actually reaching the desktop. This is hint-only data;
|
||||
// the reducer's state machine has already committed to Failed.
|
||||
var observations = newState.ScenarioObservations;
|
||||
// FirstSync-derived ESP failure-handling settings — observable hint that
|
||||
// Microsoft's ESP profile may have offered the user a "Continue anyway" button.
|
||||
if (observations?.EspSyncFailureTimeoutMinutes != null)
|
||||
{
|
||||
parameters["espSyncFailureTimeoutMinutes"] = observations.EspSyncFailureTimeoutMinutes.Value
|
||||
@@ -123,24 +358,45 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
if (observations.EspAllowContinueAnyway.Value)
|
||||
{
|
||||
parameters["mayHaveContinuedAnyway"] = "true";
|
||||
parameters["continueAnywayHint"] =
|
||||
"ESP profile allows 'Continue anyway' — the user may have dismissed the failure screen and reached the desktop; this monitor only sees the ESP terminal failure on the agent side.";
|
||||
parameters["continueAnywayHint"] = advisoryPath
|
||||
? "ESP reported a subcategory failure but the device had already progressed to AccountSetup; the agent continues monitoring instead of declaring the session failed."
|
||||
: "ESP profile allows 'Continue anyway' — the user may have dismissed the failure screen and reached the desktop; this monitor only sees the ESP terminal failure on the agent side.";
|
||||
}
|
||||
}
|
||||
|
||||
var effects = new[]
|
||||
{
|
||||
new DecisionEffect(
|
||||
DecisionEffectKind.EmitEventTimelineEntry,
|
||||
parameters: parameters,
|
||||
typedPayload: DecisionAuditTrailBuilder.Build(
|
||||
postState: newState,
|
||||
decidedStage: SessionStage.Failed,
|
||||
trigger: nameof(DecisionSignalKind.EspTerminalFailure),
|
||||
failureReason: reason)),
|
||||
};
|
||||
return parameters;
|
||||
}
|
||||
|
||||
return new DecisionStep(newState, transition, effects);
|
||||
/// <summary>
|
||||
/// Builds the audit-trail typed payload for ESP-failure timeline events by combining
|
||||
/// <see cref="DecisionAuditTrailBuilder.Build"/> with a whitelisted merge of
|
||||
/// failure-context keys from <paramref name="parameters"/>. The base builder owns
|
||||
/// scenario/signal-census/trigger fields; this helper layers in the registry-derived
|
||||
/// detail (failureType/errorCode/failedSubcategory/category) plus the conditional
|
||||
/// ContinueAnyway hint keys. Emitter-metadata keys (<c>eventType</c>, <c>severity</c>,
|
||||
/// etc.) are NOT merged — they belong on the event header. Codex review finding #18.
|
||||
/// </summary>
|
||||
private static Dictionary<string, object> BuildEspFailureAuditTrail(
|
||||
DecisionState postState,
|
||||
SessionStage decidedStage,
|
||||
string trigger,
|
||||
string failureReason,
|
||||
IReadOnlyDictionary<string, string> parameters)
|
||||
{
|
||||
var trail = DecisionAuditTrailBuilder.Build(
|
||||
postState: postState,
|
||||
decidedStage: decidedStage,
|
||||
trigger: trigger,
|
||||
failureReason: failureReason);
|
||||
for (var i = 0; i < FailureContextKeysWhitelist.Length; i++)
|
||||
{
|
||||
var key = FailureContextKeysWhitelist[i];
|
||||
if (parameters.TryGetValue(key, out var value) && !string.IsNullOrEmpty(value))
|
||||
{
|
||||
trail[key] = value;
|
||||
}
|
||||
}
|
||||
return trail;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -62,6 +62,7 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
["imeMatchedPatternId"] = SerializeFact(state.ImeMatchedPatternId, v => v),
|
||||
["helloPolicyEnabled"] = SerializeFact(state.HelloPolicyEnabled, v => (object)v),
|
||||
["lastFailureTrigger"] = SerializeFact(state.LastFailureTrigger, v => v),
|
||||
["espAdvisoryFailureRecordedUtc"] = SerializeFact(state.EspAdvisoryFailureRecordedUtc, FormatUtc),
|
||||
};
|
||||
|
||||
var scenario = new Dictionary<string, object?>(StringComparer.Ordinal)
|
||||
|
||||
@@ -60,7 +60,8 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
SignalFact<string>? lastFailureTrigger = null,
|
||||
RealmJoinFacts? realmJoinFacts = null,
|
||||
SignalFact<DateTime>? deviceSetupResolvedUtc = null,
|
||||
string? schemaVersion = null)
|
||||
string? schemaVersion = null,
|
||||
SignalFact<DateTime>? espAdvisoryFailureRecordedUtc = null)
|
||||
{
|
||||
if (string.IsNullOrEmpty(sessionId))
|
||||
{
|
||||
@@ -100,6 +101,7 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
RealmJoinFacts = realmJoinFacts ?? RealmJoinFacts.Empty;
|
||||
DeviceSetupResolvedUtc = deviceSetupResolvedUtc;
|
||||
SchemaVersion = schemaVersion ?? CurrentSchemaVersion;
|
||||
EspAdvisoryFailureRecordedUtc = espAdvisoryFailureRecordedUtc;
|
||||
}
|
||||
|
||||
public string SessionId { get; }
|
||||
@@ -249,6 +251,23 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
/// </summary>
|
||||
public SignalFact<DateTime>? DeviceSetupResolvedUtc { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Set when <c>HandleEspTerminalFailureV1</c> downgrades an incoming
|
||||
/// <see cref="Signals.DecisionSignalKind.EspTerminalFailure"/> signal to an advisory
|
||||
/// instead of transitioning to <see cref="SessionStage.Failed"/>. The downgrade applies
|
||||
/// when the ESP profile permits "Continue anyway" (<c>ScenarioObservations.EspAllowContinueAnyway</c>
|
||||
/// is <c>true</c>) AND <see cref="AccountSetupEnteredUtc"/> is already set — both facts
|
||||
/// together prove that the device has already progressed past DeviceSetup despite the
|
||||
/// reported subcategory failure, so the agent stays in monitoring instead of declaring
|
||||
/// the session failed.
|
||||
/// <para>
|
||||
/// Acts as a fire-once idempotency gate: subsequent <c>EspTerminalFailure</c> signals
|
||||
/// (e.g. duplicates from <c>ShellCoreTracker</c>) are dropped as dead-ends so the
|
||||
/// timeline does not accumulate redundant <c>esp_failure_advisory</c> events.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public SignalFact<DateTime>? EspAdvisoryFailureRecordedUtc { get; }
|
||||
|
||||
public string SchemaVersion { get; }
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -47,6 +47,7 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
RealmJoinFacts = source.RealmJoinFacts;
|
||||
DeviceSetupResolvedUtc = source.DeviceSetupResolvedUtc;
|
||||
SchemaVersion = source.SchemaVersion;
|
||||
EspAdvisoryFailureRecordedUtc = source.EspAdvisoryFailureRecordedUtc;
|
||||
}
|
||||
|
||||
public string SessionId { get; set; }
|
||||
@@ -76,6 +77,7 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
public SignalFact<string>? LastFailureTrigger { get; set; }
|
||||
public RealmJoinFacts RealmJoinFacts { get; set; } = RealmJoinFacts.Empty;
|
||||
public SignalFact<DateTime>? DeviceSetupResolvedUtc { get; set; }
|
||||
public SignalFact<DateTime>? EspAdvisoryFailureRecordedUtc { get; set; }
|
||||
public string SchemaVersion { get; set; }
|
||||
|
||||
// ---------- fluent helpers for the most common reducer operations ----------
|
||||
@@ -183,6 +185,18 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Record the UTC instant at which an incoming <c>EspTerminalFailure</c> was downgraded
|
||||
/// to an advisory by <c>HandleEspTerminalFailureV1</c> (ContinueAnyway-aware defang).
|
||||
/// Fire-once gate: subsequent <c>EspTerminalFailure</c> signals see this fact set and
|
||||
/// dead-end without further effect.
|
||||
/// </summary>
|
||||
public DecisionStateBuilder WithEspAdvisoryFailureRecorded(DateTime utc, long sourceSignalOrdinal)
|
||||
{
|
||||
EspAdvisoryFailureRecordedUtc = new SignalFact<DateTime>(utc, sourceSignalOrdinal);
|
||||
return this;
|
||||
}
|
||||
|
||||
public DecisionState Build() =>
|
||||
new DecisionState(
|
||||
sessionId: SessionId,
|
||||
@@ -212,6 +226,7 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
lastFailureTrigger: LastFailureTrigger,
|
||||
realmJoinFacts: RealmJoinFacts,
|
||||
deviceSetupResolvedUtc: DeviceSetupResolvedUtc,
|
||||
schemaVersion: SchemaVersion);
|
||||
schemaVersion: SchemaVersion,
|
||||
espAdvisoryFailureRecordedUtc: EspAdvisoryFailureRecordedUtc);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,6 +245,11 @@ namespace AutopilotMonitor.Shared
|
||||
public const string WhiteGloveComplete = "whiteglove_complete";
|
||||
public const string EnrollmentComplete = "enrollment_complete";
|
||||
public const string EnrollmentFailed = "enrollment_failed";
|
||||
// PR1 — ContinueAnyway-aware ESP terminal failure defang (Session 4fa5a2d4, 2026-05-22).
|
||||
// Emitted instead of enrollment_failed when the ESP profile permits ContinueAnyway
|
||||
// AND AccountSetup has already been entered — proves the device progressed past
|
||||
// DeviceSetup so the agent stays in monitoring instead of terminating.
|
||||
public const string EspFailureAdvisory = "esp_failure_advisory";
|
||||
public const string DesktopArrived = "desktop_arrived";
|
||||
// DAD liveness (state-change-only, max 3 events per detector lifetime, NOT periodic).
|
||||
// Distinguishes "DAD never started after reboot" vs "DAD started but timer/WMI dead" vs
|
||||
|
||||
@@ -66,6 +66,8 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [
|
||||
description: "ESP category status update from Windows Provisioning registry." },
|
||||
{ value: "esp_state_change", label: "esp_state_change", category: "esp",
|
||||
description: "Generic ESP state change event." },
|
||||
{ value: "esp_failure_advisory", label: "esp_failure_advisory", category: "esp",
|
||||
description: "ESP reported a subcategory failure but the device had already progressed to AccountSetup with ContinueAnyway enabled — non-terminal advisory, the agent continues monitoring." },
|
||||
|
||||
// -------- App installs --------
|
||||
{ value: "app_install_started", label: "app_install_started", category: "app",
|
||||
|
||||
+27
-11
@@ -479,22 +479,38 @@ function EventRow({ event, showScriptOutput }: { event: EnrollmentEvent; showScr
|
||||
);
|
||||
})()}
|
||||
{/* HRESULT badge for ESP failures (enrollment_failed via esp_terminal_failure,
|
||||
esp_provisioning_status failed-subcategory). The HRESULT is extracted from the ESP
|
||||
registry statusText by the agent (e.g. "Apps (0x87d1041c)") and surfaced as
|
||||
top-level event data so the UI can render it without parsing nested text. */}
|
||||
{(event.eventType === "enrollment_failed" || event.eventType === "esp_provisioning_status") && (() => {
|
||||
esp_provisioning_status failed-subcategory, esp_failure_advisory via ContinueAnyway
|
||||
defang). The HRESULT is extracted from the ESP registry statusText by the agent
|
||||
(e.g. "Apps (0x87d1041c)") and surfaced as top-level event data so the UI can
|
||||
render it without parsing nested text. */}
|
||||
{(event.eventType === "enrollment_failed" || event.eventType === "esp_provisioning_status" || event.eventType === "esp_failure_advisory") && (() => {
|
||||
const code = event.data?.errorCode ?? event.data?.failedSubcategoryErrorCode;
|
||||
if (!code) return null;
|
||||
const codeStr = String(code);
|
||||
const entry = getErrorCodeEntry(codeStr);
|
||||
const sub = event.data?.failedSubcategory ?? event.data?.failedSubcategories;
|
||||
const isAdvisory = event.eventType === "esp_failure_advisory";
|
||||
if (!code && !isAdvisory) return null;
|
||||
const codeStr = code ? String(code) : null;
|
||||
const entry = codeStr ? getErrorCodeEntry(codeStr) : null;
|
||||
// Advisory path uses warning-color palette (amber); the device continued past the
|
||||
// failure via ContinueAnyway, so this is not a hard error. PR1 Session 4fa5a2d4.
|
||||
const badgeBg = isAdvisory ? "bg-amber-100 text-amber-800" : "bg-red-100 text-red-800";
|
||||
const descColor = isAdvisory ? "text-amber-700" : "text-red-600";
|
||||
return (
|
||||
<div className="mt-1 flex flex-wrap items-center gap-2 text-xs">
|
||||
<span className="px-1.5 py-0.5 rounded bg-red-100 text-red-800 font-mono font-medium">
|
||||
HRESULT: {formatErrorCode(codeStr)}
|
||||
</span>
|
||||
{isAdvisory && (
|
||||
<span
|
||||
className="px-1.5 py-0.5 rounded bg-amber-100 text-amber-800 font-medium"
|
||||
title="ESP reported a subcategory failure but the device had already progressed to AccountSetup — the agent continues monitoring instead of declaring the session failed."
|
||||
>
|
||||
Advisory (ContinueAnyway)
|
||||
</span>
|
||||
)}
|
||||
{codeStr && (
|
||||
<span className={`px-1.5 py-0.5 rounded ${badgeBg} font-mono font-medium`}>
|
||||
HRESULT: {formatErrorCode(codeStr)}
|
||||
</span>
|
||||
)}
|
||||
{entry && (
|
||||
<span className="text-red-600" title={`${entry.source} (${entry.confidence} confidence)`}>
|
||||
<span className={descColor} title={`${entry.source} (${entry.confidence} confidence)`}>
|
||||
{entry.description}
|
||||
</span>
|
||||
)}
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
# Scenario: ESP Terminal Failure DEFANGED by ContinueAnyway (PR1, Session 4fa5a2d4, 2026-05-22)
|
||||
# Tenant c9787ba2 observed 199/202 sessions failing with esp_terminal_failure, even though
|
||||
# customers were demonstrably deploying 1200+ devices through the same profile. Real root cause:
|
||||
# - ESP profile sets BlockInStatusPage=6 (ContinueAnyway=true)
|
||||
# - DeviceSetup completes Apps successfully
|
||||
# - Windows transitions to AccountSetup (esp_phase_changed)
|
||||
# - Then Windows writes DeviceSetup.Certificates=failed AFTER AccountSetup has already started
|
||||
# - V2 reducer fired EspTerminalFailure → Failed, even though the device kept progressing
|
||||
#
|
||||
# PR1 defang: when ContinueAnyway=true AND AccountSetupEnteredUtc is set, the reducer downgrades
|
||||
# the EspTerminalFailure to an esp_failure_advisory effect; stage stays at EspAccountSetup,
|
||||
# EspAdvisoryFailureRecordedUtc is set, monitoring continues. The session never reaches Failed
|
||||
# in this replay (it would normally complete via Desktop+Hello in production — not modeled here).
|
||||
#
|
||||
# Coverage: validates that the engine produces the advisory effect end-to-end through the same
|
||||
# signal stream the real session 4fa5a2d4 saw.
|
||||
{"SessionSignalOrdinal":0,"SessionTraceOrdinal":0,"Kind":"SessionStarted","KindSchemaVersion":1,"OccurredAtUtc":"2026-05-22T09:00:00Z","SourceOrigin":"orchestrator","Evidence":{"Kind":"Synthetic","Identifier":"session:advisory:started","Summary":"Session start anchor"}}
|
||||
{"SessionSignalOrdinal":1,"SessionTraceOrdinal":1,"Kind":"EspConfigDetected","KindSchemaVersion":1,"OccurredAtUtc":"2026-05-22T09:00:30Z","SourceOrigin":"EnrollmentOrchestrator","Evidence":{"Kind":"Raw","Identifier":"esp_config_detected_bootstrap","Summary":"SkipUser=False, SkipDevice=False, AllowContinueAnyway=True","DerivationInputs":{"source":"registry_firstsync_bootstrap"}},"Payload":{"skipUserEsp":"false","skipDeviceEsp":"false","espAllowContinueAnyway":"true","espSyncFailureTimeoutMinutes":"30"}}
|
||||
{"SessionSignalOrdinal":2,"SessionTraceOrdinal":2,"Kind":"EspPhaseChanged","KindSchemaVersion":1,"OccurredAtUtc":"2026-05-22T09:01:00Z","SourceOrigin":"EspAndHelloTracker","Evidence":{"Kind":"Derived","Identifier":"esp-phase-detector-v1","Summary":"ESP phase=DeviceSetup","DerivationInputs":{"rawValue":"1"}},"Payload":{"phase":"DeviceSetup"}}
|
||||
{"SessionSignalOrdinal":3,"SessionTraceOrdinal":3,"Kind":"EspPhaseChanged","KindSchemaVersion":1,"OccurredAtUtc":"2026-05-22T09:16:00Z","SourceOrigin":"EspAndHelloTracker","Evidence":{"Kind":"Derived","Identifier":"esp-phase-detector-v1","Summary":"ESP phase=AccountSetup","DerivationInputs":{"rawValue":"2"}},"Payload":{"phase":"AccountSetup"}}
|
||||
{"SessionSignalOrdinal":4,"SessionTraceOrdinal":4,"Kind":"EspTerminalFailure","KindSchemaVersion":1,"OccurredAtUtc":"2026-05-22T09:29:52Z","SourceOrigin":"ProvisioningStatusTracker","Evidence":{"Kind":"Derived","Identifier":"provisioning-status-tracker-v1","Summary":"ESP terminal failure from provisioning registry (type=Provisioning_DeviceSetup_Certificates_Failed, errorCode=n/a)","DerivationInputs":{"registrySource":"HKLM\\SOFTWARE\\Microsoft\\Provisioning\\AutopilotSettings\\*.Status","failureType":"Provisioning_DeviceSetup_Certificates_Failed","failedSubcategory":"Certificates","category":"DeviceSetup"}},"Payload":{"failureType":"Provisioning_DeviceSetup_Certificates_Failed","failedSubcategory":"Certificates","category":"DeviceSetup"}}
|
||||
{"SessionSignalOrdinal":5,"SessionTraceOrdinal":5,"Kind":"EspTerminalFailure","KindSchemaVersion":1,"OccurredAtUtc":"2026-05-22T09:30:15Z","SourceOrigin":"ShellCoreTracker","Evidence":{"Kind":"Raw","Identifier":"eventlog:Shell-Core:62411:recordId:1234","Summary":"Duplicate ESP terminal failure (Event 62411) — must dead-end after advisory","RawPointer":"blob://events/shell-core/1234"},"Payload":{"reason":"esp_terminal_failure_account_setup"}}
|
||||
Reference in New Issue
Block a user