Seal pre-provisioning despite IME's AccountSetup line; report why a session never registered

WhiteGlove seal: IME reports "EspPhase: AccountSetup" from the device session
as soon as the device apps are done when the user ESP is enabled. The sealing
classifier counted that as user activity (-40) and left pre-provisioning Part 1
unsealed, so no whiteglove.complete marker was written and the Part-2 boot hit
the 48-hour emergency break. The excluder now also needs the ESP registry to
show AccountSetup progress: the adapter stamps accountSetupProgress on
WhiteGloveShellCoreSuccess, only after the tracker has actually read the
AccountSetup subcategories (unread is never "untouched"). The new excluder is a
subset of the old one, so a verdict can only rise; signals without the key
score as before. WhiteGloveShellCoreSuccess v1 gets its replay fixture.

The Part-1 exit writes whiteglove.complete as its first step instead of after
analyzers, diagnostics upload, grace and drain, where a reseal could kill the
process before it.

Registration failures: a run that exits because register-session failed
records outcome, last error, network link at start and end, the wall-clock
attempt window and the config-fetch outcome in registration-failure.json. The
emergency break sends the record as AgentErrorReport.PriorRegistrationFailure
(null for older agents); the backend bounds it and adds it to the ops details,
the App Insights dimensions and the timeline data. Retry behaviour is unchanged.

The raw registry trigger whiteglove_signal_* is renamed save_result_step_*: the
step exists in every 25H2 enrollment and the old name read as a
pre-provisioning signal.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-09-29 01:38:45 +02:00
parent fa637369e1
commit 8cb5d42edc
31 changed files with 969 additions and 32 deletions
@@ -83,6 +83,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
using var f = new Fixture();
Assert.False(f.Tracker.HasAccountSetupActivity);
Assert.False(f.Tracker.HasAccountSetupProgress);
Assert.Null(f.Tracker.AccountSetupProgressIfRead);
}
[Fact]
@@ -97,6 +98,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
Assert.True(f.Tracker.HasAccountSetupActivity);
Assert.False(f.Tracker.HasAccountSetupProgress);
// Read and untouched — the only state the WhiteGlove sealing gate treats as "false".
Assert.False(f.Tracker.AccountSetupProgressIfRead);
}
[Fact]
@@ -108,6 +111,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
Assert.True(f.Tracker.HasAccountSetupActivity);
Assert.True(f.Tracker.HasAccountSetupProgress);
Assert.True(f.Tracker.AccountSetupProgressIfRead);
}
[Theory]
@@ -138,5 +142,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
Assert.False(f.Tracker.HasAccountSetupProgress);
}
[Fact]
public void OnlyDeviceSetupRead_AccountSetupProgressIsUnknown()
{
// Adversarial for the WhiteGlove sealing gate: the tracker holds DeviceSetup data but
// has not read the AccountSetup category — that must be "unknown", never "untouched".
using var f = new Fixture();
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status",
"{\"categoryState\":\"inProgress\",\"DeviceSetup.AppsSubcategory\":{\"subcategoryState\":\"inProgress\"}}");
Assert.Null(f.Tracker.AccountSetupProgressIfRead);
Assert.False(f.Tracker.HasAccountSetupProgress);
}
}
}
@@ -157,5 +157,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
Assert.Equal(478.2, client.Reports[0].SessionAgeHours);
Assert.Null(client.Reports[1].SessionAgeHours);
}
[Fact]
public async Task Prior_registration_failure_is_carried_in_the_report_and_absent_by_default()
{
var client = new FakeApiClient();
client.Results.Enqueue(true);
client.Results.Enqueue(true);
var record = new RegistrationFailureSummary { FailedRuns = 2, Outcome = "Failed", NetworkLinkUpAtEnd = false };
await Reporter(client).TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m", priorRegistrationFailure: record);
await Reporter(client).TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
Assert.Equal(2, client.Reports.Count);
Assert.Same(record, client.Reports[0].PriorRegistrationFailure);
Assert.Null(client.Reports[1].PriorRegistrationFailure);
}
}
}
@@ -118,5 +118,88 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
Assert.Throws<ArgumentNullException>(
() => BackendSessionRegistration.Register(config, auth, http, "1.0", consoleMode: false, logger: null));
}
// ============================================================ failed-registration record
[Fact]
public void RecordRegistrationRun_records_a_failed_run_with_its_window_and_bounded_error()
{
using var tmp = new TempDirectory();
var persistence = new SessionIdPersistence(tmp.Path);
var started = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
BackendSessionRegistration.RecordRegistrationRun(
persistence, SessionRegistrationResult.Failed(new string('e', 1000)),
started, started.AddSeconds(1204.46), linkUpAtStart: false, linkUpAtEnd: true,
configFetchOutcome: "UsedDefaults", logger: NewLogger(tmp.Path));
var record = persistence.LoadRegistrationFailure();
Assert.NotNull(record);
Assert.Equal(1, record!.FailedRuns);
Assert.Equal("Failed", record.Outcome);
Assert.Equal(BackendSessionRegistration.MaxRecordedErrorLength, record.LastError!.Length);
Assert.False(record.NetworkLinkUpAtStart);
Assert.True(record.NetworkLinkUpAtEnd);
Assert.Equal(1204.5, record.AttemptWindowSeconds);
Assert.Equal("UsedDefaults", record.ConfigFetchOutcome);
Assert.Equal(started.AddSeconds(1204.46), record.LastFailedAtUtc);
}
[Fact]
public void RecordRegistrationRun_records_auth_failures_and_clears_on_success()
{
using var tmp = new TempDirectory();
var persistence = new SessionIdPersistence(tmp.Path);
var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
BackendSessionRegistration.RecordRegistrationRun(
persistence, SessionRegistrationResult.AuthFailed(401, "unauthorized"),
at, at.AddSeconds(3), true, true, "Succeeded", NewLogger(tmp.Path));
Assert.Equal("AuthFailed", persistence.LoadRegistrationFailure()!.Outcome);
BackendSessionRegistration.RecordRegistrationRun(
persistence, SessionRegistrationResult.Succeeded(new RegisterSessionResponse { Success = true }),
at, at.AddSeconds(1), true, true, "Succeeded", NewLogger(tmp.Path));
Assert.Null(persistence.LoadRegistrationFailure());
}
[Fact]
public void RecordRegistrationRun_without_persistence_is_a_no_op()
{
using var tmp = new TempDirectory();
BackendSessionRegistration.RecordRegistrationRun(
null, SessionRegistrationResult.Failed("x"), DateTime.UtcNow, DateTime.UtcNow,
null, null, null, NewLogger(tmp.Path));
}
// ============================================================ emergency-break message
[Fact]
public void Break_message_names_failed_registrations_in_invariant_culture()
{
var previous = System.Threading.Thread.CurrentThread.CurrentCulture;
try
{
System.Threading.Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("de-DE");
var record = new RegistrationFailureSummary { FailedRuns = 3, Outcome = "Failed", NetworkLinkUpAtEnd = false };
var message = EmergencyBreakReporter.BuildMessage(478.2, 48, record);
Assert.Contains("session age 478.2h (cap 48h)", message);
Assert.EndsWith(" 3 earlier start(s) failed to register (last: Failed, network link at end: down).", message);
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = previous;
}
}
[Fact]
public void Break_message_without_record_keeps_the_historical_text()
{
Assert.Equal(
"Agent absolute session-age emergency break fired at session age 50.0h (cap 48h) — cleaning up and exiting.",
EmergencyBreakReporter.BuildMessage(50.0, 48, null));
}
}
}
@@ -3,6 +3,7 @@ using System.IO;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Security;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Shared.Models;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
@@ -234,5 +235,78 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
Assert.False(File.Exists(Path.Combine(tmp.Path, "session.created")));
Assert.False(File.Exists(Path.Combine(tmp.Path, "whiteglove.complete")));
}
// ============================================================ failed-registration record
private static RegistrationFailureSummary FailedRun(DateTime at, string outcome = "Failed", bool? linkUpAtEnd = false) =>
new RegistrationFailureSummary
{
FailedRuns = 1,
FirstFailedAtUtc = at,
LastFailedAtUtc = at,
Outcome = outcome,
LastError = "No such host is known.",
NetworkLinkUpAtStart = false,
NetworkLinkUpAtEnd = linkUpAtEnd,
AttemptWindowSeconds = 91.3,
ConfigFetchOutcome = "UsedDefaults",
};
[Fact]
public void RecordRegistrationFailure_counts_runs_keeps_the_first_time_and_describes_the_latest_run()
{
using var tmp = new TempDirectory();
var sut = new SessionIdPersistence(tmp.Path);
var first = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
var second = new DateTime(2026, 9, 12, 7, 30, 0, DateTimeKind.Utc);
Assert.Null(sut.LoadRegistrationFailure());
sut.RecordRegistrationFailure(FailedRun(first));
sut.RecordRegistrationFailure(FailedRun(second, outcome: "AuthFailed", linkUpAtEnd: true));
// A fresh instance reads it back from disk, as the next boot's emergency break does.
var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailure();
Assert.NotNull(record);
Assert.Equal(2, record!.FailedRuns);
Assert.Equal(first, record.FirstFailedAtUtc);
Assert.Equal(second, record.LastFailedAtUtc);
Assert.Equal("AuthFailed", record.Outcome);
Assert.True(record.NetworkLinkUpAtEnd);
Assert.Equal(91.3, record.AttemptWindowSeconds);
Assert.Equal("UsedDefaults", record.ConfigFetchOutcome);
}
[Fact]
public void Registration_failure_record_is_cleared_by_success_delete_and_rotate()
{
using var tmp = new TempDirectory();
var sut = new SessionIdPersistence(tmp.Path);
sut.GetOrCreate();
var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
sut.RecordRegistrationFailure(FailedRun(at));
sut.ClearRegistrationFailure();
Assert.Null(sut.LoadRegistrationFailure());
sut.RecordRegistrationFailure(FailedRun(at));
sut.Rotate();
Assert.Null(sut.LoadRegistrationFailure());
sut.RecordRegistrationFailure(FailedRun(at));
sut.Delete();
Assert.Null(sut.LoadRegistrationFailure());
Assert.False(File.Exists(Path.Combine(tmp.Path, "registration-failure.json")));
}
[Theory]
[InlineData("not json")]
[InlineData("{\"FailedRuns\":0}")]
public void LoadRegistrationFailure_reports_nothing_for_a_corrupt_or_empty_record(string content)
{
using var tmp = new TempDirectory();
File.WriteAllText(Path.Combine(tmp.Path, "registration-failure.json"), content);
Assert.Null(new SessionIdPersistence(tmp.Path).LoadRegistrationFailure());
}
}
}
@@ -99,6 +99,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
Assert.Equal(DecisionSignalKind.WhiteGloveShellCoreSuccess, posted.Kind);
}
[Theory]
[InlineData(true, "true")]
[InlineData(false, "false")]
public void WhiteGloveEvent_stamps_read_AccountSetup_progress_on_payload_and_evidence(bool progress, string expected)
{
using var f = new Fixture();
using var adapter = new EspAndHelloTrackerAdapter(
f.Coordinator, f.Ingress, f.Clock, accountSetupProgressProbe: () => progress);
adapter.TriggerWhiteGloveFromTest();
var posted = Assert.Single(f.Ingress.Posted);
Assert.Equal(expected, posted.Payload![SignalPayloadKeys.AccountSetupProgress]);
Assert.Equal(expected, posted.Evidence.DerivationInputs![SignalPayloadKeys.AccountSetupProgress]);
}
[Fact]
public void WhiteGloveEvent_with_unread_AccountSetup_registry_sends_no_progress_key()
{
// Unknown must keep the sealing classifier's AccountSetup excluder — the payload key
// is absent, the evidence records why.
using var f = new Fixture();
using var adapter = new EspAndHelloTrackerAdapter(
f.Coordinator, f.Ingress, f.Clock, accountSetupProgressProbe: () => null);
adapter.TriggerWhiteGloveFromTest();
var posted = Assert.Single(f.Ingress.Posted);
Assert.True(posted.Payload == null || !posted.Payload.ContainsKey(SignalPayloadKeys.AccountSetupProgress));
Assert.Equal("unread", posted.Evidence.DerivationInputs![SignalPayloadKeys.AccountSetupProgress]);
}
[Fact]
public void WhiteGloveEvent_default_probe_on_unstarted_coordinator_reports_unread()
{
// Production probe: the provisioning tracker does not exist before Start (and never on
// Device Preparation) — that is "unread", not "untouched".
using var f = new Fixture();
using var adapter = new EspAndHelloTrackerAdapter(f.Coordinator, f.Ingress, f.Clock);
adapter.TriggerWhiteGloveFromTest();
var posted = Assert.Single(f.Ingress.Posted);
Assert.True(posted.Payload == null || !posted.Payload.ContainsKey(SignalPayloadKeys.AccountSetupProgress));
}
[Fact]
public void EspFailureEvent_emits_EspTerminalFailure_with_merged_subSource_annotation()
{
@@ -564,6 +564,37 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
Assert.False(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
}
[Fact]
public void Handle_whiteglove_part1_writes_marker_before_the_later_exit_steps()
{
// The technician's reseal can kill the process anywhere after the seal. The marker
// is what lets the next boot skip the 48-hour break and resume Part 2, so it must
// exist before collectors stop, diagnostics upload, grace and drain run.
using var rig = new Rig();
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.WhiteGloveSealed }.Build();
bool? markerPresentAtCollectorStop = null;
rig.StopPeripheralCollectorsHook = () => markerPresentAtCollectorStop = rig.SessionPersistence.IsWhiteGloveResume();
rig.Build().Handle(sender: null!,
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed));
Assert.True(markerPresentAtCollectorStop, "whiteglove.complete must be written before the peripheral collectors are stopped.");
var emitted = rig.EmittedEventTypes.ToList();
Assert.True(emitted.IndexOf("agent_shutting_down") >= 0, "agent_shutting_down must still be emitted.");
}
[Fact]
public void Handle_non_whiteglove_termination_never_writes_the_marker()
{
using var rig = new Rig();
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build();
rig.Build().Handle(sender: null!,
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed));
Assert.False(rig.SessionPersistence.IsWhiteGloveResume());
}
[Fact]
public void Handle_whiteglove_part1_stops_collectors_before_emitting_part1_complete()
{
@@ -216,6 +216,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
/// </summary>
public bool HasAccountSetupProgress => _provisioningTracker?.HasAccountSetupProgress ?? false;
/// <summary>
/// <see cref="ProvisioningStatusTracker.AccountSetupProgressIfRead"/>; <c>null</c> also when
/// the provisioning tracker is not running (Device Preparation, before Start). Stamped on
/// the WhiteGloveShellCoreSuccess signal for the sealing classifier.
/// </summary>
public bool? AccountSetupProgressIfRead => _provisioningTracker?.AccountSetupProgressIfRead;
/// <summary>
/// True once WhiteGlove start has been detected (EventID 509 or persisted from prior run).
/// Forwarded from ModernDeploymentTracker.
@@ -254,16 +254,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
/// host to tell the real user-ESP page exit from the Device→Account handoff exit; not a
/// completion input.
/// </summary>
public bool HasAccountSetupProgress
public bool HasAccountSetupProgress => AccountSetupProgressIfRead == true;
/// <summary>
/// Tri-state form of <see cref="HasAccountSetupProgress"/>: <c>null</c> until the tracker
/// has actually read AccountSetup subcategories (not started yet, category not written,
/// empty), otherwise whether one has left <c>notStarted</c>. The WhiteGlove sealing
/// classifier treats only a read <c>false</c> as "user ESP untouched" — an unread
/// registry must never pass for an untouched one.
/// </summary>
public bool? AccountSetupProgressIfRead
{
get
{
lock (_stateLock)
{
if (_lastSubcategoryStates == null)
return false;
if (!_lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs) || subs == null)
return false;
return null;
if (!_lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs)
|| subs == null
|| subs.Count == 0)
return null;
foreach (var state in subs.Values)
{
if (!string.Equals(state, "notStarted", StringComparison.OrdinalIgnoreCase))
@@ -279,7 +290,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
/// with subcategoryState=succeeded. NOT a WhiteGlove indicator: on Win11 25H2 (build
/// 26200+) Windows writes this property as a trivially-succeeding step in EVERY
/// enrollment (validated 2026-08-18 — user-driven sessions f475e697/c601a24b/4219203f
/// all carry it). Kept for observability only (whiteglove_signal_* raw dumps); must not
/// all carry it). Kept for observability only (save_result_step_* raw dumps); must not
/// be used for classification. Reliable WhiteGlove anchors are ModernDeployment
/// EventID 509 and the Shell-Core WhiteGlove-success events.
/// </summary>
@@ -966,8 +977,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_logger.Info($"ProvisioningStatusTracker: SaveWhiteGloveSuccessResult state " +
$"transition: {previousState} -> {state}");
// Neutral trigger name: the step exists in every 25H2 enrollment,
// and "whiteglove_*" was read as a pre-provisioning signal.
EmitRawRegistryDump(categoryLabel, jsonValue,
$"whiteglove_signal_{state}");
$"save_result_step_{state}");
if (string.Equals(state, "succeeded", StringComparison.OrdinalIgnoreCase))
{
@@ -81,7 +81,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
/// multiply against the per-session budget.
///
/// <paramref name="sessionAgeHours"/> is the emergency break's measured session age
/// (<see cref="AgentErrorReport.SessionAgeHours"/>); null for every other error type.
/// (<see cref="AgentErrorReport.SessionAgeHours"/>) and <paramref name="priorRegistrationFailure"/>
/// its record of earlier starts that never registered
/// (<see cref="AgentErrorReport.PriorRegistrationFailure"/>); null for every other error type.
/// </summary>
public virtual async Task TrySendAsync(
AgentErrorType errorType,
@@ -91,7 +93,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
int attempts = 1,
TimeSpan? perAttemptTimeout = null,
TimeSpan? retryDelay = null,
double? sessionAgeHours = null)
double? sessionAgeHours = null,
RegistrationFailureSummary priorRegistrationFailure = null)
{
// Deduplicate by error type + status code: same failure category sent only once per session.
// All anti-flood checks are protected by a lock to prevent race conditions when
@@ -140,6 +143,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
HttpStatusCode = httpStatusCode,
SequenceNumber = sequenceNumber,
SessionAgeHours = sessionAgeHours,
PriorRegistrationFailure = priorRegistrationFailure,
AgentVersion = _agentVersion,
Timestamp = DateTime.UtcNow
};
@@ -1,6 +1,8 @@
using System;
using System.IO;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json;
namespace AutopilotMonitor.Agent.V2.Core.Security
{
@@ -22,6 +24,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
private readonly string _sessionFilePath;
private readonly string _sessionCreatedFilePath;
private readonly string _whiteGloveMarkerPath;
private readonly string _registrationFailurePath;
private readonly object _lockObject = new object();
public SessionIdPersistence(string dataDirectory)
@@ -35,6 +38,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
_sessionFilePath = Path.Combine(dataDirectory, "session.id");
_sessionCreatedFilePath = Path.Combine(dataDirectory, "session.created");
_whiteGloveMarkerPath = Path.Combine(dataDirectory, "whiteglove.complete");
_registrationFailurePath = Path.Combine(dataDirectory, "registration-failure.json");
}
/// <summary>
@@ -164,6 +168,55 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
lock (_lockObject) TryDelete(_whiteGloveMarkerPath, logger);
}
/// <summary>
/// Folds one failed registration run into <c>registration-failure.json</c>: the run
/// counter grows, the first failure time is kept, everything else describes the latest
/// run. A run that never registers exits and leaves nothing else behind — the emergency
/// break reads this file so the backend learns why the session never appeared.
/// Best-effort; never throws.
/// </summary>
public void RecordRegistrationFailure(RegistrationFailureSummary latestRun, AgentLogger logger = null)
{
if (latestRun == null) return;
lock (_lockObject)
{
try
{
var previous = LoadRegistrationFailureInternal();
var merged = new RegistrationFailureSummary
{
FailedRuns = (previous?.FailedRuns ?? 0) + 1,
FirstFailedAtUtc = previous?.FirstFailedAtUtc ?? latestRun.LastFailedAtUtc,
LastFailedAtUtc = latestRun.LastFailedAtUtc,
Outcome = latestRun.Outcome,
LastError = latestRun.LastError,
NetworkLinkUpAtStart = latestRun.NetworkLinkUpAtStart,
NetworkLinkUpAtEnd = latestRun.NetworkLinkUpAtEnd,
AttemptWindowSeconds = latestRun.AttemptWindowSeconds,
ConfigFetchOutcome = latestRun.ConfigFetchOutcome,
};
File.WriteAllText(_registrationFailurePath, JsonConvert.SerializeObject(merged));
logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns}.");
}
catch (Exception ex)
{
logger?.Warning($"SessionIdPersistence: RecordRegistrationFailure failed: {ex.Message}");
}
}
}
/// <summary>The failed-registration record since the last success, or <c>null</c> (none, unreadable).</summary>
public RegistrationFailureSummary LoadRegistrationFailure()
{
lock (_lockObject) return LoadRegistrationFailureInternal();
}
/// <summary>Drops the failed-registration record — called once registration succeeds.</summary>
public void ClearRegistrationFailure(AgentLogger logger = null)
{
lock (_lockObject) TryDelete(_registrationFailurePath, logger);
}
/// <summary>Deletes the persisted SessionId. Next <see cref="GetOrCreate"/> starts a fresh session.</summary>
public void Delete(AgentLogger logger = null)
{
@@ -172,6 +225,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
TryDelete(_sessionFilePath, logger);
TryDelete(_sessionCreatedFilePath, logger);
TryDelete(_whiteGloveMarkerPath, logger);
TryDelete(_registrationFailurePath, logger);
}
}
@@ -191,11 +245,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
var fresh = Guid.NewGuid().ToString();
WriteAtomic(fresh, logger);
SaveSessionCreatedAtInternal(DateTime.UtcNow);
// Earlier failed runs belonged to the refused session id.
TryDelete(_registrationFailurePath, logger);
logger?.Info($"SessionIdPersistence: rotated SessionId to {fresh}.");
return fresh;
}
}
private RegistrationFailureSummary LoadRegistrationFailureInternal()
{
try
{
if (!File.Exists(_registrationFailurePath)) return null;
var record = JsonConvert.DeserializeObject<RegistrationFailureSummary>(File.ReadAllText(_registrationFailurePath));
return record != null && record.FailedRuns > 0 ? record : null;
}
catch
{
// Corrupt record: report nothing rather than a half-read account.
return null;
}
}
private static void TryDelete(string path, AgentLogger logger)
{
try { if (File.Exists(path)) File.Delete(path); }
@@ -24,7 +24,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
/// <list type="bullet">
/// <item><c>HelloCompleted</c> → <see cref="DecisionSignalKind.HelloResolved"/></item>
/// <item><c>FinalizingSetupPhaseTriggered</c> → <see cref="DecisionSignalKind.EspPhaseChanged"/> (phase=FinalizingSetup)</item>
/// <item><c>WhiteGloveCompleted</c> → <see cref="DecisionSignalKind.WhiteGloveShellCoreSuccess"/></item>
/// <item><c>WhiteGloveCompleted</c> → <see cref="DecisionSignalKind.WhiteGloveShellCoreSuccess"/> (payload: registry AccountSetup progress at that moment)</item>
/// <item><c>EspFailureDetected</c> → <see cref="DecisionSignalKind.EspTerminalFailure"/> (merged aus ShellCore + Provisioning)</item>
/// <item><c>DeviceSetupProvisioningComplete</c> → <see cref="DecisionSignalKind.DeviceSetupProvisioningComplete"/></item>
/// <item><c>AccountSetupProvisioningComplete</c> → <see cref="DecisionSignalKind.AccountSetupProvisioningComplete"/> (session 330f73f3 fix — strong AccountSetup-done gate)</item>
@@ -46,6 +46,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
private readonly EspAndHelloTracker _coordinator;
private readonly ISignalIngressSink _ingress;
private readonly IClock _clock;
private readonly Func<bool?> _accountSetupProgressProbe;
private bool _helloPosted;
private bool _helloWizardStartedPosted;
@@ -60,11 +61,14 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
public EspAndHelloTrackerAdapter(
EspAndHelloTracker coordinator,
ISignalIngressSink ingress,
IClock clock)
IClock clock,
Func<bool?>? accountSetupProgressProbe = null)
{
_coordinator = coordinator ?? throw new ArgumentNullException(nameof(coordinator));
_ingress = ingress ?? throw new ArgumentNullException(nameof(ingress));
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
// Test seam; production reads the provisioning tracker's registry-derived flag.
_accountSetupProgressProbe = accountSetupProgressProbe ?? (() => _coordinator.AccountSetupProgressIfRead);
_coordinator.HelloCompleted += OnHelloCompleted;
_coordinator.FinalizingSetupPhaseTriggered += OnFinalizing;
@@ -216,13 +220,29 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
if (_whiteGloveSuccessPosted) return;
_whiteGloveSuccessPosted = true;
// Registry AccountSetup progress at the moment of the success page. The sealing
// classifier lifts its AccountSetup excluder only on an explicit "false" — IME logs
// AccountSetup from the device session once the device apps are done, which is no
// user activity. Sampled here so replay reads it from the SignalLog (L.2/L.5). An
// unread registry sends no key: unknown must keep the excluder, never lift it.
var progress = _accountSetupProgressProbe();
var progressText = progress.HasValue ? (progress.Value ? "true" : "false") : null;
var now = ResolveOccurredAt(out var derivedFromClock);
var derivationInputs = new Dictionary<string, string>(StringComparer.Ordinal)
{
["subSource"] = "ShellCoreTracker",
[SignalPayloadKeys.AccountSetupProgress] = progressText ?? "unread",
};
TagDerivedTimestamp(derivationInputs, derivedFromClock);
var payload = progressText == null
? null
: new Dictionary<string, string>(StringComparer.Ordinal)
{
[SignalPayloadKeys.AccountSetupProgress] = progressText,
};
_ingress.Post(
kind: DecisionSignalKind.WhiteGloveShellCoreSuccess,
occurredAtUtc: now,
@@ -231,7 +251,8 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
kind: EvidenceKind.Derived,
identifier: DetectorId,
summary: "WhiteGlove sealing success (coordinator-forwarded)",
derivationInputs: derivationInputs));
derivationInputs: derivationInputs),
payload: payload);
}
private void EmitEspFailure(EspFailureDetectedEventArgs args)
@@ -169,6 +169,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
_logger.Info(
$"EnrollmentTerminationHandler: handling Terminated (reason={args.Reason}, outcome={args.Outcome}, stage={args.StageName}).");
// The seal is already persisted in the decision snapshot. The marker goes down
// before anything else: the technician's reseal can kill this process at any
// later step, and a missing marker makes the next boot run the 48-hour break
// against the Part-1 session age instead of resuming Part 2.
if (isWhiteGlovePart1)
{
TrySaveWhiteGloveComplete();
}
var state = TryGetCurrentState();
// M4.6.δ — shutdown analyzers run BEFORE the dialog / diagnostics upload so
@@ -254,7 +263,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
// WhiteGlove Part-1 exit: keep the session alive, but announce the handoff so
// the timeline clearly marks the transition. The `whiteglove.complete` marker
// lets the next agent boot classify itself as a Part-2 resume.
// (written at the top of this handler) lets the next agent boot classify itself
// as a Part-2 resume.
if (isWhiteGlovePart1)
{
// Stop peripheral collectors BEFORE the Part-1-complete marker so their
@@ -298,7 +308,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
DelayLateEventGrace();
DrainSpool();
TrySaveWhiteGloveComplete();
// Option 2 (WG Part 1 graceful-exit hardening, 2026-04-30): write the
// clean-exit marker BEFORE _signalShutdown returns control to the main
// thread. The AppDomain.ProcessExit handler still writes it as a second
@@ -273,7 +273,9 @@ namespace AutopilotMonitor.Agent.V2
var registration = Runtime.BackendSessionRegistration.Register(
bootstrap.AgentConfig, auth, telemetry.MtlsHttpClient, GetAgentVersion(), consoleMode, logger,
rotateSession: () => Runtime.BackendSessionRegistration.RotateSession(
bootstrap.AgentConfig, bootstrap.SessionPersistence, auth, transportDir, logger));
bootstrap.AgentConfig, bootstrap.SessionPersistence, auth, transportDir, logger),
sessionPersistence: bootstrap.SessionPersistence,
configFetchOutcome: runtimeConfig.RemoteConfigService.LastFetchOutcome.ToString());
if (registration.ShouldExit)
{
return registration.ExitCode;
@@ -87,8 +87,11 @@ namespace AutopilotMonitor.Agent.V2.Runtime
cleanupServiceFactory, logger, consoleMode,
// Best-effort: surface the otherwise-silent 48h break to the backend before cleanup
// (tasks/enrollment-status-reclassification.md). Never blocks the exit.
// The failed-registration record is read here, before the break deletes the
// session files.
onBreakFired: sessionAgeHours => EmergencyBreakReporter.TrySend(
agentConfig, Program.GetAgentVersion(), sessionAgeHours, logger)))
agentConfig, Program.GetAgentVersion(), sessionAgeHours,
sessionPersistence.LoadRegistrationFailure(), logger)))
{
logger.Info("Emergency break fired — agent exiting.");
return BootstrapResult.Exit(0);
@@ -1,6 +1,7 @@
using System;
using System.Net;
using System.Net.Http;
using System.Net.NetworkInformation;
using System.Threading.Tasks;
using AutopilotMonitor.Agent.V2.Core.Configuration;
using AutopilotMonitor.Agent.V2.Core.Logging;
@@ -17,7 +18,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
/// <c>IncrementSessionEventCountAsync</c> / <c>UpdateSessionStatusAsync</c> silently
/// no-op and session status / phase / admin-overrides / validator reconcile all break.
/// On registration failure we follow V1's rule: collectors MUST NOT start and the
/// agent exits cleanly so the next Scheduled-Task tick can retry.
/// agent exits cleanly; the Scheduled Task has only a boot trigger, so the next attempt is
/// the next boot. The failed run is recorded for the emergency break report.
/// </summary>
internal static class BackendSessionRegistration
{
@@ -28,12 +30,19 @@ namespace AutopilotMonitor.Agent.V2.Runtime
string agentVersion,
bool consoleMode,
AgentLogger logger,
Func<string> rotateSession = null)
Func<string> rotateSession = null,
SessionIdPersistence sessionPersistence = null,
string configFetchOutcome = null)
{
if (agentConfig == null) throw new ArgumentNullException(nameof(agentConfig));
if (auth == null) throw new ArgumentNullException(nameof(auth));
if (logger == null) throw new ArgumentNullException(nameof(logger));
// Wall clock on purpose: a window far above the retry schedule is the evidence that
// the device slept or was suspended while this run tried to register.
var startedUtc = DateTime.UtcNow;
var linkUpAtStart = TryIsNetworkLinkUp();
var registrationResult = SessionRegistrationHelper.RegisterWithRetryAsync(
apiClient: auth.BackendApiClient,
agentConfig: agentConfig,
@@ -48,6 +57,10 @@ namespace AutopilotMonitor.Agent.V2.Runtime
rotateSession: rotateSession)
.GetAwaiter().GetResult();
RecordRegistrationRun(
sessionPersistence, registrationResult, startedUtc, DateTime.UtcNow,
linkUpAtStart, TryIsNetworkLinkUp(), configFetchOutcome, logger);
if (registrationResult.Outcome != SessionRegistrationOutcome.Succeeded)
{
logger.Error(
@@ -65,6 +78,62 @@ namespace AutopilotMonitor.Agent.V2.Runtime
return SessionRegistrationOutcomeResult.Continue(registrationResult);
}
/// <summary>Longest <see cref="RegistrationFailureSummary.LastError"/> kept on disk and on the wire.</summary>
internal const int MaxRecordedErrorLength = 256;
/// <summary>
/// Keeps <c>registration-failure.json</c> in step with this run: a failed run is folded
/// into the record (the emergency break later reports it — for a session that never
/// registered, the only account of why), a successful one clears it. Never throws.
/// </summary>
internal static void RecordRegistrationRun(
SessionIdPersistence sessionPersistence,
SessionRegistrationResult registrationResult,
DateTime startedUtc,
DateTime endedUtc,
bool? linkUpAtStart,
bool? linkUpAtEnd,
string configFetchOutcome,
AgentLogger logger)
{
if (sessionPersistence == null || registrationResult == null) return;
try
{
if (registrationResult.Outcome == SessionRegistrationOutcome.Succeeded)
{
sessionPersistence.ClearRegistrationFailure(logger);
return;
}
var error = registrationResult.ErrorMessage;
if (error != null && error.Length > MaxRecordedErrorLength)
error = error.Substring(0, MaxRecordedErrorLength);
sessionPersistence.RecordRegistrationFailure(new RegistrationFailureSummary
{
FailedRuns = 1,
FirstFailedAtUtc = endedUtc,
LastFailedAtUtc = endedUtc,
Outcome = registrationResult.Outcome.ToString(),
LastError = error,
NetworkLinkUpAtStart = linkUpAtStart,
NetworkLinkUpAtEnd = linkUpAtEnd,
AttemptWindowSeconds = Math.Round(Math.Max(0, (endedUtc - startedUtc).TotalSeconds), 1),
ConfigFetchOutcome = configFetchOutcome,
}, logger);
}
catch (Exception ex)
{
logger?.Warning($"Registration run record failed (best-effort): {ex.Message}");
}
}
private static bool? TryIsNetworkLinkUp()
{
try { return NetworkInterface.GetIsNetworkAvailable(); }
catch { return null; }
}
/// <summary>
/// SESSION-OWNER-BINDING: the one place that turns "backend says this session id is not
/// ours" into a new session. Rotates the persisted id, points the runtime config at it,
@@ -36,18 +36,20 @@ namespace AutopilotMonitor.Agent.V2.Runtime
// Upper bound for the blocking wait: NIC wait + 3×15s attempts + 2×5s delays + slack.
private static readonly TimeSpan OverallBudget = TimeSpan.FromSeconds(75);
public static void TrySend(AgentConfiguration agentConfig, string agentVersion, double sessionAgeHours, AgentLogger logger)
public static void TrySend(
AgentConfiguration agentConfig,
string agentVersion,
double sessionAgeHours,
RegistrationFailureSummary priorRegistrationFailure,
AgentLogger logger)
{
try
{
WaitForNetwork(logger);
var auth = BackendClientFactory.BuildAuthClients(agentConfig, agentVersion, logger);
// Invariant: the text lands in the backend's ops feed and timeline; a device locale
// would otherwise print "478,2h".
var roundedAgeHours = Math.Round(sessionAgeHours, 1);
var message = FormattableString.Invariant(
$"Agent absolute session-age emergency break fired at session age {roundedAgeHours:F1}h (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting.");
var message = BuildMessage(roundedAgeHours, agentConfig.AbsoluteMaxSessionHours, priorRegistrationFailure);
// The process is about to exit, so block on the otherwise fire-and-forget send
// rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP
@@ -60,7 +62,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
attempts: SendAttempts,
perAttemptTimeout: PerAttemptTimeout,
retryDelay: RetryDelay,
sessionAgeHours: roundedAgeHours)
sessionAgeHours: roundedAgeHours,
priorRegistrationFailure: priorRegistrationFailure)
.Wait(OverallBudget);
}
catch (Exception ex)
@@ -69,6 +72,25 @@ namespace AutopilotMonitor.Agent.V2.Runtime
}
}
/// <summary>
/// Invariant culture: the text lands in the backend's ops feed and timeline, and a device
/// locale would otherwise print "478,2h". The registration clause names what the ops feed
/// cannot otherwise show for a session that never registered.
/// </summary>
internal static string BuildMessage(double roundedAgeHours, int capHours, RegistrationFailureSummary priorRegistrationFailure)
{
var message = FormattableString.Invariant(
$"Agent absolute session-age emergency break fired at session age {roundedAgeHours:F1}h (cap {capHours}h) — cleaning up and exiting.");
if (priorRegistrationFailure != null && priorRegistrationFailure.FailedRuns > 0)
{
message += FormattableString.Invariant(
$" {priorRegistrationFailure.FailedRuns} earlier start(s) failed to register (last: {priorRegistrationFailure.Outcome}, network link at end: {FormatLink(priorRegistrationFailure.NetworkLinkUpAtEnd)}).");
}
return message;
}
private static string FormatLink(bool? up) => up == null ? "unknown" : (up.Value ? "up" : "down");
/// <summary>
/// Boot-time NIC grace: the break fires during bootstrap, often seconds after boot while
/// Wi-Fi is still associating. Polls the cheap link-level signal — it cannot prove backend
@@ -522,7 +522,8 @@ namespace AutopilotMonitor.DecisionCore.Tests
.WithSkipUserEsp(value: true, sourceSignalOrdinal: 4)
.WithEspSyncFailureTimeoutMinutes(value: 90, sourceSignalOrdinal: 5)
.WithRegistrySelfDeployingProfile(value: false, sourceSignalOrdinal: 6)
.WithCloudPc(value: true, sourceSignalOrdinal: 7))
.WithCloudPc(value: true, sourceSignalOrdinal: 7)
.WithAccountSetupProgressAtWhiteGloveSuccess(value: false, sourceSignalOrdinal: 8))
.Build();
var json = StateSerializer.Serialize(state);
@@ -538,6 +539,8 @@ namespace AutopilotMonitor.DecisionCore.Tests
Assert.Equal(6, o.RegistrySelfDeployingProfile!.SourceSignalOrdinal);
Assert.True(o.CloudPc!.Value);
Assert.Equal(7, o.CloudPc!.SourceSignalOrdinal);
Assert.False(o.AccountSetupProgressAtWhiteGloveSuccess!.Value);
Assert.Equal(8, o.AccountSetupProgressAtWhiteGloveSuccess!.SourceSignalOrdinal);
// Never-observed facts must rehydrate as null (null ≠ veto — session 62e603c9).
Assert.Null(o.WhiteGloveSealingPatternSeen);
@@ -31,7 +31,6 @@ namespace AutopilotMonitor.DecisionCore.Tests
DecisionSignalKind.HelloResolved,
DecisionSignalKind.ImeUserSessionCompleted,
DecisionSignalKind.DeviceSetupProvisioningComplete,
DecisionSignalKind.WhiteGloveShellCoreSuccess,
DecisionSignalKind.WhiteGloveSealingPatternDetected,
DecisionSignalKind.AadUserJoinedLate,
DecisionSignalKind.DeviceInfoCollected,
@@ -0,0 +1,155 @@
using System;
using System.Collections.Generic;
using AutopilotMonitor.DecisionCore.Classifiers;
using AutopilotMonitor.DecisionCore.Engine;
using AutopilotMonitor.DecisionCore.Serialization;
using AutopilotMonitor.DecisionCore.Signals;
using AutopilotMonitor.DecisionCore.State;
using Xunit;
namespace AutopilotMonitor.DecisionCore.Tests
{
/// <summary>
/// The sealing classifier's AccountSetup excluder (-40) applies only when the IME reported
/// the AccountSetup phase AND the WhiteGlove_Success signal did not report an untouched
/// AccountSetup registry. IME logs "EspPhase: AccountSetup" from the device session once
/// the device apps are done — on pre-provisioning with the user ESP enabled that line
/// routinely precedes the technician's success page and used to leave Part 1 unsealed.
/// The gate may only ever lift the excluder, never add one.
/// </summary>
public sealed class WhiteGloveAccountSetupProgressGateTests
{
private static readonly DateTime T0 = new DateTime(2026, 9, 29, 7, 25, 9, DateTimeKind.Utc);
[Fact]
public void Ime_account_setup_with_untouched_registry_seals_on_the_fast_path()
{
var step = new DecisionEngine().Reduce(
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
Assert.Equal(SessionStage.WhiteGloveSealed, step.NewState.Stage);
Assert.Equal(SessionOutcome.WhiteGlovePart1Sealed, step.NewState.Outcome);
Assert.Equal(HypothesisLevel.Confirmed, step.NewState.ClassifierOutcomes.WhiteGloveSealing.Level);
Assert.Equal(WhiteGloveSealingClassifier.WeightShellCoreWhiteGloveSuccess,
step.NewState.ClassifierOutcomes.WhiteGloveSealing.Score);
var fact = step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess;
Assert.NotNull(fact);
Assert.False(fact!.Value);
Assert.Equal(3, fact.SourceSignalOrdinal);
}
[Fact]
public void Ime_account_setup_with_registry_progress_keeps_the_excluder()
{
var step = new DecisionEngine().Reduce(
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "true"));
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
Assert.Contains(step.Effects, e => e.Kind == DecisionEffectKind.RunClassifier);
Assert.True(DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState).HasAccountSetupActivity);
}
[Fact]
public void Missing_progress_payload_keeps_the_excluder_for_older_agents()
{
var step = new DecisionEngine().Reduce(
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: null));
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
Assert.Null(step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess);
Assert.True(DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState).HasAccountSetupActivity);
}
[Theory]
[InlineData("")]
[InlineData("yes")]
public void Unparseable_progress_payload_is_treated_as_missing(string raw)
{
var step = new DecisionEngine().Reduce(
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: raw));
Assert.Null(step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess);
Assert.NotEqual(SessionStage.WhiteGloveSealed, step.NewState.Stage);
}
[Fact]
public void Registry_progress_without_ime_account_setup_adds_no_excluder()
{
var state = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
.WithStage(SessionStage.EspDeviceSetup)
.WithStepIndex(1)
.WithLastAppliedSignalOrdinal(0)
.Build();
var step = new DecisionEngine().Reduce(state, MakeShellCoreSignal(ordinal: 1, accountSetupProgress: "true"));
Assert.Equal(SessionStage.WhiteGloveSealed, step.NewState.Stage);
}
[Fact]
public void Untouched_registry_does_not_lift_a_hard_excluder()
{
var builder = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
.WithStage(SessionStage.EspAccountSetup)
.WithStepIndex(3)
.WithLastAppliedSignalOrdinal(2);
builder.AccountSetupEnteredUtc = new SignalFact<DateTime>(T0.AddMinutes(-3), 1);
builder.DesktopArrivedUtc = new SignalFact<DateTime>(T0.AddMinutes(-1), 2);
var step = new DecisionEngine().Reduce(builder.Build(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
Assert.NotEqual(SessionStage.WhiteGloveSealed, step.NewState.Stage);
var snapshot = DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState);
Assert.False(snapshot.HasAccountSetupActivity);
Assert.True(snapshot.DesktopArrived);
}
[Fact]
public void Progress_fact_survives_the_state_serializer()
{
var step = new DecisionEngine().Reduce(
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
var roundTripped = StateSerializer.Deserialize(StateSerializer.Serialize(step.NewState));
var fact = roundTripped.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess;
Assert.NotNull(fact);
Assert.False(fact!.Value);
Assert.Equal(3, fact.SourceSignalOrdinal);
}
private static DecisionState StateWithImeAccountSetup()
{
// The IME device-session line: AccountSetup entered before the success page.
var builder = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
.WithStage(SessionStage.EspAccountSetup)
.WithStepIndex(3)
.WithLastAppliedSignalOrdinal(2);
builder.AccountSetupEnteredUtc = new SignalFact<DateTime>(T0.AddMinutes(-3), 2);
return builder.Build();
}
private static DecisionSignal MakeShellCoreSignal(long ordinal, string? accountSetupProgress)
{
var payload = new Dictionary<string, string>(StringComparer.Ordinal);
if (accountSetupProgress != null)
{
payload[SignalPayloadKeys.AccountSetupProgress] = accountSetupProgress;
}
return new DecisionSignal(
sessionSignalOrdinal: ordinal,
sessionTraceOrdinal: ordinal,
kind: DecisionSignalKind.WhiteGloveShellCoreSuccess,
kindSchemaVersion: 1,
occurredAtUtc: T0,
sourceOrigin: "EspAndHelloTracker",
evidence: new Evidence(
kind: EvidenceKind.Raw,
identifier: "ShellCore-62407",
summary: "WhiteGlove_Success"),
payload: payload);
}
}
}
@@ -330,6 +330,95 @@ public class ReportAgentErrorFunctionTests
It.Is<List<EnrollmentEvent>>(e => (double)e[0].Data["sessionAgeHours"] == 478.2)), Times.Once);
}
// A session that never registered leaves the backend nothing but this report; the agent's
// record of its failed starts is the only account of why.
[Fact]
public async Task Prior_registration_failure_lands_in_ops_details_even_without_a_row()
{
var h = new Harness();
var report = Report(Now);
report.PriorRegistrationFailure = ReportAgentErrorFunction.SanitizeRegistrationFailure(Failure());
await h.RunAsync(report, Read(null));
var details = Assert.Single(h.OpsEvents).Details;
Assert.Contains("\"priorRegistrationFailure\":{\"failedRuns\":3", details);
Assert.Contains("\"networkLinkUpAtEnd\":false", details);
Assert.Contains("\"attemptWindowSeconds\":1204.5", details);
Assert.Contains("\"configFetchOutcome\":\"UsedDefaults\"", details);
h.VerifyNothingWrittenIntoTheSession();
}
[Fact]
public async Task Prior_registration_failure_lands_in_the_timeline_data_of_a_registered_session()
{
var h = new Harness();
var report = Report(Now);
report.PriorRegistrationFailure = ReportAgentErrorFunction.SanitizeRegistrationFailure(Failure());
await h.RunAsync(report);
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
It.Is<List<EnrollmentEvent>>(e =>
((Dictionary<string, object?>)e[0].Data["priorRegistrationFailure"])["failedRuns"]!.Equals(3))), Times.Once);
}
[Fact]
public async Task Agent_without_registration_record_leaves_it_null_and_out_of_the_timeline()
{
var h = new Harness();
await h.RunAsync(Report(Now));
Assert.Contains("\"priorRegistrationFailure\":null", Assert.Single(h.OpsEvents).Details);
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
It.Is<List<EnrollmentEvent>>(e => !e[0].Data.ContainsKey("priorRegistrationFailure"))), Times.Once);
}
[Fact]
public void SanitizeRegistrationFailure_bounds_device_input()
{
var raw = Failure();
raw.LastError = new string('x', 5000);
raw.Outcome = new string('o', 500);
raw.ConfigFetchOutcome = new string('c', 500);
raw.AttemptWindowSeconds = -3;
var clean = ReportAgentErrorFunction.SanitizeRegistrationFailure(raw)!;
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationErrorLength, clean.LastError!.Length);
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationTokenLength, clean.Outcome.Length);
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationTokenLength, clean.ConfigFetchOutcome!.Length);
Assert.Equal(0, clean.AttemptWindowSeconds);
Assert.Equal(3, clean.FailedRuns);
Assert.NotSame(raw, clean);
}
[Theory]
[InlineData(0)]
[InlineData(-1)]
public void SanitizeRegistrationFailure_drops_a_record_without_failed_runs(int failedRuns)
{
var raw = Failure();
raw.FailedRuns = failedRuns;
Assert.Null(ReportAgentErrorFunction.SanitizeRegistrationFailure(raw));
Assert.Null(ReportAgentErrorFunction.SanitizeRegistrationFailure(null));
}
private static RegistrationFailureSummary Failure() => new()
{
FailedRuns = 3,
FirstFailedAtUtc = Now.AddDays(-20),
LastFailedAtUtc = Now.AddDays(-19),
Outcome = "Failed",
LastError = "No such host is known.",
NetworkLinkUpAtStart = false,
NetworkLinkUpAtEnd = false,
AttemptWindowSeconds = 1204.5,
ConfigFetchOutcome = "UsedDefaults",
};
[Fact]
public async Task Agent_without_session_age_leaves_it_null_and_out_of_the_timeline()
{
@@ -167,6 +167,9 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
return req.CreateResponse(HttpStatusCode.OK);
}
report.PriorRegistrationFailure = SanitizeRegistrationFailure(report.PriorRegistrationFailure);
var priorRegistration = report.PriorRegistrationFailure;
// Emit a structured log entry (captured by App Insights as a trace)
_logger.LogCritical(
"AgentEmergencyError [{ErrorType}] tenant={TenantId} session={SessionId} http={HttpStatusCode} seq={SequenceNumber} ver={AgentVersion}: {Message}",
@@ -191,6 +194,15 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
["Message"] = report.Message ?? string.Empty,
["AgentTimestamp"] = report.Timestamp.ToString("O"),
["SessionAgeHours"] = report.SessionAgeHours?.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
["PriorRegistrationFailedRuns"] = priorRegistration?.FailedRuns.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
["PriorRegistrationOutcome"] = priorRegistration?.Outcome ?? string.Empty,
["PriorRegistrationLastError"] = priorRegistration?.LastError ?? string.Empty,
["PriorRegistrationLinkUpAtStart"] = FormatNullableBool(priorRegistration?.NetworkLinkUpAtStart),
["PriorRegistrationLinkUpAtEnd"] = FormatNullableBool(priorRegistration?.NetworkLinkUpAtEnd),
["PriorRegistrationWindowSeconds"] = priorRegistration?.AttemptWindowSeconds.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
["PriorRegistrationConfigFetch"] = priorRegistration?.ConfigFetchOutcome ?? string.Empty,
["PriorRegistrationFirstFailedAtUtc"] = priorRegistration?.FirstFailedAtUtc.ToString("O") ?? string.Empty,
["PriorRegistrationLastFailedAtUtc"] = priorRegistration?.LastFailedAtUtc.ToString("O") ?? string.Empty,
});
// One Sessions-row point-read serves the owner binding and the emergency-break verdict.
@@ -396,7 +408,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
await opsEventService.RecordAgentEmergencyBreakAsync(
tenantId, report.SessionId, report.AgentVersion, message,
verdict.Severity, verdict.SessionStatusAtBreak, verdict.LateCleanup, verdict.Context,
report.SessionAgeHours);
report.SessionAgeHours, report.PriorRegistrationFailure);
}
catch (Exception ex)
{
@@ -467,6 +479,39 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
? "Agent absolute session-age emergency break fired — agent cleaned up and exited"
: report.Message;
internal const int MaxRegistrationErrorLength = 256;
internal const int MaxRegistrationTokenLength = 32;
/// <summary>
/// Bounds the device-supplied failed-registration record before it reaches App Insights, the
/// ops feed or a timeline: strings are truncated, a record without a positive run count is
/// dropped, negative windows are clamped. Returns a copy; null in, null out.
/// </summary>
internal static RegistrationFailureSummary? SanitizeRegistrationFailure(RegistrationFailureSummary? source)
{
if (source == null || source.FailedRuns <= 0) return null;
return new RegistrationFailureSummary
{
FailedRuns = source.FailedRuns,
FirstFailedAtUtc = source.FirstFailedAtUtc,
LastFailedAtUtc = source.LastFailedAtUtc,
Outcome = Truncate(source.Outcome, MaxRegistrationTokenLength) ?? string.Empty,
LastError = Truncate(source.LastError, MaxRegistrationErrorLength),
NetworkLinkUpAtStart = source.NetworkLinkUpAtStart,
NetworkLinkUpAtEnd = source.NetworkLinkUpAtEnd,
AttemptWindowSeconds = source.AttemptWindowSeconds < 0 || double.IsNaN(source.AttemptWindowSeconds)
? 0
: source.AttemptWindowSeconds,
ConfigFetchOutcome = Truncate(source.ConfigFetchOutcome, MaxRegistrationTokenLength),
};
}
private static string? Truncate(string? value, int max) =>
value == null || value.Length <= max ? value : value.Substring(0, max);
private static string FormatNullableBool(bool? value) =>
value == null ? string.Empty : (value.Value ? "true" : "false");
/// <summary>
/// Builds the backend-materialized <c>agent_emergency_break</c> timeline event from the agent's
/// best-effort emergency report. Static + pure (analog to
@@ -497,6 +542,21 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
};
if (report.SessionAgeHours.HasValue)
data["sessionAgeHours"] = report.SessionAgeHours.Value;
if (report.PriorRegistrationFailure is { } prior)
{
data["priorRegistrationFailure"] = new Dictionary<string, object?>
{
["failedRuns"] = prior.FailedRuns,
["firstFailedAtUtc"] = prior.FirstFailedAtUtc.ToString("o"),
["lastFailedAtUtc"] = prior.LastFailedAtUtc.ToString("o"),
["outcome"] = prior.Outcome,
["lastError"] = prior.LastError,
["networkLinkUpAtStart"] = prior.NetworkLinkUpAtStart,
["networkLinkUpAtEnd"] = prior.NetworkLinkUpAtEnd,
["attemptWindowSeconds"] = prior.AttemptWindowSeconds,
["configFetchOutcome"] = prior.ConfigFetchOutcome,
};
}
return new EnrollmentEvent
{
@@ -965,15 +965,39 @@ namespace AutopilotMonitor.Functions.Services
/// remaining Warnings named sessions that never registered; as Warnings both buried the real
/// ones. <paramref name="sessionAgeHours"/> is the agent-measured age (null for agents that
/// predate the field) — it separates a break just past the cap from a device that slept for
/// weeks, which the backend cannot tell from its own rows.
/// weeks, which the backend cannot tell from its own rows. <paramref name="priorRegistrationFailure"/>
/// is the agent's record of earlier starts that never registered (bounded by the caller;
/// null when none or for older agents) — for a session without a row the only account of why.
/// </summary>
public Task RecordAgentEmergencyBreakAsync(
string tenantId, string sessionId, string? agentVersion, string message,
string severity, string sessionStatusAtBreak, bool lateCleanup, string context, double? sessionAgeHours)
string severity, string sessionStatusAtBreak, bool lateCleanup, string context, double? sessionAgeHours,
RegistrationFailureSummary? priorRegistrationFailure = null)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
$"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}, {context}): {message}",
tenantId, "System.EmergencyChannel",
new { sessionId, agentVersion, sessionStatusAtBreak, lateCleanup, sessionAgeHours });
new
{
sessionId, agentVersion, sessionStatusAtBreak, lateCleanup, sessionAgeHours,
priorRegistrationFailure = DescribeRegistrationFailure(priorRegistrationFailure),
});
// camelCase like the rest of the details; the class itself would serialize in PascalCase.
private static object? DescribeRegistrationFailure(RegistrationFailureSummary? failure) =>
failure == null
? null
: new
{
failedRuns = failure.FailedRuns,
firstFailedAtUtc = failure.FirstFailedAtUtc,
lastFailedAtUtc = failure.LastFailedAtUtc,
outcome = failure.Outcome,
lastError = failure.LastError,
networkLinkUpAtStart = failure.NetworkLinkUpAtStart,
networkLinkUpAtEnd = failure.NetworkLinkUpAtEnd,
attemptWindowSeconds = failure.AttemptWindowSeconds,
configFetchOutcome = failure.ConfigFetchOutcome,
};
/// <summary>
/// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend
@@ -104,6 +104,15 @@ namespace AutopilotMonitor.DecisionCore.Engine
/// <summary>On <c>AadUserJoinedLate</c>: user presence indicator ("true" / "false").</summary>
public const string AadJoinedWithUser = "aadJoinedWithUser";
/// <summary>
/// On <c>WhiteGloveShellCoreSuccess</c>: "true" / "false" — whether the ESP registry
/// showed an AccountSetup subcategory past <c>notStarted</c> when the agent observed the
/// Shell-Core WhiteGlove_Success event. Missing (registry not read yet, or agents
/// predating the key) → the fact stays unset and the sealing classifier keeps the
/// IME-phase AccountSetup excluder.
/// </summary>
public const string AccountSetupProgress = "accountSetupProgress";
/// <summary>On <c>ImeUserSessionCompleted</c>: matched pattern id.</summary>
public const string ImePatternId = "patternId";
@@ -53,6 +53,16 @@ namespace AutopilotMonitor.DecisionCore.Engine
.WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal);
builder.ScenarioObservations = builder.ScenarioObservations.WithShellCoreWhiteGloveSuccessSeen(signal.SessionSignalOrdinal);
// Recorded before the inline classifier runs: it decides whether the AccountSetup
// excluder applies (see BuildWhiteGloveSealingSnapshot).
if (signal.Payload != null
&& signal.Payload.TryGetValue(SignalPayloadKeys.AccountSetupProgress, out var rawProgress)
&& bool.TryParse(rawProgress, out var accountSetupProgress))
{
builder.ScenarioObservations = builder.ScenarioObservations
.WithAccountSetupProgressAtWhiteGloveSuccess(accountSetupProgress, signal.SessionSignalOrdinal);
}
// Option 3 fast-path: inline classifier evaluation on the strong WG signal.
var afterObservation = builder.Build();
var inlineSnapshot = BuildWhiteGloveSealingSnapshot(afterObservation);
@@ -362,6 +372,15 @@ namespace AutopilotMonitor.DecisionCore.Engine
/// <see cref="WhiteGloveSealingSnapshot.ComputeInputHash"/> is therefore byte-stable
/// across the refactor, which preserves the anti-loop semantics.
/// </para>
/// <para>
/// <c>hasAccountSetupActivity</c> needs the IME AccountSetup phase AND no explicit
/// "registry shows no AccountSetup progress" on the WhiteGlove_Success signal. IME
/// reports AccountSetup from the device session as soon as the device apps are done, so
/// on pre-provisioning with the user ESP enabled that line routinely precedes the
/// technician's success page. The conjunction keeps the excluder a subset of the old
/// one: a verdict can only rise, and a missing payload (older agent) scores exactly as
/// before.
/// </para>
/// </summary>
internal static WhiteGloveSealingSnapshot BuildWhiteGloveSealingSnapshot(DecisionState state) =>
new WhiteGloveSealingSnapshot(
@@ -370,7 +389,8 @@ namespace AutopilotMonitor.DecisionCore.Engine
aadJoinedWithUser: state.ScenarioObservations.AadUserJoinWithUserObserved?.Value == true,
desktopArrived: state.DesktopArrivedUtc != null,
helloResolved: state.HelloResolvedUtc != null,
hasAccountSetupActivity: state.AccountSetupEnteredUtc != null,
hasAccountSetupActivity: state.AccountSetupEnteredUtc != null
&& state.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess?.Value != false,
isDeviceOnlyDeploymentHypothesis:
state.ClassifierOutcomes.DeviceOnlyDeployment.Level >= HypothesisLevel.Strong &&
state.ClassifierOutcomes.DeviceOnlyDeployment.Reason == DeviceOnlyReasons.DeviceOnly,
@@ -144,6 +144,20 @@ namespace AutopilotMonitor.DecisionCore.Engine
seen.Add("whiteglove_sealing_pattern");
evidence["whiteGloveSealingPattern"] = OrdinalEvidence(obs.WhiteGloveSealingPatternSeen.SourceSignalOrdinal);
}
// Registry AccountSetup progress stamped on the WhiteGlove_Success signal — the gate of
// the sealing classifier's AccountSetup excluder, so a Part-1 audit trail shows why
// an IME AccountSetup phase did or did not count.
if (obs.AccountSetupProgressAtWhiteGloveSuccess != null)
{
seen.Add(obs.AccountSetupProgressAtWhiteGloveSuccess.Value
? "whiteglove_account_setup_progress"
: "whiteglove_account_setup_untouched");
evidence["accountSetupProgressAtWhiteGloveSuccess"] = new Dictionary<string, object>(capacity: 2, StringComparer.Ordinal)
{
["ordinal"] = obs.AccountSetupProgressAtWhiteGloveSuccess.SourceSignalOrdinal,
["progress"] = obs.AccountSetupProgressAtWhiteGloveSuccess.Value,
};
}
if (obs.AadUserJoinWithUserObserved != null)
{
seen.Add(obs.AadUserJoinWithUserObserved.Value
@@ -33,6 +33,17 @@ namespace AutopilotMonitor.DecisionCore.State
/// <summary>True once <see cref="Signals.DecisionSignalKind.WhiteGloveSealingPatternDetected"/> has fired.</summary>
public SignalFact<bool>? WhiteGloveSealingPatternSeen { get; init; }
/// <summary>
/// ESP-registry AccountSetup progress as stamped on the
/// <see cref="Signals.DecisionSignalKind.WhiteGloveShellCoreSuccess"/> payload:
/// <c>true</c> = an AccountSetup subcategory had left <c>notStarted</c>, <c>false</c> =
/// none had. <c>null</c> = not reported (registry not read yet, or an agent predating the
/// payload key) and must be read as "unknown", never as "no progress". Gates the sealing classifier's
/// AccountSetup excluder: IME logs "EspPhase: AccountSetup" from the device session as
/// soon as the device apps are done, so only an explicit <c>false</c> here lifts it.
/// </summary>
public SignalFact<bool>? AccountSetupProgressAtWhiteGloveSuccess { get; init; }
/// <summary>
/// Payload-carrying observation from <see cref="Signals.DecisionSignalKind.AadUserJoinedLate"/>.
/// <c>true</c> = late AADJ observed with a user-side principal (hard-excluder for
@@ -113,6 +124,11 @@ namespace AutopilotMonitor.DecisionCore.State
? this
: this with { WhiteGloveSealingPatternSeen = new SignalFact<bool>(true, sourceSignalOrdinal) };
public EnrollmentScenarioObservations WithAccountSetupProgressAtWhiteGloveSuccess(bool value, long sourceSignalOrdinal) =>
AccountSetupProgressAtWhiteGloveSuccess != null
? this
: this with { AccountSetupProgressAtWhiteGloveSuccess = new SignalFact<bool>(value, sourceSignalOrdinal) };
public EnrollmentScenarioObservations WithAadUserJoinWithUserObserved(bool value, long sourceSignalOrdinal) =>
AadUserJoinWithUserObserved != null
? this
@@ -90,6 +90,13 @@ namespace AutopilotMonitor.Shared.Models
/// </summary>
public double? SessionAgeHours { get; set; }
/// <summary>
/// Why earlier process starts of this session never registered
/// (<see cref="AgentErrorType.SessionAgeEmergencyBreak"/> only). Null when every start
/// registered, for other error types, and for agents that predate the field.
/// </summary>
public RegistrationFailureSummary? PriorRegistrationFailure { get; set; }
/// <summary>
/// Agent version string for diagnosing version-specific issues.
/// </summary>
@@ -0,0 +1,44 @@
using System;
namespace AutopilotMonitor.Shared.Models
{
/// <summary>
/// What the agent recorded locally each time a process start ended without a registered
/// session (register-session failed after all retries, the agent exited). Carried on the
/// emergency-break report (<see cref="AgentErrorReport.PriorRegistrationFailure"/>): for a
/// session that never registered it is the only account of why that ever reaches the
/// backend. Cleared by the next successful registration.
/// </summary>
public class RegistrationFailureSummary
{
/// <summary>Process starts whose registration failed since the last success.</summary>
public int FailedRuns { get; set; }
/// <summary>UTC time the first of these runs gave up.</summary>
public DateTime FirstFailedAtUtc { get; set; }
/// <summary>UTC time the latest of these runs gave up.</summary>
public DateTime LastFailedAtUtc { get; set; }
/// <summary>Registration outcome of the latest failed run (e.g. <c>Failed</c>, <c>AuthFailed</c>).</summary>
public string Outcome { get; set; } = default!;
/// <summary>Last error message of the latest failed run, truncated by the agent.</summary>
public string? LastError { get; set; }
/// <summary>Network link up (any interface) when the latest failed run started registering.</summary>
public bool? NetworkLinkUpAtStart { get; set; }
/// <summary>Network link up when the latest failed run gave up.</summary>
public bool? NetworkLinkUpAtEnd { get; set; }
/// <summary>
/// Wall-clock seconds the latest failed run spent registering (link wait plus all attempts).
/// Far above the retry schedule means the device slept or was suspended meanwhile.
/// </summary>
public double AttemptWindowSeconds { get; set; }
/// <summary>Outcome of the remote-config fetch in the latest failed run (e.g. <c>Succeeded</c>, <c>UsedDefaults</c>).</summary>
public string? ConfigFetchOutcome { get; set; }
}
}
@@ -0,0 +1,9 @@
# WhiteGloveShellCoreSuccess v1 — Shell-Core 62407 "CommercialOOBE_ESPProgress_WhiteGlove_Success".
# Posted by EspAndHelloTrackerAdapter (fire-once) when the ShellCoreTracker observes the
# technician-flow success page. Payload `accountSetupProgress` (added 2026-09-29) carries the ESP
# registry's AccountSetup progress at that moment ("true" = a subcategory had left notStarted);
# signals from older agents carry no payload.
# Reducer: HandleWhiteGloveShellCoreSuccessV1 → records ShellCoreWhiteGloveSuccessSeen and, when
# the payload parses, AccountSetupProgressAtWhiteGloveSuccess; runs the sealing classifier inline.
# The IME AccountSetup excluder (-40) is lifted only by an explicit "false".
{"SessionSignalOrdinal":1,"SessionTraceOrdinal":1,"Kind":"WhiteGloveShellCoreSuccess","KindSchemaVersion":1,"OccurredAtUtc":"2026-09-14T07:25:09.7628703Z","SourceOrigin":"EspAndHelloTracker","Evidence":{"Kind":"Derived","Identifier":"esp-and-hello-tracker-v1","Summary":"WhiteGlove sealing success (coordinator-forwarded)","DerivationInputs":{"subSource":"ShellCoreTracker","accountSetupProgress":"false"}},"Payload":{"accountSetupProgress":"false"}}