mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
Seal pre-provisioning despite IME's AccountSetup line; report why a session never registered
WhiteGlove seal: IME reports "EspPhase: AccountSetup" from the device session as soon as the device apps are done when the user ESP is enabled. The sealing classifier counted that as user activity (-40) and left pre-provisioning Part 1 unsealed, so no whiteglove.complete marker was written and the Part-2 boot hit the 48-hour emergency break. The excluder now also needs the ESP registry to show AccountSetup progress: the adapter stamps accountSetupProgress on WhiteGloveShellCoreSuccess, only after the tracker has actually read the AccountSetup subcategories (unread is never "untouched"). The new excluder is a subset of the old one, so a verdict can only rise; signals without the key score as before. WhiteGloveShellCoreSuccess v1 gets its replay fixture. The Part-1 exit writes whiteglove.complete as its first step instead of after analyzers, diagnostics upload, grace and drain, where a reseal could kill the process before it. Registration failures: a run that exits because register-session failed records outcome, last error, network link at start and end, the wall-clock attempt window and the config-fetch outcome in registration-failure.json. The emergency break sends the record as AgentErrorReport.PriorRegistrationFailure (null for older agents); the backend bounds it and adds it to the ops details, the App Insights dimensions and the timeline data. Retry behaviour is unchanged. The raw registry trigger whiteglove_signal_* is renamed save_result_step_*: the step exists in every 25H2 enrollment and the old name read as a pre-provisioning signal. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+1
-1
Submodule .claude/commands updated: 88fa1b0274...2ace35ae7d
+1
-1
Submodule internal updated: 9ce17a7ed0...c2cf529c83
+17
@@ -83,6 +83,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
using var f = new Fixture();
|
||||
Assert.False(f.Tracker.HasAccountSetupActivity);
|
||||
Assert.False(f.Tracker.HasAccountSetupProgress);
|
||||
Assert.Null(f.Tracker.AccountSetupProgressIfRead);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -97,6 +98,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
|
||||
Assert.True(f.Tracker.HasAccountSetupActivity);
|
||||
Assert.False(f.Tracker.HasAccountSetupProgress);
|
||||
// Read and untouched — the only state the WhiteGlove sealing gate treats as "false".
|
||||
Assert.False(f.Tracker.AccountSetupProgressIfRead);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -108,6 +111,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
|
||||
Assert.True(f.Tracker.HasAccountSetupActivity);
|
||||
Assert.True(f.Tracker.HasAccountSetupProgress);
|
||||
Assert.True(f.Tracker.AccountSetupProgressIfRead);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -138,5 +142,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
|
||||
Assert.False(f.Tracker.HasAccountSetupProgress);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OnlyDeviceSetupRead_AccountSetupProgressIsUnknown()
|
||||
{
|
||||
// Adversarial for the WhiteGlove sealing gate: the tracker holds DeviceSetup data but
|
||||
// has not read the AccountSetup category — that must be "unknown", never "untouched".
|
||||
using var f = new Fixture();
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status",
|
||||
"{\"categoryState\":\"inProgress\",\"DeviceSetup.AppsSubcategory\":{\"subcategoryState\":\"inProgress\"}}");
|
||||
|
||||
Assert.Null(f.Tracker.AccountSetupProgressIfRead);
|
||||
Assert.False(f.Tracker.HasAccountSetupProgress);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+16
@@ -157,5 +157,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
Assert.Equal(478.2, client.Reports[0].SessionAgeHours);
|
||||
Assert.Null(client.Reports[1].SessionAgeHours);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prior_registration_failure_is_carried_in_the_report_and_absent_by_default()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
var record = new RegistrationFailureSummary { FailedRuns = 2, Outcome = "Failed", NetworkLinkUpAtEnd = false };
|
||||
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m", priorRegistrationFailure: record);
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
|
||||
Assert.Equal(2, client.Reports.Count);
|
||||
Assert.Same(record, client.Reports[0].PriorRegistrationFailure);
|
||||
Assert.Null(client.Reports[1].PriorRegistrationFailure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+83
@@ -118,5 +118,88 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.Throws<ArgumentNullException>(
|
||||
() => BackendSessionRegistration.Register(config, auth, http, "1.0", consoleMode: false, logger: null));
|
||||
}
|
||||
|
||||
// ============================================================ failed-registration record
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_records_a_failed_run_with_its_window_and_bounded_error()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var persistence = new SessionIdPersistence(tmp.Path);
|
||||
var started = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
persistence, SessionRegistrationResult.Failed(new string('e', 1000)),
|
||||
started, started.AddSeconds(1204.46), linkUpAtStart: false, linkUpAtEnd: true,
|
||||
configFetchOutcome: "UsedDefaults", logger: NewLogger(tmp.Path));
|
||||
|
||||
var record = persistence.LoadRegistrationFailure();
|
||||
Assert.NotNull(record);
|
||||
Assert.Equal(1, record!.FailedRuns);
|
||||
Assert.Equal("Failed", record.Outcome);
|
||||
Assert.Equal(BackendSessionRegistration.MaxRecordedErrorLength, record.LastError!.Length);
|
||||
Assert.False(record.NetworkLinkUpAtStart);
|
||||
Assert.True(record.NetworkLinkUpAtEnd);
|
||||
Assert.Equal(1204.5, record.AttemptWindowSeconds);
|
||||
Assert.Equal("UsedDefaults", record.ConfigFetchOutcome);
|
||||
Assert.Equal(started.AddSeconds(1204.46), record.LastFailedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_records_auth_failures_and_clears_on_success()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var persistence = new SessionIdPersistence(tmp.Path);
|
||||
var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
persistence, SessionRegistrationResult.AuthFailed(401, "unauthorized"),
|
||||
at, at.AddSeconds(3), true, true, "Succeeded", NewLogger(tmp.Path));
|
||||
Assert.Equal("AuthFailed", persistence.LoadRegistrationFailure()!.Outcome);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
persistence, SessionRegistrationResult.Succeeded(new RegisterSessionResponse { Success = true }),
|
||||
at, at.AddSeconds(1), true, true, "Succeeded", NewLogger(tmp.Path));
|
||||
Assert.Null(persistence.LoadRegistrationFailure());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_without_persistence_is_a_no_op()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
null, SessionRegistrationResult.Failed("x"), DateTime.UtcNow, DateTime.UtcNow,
|
||||
null, null, null, NewLogger(tmp.Path));
|
||||
}
|
||||
|
||||
// ============================================================ emergency-break message
|
||||
|
||||
[Fact]
|
||||
public void Break_message_names_failed_registrations_in_invariant_culture()
|
||||
{
|
||||
var previous = System.Threading.Thread.CurrentThread.CurrentCulture;
|
||||
try
|
||||
{
|
||||
System.Threading.Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("de-DE");
|
||||
var record = new RegistrationFailureSummary { FailedRuns = 3, Outcome = "Failed", NetworkLinkUpAtEnd = false };
|
||||
|
||||
var message = EmergencyBreakReporter.BuildMessage(478.2, 48, record);
|
||||
|
||||
Assert.Contains("session age 478.2h (cap 48h)", message);
|
||||
Assert.EndsWith(" 3 earlier start(s) failed to register (last: Failed, network link at end: down).", message);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Threading.Thread.CurrentThread.CurrentCulture = previous;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Break_message_without_record_keeps_the_historical_text()
|
||||
{
|
||||
Assert.Equal(
|
||||
"Agent absolute session-age emergency break fired at session age 50.0h (cap 48h) — cleaning up and exiting.",
|
||||
EmergencyBreakReporter.BuildMessage(50.0, 48, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.IO;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
@@ -234,5 +235,78 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.False(File.Exists(Path.Combine(tmp.Path, "session.created")));
|
||||
Assert.False(File.Exists(Path.Combine(tmp.Path, "whiteglove.complete")));
|
||||
}
|
||||
|
||||
// ============================================================ failed-registration record
|
||||
|
||||
private static RegistrationFailureSummary FailedRun(DateTime at, string outcome = "Failed", bool? linkUpAtEnd = false) =>
|
||||
new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = 1,
|
||||
FirstFailedAtUtc = at,
|
||||
LastFailedAtUtc = at,
|
||||
Outcome = outcome,
|
||||
LastError = "No such host is known.",
|
||||
NetworkLinkUpAtStart = false,
|
||||
NetworkLinkUpAtEnd = linkUpAtEnd,
|
||||
AttemptWindowSeconds = 91.3,
|
||||
ConfigFetchOutcome = "UsedDefaults",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationFailure_counts_runs_keeps_the_first_time_and_describes_the_latest_run()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var sut = new SessionIdPersistence(tmp.Path);
|
||||
var first = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
var second = new DateTime(2026, 9, 12, 7, 30, 0, DateTimeKind.Utc);
|
||||
|
||||
Assert.Null(sut.LoadRegistrationFailure());
|
||||
sut.RecordRegistrationFailure(FailedRun(first));
|
||||
sut.RecordRegistrationFailure(FailedRun(second, outcome: "AuthFailed", linkUpAtEnd: true));
|
||||
|
||||
// A fresh instance reads it back from disk, as the next boot's emergency break does.
|
||||
var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailure();
|
||||
Assert.NotNull(record);
|
||||
Assert.Equal(2, record!.FailedRuns);
|
||||
Assert.Equal(first, record.FirstFailedAtUtc);
|
||||
Assert.Equal(second, record.LastFailedAtUtc);
|
||||
Assert.Equal("AuthFailed", record.Outcome);
|
||||
Assert.True(record.NetworkLinkUpAtEnd);
|
||||
Assert.Equal(91.3, record.AttemptWindowSeconds);
|
||||
Assert.Equal("UsedDefaults", record.ConfigFetchOutcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Registration_failure_record_is_cleared_by_success_delete_and_rotate()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var sut = new SessionIdPersistence(tmp.Path);
|
||||
sut.GetOrCreate();
|
||||
var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
sut.RecordRegistrationFailure(FailedRun(at));
|
||||
sut.ClearRegistrationFailure();
|
||||
Assert.Null(sut.LoadRegistrationFailure());
|
||||
|
||||
sut.RecordRegistrationFailure(FailedRun(at));
|
||||
sut.Rotate();
|
||||
Assert.Null(sut.LoadRegistrationFailure());
|
||||
|
||||
sut.RecordRegistrationFailure(FailedRun(at));
|
||||
sut.Delete();
|
||||
Assert.Null(sut.LoadRegistrationFailure());
|
||||
Assert.False(File.Exists(Path.Combine(tmp.Path, "registration-failure.json")));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not json")]
|
||||
[InlineData("{\"FailedRuns\":0}")]
|
||||
public void LoadRegistrationFailure_reports_nothing_for_a_corrupt_or_empty_record(string content)
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
File.WriteAllText(Path.Combine(tmp.Path, "registration-failure.json"), content);
|
||||
|
||||
Assert.Null(new SessionIdPersistence(tmp.Path).LoadRegistrationFailure());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+46
@@ -99,6 +99,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
Assert.Equal(DecisionSignalKind.WhiteGloveShellCoreSuccess, posted.Kind);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, "true")]
|
||||
[InlineData(false, "false")]
|
||||
public void WhiteGloveEvent_stamps_read_AccountSetup_progress_on_payload_and_evidence(bool progress, string expected)
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var adapter = new EspAndHelloTrackerAdapter(
|
||||
f.Coordinator, f.Ingress, f.Clock, accountSetupProgressProbe: () => progress);
|
||||
|
||||
adapter.TriggerWhiteGloveFromTest();
|
||||
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.Equal(expected, posted.Payload![SignalPayloadKeys.AccountSetupProgress]);
|
||||
Assert.Equal(expected, posted.Evidence.DerivationInputs![SignalPayloadKeys.AccountSetupProgress]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhiteGloveEvent_with_unread_AccountSetup_registry_sends_no_progress_key()
|
||||
{
|
||||
// Unknown must keep the sealing classifier's AccountSetup excluder — the payload key
|
||||
// is absent, the evidence records why.
|
||||
using var f = new Fixture();
|
||||
using var adapter = new EspAndHelloTrackerAdapter(
|
||||
f.Coordinator, f.Ingress, f.Clock, accountSetupProgressProbe: () => null);
|
||||
|
||||
adapter.TriggerWhiteGloveFromTest();
|
||||
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.True(posted.Payload == null || !posted.Payload.ContainsKey(SignalPayloadKeys.AccountSetupProgress));
|
||||
Assert.Equal("unread", posted.Evidence.DerivationInputs![SignalPayloadKeys.AccountSetupProgress]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhiteGloveEvent_default_probe_on_unstarted_coordinator_reports_unread()
|
||||
{
|
||||
// Production probe: the provisioning tracker does not exist before Start (and never on
|
||||
// Device Preparation) — that is "unread", not "untouched".
|
||||
using var f = new Fixture();
|
||||
using var adapter = new EspAndHelloTrackerAdapter(f.Coordinator, f.Ingress, f.Clock);
|
||||
|
||||
adapter.TriggerWhiteGloveFromTest();
|
||||
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.True(posted.Payload == null || !posted.Payload.ContainsKey(SignalPayloadKeys.AccountSetupProgress));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspFailureEvent_emits_EspTerminalFailure_with_merged_subSource_annotation()
|
||||
{
|
||||
|
||||
+31
@@ -564,6 +564,37 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.False(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_whiteglove_part1_writes_marker_before_the_later_exit_steps()
|
||||
{
|
||||
// The technician's reseal can kill the process anywhere after the seal. The marker
|
||||
// is what lets the next boot skip the 48-hour break and resume Part 2, so it must
|
||||
// exist before collectors stop, diagnostics upload, grace and drain run.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.WhiteGloveSealed }.Build();
|
||||
bool? markerPresentAtCollectorStop = null;
|
||||
rig.StopPeripheralCollectorsHook = () => markerPresentAtCollectorStop = rig.SessionPersistence.IsWhiteGloveResume();
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed));
|
||||
|
||||
Assert.True(markerPresentAtCollectorStop, "whiteglove.complete must be written before the peripheral collectors are stopped.");
|
||||
var emitted = rig.EmittedEventTypes.ToList();
|
||||
Assert.True(emitted.IndexOf("agent_shutting_down") >= 0, "agent_shutting_down must still be emitted.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_non_whiteglove_termination_never_writes_the_marker()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build();
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed));
|
||||
|
||||
Assert.False(rig.SessionPersistence.IsWhiteGloveResume());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_whiteglove_part1_stops_collectors_before_emitting_part1_complete()
|
||||
{
|
||||
|
||||
+7
@@ -216,6 +216,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// </summary>
|
||||
public bool HasAccountSetupProgress => _provisioningTracker?.HasAccountSetupProgress ?? false;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ProvisioningStatusTracker.AccountSetupProgressIfRead"/>; <c>null</c> also when
|
||||
/// the provisioning tracker is not running (Device Preparation, before Start). Stamped on
|
||||
/// the WhiteGloveShellCoreSuccess signal for the sealing classifier.
|
||||
/// </summary>
|
||||
public bool? AccountSetupProgressIfRead => _provisioningTracker?.AccountSetupProgressIfRead;
|
||||
|
||||
/// <summary>
|
||||
/// True once WhiteGlove start has been detected (EventID 509 or persisted from prior run).
|
||||
/// Forwarded from ModernDeploymentTracker.
|
||||
|
||||
+19
-6
@@ -254,16 +254,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// host to tell the real user-ESP page exit from the Device→Account handoff exit; not a
|
||||
/// completion input.
|
||||
/// </summary>
|
||||
public bool HasAccountSetupProgress
|
||||
public bool HasAccountSetupProgress => AccountSetupProgressIfRead == true;
|
||||
|
||||
/// <summary>
|
||||
/// Tri-state form of <see cref="HasAccountSetupProgress"/>: <c>null</c> until the tracker
|
||||
/// has actually read AccountSetup subcategories (not started yet, category not written,
|
||||
/// empty), otherwise whether one has left <c>notStarted</c>. The WhiteGlove sealing
|
||||
/// classifier treats only a read <c>false</c> as "user ESP untouched" — an unread
|
||||
/// registry must never pass for an untouched one.
|
||||
/// </summary>
|
||||
public bool? AccountSetupProgressIfRead
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_stateLock)
|
||||
{
|
||||
if (_lastSubcategoryStates == null)
|
||||
return false;
|
||||
if (!_lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs) || subs == null)
|
||||
return false;
|
||||
return null;
|
||||
if (!_lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs)
|
||||
|| subs == null
|
||||
|| subs.Count == 0)
|
||||
return null;
|
||||
foreach (var state in subs.Values)
|
||||
{
|
||||
if (!string.Equals(state, "notStarted", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -279,7 +290,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// with subcategoryState=succeeded. NOT a WhiteGlove indicator: on Win11 25H2 (build
|
||||
/// 26200+) Windows writes this property as a trivially-succeeding step in EVERY
|
||||
/// enrollment (validated 2026-08-18 — user-driven sessions f475e697/c601a24b/4219203f
|
||||
/// all carry it). Kept for observability only (whiteglove_signal_* raw dumps); must not
|
||||
/// all carry it). Kept for observability only (save_result_step_* raw dumps); must not
|
||||
/// be used for classification. Reliable WhiteGlove anchors are ModernDeployment
|
||||
/// EventID 509 and the Shell-Core WhiteGlove-success events.
|
||||
/// </summary>
|
||||
@@ -966,8 +977,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_logger.Info($"ProvisioningStatusTracker: SaveWhiteGloveSuccessResult state " +
|
||||
$"transition: {previousState} -> {state}");
|
||||
|
||||
// Neutral trigger name: the step exists in every 25H2 enrollment,
|
||||
// and "whiteglove_*" was read as a pre-provisioning signal.
|
||||
EmitRawRegistryDump(categoryLabel, jsonValue,
|
||||
$"whiteglove_signal_{state}");
|
||||
$"save_result_step_{state}");
|
||||
|
||||
if (string.Equals(state, "succeeded", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
|
||||
@@ -81,7 +81,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// multiply against the per-session budget.
|
||||
///
|
||||
/// <paramref name="sessionAgeHours"/> is the emergency break's measured session age
|
||||
/// (<see cref="AgentErrorReport.SessionAgeHours"/>); null for every other error type.
|
||||
/// (<see cref="AgentErrorReport.SessionAgeHours"/>) and <paramref name="priorRegistrationFailure"/>
|
||||
/// its record of earlier starts that never registered
|
||||
/// (<see cref="AgentErrorReport.PriorRegistrationFailure"/>); null for every other error type.
|
||||
/// </summary>
|
||||
public virtual async Task TrySendAsync(
|
||||
AgentErrorType errorType,
|
||||
@@ -91,7 +93,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
int attempts = 1,
|
||||
TimeSpan? perAttemptTimeout = null,
|
||||
TimeSpan? retryDelay = null,
|
||||
double? sessionAgeHours = null)
|
||||
double? sessionAgeHours = null,
|
||||
RegistrationFailureSummary priorRegistrationFailure = null)
|
||||
{
|
||||
// Deduplicate by error type + status code: same failure category sent only once per session.
|
||||
// All anti-flood checks are protected by a lock to prevent race conditions when
|
||||
@@ -140,6 +143,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
HttpStatusCode = httpStatusCode,
|
||||
SequenceNumber = sequenceNumber,
|
||||
SessionAgeHours = sessionAgeHours,
|
||||
PriorRegistrationFailure = priorRegistrationFailure,
|
||||
AgentVersion = _agentVersion,
|
||||
Timestamp = DateTime.UtcNow
|
||||
};
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
@@ -22,6 +24,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
private readonly string _sessionFilePath;
|
||||
private readonly string _sessionCreatedFilePath;
|
||||
private readonly string _whiteGloveMarkerPath;
|
||||
private readonly string _registrationFailurePath;
|
||||
private readonly object _lockObject = new object();
|
||||
|
||||
public SessionIdPersistence(string dataDirectory)
|
||||
@@ -35,6 +38,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
_sessionFilePath = Path.Combine(dataDirectory, "session.id");
|
||||
_sessionCreatedFilePath = Path.Combine(dataDirectory, "session.created");
|
||||
_whiteGloveMarkerPath = Path.Combine(dataDirectory, "whiteglove.complete");
|
||||
_registrationFailurePath = Path.Combine(dataDirectory, "registration-failure.json");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -164,6 +168,55 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
lock (_lockObject) TryDelete(_whiteGloveMarkerPath, logger);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Folds one failed registration run into <c>registration-failure.json</c>: the run
|
||||
/// counter grows, the first failure time is kept, everything else describes the latest
|
||||
/// run. A run that never registers exits and leaves nothing else behind — the emergency
|
||||
/// break reads this file so the backend learns why the session never appeared.
|
||||
/// Best-effort; never throws.
|
||||
/// </summary>
|
||||
public void RecordRegistrationFailure(RegistrationFailureSummary latestRun, AgentLogger logger = null)
|
||||
{
|
||||
if (latestRun == null) return;
|
||||
lock (_lockObject)
|
||||
{
|
||||
try
|
||||
{
|
||||
var previous = LoadRegistrationFailureInternal();
|
||||
var merged = new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = (previous?.FailedRuns ?? 0) + 1,
|
||||
FirstFailedAtUtc = previous?.FirstFailedAtUtc ?? latestRun.LastFailedAtUtc,
|
||||
LastFailedAtUtc = latestRun.LastFailedAtUtc,
|
||||
Outcome = latestRun.Outcome,
|
||||
LastError = latestRun.LastError,
|
||||
NetworkLinkUpAtStart = latestRun.NetworkLinkUpAtStart,
|
||||
NetworkLinkUpAtEnd = latestRun.NetworkLinkUpAtEnd,
|
||||
AttemptWindowSeconds = latestRun.AttemptWindowSeconds,
|
||||
ConfigFetchOutcome = latestRun.ConfigFetchOutcome,
|
||||
};
|
||||
File.WriteAllText(_registrationFailurePath, JsonConvert.SerializeObject(merged));
|
||||
logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns}.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"SessionIdPersistence: RecordRegistrationFailure failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The failed-registration record since the last success, or <c>null</c> (none, unreadable).</summary>
|
||||
public RegistrationFailureSummary LoadRegistrationFailure()
|
||||
{
|
||||
lock (_lockObject) return LoadRegistrationFailureInternal();
|
||||
}
|
||||
|
||||
/// <summary>Drops the failed-registration record — called once registration succeeds.</summary>
|
||||
public void ClearRegistrationFailure(AgentLogger logger = null)
|
||||
{
|
||||
lock (_lockObject) TryDelete(_registrationFailurePath, logger);
|
||||
}
|
||||
|
||||
/// <summary>Deletes the persisted SessionId. Next <see cref="GetOrCreate"/> starts a fresh session.</summary>
|
||||
public void Delete(AgentLogger logger = null)
|
||||
{
|
||||
@@ -172,6 +225,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
TryDelete(_sessionFilePath, logger);
|
||||
TryDelete(_sessionCreatedFilePath, logger);
|
||||
TryDelete(_whiteGloveMarkerPath, logger);
|
||||
TryDelete(_registrationFailurePath, logger);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,11 +245,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
var fresh = Guid.NewGuid().ToString();
|
||||
WriteAtomic(fresh, logger);
|
||||
SaveSessionCreatedAtInternal(DateTime.UtcNow);
|
||||
// Earlier failed runs belonged to the refused session id.
|
||||
TryDelete(_registrationFailurePath, logger);
|
||||
logger?.Info($"SessionIdPersistence: rotated SessionId to {fresh}.");
|
||||
return fresh;
|
||||
}
|
||||
}
|
||||
|
||||
private RegistrationFailureSummary LoadRegistrationFailureInternal()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(_registrationFailurePath)) return null;
|
||||
var record = JsonConvert.DeserializeObject<RegistrationFailureSummary>(File.ReadAllText(_registrationFailurePath));
|
||||
return record != null && record.FailedRuns > 0 ? record : null;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Corrupt record: report nothing rather than a half-read account.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path, AgentLogger logger)
|
||||
{
|
||||
try { if (File.Exists(path)) File.Delete(path); }
|
||||
|
||||
+24
-3
@@ -24,7 +24,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
/// <list type="bullet">
|
||||
/// <item><c>HelloCompleted</c> → <see cref="DecisionSignalKind.HelloResolved"/></item>
|
||||
/// <item><c>FinalizingSetupPhaseTriggered</c> → <see cref="DecisionSignalKind.EspPhaseChanged"/> (phase=FinalizingSetup)</item>
|
||||
/// <item><c>WhiteGloveCompleted</c> → <see cref="DecisionSignalKind.WhiteGloveShellCoreSuccess"/></item>
|
||||
/// <item><c>WhiteGloveCompleted</c> → <see cref="DecisionSignalKind.WhiteGloveShellCoreSuccess"/> (payload: registry AccountSetup progress at that moment)</item>
|
||||
/// <item><c>EspFailureDetected</c> → <see cref="DecisionSignalKind.EspTerminalFailure"/> (merged aus ShellCore + Provisioning)</item>
|
||||
/// <item><c>DeviceSetupProvisioningComplete</c> → <see cref="DecisionSignalKind.DeviceSetupProvisioningComplete"/></item>
|
||||
/// <item><c>AccountSetupProvisioningComplete</c> → <see cref="DecisionSignalKind.AccountSetupProvisioningComplete"/> (session 330f73f3 fix — strong AccountSetup-done gate)</item>
|
||||
@@ -46,6 +46,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
private readonly EspAndHelloTracker _coordinator;
|
||||
private readonly ISignalIngressSink _ingress;
|
||||
private readonly IClock _clock;
|
||||
private readonly Func<bool?> _accountSetupProgressProbe;
|
||||
|
||||
private bool _helloPosted;
|
||||
private bool _helloWizardStartedPosted;
|
||||
@@ -60,11 +61,14 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
public EspAndHelloTrackerAdapter(
|
||||
EspAndHelloTracker coordinator,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock)
|
||||
IClock clock,
|
||||
Func<bool?>? accountSetupProgressProbe = null)
|
||||
{
|
||||
_coordinator = coordinator ?? throw new ArgumentNullException(nameof(coordinator));
|
||||
_ingress = ingress ?? throw new ArgumentNullException(nameof(ingress));
|
||||
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
// Test seam; production reads the provisioning tracker's registry-derived flag.
|
||||
_accountSetupProgressProbe = accountSetupProgressProbe ?? (() => _coordinator.AccountSetupProgressIfRead);
|
||||
|
||||
_coordinator.HelloCompleted += OnHelloCompleted;
|
||||
_coordinator.FinalizingSetupPhaseTriggered += OnFinalizing;
|
||||
@@ -216,13 +220,29 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
if (_whiteGloveSuccessPosted) return;
|
||||
_whiteGloveSuccessPosted = true;
|
||||
|
||||
// Registry AccountSetup progress at the moment of the success page. The sealing
|
||||
// classifier lifts its AccountSetup excluder only on an explicit "false" — IME logs
|
||||
// AccountSetup from the device session once the device apps are done, which is no
|
||||
// user activity. Sampled here so replay reads it from the SignalLog (L.2/L.5). An
|
||||
// unread registry sends no key: unknown must keep the excluder, never lift it.
|
||||
var progress = _accountSetupProgressProbe();
|
||||
var progressText = progress.HasValue ? (progress.Value ? "true" : "false") : null;
|
||||
|
||||
var now = ResolveOccurredAt(out var derivedFromClock);
|
||||
var derivationInputs = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["subSource"] = "ShellCoreTracker",
|
||||
[SignalPayloadKeys.AccountSetupProgress] = progressText ?? "unread",
|
||||
};
|
||||
TagDerivedTimestamp(derivationInputs, derivedFromClock);
|
||||
|
||||
var payload = progressText == null
|
||||
? null
|
||||
: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
[SignalPayloadKeys.AccountSetupProgress] = progressText,
|
||||
};
|
||||
|
||||
_ingress.Post(
|
||||
kind: DecisionSignalKind.WhiteGloveShellCoreSuccess,
|
||||
occurredAtUtc: now,
|
||||
@@ -231,7 +251,8 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
kind: EvidenceKind.Derived,
|
||||
identifier: DetectorId,
|
||||
summary: "WhiteGlove sealing success (coordinator-forwarded)",
|
||||
derivationInputs: derivationInputs));
|
||||
derivationInputs: derivationInputs),
|
||||
payload: payload);
|
||||
}
|
||||
|
||||
private void EmitEspFailure(EspFailureDetectedEventArgs args)
|
||||
|
||||
+11
-2
@@ -169,6 +169,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
_logger.Info(
|
||||
$"EnrollmentTerminationHandler: handling Terminated (reason={args.Reason}, outcome={args.Outcome}, stage={args.StageName}).");
|
||||
|
||||
// The seal is already persisted in the decision snapshot. The marker goes down
|
||||
// before anything else: the technician's reseal can kill this process at any
|
||||
// later step, and a missing marker makes the next boot run the 48-hour break
|
||||
// against the Part-1 session age instead of resuming Part 2.
|
||||
if (isWhiteGlovePart1)
|
||||
{
|
||||
TrySaveWhiteGloveComplete();
|
||||
}
|
||||
|
||||
var state = TryGetCurrentState();
|
||||
|
||||
// M4.6.δ — shutdown analyzers run BEFORE the dialog / diagnostics upload so
|
||||
@@ -254,7 +263,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
|
||||
// WhiteGlove Part-1 exit: keep the session alive, but announce the handoff so
|
||||
// the timeline clearly marks the transition. The `whiteglove.complete` marker
|
||||
// lets the next agent boot classify itself as a Part-2 resume.
|
||||
// (written at the top of this handler) lets the next agent boot classify itself
|
||||
// as a Part-2 resume.
|
||||
if (isWhiteGlovePart1)
|
||||
{
|
||||
// Stop peripheral collectors BEFORE the Part-1-complete marker so their
|
||||
@@ -298,7 +308,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
DelayLateEventGrace();
|
||||
DrainSpool();
|
||||
|
||||
TrySaveWhiteGloveComplete();
|
||||
// Option 2 (WG Part 1 graceful-exit hardening, 2026-04-30): write the
|
||||
// clean-exit marker BEFORE _signalShutdown returns control to the main
|
||||
// thread. The AppDomain.ProcessExit handler still writes it as a second
|
||||
|
||||
@@ -273,7 +273,9 @@ namespace AutopilotMonitor.Agent.V2
|
||||
var registration = Runtime.BackendSessionRegistration.Register(
|
||||
bootstrap.AgentConfig, auth, telemetry.MtlsHttpClient, GetAgentVersion(), consoleMode, logger,
|
||||
rotateSession: () => Runtime.BackendSessionRegistration.RotateSession(
|
||||
bootstrap.AgentConfig, bootstrap.SessionPersistence, auth, transportDir, logger));
|
||||
bootstrap.AgentConfig, bootstrap.SessionPersistence, auth, transportDir, logger),
|
||||
sessionPersistence: bootstrap.SessionPersistence,
|
||||
configFetchOutcome: runtimeConfig.RemoteConfigService.LastFetchOutcome.ToString());
|
||||
if (registration.ShouldExit)
|
||||
{
|
||||
return registration.ExitCode;
|
||||
|
||||
@@ -87,8 +87,11 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
cleanupServiceFactory, logger, consoleMode,
|
||||
// Best-effort: surface the otherwise-silent 48h break to the backend before cleanup
|
||||
// (tasks/enrollment-status-reclassification.md). Never blocks the exit.
|
||||
// The failed-registration record is read here, before the break deletes the
|
||||
// session files.
|
||||
onBreakFired: sessionAgeHours => EmergencyBreakReporter.TrySend(
|
||||
agentConfig, Program.GetAgentVersion(), sessionAgeHours, logger)))
|
||||
agentConfig, Program.GetAgentVersion(), sessionAgeHours,
|
||||
sessionPersistence.LoadRegistrationFailure(), logger)))
|
||||
{
|
||||
logger.Info("Emergency break fired — agent exiting.");
|
||||
return BootstrapResult.Exit(0);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
@@ -17,7 +18,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// <c>IncrementSessionEventCountAsync</c> / <c>UpdateSessionStatusAsync</c> silently
|
||||
/// no-op and session status / phase / admin-overrides / validator reconcile all break.
|
||||
/// On registration failure we follow V1's rule: collectors MUST NOT start and the
|
||||
/// agent exits cleanly so the next Scheduled-Task tick can retry.
|
||||
/// agent exits cleanly; the Scheduled Task has only a boot trigger, so the next attempt is
|
||||
/// the next boot. The failed run is recorded for the emergency break report.
|
||||
/// </summary>
|
||||
internal static class BackendSessionRegistration
|
||||
{
|
||||
@@ -28,12 +30,19 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
string agentVersion,
|
||||
bool consoleMode,
|
||||
AgentLogger logger,
|
||||
Func<string> rotateSession = null)
|
||||
Func<string> rotateSession = null,
|
||||
SessionIdPersistence sessionPersistence = null,
|
||||
string configFetchOutcome = null)
|
||||
{
|
||||
if (agentConfig == null) throw new ArgumentNullException(nameof(agentConfig));
|
||||
if (auth == null) throw new ArgumentNullException(nameof(auth));
|
||||
if (logger == null) throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
// Wall clock on purpose: a window far above the retry schedule is the evidence that
|
||||
// the device slept or was suspended while this run tried to register.
|
||||
var startedUtc = DateTime.UtcNow;
|
||||
var linkUpAtStart = TryIsNetworkLinkUp();
|
||||
|
||||
var registrationResult = SessionRegistrationHelper.RegisterWithRetryAsync(
|
||||
apiClient: auth.BackendApiClient,
|
||||
agentConfig: agentConfig,
|
||||
@@ -48,6 +57,10 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
rotateSession: rotateSession)
|
||||
.GetAwaiter().GetResult();
|
||||
|
||||
RecordRegistrationRun(
|
||||
sessionPersistence, registrationResult, startedUtc, DateTime.UtcNow,
|
||||
linkUpAtStart, TryIsNetworkLinkUp(), configFetchOutcome, logger);
|
||||
|
||||
if (registrationResult.Outcome != SessionRegistrationOutcome.Succeeded)
|
||||
{
|
||||
logger.Error(
|
||||
@@ -65,6 +78,62 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
return SessionRegistrationOutcomeResult.Continue(registrationResult);
|
||||
}
|
||||
|
||||
/// <summary>Longest <see cref="RegistrationFailureSummary.LastError"/> kept on disk and on the wire.</summary>
|
||||
internal const int MaxRecordedErrorLength = 256;
|
||||
|
||||
/// <summary>
|
||||
/// Keeps <c>registration-failure.json</c> in step with this run: a failed run is folded
|
||||
/// into the record (the emergency break later reports it — for a session that never
|
||||
/// registered, the only account of why), a successful one clears it. Never throws.
|
||||
/// </summary>
|
||||
internal static void RecordRegistrationRun(
|
||||
SessionIdPersistence sessionPersistence,
|
||||
SessionRegistrationResult registrationResult,
|
||||
DateTime startedUtc,
|
||||
DateTime endedUtc,
|
||||
bool? linkUpAtStart,
|
||||
bool? linkUpAtEnd,
|
||||
string configFetchOutcome,
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (sessionPersistence == null || registrationResult == null) return;
|
||||
try
|
||||
{
|
||||
if (registrationResult.Outcome == SessionRegistrationOutcome.Succeeded)
|
||||
{
|
||||
sessionPersistence.ClearRegistrationFailure(logger);
|
||||
return;
|
||||
}
|
||||
|
||||
var error = registrationResult.ErrorMessage;
|
||||
if (error != null && error.Length > MaxRecordedErrorLength)
|
||||
error = error.Substring(0, MaxRecordedErrorLength);
|
||||
|
||||
sessionPersistence.RecordRegistrationFailure(new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = 1,
|
||||
FirstFailedAtUtc = endedUtc,
|
||||
LastFailedAtUtc = endedUtc,
|
||||
Outcome = registrationResult.Outcome.ToString(),
|
||||
LastError = error,
|
||||
NetworkLinkUpAtStart = linkUpAtStart,
|
||||
NetworkLinkUpAtEnd = linkUpAtEnd,
|
||||
AttemptWindowSeconds = Math.Round(Math.Max(0, (endedUtc - startedUtc).TotalSeconds), 1),
|
||||
ConfigFetchOutcome = configFetchOutcome,
|
||||
}, logger);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"Registration run record failed (best-effort): {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static bool? TryIsNetworkLinkUp()
|
||||
{
|
||||
try { return NetworkInterface.GetIsNetworkAvailable(); }
|
||||
catch { return null; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SESSION-OWNER-BINDING: the one place that turns "backend says this session id is not
|
||||
/// ours" into a new session. Rotates the persisted id, points the runtime config at it,
|
||||
|
||||
@@ -36,18 +36,20 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// Upper bound for the blocking wait: NIC wait + 3×15s attempts + 2×5s delays + slack.
|
||||
private static readonly TimeSpan OverallBudget = TimeSpan.FromSeconds(75);
|
||||
|
||||
public static void TrySend(AgentConfiguration agentConfig, string agentVersion, double sessionAgeHours, AgentLogger logger)
|
||||
public static void TrySend(
|
||||
AgentConfiguration agentConfig,
|
||||
string agentVersion,
|
||||
double sessionAgeHours,
|
||||
RegistrationFailureSummary priorRegistrationFailure,
|
||||
AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
WaitForNetwork(logger);
|
||||
|
||||
var auth = BackendClientFactory.BuildAuthClients(agentConfig, agentVersion, logger);
|
||||
// Invariant: the text lands in the backend's ops feed and timeline; a device locale
|
||||
// would otherwise print "478,2h".
|
||||
var roundedAgeHours = Math.Round(sessionAgeHours, 1);
|
||||
var message = FormattableString.Invariant(
|
||||
$"Agent absolute session-age emergency break fired at session age {roundedAgeHours:F1}h (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting.");
|
||||
var message = BuildMessage(roundedAgeHours, agentConfig.AbsoluteMaxSessionHours, priorRegistrationFailure);
|
||||
|
||||
// The process is about to exit, so block on the otherwise fire-and-forget send
|
||||
// rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP
|
||||
@@ -60,7 +62,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
attempts: SendAttempts,
|
||||
perAttemptTimeout: PerAttemptTimeout,
|
||||
retryDelay: RetryDelay,
|
||||
sessionAgeHours: roundedAgeHours)
|
||||
sessionAgeHours: roundedAgeHours,
|
||||
priorRegistrationFailure: priorRegistrationFailure)
|
||||
.Wait(OverallBudget);
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -69,6 +72,25 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invariant culture: the text lands in the backend's ops feed and timeline, and a device
|
||||
/// locale would otherwise print "478,2h". The registration clause names what the ops feed
|
||||
/// cannot otherwise show for a session that never registered.
|
||||
/// </summary>
|
||||
internal static string BuildMessage(double roundedAgeHours, int capHours, RegistrationFailureSummary priorRegistrationFailure)
|
||||
{
|
||||
var message = FormattableString.Invariant(
|
||||
$"Agent absolute session-age emergency break fired at session age {roundedAgeHours:F1}h (cap {capHours}h) — cleaning up and exiting.");
|
||||
if (priorRegistrationFailure != null && priorRegistrationFailure.FailedRuns > 0)
|
||||
{
|
||||
message += FormattableString.Invariant(
|
||||
$" {priorRegistrationFailure.FailedRuns} earlier start(s) failed to register (last: {priorRegistrationFailure.Outcome}, network link at end: {FormatLink(priorRegistrationFailure.NetworkLinkUpAtEnd)}).");
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
private static string FormatLink(bool? up) => up == null ? "unknown" : (up.Value ? "up" : "down");
|
||||
|
||||
/// <summary>
|
||||
/// Boot-time NIC grace: the break fires during bootstrap, often seconds after boot while
|
||||
/// Wi-Fi is still associating. Polls the cheap link-level signal — it cannot prove backend
|
||||
|
||||
@@ -522,7 +522,8 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
.WithSkipUserEsp(value: true, sourceSignalOrdinal: 4)
|
||||
.WithEspSyncFailureTimeoutMinutes(value: 90, sourceSignalOrdinal: 5)
|
||||
.WithRegistrySelfDeployingProfile(value: false, sourceSignalOrdinal: 6)
|
||||
.WithCloudPc(value: true, sourceSignalOrdinal: 7))
|
||||
.WithCloudPc(value: true, sourceSignalOrdinal: 7)
|
||||
.WithAccountSetupProgressAtWhiteGloveSuccess(value: false, sourceSignalOrdinal: 8))
|
||||
.Build();
|
||||
|
||||
var json = StateSerializer.Serialize(state);
|
||||
@@ -538,6 +539,8 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
Assert.Equal(6, o.RegistrySelfDeployingProfile!.SourceSignalOrdinal);
|
||||
Assert.True(o.CloudPc!.Value);
|
||||
Assert.Equal(7, o.CloudPc!.SourceSignalOrdinal);
|
||||
Assert.False(o.AccountSetupProgressAtWhiteGloveSuccess!.Value);
|
||||
Assert.Equal(8, o.AccountSetupProgressAtWhiteGloveSuccess!.SourceSignalOrdinal);
|
||||
|
||||
// Never-observed facts must rehydrate as null (null ≠ veto — session 62e603c9).
|
||||
Assert.Null(o.WhiteGloveSealingPatternSeen);
|
||||
|
||||
@@ -31,7 +31,6 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
DecisionSignalKind.HelloResolved,
|
||||
DecisionSignalKind.ImeUserSessionCompleted,
|
||||
DecisionSignalKind.DeviceSetupProvisioningComplete,
|
||||
DecisionSignalKind.WhiteGloveShellCoreSuccess,
|
||||
DecisionSignalKind.WhiteGloveSealingPatternDetected,
|
||||
DecisionSignalKind.AadUserJoinedLate,
|
||||
DecisionSignalKind.DeviceInfoCollected,
|
||||
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.DecisionCore.Classifiers;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Serialization;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.DecisionCore.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// The sealing classifier's AccountSetup excluder (-40) applies only when the IME reported
|
||||
/// the AccountSetup phase AND the WhiteGlove_Success signal did not report an untouched
|
||||
/// AccountSetup registry. IME logs "EspPhase: AccountSetup" from the device session once
|
||||
/// the device apps are done — on pre-provisioning with the user ESP enabled that line
|
||||
/// routinely precedes the technician's success page and used to leave Part 1 unsealed.
|
||||
/// The gate may only ever lift the excluder, never add one.
|
||||
/// </summary>
|
||||
public sealed class WhiteGloveAccountSetupProgressGateTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 9, 29, 7, 25, 9, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public void Ime_account_setup_with_untouched_registry_seals_on_the_fast_path()
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
|
||||
|
||||
Assert.Equal(SessionStage.WhiteGloveSealed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.WhiteGlovePart1Sealed, step.NewState.Outcome);
|
||||
Assert.Equal(HypothesisLevel.Confirmed, step.NewState.ClassifierOutcomes.WhiteGloveSealing.Level);
|
||||
Assert.Equal(WhiteGloveSealingClassifier.WeightShellCoreWhiteGloveSuccess,
|
||||
step.NewState.ClassifierOutcomes.WhiteGloveSealing.Score);
|
||||
|
||||
var fact = step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess;
|
||||
Assert.NotNull(fact);
|
||||
Assert.False(fact!.Value);
|
||||
Assert.Equal(3, fact.SourceSignalOrdinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ime_account_setup_with_registry_progress_keeps_the_excluder()
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "true"));
|
||||
|
||||
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
|
||||
Assert.Contains(step.Effects, e => e.Kind == DecisionEffectKind.RunClassifier);
|
||||
Assert.True(DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState).HasAccountSetupActivity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Missing_progress_payload_keeps_the_excluder_for_older_agents()
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: null));
|
||||
|
||||
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
|
||||
Assert.Null(step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess);
|
||||
Assert.True(DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState).HasAccountSetupActivity);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData("yes")]
|
||||
public void Unparseable_progress_payload_is_treated_as_missing(string raw)
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: raw));
|
||||
|
||||
Assert.Null(step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess);
|
||||
Assert.NotEqual(SessionStage.WhiteGloveSealed, step.NewState.Stage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Registry_progress_without_ime_account_setup_adds_no_excluder()
|
||||
{
|
||||
var state = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
|
||||
.WithStage(SessionStage.EspDeviceSetup)
|
||||
.WithStepIndex(1)
|
||||
.WithLastAppliedSignalOrdinal(0)
|
||||
.Build();
|
||||
|
||||
var step = new DecisionEngine().Reduce(state, MakeShellCoreSignal(ordinal: 1, accountSetupProgress: "true"));
|
||||
|
||||
Assert.Equal(SessionStage.WhiteGloveSealed, step.NewState.Stage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Untouched_registry_does_not_lift_a_hard_excluder()
|
||||
{
|
||||
var builder = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
|
||||
.WithStage(SessionStage.EspAccountSetup)
|
||||
.WithStepIndex(3)
|
||||
.WithLastAppliedSignalOrdinal(2);
|
||||
builder.AccountSetupEnteredUtc = new SignalFact<DateTime>(T0.AddMinutes(-3), 1);
|
||||
builder.DesktopArrivedUtc = new SignalFact<DateTime>(T0.AddMinutes(-1), 2);
|
||||
|
||||
var step = new DecisionEngine().Reduce(builder.Build(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
|
||||
|
||||
Assert.NotEqual(SessionStage.WhiteGloveSealed, step.NewState.Stage);
|
||||
var snapshot = DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState);
|
||||
Assert.False(snapshot.HasAccountSetupActivity);
|
||||
Assert.True(snapshot.DesktopArrived);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Progress_fact_survives_the_state_serializer()
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
|
||||
|
||||
var roundTripped = StateSerializer.Deserialize(StateSerializer.Serialize(step.NewState));
|
||||
|
||||
var fact = roundTripped.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess;
|
||||
Assert.NotNull(fact);
|
||||
Assert.False(fact!.Value);
|
||||
Assert.Equal(3, fact.SourceSignalOrdinal);
|
||||
}
|
||||
|
||||
private static DecisionState StateWithImeAccountSetup()
|
||||
{
|
||||
// The IME device-session line: AccountSetup entered before the success page.
|
||||
var builder = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
|
||||
.WithStage(SessionStage.EspAccountSetup)
|
||||
.WithStepIndex(3)
|
||||
.WithLastAppliedSignalOrdinal(2);
|
||||
builder.AccountSetupEnteredUtc = new SignalFact<DateTime>(T0.AddMinutes(-3), 2);
|
||||
return builder.Build();
|
||||
}
|
||||
|
||||
private static DecisionSignal MakeShellCoreSignal(long ordinal, string? accountSetupProgress)
|
||||
{
|
||||
var payload = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
if (accountSetupProgress != null)
|
||||
{
|
||||
payload[SignalPayloadKeys.AccountSetupProgress] = accountSetupProgress;
|
||||
}
|
||||
|
||||
return new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: DecisionSignalKind.WhiteGloveShellCoreSuccess,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: T0,
|
||||
sourceOrigin: "EspAndHelloTracker",
|
||||
evidence: new Evidence(
|
||||
kind: EvidenceKind.Raw,
|
||||
identifier: "ShellCore-62407",
|
||||
summary: "WhiteGlove_Success"),
|
||||
payload: payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -330,6 +330,95 @@ public class ReportAgentErrorFunctionTests
|
||||
It.Is<List<EnrollmentEvent>>(e => (double)e[0].Data["sessionAgeHours"] == 478.2)), Times.Once);
|
||||
}
|
||||
|
||||
// A session that never registered leaves the backend nothing but this report; the agent's
|
||||
// record of its failed starts is the only account of why.
|
||||
[Fact]
|
||||
public async Task Prior_registration_failure_lands_in_ops_details_even_without_a_row()
|
||||
{
|
||||
var h = new Harness();
|
||||
var report = Report(Now);
|
||||
report.PriorRegistrationFailure = ReportAgentErrorFunction.SanitizeRegistrationFailure(Failure());
|
||||
|
||||
await h.RunAsync(report, Read(null));
|
||||
|
||||
var details = Assert.Single(h.OpsEvents).Details;
|
||||
Assert.Contains("\"priorRegistrationFailure\":{\"failedRuns\":3", details);
|
||||
Assert.Contains("\"networkLinkUpAtEnd\":false", details);
|
||||
Assert.Contains("\"attemptWindowSeconds\":1204.5", details);
|
||||
Assert.Contains("\"configFetchOutcome\":\"UsedDefaults\"", details);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prior_registration_failure_lands_in_the_timeline_data_of_a_registered_session()
|
||||
{
|
||||
var h = new Harness();
|
||||
var report = Report(Now);
|
||||
report.PriorRegistrationFailure = ReportAgentErrorFunction.SanitizeRegistrationFailure(Failure());
|
||||
|
||||
await h.RunAsync(report);
|
||||
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e =>
|
||||
((Dictionary<string, object?>)e[0].Data["priorRegistrationFailure"])["failedRuns"]!.Equals(3))), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_without_registration_record_leaves_it_null_and_out_of_the_timeline()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
Assert.Contains("\"priorRegistrationFailure\":null", Assert.Single(h.OpsEvents).Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => !e[0].Data.ContainsKey("priorRegistrationFailure"))), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SanitizeRegistrationFailure_bounds_device_input()
|
||||
{
|
||||
var raw = Failure();
|
||||
raw.LastError = new string('x', 5000);
|
||||
raw.Outcome = new string('o', 500);
|
||||
raw.ConfigFetchOutcome = new string('c', 500);
|
||||
raw.AttemptWindowSeconds = -3;
|
||||
|
||||
var clean = ReportAgentErrorFunction.SanitizeRegistrationFailure(raw)!;
|
||||
|
||||
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationErrorLength, clean.LastError!.Length);
|
||||
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationTokenLength, clean.Outcome.Length);
|
||||
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationTokenLength, clean.ConfigFetchOutcome!.Length);
|
||||
Assert.Equal(0, clean.AttemptWindowSeconds);
|
||||
Assert.Equal(3, clean.FailedRuns);
|
||||
Assert.NotSame(raw, clean);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(-1)]
|
||||
public void SanitizeRegistrationFailure_drops_a_record_without_failed_runs(int failedRuns)
|
||||
{
|
||||
var raw = Failure();
|
||||
raw.FailedRuns = failedRuns;
|
||||
|
||||
Assert.Null(ReportAgentErrorFunction.SanitizeRegistrationFailure(raw));
|
||||
Assert.Null(ReportAgentErrorFunction.SanitizeRegistrationFailure(null));
|
||||
}
|
||||
|
||||
private static RegistrationFailureSummary Failure() => new()
|
||||
{
|
||||
FailedRuns = 3,
|
||||
FirstFailedAtUtc = Now.AddDays(-20),
|
||||
LastFailedAtUtc = Now.AddDays(-19),
|
||||
Outcome = "Failed",
|
||||
LastError = "No such host is known.",
|
||||
NetworkLinkUpAtStart = false,
|
||||
NetworkLinkUpAtEnd = false,
|
||||
AttemptWindowSeconds = 1204.5,
|
||||
ConfigFetchOutcome = "UsedDefaults",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_without_session_age_leaves_it_null_and_out_of_the_timeline()
|
||||
{
|
||||
|
||||
+61
-1
@@ -167,6 +167,9 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
return req.CreateResponse(HttpStatusCode.OK);
|
||||
}
|
||||
|
||||
report.PriorRegistrationFailure = SanitizeRegistrationFailure(report.PriorRegistrationFailure);
|
||||
var priorRegistration = report.PriorRegistrationFailure;
|
||||
|
||||
// Emit a structured log entry (captured by App Insights as a trace)
|
||||
_logger.LogCritical(
|
||||
"AgentEmergencyError [{ErrorType}] tenant={TenantId} session={SessionId} http={HttpStatusCode} seq={SequenceNumber} ver={AgentVersion}: {Message}",
|
||||
@@ -191,6 +194,15 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
["Message"] = report.Message ?? string.Empty,
|
||||
["AgentTimestamp"] = report.Timestamp.ToString("O"),
|
||||
["SessionAgeHours"] = report.SessionAgeHours?.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationFailedRuns"] = priorRegistration?.FailedRuns.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationOutcome"] = priorRegistration?.Outcome ?? string.Empty,
|
||||
["PriorRegistrationLastError"] = priorRegistration?.LastError ?? string.Empty,
|
||||
["PriorRegistrationLinkUpAtStart"] = FormatNullableBool(priorRegistration?.NetworkLinkUpAtStart),
|
||||
["PriorRegistrationLinkUpAtEnd"] = FormatNullableBool(priorRegistration?.NetworkLinkUpAtEnd),
|
||||
["PriorRegistrationWindowSeconds"] = priorRegistration?.AttemptWindowSeconds.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationConfigFetch"] = priorRegistration?.ConfigFetchOutcome ?? string.Empty,
|
||||
["PriorRegistrationFirstFailedAtUtc"] = priorRegistration?.FirstFailedAtUtc.ToString("O") ?? string.Empty,
|
||||
["PriorRegistrationLastFailedAtUtc"] = priorRegistration?.LastFailedAtUtc.ToString("O") ?? string.Empty,
|
||||
});
|
||||
|
||||
// One Sessions-row point-read serves the owner binding and the emergency-break verdict.
|
||||
@@ -396,7 +408,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
await opsEventService.RecordAgentEmergencyBreakAsync(
|
||||
tenantId, report.SessionId, report.AgentVersion, message,
|
||||
verdict.Severity, verdict.SessionStatusAtBreak, verdict.LateCleanup, verdict.Context,
|
||||
report.SessionAgeHours);
|
||||
report.SessionAgeHours, report.PriorRegistrationFailure);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -467,6 +479,39 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
? "Agent absolute session-age emergency break fired — agent cleaned up and exited"
|
||||
: report.Message;
|
||||
|
||||
internal const int MaxRegistrationErrorLength = 256;
|
||||
internal const int MaxRegistrationTokenLength = 32;
|
||||
|
||||
/// <summary>
|
||||
/// Bounds the device-supplied failed-registration record before it reaches App Insights, the
|
||||
/// ops feed or a timeline: strings are truncated, a record without a positive run count is
|
||||
/// dropped, negative windows are clamped. Returns a copy; null in, null out.
|
||||
/// </summary>
|
||||
internal static RegistrationFailureSummary? SanitizeRegistrationFailure(RegistrationFailureSummary? source)
|
||||
{
|
||||
if (source == null || source.FailedRuns <= 0) return null;
|
||||
return new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = source.FailedRuns,
|
||||
FirstFailedAtUtc = source.FirstFailedAtUtc,
|
||||
LastFailedAtUtc = source.LastFailedAtUtc,
|
||||
Outcome = Truncate(source.Outcome, MaxRegistrationTokenLength) ?? string.Empty,
|
||||
LastError = Truncate(source.LastError, MaxRegistrationErrorLength),
|
||||
NetworkLinkUpAtStart = source.NetworkLinkUpAtStart,
|
||||
NetworkLinkUpAtEnd = source.NetworkLinkUpAtEnd,
|
||||
AttemptWindowSeconds = source.AttemptWindowSeconds < 0 || double.IsNaN(source.AttemptWindowSeconds)
|
||||
? 0
|
||||
: source.AttemptWindowSeconds,
|
||||
ConfigFetchOutcome = Truncate(source.ConfigFetchOutcome, MaxRegistrationTokenLength),
|
||||
};
|
||||
}
|
||||
|
||||
private static string? Truncate(string? value, int max) =>
|
||||
value == null || value.Length <= max ? value : value.Substring(0, max);
|
||||
|
||||
private static string FormatNullableBool(bool? value) =>
|
||||
value == null ? string.Empty : (value.Value ? "true" : "false");
|
||||
|
||||
/// <summary>
|
||||
/// Builds the backend-materialized <c>agent_emergency_break</c> timeline event from the agent's
|
||||
/// best-effort emergency report. Static + pure (analog to
|
||||
@@ -497,6 +542,21 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
};
|
||||
if (report.SessionAgeHours.HasValue)
|
||||
data["sessionAgeHours"] = report.SessionAgeHours.Value;
|
||||
if (report.PriorRegistrationFailure is { } prior)
|
||||
{
|
||||
data["priorRegistrationFailure"] = new Dictionary<string, object?>
|
||||
{
|
||||
["failedRuns"] = prior.FailedRuns,
|
||||
["firstFailedAtUtc"] = prior.FirstFailedAtUtc.ToString("o"),
|
||||
["lastFailedAtUtc"] = prior.LastFailedAtUtc.ToString("o"),
|
||||
["outcome"] = prior.Outcome,
|
||||
["lastError"] = prior.LastError,
|
||||
["networkLinkUpAtStart"] = prior.NetworkLinkUpAtStart,
|
||||
["networkLinkUpAtEnd"] = prior.NetworkLinkUpAtEnd,
|
||||
["attemptWindowSeconds"] = prior.AttemptWindowSeconds,
|
||||
["configFetchOutcome"] = prior.ConfigFetchOutcome,
|
||||
};
|
||||
}
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
|
||||
@@ -965,15 +965,39 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// remaining Warnings named sessions that never registered; as Warnings both buried the real
|
||||
/// ones. <paramref name="sessionAgeHours"/> is the agent-measured age (null for agents that
|
||||
/// predate the field) — it separates a break just past the cap from a device that slept for
|
||||
/// weeks, which the backend cannot tell from its own rows.
|
||||
/// weeks, which the backend cannot tell from its own rows. <paramref name="priorRegistrationFailure"/>
|
||||
/// is the agent's record of earlier starts that never registered (bounded by the caller;
|
||||
/// null when none or for older agents) — for a session without a row the only account of why.
|
||||
/// </summary>
|
||||
public Task RecordAgentEmergencyBreakAsync(
|
||||
string tenantId, string sessionId, string? agentVersion, string message,
|
||||
string severity, string sessionStatusAtBreak, bool lateCleanup, string context, double? sessionAgeHours)
|
||||
string severity, string sessionStatusAtBreak, bool lateCleanup, string context, double? sessionAgeHours,
|
||||
RegistrationFailureSummary? priorRegistrationFailure = null)
|
||||
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
|
||||
$"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}, {context}): {message}",
|
||||
tenantId, "System.EmergencyChannel",
|
||||
new { sessionId, agentVersion, sessionStatusAtBreak, lateCleanup, sessionAgeHours });
|
||||
new
|
||||
{
|
||||
sessionId, agentVersion, sessionStatusAtBreak, lateCleanup, sessionAgeHours,
|
||||
priorRegistrationFailure = DescribeRegistrationFailure(priorRegistrationFailure),
|
||||
});
|
||||
|
||||
// camelCase like the rest of the details; the class itself would serialize in PascalCase.
|
||||
private static object? DescribeRegistrationFailure(RegistrationFailureSummary? failure) =>
|
||||
failure == null
|
||||
? null
|
||||
: new
|
||||
{
|
||||
failedRuns = failure.FailedRuns,
|
||||
firstFailedAtUtc = failure.FirstFailedAtUtc,
|
||||
lastFailedAtUtc = failure.LastFailedAtUtc,
|
||||
outcome = failure.Outcome,
|
||||
lastError = failure.LastError,
|
||||
networkLinkUpAtStart = failure.NetworkLinkUpAtStart,
|
||||
networkLinkUpAtEnd = failure.NetworkLinkUpAtEnd,
|
||||
attemptWindowSeconds = failure.AttemptWindowSeconds,
|
||||
configFetchOutcome = failure.ConfigFetchOutcome,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend
|
||||
|
||||
@@ -104,6 +104,15 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
/// <summary>On <c>AadUserJoinedLate</c>: user presence indicator ("true" / "false").</summary>
|
||||
public const string AadJoinedWithUser = "aadJoinedWithUser";
|
||||
|
||||
/// <summary>
|
||||
/// On <c>WhiteGloveShellCoreSuccess</c>: "true" / "false" — whether the ESP registry
|
||||
/// showed an AccountSetup subcategory past <c>notStarted</c> when the agent observed the
|
||||
/// Shell-Core WhiteGlove_Success event. Missing (registry not read yet, or agents
|
||||
/// predating the key) → the fact stays unset and the sealing classifier keeps the
|
||||
/// IME-phase AccountSetup excluder.
|
||||
/// </summary>
|
||||
public const string AccountSetupProgress = "accountSetupProgress";
|
||||
|
||||
/// <summary>On <c>ImeUserSessionCompleted</c>: matched pattern id.</summary>
|
||||
public const string ImePatternId = "patternId";
|
||||
|
||||
|
||||
@@ -53,6 +53,16 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
.WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal);
|
||||
builder.ScenarioObservations = builder.ScenarioObservations.WithShellCoreWhiteGloveSuccessSeen(signal.SessionSignalOrdinal);
|
||||
|
||||
// Recorded before the inline classifier runs: it decides whether the AccountSetup
|
||||
// excluder applies (see BuildWhiteGloveSealingSnapshot).
|
||||
if (signal.Payload != null
|
||||
&& signal.Payload.TryGetValue(SignalPayloadKeys.AccountSetupProgress, out var rawProgress)
|
||||
&& bool.TryParse(rawProgress, out var accountSetupProgress))
|
||||
{
|
||||
builder.ScenarioObservations = builder.ScenarioObservations
|
||||
.WithAccountSetupProgressAtWhiteGloveSuccess(accountSetupProgress, signal.SessionSignalOrdinal);
|
||||
}
|
||||
|
||||
// Option 3 fast-path: inline classifier evaluation on the strong WG signal.
|
||||
var afterObservation = builder.Build();
|
||||
var inlineSnapshot = BuildWhiteGloveSealingSnapshot(afterObservation);
|
||||
@@ -362,6 +372,15 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
/// <see cref="WhiteGloveSealingSnapshot.ComputeInputHash"/> is therefore byte-stable
|
||||
/// across the refactor, which preserves the anti-loop semantics.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <c>hasAccountSetupActivity</c> needs the IME AccountSetup phase AND no explicit
|
||||
/// "registry shows no AccountSetup progress" on the WhiteGlove_Success signal. IME
|
||||
/// reports AccountSetup from the device session as soon as the device apps are done, so
|
||||
/// on pre-provisioning with the user ESP enabled that line routinely precedes the
|
||||
/// technician's success page. The conjunction keeps the excluder a subset of the old
|
||||
/// one: a verdict can only rise, and a missing payload (older agent) scores exactly as
|
||||
/// before.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static WhiteGloveSealingSnapshot BuildWhiteGloveSealingSnapshot(DecisionState state) =>
|
||||
new WhiteGloveSealingSnapshot(
|
||||
@@ -370,7 +389,8 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
aadJoinedWithUser: state.ScenarioObservations.AadUserJoinWithUserObserved?.Value == true,
|
||||
desktopArrived: state.DesktopArrivedUtc != null,
|
||||
helloResolved: state.HelloResolvedUtc != null,
|
||||
hasAccountSetupActivity: state.AccountSetupEnteredUtc != null,
|
||||
hasAccountSetupActivity: state.AccountSetupEnteredUtc != null
|
||||
&& state.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess?.Value != false,
|
||||
isDeviceOnlyDeploymentHypothesis:
|
||||
state.ClassifierOutcomes.DeviceOnlyDeployment.Level >= HypothesisLevel.Strong &&
|
||||
state.ClassifierOutcomes.DeviceOnlyDeployment.Reason == DeviceOnlyReasons.DeviceOnly,
|
||||
|
||||
@@ -144,6 +144,20 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
seen.Add("whiteglove_sealing_pattern");
|
||||
evidence["whiteGloveSealingPattern"] = OrdinalEvidence(obs.WhiteGloveSealingPatternSeen.SourceSignalOrdinal);
|
||||
}
|
||||
// Registry AccountSetup progress stamped on the WhiteGlove_Success signal — the gate of
|
||||
// the sealing classifier's AccountSetup excluder, so a Part-1 audit trail shows why
|
||||
// an IME AccountSetup phase did or did not count.
|
||||
if (obs.AccountSetupProgressAtWhiteGloveSuccess != null)
|
||||
{
|
||||
seen.Add(obs.AccountSetupProgressAtWhiteGloveSuccess.Value
|
||||
? "whiteglove_account_setup_progress"
|
||||
: "whiteglove_account_setup_untouched");
|
||||
evidence["accountSetupProgressAtWhiteGloveSuccess"] = new Dictionary<string, object>(capacity: 2, StringComparer.Ordinal)
|
||||
{
|
||||
["ordinal"] = obs.AccountSetupProgressAtWhiteGloveSuccess.SourceSignalOrdinal,
|
||||
["progress"] = obs.AccountSetupProgressAtWhiteGloveSuccess.Value,
|
||||
};
|
||||
}
|
||||
if (obs.AadUserJoinWithUserObserved != null)
|
||||
{
|
||||
seen.Add(obs.AadUserJoinWithUserObserved.Value
|
||||
|
||||
@@ -33,6 +33,17 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
/// <summary>True once <see cref="Signals.DecisionSignalKind.WhiteGloveSealingPatternDetected"/> has fired.</summary>
|
||||
public SignalFact<bool>? WhiteGloveSealingPatternSeen { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// ESP-registry AccountSetup progress as stamped on the
|
||||
/// <see cref="Signals.DecisionSignalKind.WhiteGloveShellCoreSuccess"/> payload:
|
||||
/// <c>true</c> = an AccountSetup subcategory had left <c>notStarted</c>, <c>false</c> =
|
||||
/// none had. <c>null</c> = not reported (registry not read yet, or an agent predating the
|
||||
/// payload key) and must be read as "unknown", never as "no progress". Gates the sealing classifier's
|
||||
/// AccountSetup excluder: IME logs "EspPhase: AccountSetup" from the device session as
|
||||
/// soon as the device apps are done, so only an explicit <c>false</c> here lifts it.
|
||||
/// </summary>
|
||||
public SignalFact<bool>? AccountSetupProgressAtWhiteGloveSuccess { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Payload-carrying observation from <see cref="Signals.DecisionSignalKind.AadUserJoinedLate"/>.
|
||||
/// <c>true</c> = late AADJ observed with a user-side principal (hard-excluder for
|
||||
@@ -113,6 +124,11 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
? this
|
||||
: this with { WhiteGloveSealingPatternSeen = new SignalFact<bool>(true, sourceSignalOrdinal) };
|
||||
|
||||
public EnrollmentScenarioObservations WithAccountSetupProgressAtWhiteGloveSuccess(bool value, long sourceSignalOrdinal) =>
|
||||
AccountSetupProgressAtWhiteGloveSuccess != null
|
||||
? this
|
||||
: this with { AccountSetupProgressAtWhiteGloveSuccess = new SignalFact<bool>(value, sourceSignalOrdinal) };
|
||||
|
||||
public EnrollmentScenarioObservations WithAadUserJoinWithUserObserved(bool value, long sourceSignalOrdinal) =>
|
||||
AadUserJoinWithUserObserved != null
|
||||
? this
|
||||
|
||||
@@ -90,6 +90,13 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public double? SessionAgeHours { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Why earlier process starts of this session never registered
|
||||
/// (<see cref="AgentErrorType.SessionAgeEmergencyBreak"/> only). Null when every start
|
||||
/// registered, for other error types, and for agents that predate the field.
|
||||
/// </summary>
|
||||
public RegistrationFailureSummary? PriorRegistrationFailure { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Agent version string for diagnosing version-specific issues.
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// What the agent recorded locally each time a process start ended without a registered
|
||||
/// session (register-session failed after all retries, the agent exited). Carried on the
|
||||
/// emergency-break report (<see cref="AgentErrorReport.PriorRegistrationFailure"/>): for a
|
||||
/// session that never registered it is the only account of why that ever reaches the
|
||||
/// backend. Cleared by the next successful registration.
|
||||
/// </summary>
|
||||
public class RegistrationFailureSummary
|
||||
{
|
||||
/// <summary>Process starts whose registration failed since the last success.</summary>
|
||||
public int FailedRuns { get; set; }
|
||||
|
||||
/// <summary>UTC time the first of these runs gave up.</summary>
|
||||
public DateTime FirstFailedAtUtc { get; set; }
|
||||
|
||||
/// <summary>UTC time the latest of these runs gave up.</summary>
|
||||
public DateTime LastFailedAtUtc { get; set; }
|
||||
|
||||
/// <summary>Registration outcome of the latest failed run (e.g. <c>Failed</c>, <c>AuthFailed</c>).</summary>
|
||||
public string Outcome { get; set; } = default!;
|
||||
|
||||
/// <summary>Last error message of the latest failed run, truncated by the agent.</summary>
|
||||
public string? LastError { get; set; }
|
||||
|
||||
/// <summary>Network link up (any interface) when the latest failed run started registering.</summary>
|
||||
public bool? NetworkLinkUpAtStart { get; set; }
|
||||
|
||||
/// <summary>Network link up when the latest failed run gave up.</summary>
|
||||
public bool? NetworkLinkUpAtEnd { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Wall-clock seconds the latest failed run spent registering (link wait plus all attempts).
|
||||
/// Far above the retry schedule means the device slept or was suspended meanwhile.
|
||||
/// </summary>
|
||||
public double AttemptWindowSeconds { get; set; }
|
||||
|
||||
/// <summary>Outcome of the remote-config fetch in the latest failed run (e.g. <c>Succeeded</c>, <c>UsedDefaults</c>).</summary>
|
||||
public string? ConfigFetchOutcome { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# WhiteGloveShellCoreSuccess v1 — Shell-Core 62407 "CommercialOOBE_ESPProgress_WhiteGlove_Success".
|
||||
# Posted by EspAndHelloTrackerAdapter (fire-once) when the ShellCoreTracker observes the
|
||||
# technician-flow success page. Payload `accountSetupProgress` (added 2026-09-29) carries the ESP
|
||||
# registry's AccountSetup progress at that moment ("true" = a subcategory had left notStarted);
|
||||
# signals from older agents carry no payload.
|
||||
# Reducer: HandleWhiteGloveShellCoreSuccessV1 → records ShellCoreWhiteGloveSuccessSeen and, when
|
||||
# the payload parses, AccountSetupProgressAtWhiteGloveSuccess; runs the sealing classifier inline.
|
||||
# The IME AccountSetup excluder (-40) is lifted only by an explicit "false".
|
||||
{"SessionSignalOrdinal":1,"SessionTraceOrdinal":1,"Kind":"WhiteGloveShellCoreSuccess","KindSchemaVersion":1,"OccurredAtUtc":"2026-09-14T07:25:09.7628703Z","SourceOrigin":"EspAndHelloTracker","Evidence":{"Kind":"Derived","Identifier":"esp-and-hello-tracker-v1","Summary":"WhiteGlove sealing success (coordinator-forwarded)","DerivationInputs":{"subSource":"ShellCoreTracker","accountSetupProgress":"false"}},"Payload":{"accountSetupProgress":"false"}}
|
||||
Reference in New Issue
Block a user