mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-26 05:29:43 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8cb5d42edc | |||
| fa637369e1 | |||
| edf30d35e0 | |||
| 3b4228e3a1 |
+1
-1
Submodule .claude/commands updated: 313fe61e98...2ace35ae7d
+1
-1
Submodule internal updated: df3667d672...c2cf529c83
+17
@@ -83,6 +83,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
using var f = new Fixture();
|
||||
Assert.False(f.Tracker.HasAccountSetupActivity);
|
||||
Assert.False(f.Tracker.HasAccountSetupProgress);
|
||||
Assert.Null(f.Tracker.AccountSetupProgressIfRead);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -97,6 +98,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
|
||||
Assert.True(f.Tracker.HasAccountSetupActivity);
|
||||
Assert.False(f.Tracker.HasAccountSetupProgress);
|
||||
// Read and untouched — the only state the WhiteGlove sealing gate treats as "false".
|
||||
Assert.False(f.Tracker.AccountSetupProgressIfRead);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -108,6 +111,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
|
||||
Assert.True(f.Tracker.HasAccountSetupActivity);
|
||||
Assert.True(f.Tracker.HasAccountSetupProgress);
|
||||
Assert.True(f.Tracker.AccountSetupProgressIfRead);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -138,5 +142,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
|
||||
Assert.False(f.Tracker.HasAccountSetupProgress);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OnlyDeviceSetupRead_AccountSetupProgressIsUnknown()
|
||||
{
|
||||
// Adversarial for the WhiteGlove sealing gate: the tracker holds DeviceSetup data but
|
||||
// has not read the AccountSetup category — that must be "unknown", never "untouched".
|
||||
using var f = new Fixture();
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status",
|
||||
"{\"categoryState\":\"inProgress\",\"DeviceSetup.AppsSubcategory\":{\"subcategoryState\":\"inProgress\"}}");
|
||||
|
||||
Assert.Null(f.Tracker.AccountSetupProgressIfRead);
|
||||
Assert.False(f.Tracker.HasAccountSetupProgress);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+34
@@ -20,6 +20,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
private sealed class FakeApiClient : BackendApiClient
|
||||
{
|
||||
public readonly List<TimeSpan?> Timeouts = new List<TimeSpan?>();
|
||||
public readonly List<AgentErrorReport> Reports = new List<AgentErrorReport>();
|
||||
public readonly Queue<bool> Results = new Queue<bool>();
|
||||
|
||||
public FakeApiClient() : base(
|
||||
@@ -40,6 +41,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
public override Task<bool> ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null)
|
||||
{
|
||||
Timeouts.Add(timeout);
|
||||
Reports.Add(report);
|
||||
return Task.FromResult(Results.Count > 0 && Results.Dequeue());
|
||||
}
|
||||
}
|
||||
@@ -139,5 +141,37 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_age_is_carried_in_the_report_and_absent_by_default()
|
||||
{
|
||||
// One reporter per report: a delivered report starts the 10-minute cooldown.
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m", sessionAgeHours: 478.2);
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
|
||||
Assert.Equal(2, client.Reports.Count);
|
||||
Assert.Equal(478.2, client.Reports[0].SessionAgeHours);
|
||||
Assert.Null(client.Reports[1].SessionAgeHours);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prior_registration_failure_is_carried_in_the_report_and_absent_by_default()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
var record = new RegistrationFailureSummary { FailedRuns = 2, Outcome = "Failed", NetworkLinkUpAtEnd = false };
|
||||
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m", priorRegistrationFailure: record);
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
|
||||
Assert.Equal(2, client.Reports.Count);
|
||||
Assert.Same(record, client.Reports[0].PriorRegistrationFailure);
|
||||
Assert.Null(client.Reports[1].PriorRegistrationFailure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+83
@@ -118,5 +118,88 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.Throws<ArgumentNullException>(
|
||||
() => BackendSessionRegistration.Register(config, auth, http, "1.0", consoleMode: false, logger: null));
|
||||
}
|
||||
|
||||
// ============================================================ failed-registration record
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_records_a_failed_run_with_its_window_and_bounded_error()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var persistence = new SessionIdPersistence(tmp.Path);
|
||||
var started = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
persistence, SessionRegistrationResult.Failed(new string('e', 1000)),
|
||||
started, started.AddSeconds(1204.46), linkUpAtStart: false, linkUpAtEnd: true,
|
||||
configFetchOutcome: "UsedDefaults", logger: NewLogger(tmp.Path));
|
||||
|
||||
var record = persistence.LoadRegistrationFailure();
|
||||
Assert.NotNull(record);
|
||||
Assert.Equal(1, record!.FailedRuns);
|
||||
Assert.Equal("Failed", record.Outcome);
|
||||
Assert.Equal(BackendSessionRegistration.MaxRecordedErrorLength, record.LastError!.Length);
|
||||
Assert.False(record.NetworkLinkUpAtStart);
|
||||
Assert.True(record.NetworkLinkUpAtEnd);
|
||||
Assert.Equal(1204.5, record.AttemptWindowSeconds);
|
||||
Assert.Equal("UsedDefaults", record.ConfigFetchOutcome);
|
||||
Assert.Equal(started.AddSeconds(1204.46), record.LastFailedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_records_auth_failures_and_clears_on_success()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var persistence = new SessionIdPersistence(tmp.Path);
|
||||
var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
persistence, SessionRegistrationResult.AuthFailed(401, "unauthorized"),
|
||||
at, at.AddSeconds(3), true, true, "Succeeded", NewLogger(tmp.Path));
|
||||
Assert.Equal("AuthFailed", persistence.LoadRegistrationFailure()!.Outcome);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
persistence, SessionRegistrationResult.Succeeded(new RegisterSessionResponse { Success = true }),
|
||||
at, at.AddSeconds(1), true, true, "Succeeded", NewLogger(tmp.Path));
|
||||
Assert.Null(persistence.LoadRegistrationFailure());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_without_persistence_is_a_no_op()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
null, SessionRegistrationResult.Failed("x"), DateTime.UtcNow, DateTime.UtcNow,
|
||||
null, null, null, NewLogger(tmp.Path));
|
||||
}
|
||||
|
||||
// ============================================================ emergency-break message
|
||||
|
||||
[Fact]
|
||||
public void Break_message_names_failed_registrations_in_invariant_culture()
|
||||
{
|
||||
var previous = System.Threading.Thread.CurrentThread.CurrentCulture;
|
||||
try
|
||||
{
|
||||
System.Threading.Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("de-DE");
|
||||
var record = new RegistrationFailureSummary { FailedRuns = 3, Outcome = "Failed", NetworkLinkUpAtEnd = false };
|
||||
|
||||
var message = EmergencyBreakReporter.BuildMessage(478.2, 48, record);
|
||||
|
||||
Assert.Contains("session age 478.2h (cap 48h)", message);
|
||||
Assert.EndsWith(" 3 earlier start(s) failed to register (last: Failed, network link at end: down).", message);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Threading.Thread.CurrentThread.CurrentCulture = previous;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Break_message_without_record_keeps_the_historical_text()
|
||||
{
|
||||
Assert.Equal(
|
||||
"Agent absolute session-age emergency break fired at session age 50.0h (cap 48h) — cleaning up and exiting.",
|
||||
EmergencyBreakReporter.BuildMessage(50.0, 48, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
persistence.SaveSessionCreatedAt(DateTime.UtcNow.AddHours(-100));
|
||||
|
||||
var stateDir = Path.Combine(tmp.Path, "State");
|
||||
double? reportedAgeHours = null;
|
||||
|
||||
var tripped = AutopilotMonitor.Agent.V2.Program.CheckSessionAgeEmergencyBreak(
|
||||
dataDirectory: tmp.Path,
|
||||
@@ -157,9 +158,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
selfDestructOnComplete: false,
|
||||
cleanupServiceFactory: null,
|
||||
logger: logger,
|
||||
consoleMode: false);
|
||||
consoleMode: false,
|
||||
onBreakFired: ageHours => reportedAgeHours = ageHours);
|
||||
|
||||
Assert.True(tripped);
|
||||
// The measured age reaches the break report (the backend cannot derive it).
|
||||
Assert.NotNull(reportedAgeHours);
|
||||
Assert.InRange(reportedAgeHours.Value, 99.9, 100.1);
|
||||
// Marker must have been written so the next restart exits cleanly.
|
||||
Assert.True(File.Exists(Path.Combine(stateDir, "enrollment-complete.marker")));
|
||||
// Session must have been cleared.
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.IO;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
@@ -234,5 +235,78 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.False(File.Exists(Path.Combine(tmp.Path, "session.created")));
|
||||
Assert.False(File.Exists(Path.Combine(tmp.Path, "whiteglove.complete")));
|
||||
}
|
||||
|
||||
// ============================================================ failed-registration record
|
||||
|
||||
private static RegistrationFailureSummary FailedRun(DateTime at, string outcome = "Failed", bool? linkUpAtEnd = false) =>
|
||||
new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = 1,
|
||||
FirstFailedAtUtc = at,
|
||||
LastFailedAtUtc = at,
|
||||
Outcome = outcome,
|
||||
LastError = "No such host is known.",
|
||||
NetworkLinkUpAtStart = false,
|
||||
NetworkLinkUpAtEnd = linkUpAtEnd,
|
||||
AttemptWindowSeconds = 91.3,
|
||||
ConfigFetchOutcome = "UsedDefaults",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationFailure_counts_runs_keeps_the_first_time_and_describes_the_latest_run()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var sut = new SessionIdPersistence(tmp.Path);
|
||||
var first = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
var second = new DateTime(2026, 9, 12, 7, 30, 0, DateTimeKind.Utc);
|
||||
|
||||
Assert.Null(sut.LoadRegistrationFailure());
|
||||
sut.RecordRegistrationFailure(FailedRun(first));
|
||||
sut.RecordRegistrationFailure(FailedRun(second, outcome: "AuthFailed", linkUpAtEnd: true));
|
||||
|
||||
// A fresh instance reads it back from disk, as the next boot's emergency break does.
|
||||
var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailure();
|
||||
Assert.NotNull(record);
|
||||
Assert.Equal(2, record!.FailedRuns);
|
||||
Assert.Equal(first, record.FirstFailedAtUtc);
|
||||
Assert.Equal(second, record.LastFailedAtUtc);
|
||||
Assert.Equal("AuthFailed", record.Outcome);
|
||||
Assert.True(record.NetworkLinkUpAtEnd);
|
||||
Assert.Equal(91.3, record.AttemptWindowSeconds);
|
||||
Assert.Equal("UsedDefaults", record.ConfigFetchOutcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Registration_failure_record_is_cleared_by_success_delete_and_rotate()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var sut = new SessionIdPersistence(tmp.Path);
|
||||
sut.GetOrCreate();
|
||||
var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
sut.RecordRegistrationFailure(FailedRun(at));
|
||||
sut.ClearRegistrationFailure();
|
||||
Assert.Null(sut.LoadRegistrationFailure());
|
||||
|
||||
sut.RecordRegistrationFailure(FailedRun(at));
|
||||
sut.Rotate();
|
||||
Assert.Null(sut.LoadRegistrationFailure());
|
||||
|
||||
sut.RecordRegistrationFailure(FailedRun(at));
|
||||
sut.Delete();
|
||||
Assert.Null(sut.LoadRegistrationFailure());
|
||||
Assert.False(File.Exists(Path.Combine(tmp.Path, "registration-failure.json")));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not json")]
|
||||
[InlineData("{\"FailedRuns\":0}")]
|
||||
public void LoadRegistrationFailure_reports_nothing_for_a_corrupt_or_empty_record(string content)
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
File.WriteAllText(Path.Combine(tmp.Path, "registration-failure.json"), content);
|
||||
|
||||
Assert.Null(new SessionIdPersistence(tmp.Path).LoadRegistrationFailure());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+46
@@ -99,6 +99,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
Assert.Equal(DecisionSignalKind.WhiteGloveShellCoreSuccess, posted.Kind);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, "true")]
|
||||
[InlineData(false, "false")]
|
||||
public void WhiteGloveEvent_stamps_read_AccountSetup_progress_on_payload_and_evidence(bool progress, string expected)
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var adapter = new EspAndHelloTrackerAdapter(
|
||||
f.Coordinator, f.Ingress, f.Clock, accountSetupProgressProbe: () => progress);
|
||||
|
||||
adapter.TriggerWhiteGloveFromTest();
|
||||
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.Equal(expected, posted.Payload![SignalPayloadKeys.AccountSetupProgress]);
|
||||
Assert.Equal(expected, posted.Evidence.DerivationInputs![SignalPayloadKeys.AccountSetupProgress]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhiteGloveEvent_with_unread_AccountSetup_registry_sends_no_progress_key()
|
||||
{
|
||||
// Unknown must keep the sealing classifier's AccountSetup excluder — the payload key
|
||||
// is absent, the evidence records why.
|
||||
using var f = new Fixture();
|
||||
using var adapter = new EspAndHelloTrackerAdapter(
|
||||
f.Coordinator, f.Ingress, f.Clock, accountSetupProgressProbe: () => null);
|
||||
|
||||
adapter.TriggerWhiteGloveFromTest();
|
||||
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.True(posted.Payload == null || !posted.Payload.ContainsKey(SignalPayloadKeys.AccountSetupProgress));
|
||||
Assert.Equal("unread", posted.Evidence.DerivationInputs![SignalPayloadKeys.AccountSetupProgress]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhiteGloveEvent_default_probe_on_unstarted_coordinator_reports_unread()
|
||||
{
|
||||
// Production probe: the provisioning tracker does not exist before Start (and never on
|
||||
// Device Preparation) — that is "unread", not "untouched".
|
||||
using var f = new Fixture();
|
||||
using var adapter = new EspAndHelloTrackerAdapter(f.Coordinator, f.Ingress, f.Clock);
|
||||
|
||||
adapter.TriggerWhiteGloveFromTest();
|
||||
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.True(posted.Payload == null || !posted.Payload.ContainsKey(SignalPayloadKeys.AccountSetupProgress));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EspFailureEvent_emits_EspTerminalFailure_with_merged_subSource_annotation()
|
||||
{
|
||||
|
||||
+31
@@ -564,6 +564,37 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.False(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_whiteglove_part1_writes_marker_before_the_later_exit_steps()
|
||||
{
|
||||
// The technician's reseal can kill the process anywhere after the seal. The marker
|
||||
// is what lets the next boot skip the 48-hour break and resume Part 2, so it must
|
||||
// exist before collectors stop, diagnostics upload, grace and drain run.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.WhiteGloveSealed }.Build();
|
||||
bool? markerPresentAtCollectorStop = null;
|
||||
rig.StopPeripheralCollectorsHook = () => markerPresentAtCollectorStop = rig.SessionPersistence.IsWhiteGloveResume();
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed));
|
||||
|
||||
Assert.True(markerPresentAtCollectorStop, "whiteglove.complete must be written before the peripheral collectors are stopped.");
|
||||
var emitted = rig.EmittedEventTypes.ToList();
|
||||
Assert.True(emitted.IndexOf("agent_shutting_down") >= 0, "agent_shutting_down must still be emitted.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_non_whiteglove_termination_never_writes_the_marker()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build();
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed));
|
||||
|
||||
Assert.False(rig.SessionPersistence.IsWhiteGloveResume());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_whiteglove_part1_stops_collectors_before_emitting_part1_complete()
|
||||
{
|
||||
|
||||
+7
@@ -216,6 +216,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// </summary>
|
||||
public bool HasAccountSetupProgress => _provisioningTracker?.HasAccountSetupProgress ?? false;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ProvisioningStatusTracker.AccountSetupProgressIfRead"/>; <c>null</c> also when
|
||||
/// the provisioning tracker is not running (Device Preparation, before Start). Stamped on
|
||||
/// the WhiteGloveShellCoreSuccess signal for the sealing classifier.
|
||||
/// </summary>
|
||||
public bool? AccountSetupProgressIfRead => _provisioningTracker?.AccountSetupProgressIfRead;
|
||||
|
||||
/// <summary>
|
||||
/// True once WhiteGlove start has been detected (EventID 509 or persisted from prior run).
|
||||
/// Forwarded from ModernDeploymentTracker.
|
||||
|
||||
+19
-6
@@ -254,16 +254,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// host to tell the real user-ESP page exit from the Device→Account handoff exit; not a
|
||||
/// completion input.
|
||||
/// </summary>
|
||||
public bool HasAccountSetupProgress
|
||||
public bool HasAccountSetupProgress => AccountSetupProgressIfRead == true;
|
||||
|
||||
/// <summary>
|
||||
/// Tri-state form of <see cref="HasAccountSetupProgress"/>: <c>null</c> until the tracker
|
||||
/// has actually read AccountSetup subcategories (not started yet, category not written,
|
||||
/// empty), otherwise whether one has left <c>notStarted</c>. The WhiteGlove sealing
|
||||
/// classifier treats only a read <c>false</c> as "user ESP untouched" — an unread
|
||||
/// registry must never pass for an untouched one.
|
||||
/// </summary>
|
||||
public bool? AccountSetupProgressIfRead
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_stateLock)
|
||||
{
|
||||
if (_lastSubcategoryStates == null)
|
||||
return false;
|
||||
if (!_lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs) || subs == null)
|
||||
return false;
|
||||
return null;
|
||||
if (!_lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs)
|
||||
|| subs == null
|
||||
|| subs.Count == 0)
|
||||
return null;
|
||||
foreach (var state in subs.Values)
|
||||
{
|
||||
if (!string.Equals(state, "notStarted", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -279,7 +290,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// with subcategoryState=succeeded. NOT a WhiteGlove indicator: on Win11 25H2 (build
|
||||
/// 26200+) Windows writes this property as a trivially-succeeding step in EVERY
|
||||
/// enrollment (validated 2026-08-18 — user-driven sessions f475e697/c601a24b/4219203f
|
||||
/// all carry it). Kept for observability only (whiteglove_signal_* raw dumps); must not
|
||||
/// all carry it). Kept for observability only (save_result_step_* raw dumps); must not
|
||||
/// be used for classification. Reliable WhiteGlove anchors are ModernDeployment
|
||||
/// EventID 509 and the Shell-Core WhiteGlove-success events.
|
||||
/// </summary>
|
||||
@@ -966,8 +977,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_logger.Info($"ProvisioningStatusTracker: SaveWhiteGloveSuccessResult state " +
|
||||
$"transition: {previousState} -> {state}");
|
||||
|
||||
// Neutral trigger name: the step exists in every 25H2 enrollment,
|
||||
// and "whiteglove_*" was read as a pre-provisioning signal.
|
||||
EmitRawRegistryDump(categoryLabel, jsonValue,
|
||||
$"whiteglove_signal_{state}");
|
||||
$"save_result_step_{state}");
|
||||
|
||||
if (string.Equals(state, "succeeded", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
|
||||
@@ -79,6 +79,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// exist for callers whose process is about to exit and cannot come back later
|
||||
/// (emergency break): retries stay INSIDE the one reservation, so they never
|
||||
/// multiply against the per-session budget.
|
||||
///
|
||||
/// <paramref name="sessionAgeHours"/> is the emergency break's measured session age
|
||||
/// (<see cref="AgentErrorReport.SessionAgeHours"/>) and <paramref name="priorRegistrationFailure"/>
|
||||
/// its record of earlier starts that never registered
|
||||
/// (<see cref="AgentErrorReport.PriorRegistrationFailure"/>); null for every other error type.
|
||||
/// </summary>
|
||||
public virtual async Task TrySendAsync(
|
||||
AgentErrorType errorType,
|
||||
@@ -87,7 +92,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
long? sequenceNumber = null,
|
||||
int attempts = 1,
|
||||
TimeSpan? perAttemptTimeout = null,
|
||||
TimeSpan? retryDelay = null)
|
||||
TimeSpan? retryDelay = null,
|
||||
double? sessionAgeHours = null,
|
||||
RegistrationFailureSummary priorRegistrationFailure = null)
|
||||
{
|
||||
// Deduplicate by error type + status code: same failure category sent only once per session.
|
||||
// All anti-flood checks are protected by a lock to prevent race conditions when
|
||||
@@ -135,6 +142,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
Message = message,
|
||||
HttpStatusCode = httpStatusCode,
|
||||
SequenceNumber = sequenceNumber,
|
||||
SessionAgeHours = sessionAgeHours,
|
||||
PriorRegistrationFailure = priorRegistrationFailure,
|
||||
AgentVersion = _agentVersion,
|
||||
Timestamp = DateTime.UtcNow
|
||||
};
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
@@ -22,6 +24,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
private readonly string _sessionFilePath;
|
||||
private readonly string _sessionCreatedFilePath;
|
||||
private readonly string _whiteGloveMarkerPath;
|
||||
private readonly string _registrationFailurePath;
|
||||
private readonly object _lockObject = new object();
|
||||
|
||||
public SessionIdPersistence(string dataDirectory)
|
||||
@@ -35,6 +38,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
_sessionFilePath = Path.Combine(dataDirectory, "session.id");
|
||||
_sessionCreatedFilePath = Path.Combine(dataDirectory, "session.created");
|
||||
_whiteGloveMarkerPath = Path.Combine(dataDirectory, "whiteglove.complete");
|
||||
_registrationFailurePath = Path.Combine(dataDirectory, "registration-failure.json");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -164,6 +168,55 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
lock (_lockObject) TryDelete(_whiteGloveMarkerPath, logger);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Folds one failed registration run into <c>registration-failure.json</c>: the run
|
||||
/// counter grows, the first failure time is kept, everything else describes the latest
|
||||
/// run. A run that never registers exits and leaves nothing else behind — the emergency
|
||||
/// break reads this file so the backend learns why the session never appeared.
|
||||
/// Best-effort; never throws.
|
||||
/// </summary>
|
||||
public void RecordRegistrationFailure(RegistrationFailureSummary latestRun, AgentLogger logger = null)
|
||||
{
|
||||
if (latestRun == null) return;
|
||||
lock (_lockObject)
|
||||
{
|
||||
try
|
||||
{
|
||||
var previous = LoadRegistrationFailureInternal();
|
||||
var merged = new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = (previous?.FailedRuns ?? 0) + 1,
|
||||
FirstFailedAtUtc = previous?.FirstFailedAtUtc ?? latestRun.LastFailedAtUtc,
|
||||
LastFailedAtUtc = latestRun.LastFailedAtUtc,
|
||||
Outcome = latestRun.Outcome,
|
||||
LastError = latestRun.LastError,
|
||||
NetworkLinkUpAtStart = latestRun.NetworkLinkUpAtStart,
|
||||
NetworkLinkUpAtEnd = latestRun.NetworkLinkUpAtEnd,
|
||||
AttemptWindowSeconds = latestRun.AttemptWindowSeconds,
|
||||
ConfigFetchOutcome = latestRun.ConfigFetchOutcome,
|
||||
};
|
||||
File.WriteAllText(_registrationFailurePath, JsonConvert.SerializeObject(merged));
|
||||
logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns}.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"SessionIdPersistence: RecordRegistrationFailure failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The failed-registration record since the last success, or <c>null</c> (none, unreadable).</summary>
|
||||
public RegistrationFailureSummary LoadRegistrationFailure()
|
||||
{
|
||||
lock (_lockObject) return LoadRegistrationFailureInternal();
|
||||
}
|
||||
|
||||
/// <summary>Drops the failed-registration record — called once registration succeeds.</summary>
|
||||
public void ClearRegistrationFailure(AgentLogger logger = null)
|
||||
{
|
||||
lock (_lockObject) TryDelete(_registrationFailurePath, logger);
|
||||
}
|
||||
|
||||
/// <summary>Deletes the persisted SessionId. Next <see cref="GetOrCreate"/> starts a fresh session.</summary>
|
||||
public void Delete(AgentLogger logger = null)
|
||||
{
|
||||
@@ -172,6 +225,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
TryDelete(_sessionFilePath, logger);
|
||||
TryDelete(_sessionCreatedFilePath, logger);
|
||||
TryDelete(_whiteGloveMarkerPath, logger);
|
||||
TryDelete(_registrationFailurePath, logger);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,11 +245,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
var fresh = Guid.NewGuid().ToString();
|
||||
WriteAtomic(fresh, logger);
|
||||
SaveSessionCreatedAtInternal(DateTime.UtcNow);
|
||||
// Earlier failed runs belonged to the refused session id.
|
||||
TryDelete(_registrationFailurePath, logger);
|
||||
logger?.Info($"SessionIdPersistence: rotated SessionId to {fresh}.");
|
||||
return fresh;
|
||||
}
|
||||
}
|
||||
|
||||
private RegistrationFailureSummary LoadRegistrationFailureInternal()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(_registrationFailurePath)) return null;
|
||||
var record = JsonConvert.DeserializeObject<RegistrationFailureSummary>(File.ReadAllText(_registrationFailurePath));
|
||||
return record != null && record.FailedRuns > 0 ? record : null;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Corrupt record: report nothing rather than a half-read account.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path, AgentLogger logger)
|
||||
{
|
||||
try { if (File.Exists(path)) File.Delete(path); }
|
||||
|
||||
+24
-3
@@ -24,7 +24,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
/// <list type="bullet">
|
||||
/// <item><c>HelloCompleted</c> → <see cref="DecisionSignalKind.HelloResolved"/></item>
|
||||
/// <item><c>FinalizingSetupPhaseTriggered</c> → <see cref="DecisionSignalKind.EspPhaseChanged"/> (phase=FinalizingSetup)</item>
|
||||
/// <item><c>WhiteGloveCompleted</c> → <see cref="DecisionSignalKind.WhiteGloveShellCoreSuccess"/></item>
|
||||
/// <item><c>WhiteGloveCompleted</c> → <see cref="DecisionSignalKind.WhiteGloveShellCoreSuccess"/> (payload: registry AccountSetup progress at that moment)</item>
|
||||
/// <item><c>EspFailureDetected</c> → <see cref="DecisionSignalKind.EspTerminalFailure"/> (merged aus ShellCore + Provisioning)</item>
|
||||
/// <item><c>DeviceSetupProvisioningComplete</c> → <see cref="DecisionSignalKind.DeviceSetupProvisioningComplete"/></item>
|
||||
/// <item><c>AccountSetupProvisioningComplete</c> → <see cref="DecisionSignalKind.AccountSetupProvisioningComplete"/> (session 330f73f3 fix — strong AccountSetup-done gate)</item>
|
||||
@@ -46,6 +46,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
private readonly EspAndHelloTracker _coordinator;
|
||||
private readonly ISignalIngressSink _ingress;
|
||||
private readonly IClock _clock;
|
||||
private readonly Func<bool?> _accountSetupProgressProbe;
|
||||
|
||||
private bool _helloPosted;
|
||||
private bool _helloWizardStartedPosted;
|
||||
@@ -60,11 +61,14 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
public EspAndHelloTrackerAdapter(
|
||||
EspAndHelloTracker coordinator,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock)
|
||||
IClock clock,
|
||||
Func<bool?>? accountSetupProgressProbe = null)
|
||||
{
|
||||
_coordinator = coordinator ?? throw new ArgumentNullException(nameof(coordinator));
|
||||
_ingress = ingress ?? throw new ArgumentNullException(nameof(ingress));
|
||||
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
// Test seam; production reads the provisioning tracker's registry-derived flag.
|
||||
_accountSetupProgressProbe = accountSetupProgressProbe ?? (() => _coordinator.AccountSetupProgressIfRead);
|
||||
|
||||
_coordinator.HelloCompleted += OnHelloCompleted;
|
||||
_coordinator.FinalizingSetupPhaseTriggered += OnFinalizing;
|
||||
@@ -216,13 +220,29 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
if (_whiteGloveSuccessPosted) return;
|
||||
_whiteGloveSuccessPosted = true;
|
||||
|
||||
// Registry AccountSetup progress at the moment of the success page. The sealing
|
||||
// classifier lifts its AccountSetup excluder only on an explicit "false" — IME logs
|
||||
// AccountSetup from the device session once the device apps are done, which is no
|
||||
// user activity. Sampled here so replay reads it from the SignalLog (L.2/L.5). An
|
||||
// unread registry sends no key: unknown must keep the excluder, never lift it.
|
||||
var progress = _accountSetupProgressProbe();
|
||||
var progressText = progress.HasValue ? (progress.Value ? "true" : "false") : null;
|
||||
|
||||
var now = ResolveOccurredAt(out var derivedFromClock);
|
||||
var derivationInputs = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["subSource"] = "ShellCoreTracker",
|
||||
[SignalPayloadKeys.AccountSetupProgress] = progressText ?? "unread",
|
||||
};
|
||||
TagDerivedTimestamp(derivationInputs, derivedFromClock);
|
||||
|
||||
var payload = progressText == null
|
||||
? null
|
||||
: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
[SignalPayloadKeys.AccountSetupProgress] = progressText,
|
||||
};
|
||||
|
||||
_ingress.Post(
|
||||
kind: DecisionSignalKind.WhiteGloveShellCoreSuccess,
|
||||
occurredAtUtc: now,
|
||||
@@ -231,7 +251,8 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
kind: EvidenceKind.Derived,
|
||||
identifier: DetectorId,
|
||||
summary: "WhiteGlove sealing success (coordinator-forwarded)",
|
||||
derivationInputs: derivationInputs));
|
||||
derivationInputs: derivationInputs),
|
||||
payload: payload);
|
||||
}
|
||||
|
||||
private void EmitEspFailure(EspFailureDetectedEventArgs args)
|
||||
|
||||
+11
-2
@@ -169,6 +169,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
_logger.Info(
|
||||
$"EnrollmentTerminationHandler: handling Terminated (reason={args.Reason}, outcome={args.Outcome}, stage={args.StageName}).");
|
||||
|
||||
// The seal is already persisted in the decision snapshot. The marker goes down
|
||||
// before anything else: the technician's reseal can kill this process at any
|
||||
// later step, and a missing marker makes the next boot run the 48-hour break
|
||||
// against the Part-1 session age instead of resuming Part 2.
|
||||
if (isWhiteGlovePart1)
|
||||
{
|
||||
TrySaveWhiteGloveComplete();
|
||||
}
|
||||
|
||||
var state = TryGetCurrentState();
|
||||
|
||||
// M4.6.δ — shutdown analyzers run BEFORE the dialog / diagnostics upload so
|
||||
@@ -254,7 +263,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
|
||||
// WhiteGlove Part-1 exit: keep the session alive, but announce the handoff so
|
||||
// the timeline clearly marks the transition. The `whiteglove.complete` marker
|
||||
// lets the next agent boot classify itself as a Part-2 resume.
|
||||
// (written at the top of this handler) lets the next agent boot classify itself
|
||||
// as a Part-2 resume.
|
||||
if (isWhiteGlovePart1)
|
||||
{
|
||||
// Stop peripheral collectors BEFORE the Part-1-complete marker so their
|
||||
@@ -298,7 +308,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
DelayLateEventGrace();
|
||||
DrainSpool();
|
||||
|
||||
TrySaveWhiteGloveComplete();
|
||||
// Option 2 (WG Part 1 graceful-exit hardening, 2026-04-30): write the
|
||||
// clean-exit marker BEFORE _signalShutdown returns control to the main
|
||||
// thread. The AppDomain.ProcessExit handler still writes it as a second
|
||||
|
||||
@@ -308,7 +308,7 @@ namespace AutopilotMonitor.Agent.V2
|
||||
Func<CleanupService> cleanupServiceFactory,
|
||||
AgentLogger logger,
|
||||
bool consoleMode,
|
||||
Action onBreakFired = null)
|
||||
Action<double> onBreakFired = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -347,7 +347,7 @@ namespace AutopilotMonitor.Agent.V2
|
||||
// (tasks/enrollment-status-reclassification.md). Fired BEFORE cleanup, while the
|
||||
// session state and network are still intact. Never throws — a send failure (e.g. no
|
||||
// network) must not block the cleanup/exit that is the whole point of this guard.
|
||||
try { onBreakFired?.Invoke(); }
|
||||
try { onBreakFired?.Invoke(sessionAgeHours); }
|
||||
catch (Exception cbEx) { logger.Debug($"Emergency-break notify callback failed: {cbEx.Message}"); }
|
||||
|
||||
// Write enrollment-complete.marker so the next start exits cleanly even if
|
||||
|
||||
@@ -273,7 +273,9 @@ namespace AutopilotMonitor.Agent.V2
|
||||
var registration = Runtime.BackendSessionRegistration.Register(
|
||||
bootstrap.AgentConfig, auth, telemetry.MtlsHttpClient, GetAgentVersion(), consoleMode, logger,
|
||||
rotateSession: () => Runtime.BackendSessionRegistration.RotateSession(
|
||||
bootstrap.AgentConfig, bootstrap.SessionPersistence, auth, transportDir, logger));
|
||||
bootstrap.AgentConfig, bootstrap.SessionPersistence, auth, transportDir, logger),
|
||||
sessionPersistence: bootstrap.SessionPersistence,
|
||||
configFetchOutcome: runtimeConfig.RemoteConfigService.LastFetchOutcome.ToString());
|
||||
if (registration.ShouldExit)
|
||||
{
|
||||
return registration.ExitCode;
|
||||
|
||||
@@ -87,7 +87,11 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
cleanupServiceFactory, logger, consoleMode,
|
||||
// Best-effort: surface the otherwise-silent 48h break to the backend before cleanup
|
||||
// (tasks/enrollment-status-reclassification.md). Never blocks the exit.
|
||||
onBreakFired: () => EmergencyBreakReporter.TrySend(agentConfig, Program.GetAgentVersion(), logger)))
|
||||
// The failed-registration record is read here, before the break deletes the
|
||||
// session files.
|
||||
onBreakFired: sessionAgeHours => EmergencyBreakReporter.TrySend(
|
||||
agentConfig, Program.GetAgentVersion(), sessionAgeHours,
|
||||
sessionPersistence.LoadRegistrationFailure(), logger)))
|
||||
{
|
||||
logger.Info("Emergency break fired — agent exiting.");
|
||||
return BootstrapResult.Exit(0);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
@@ -17,7 +18,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// <c>IncrementSessionEventCountAsync</c> / <c>UpdateSessionStatusAsync</c> silently
|
||||
/// no-op and session status / phase / admin-overrides / validator reconcile all break.
|
||||
/// On registration failure we follow V1's rule: collectors MUST NOT start and the
|
||||
/// agent exits cleanly so the next Scheduled-Task tick can retry.
|
||||
/// agent exits cleanly; the Scheduled Task has only a boot trigger, so the next attempt is
|
||||
/// the next boot. The failed run is recorded for the emergency break report.
|
||||
/// </summary>
|
||||
internal static class BackendSessionRegistration
|
||||
{
|
||||
@@ -28,12 +30,19 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
string agentVersion,
|
||||
bool consoleMode,
|
||||
AgentLogger logger,
|
||||
Func<string> rotateSession = null)
|
||||
Func<string> rotateSession = null,
|
||||
SessionIdPersistence sessionPersistence = null,
|
||||
string configFetchOutcome = null)
|
||||
{
|
||||
if (agentConfig == null) throw new ArgumentNullException(nameof(agentConfig));
|
||||
if (auth == null) throw new ArgumentNullException(nameof(auth));
|
||||
if (logger == null) throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
// Wall clock on purpose: a window far above the retry schedule is the evidence that
|
||||
// the device slept or was suspended while this run tried to register.
|
||||
var startedUtc = DateTime.UtcNow;
|
||||
var linkUpAtStart = TryIsNetworkLinkUp();
|
||||
|
||||
var registrationResult = SessionRegistrationHelper.RegisterWithRetryAsync(
|
||||
apiClient: auth.BackendApiClient,
|
||||
agentConfig: agentConfig,
|
||||
@@ -48,6 +57,10 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
rotateSession: rotateSession)
|
||||
.GetAwaiter().GetResult();
|
||||
|
||||
RecordRegistrationRun(
|
||||
sessionPersistence, registrationResult, startedUtc, DateTime.UtcNow,
|
||||
linkUpAtStart, TryIsNetworkLinkUp(), configFetchOutcome, logger);
|
||||
|
||||
if (registrationResult.Outcome != SessionRegistrationOutcome.Succeeded)
|
||||
{
|
||||
logger.Error(
|
||||
@@ -65,6 +78,62 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
return SessionRegistrationOutcomeResult.Continue(registrationResult);
|
||||
}
|
||||
|
||||
/// <summary>Longest <see cref="RegistrationFailureSummary.LastError"/> kept on disk and on the wire.</summary>
|
||||
internal const int MaxRecordedErrorLength = 256;
|
||||
|
||||
/// <summary>
|
||||
/// Keeps <c>registration-failure.json</c> in step with this run: a failed run is folded
|
||||
/// into the record (the emergency break later reports it — for a session that never
|
||||
/// registered, the only account of why), a successful one clears it. Never throws.
|
||||
/// </summary>
|
||||
internal static void RecordRegistrationRun(
|
||||
SessionIdPersistence sessionPersistence,
|
||||
SessionRegistrationResult registrationResult,
|
||||
DateTime startedUtc,
|
||||
DateTime endedUtc,
|
||||
bool? linkUpAtStart,
|
||||
bool? linkUpAtEnd,
|
||||
string configFetchOutcome,
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (sessionPersistence == null || registrationResult == null) return;
|
||||
try
|
||||
{
|
||||
if (registrationResult.Outcome == SessionRegistrationOutcome.Succeeded)
|
||||
{
|
||||
sessionPersistence.ClearRegistrationFailure(logger);
|
||||
return;
|
||||
}
|
||||
|
||||
var error = registrationResult.ErrorMessage;
|
||||
if (error != null && error.Length > MaxRecordedErrorLength)
|
||||
error = error.Substring(0, MaxRecordedErrorLength);
|
||||
|
||||
sessionPersistence.RecordRegistrationFailure(new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = 1,
|
||||
FirstFailedAtUtc = endedUtc,
|
||||
LastFailedAtUtc = endedUtc,
|
||||
Outcome = registrationResult.Outcome.ToString(),
|
||||
LastError = error,
|
||||
NetworkLinkUpAtStart = linkUpAtStart,
|
||||
NetworkLinkUpAtEnd = linkUpAtEnd,
|
||||
AttemptWindowSeconds = Math.Round(Math.Max(0, (endedUtc - startedUtc).TotalSeconds), 1),
|
||||
ConfigFetchOutcome = configFetchOutcome,
|
||||
}, logger);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"Registration run record failed (best-effort): {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static bool? TryIsNetworkLinkUp()
|
||||
{
|
||||
try { return NetworkInterface.GetIsNetworkAvailable(); }
|
||||
catch { return null; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SESSION-OWNER-BINDING: the one place that turns "backend says this session id is not
|
||||
/// ours" into a new session. Rotates the persisted id, points the runtime config at it,
|
||||
|
||||
@@ -36,15 +36,20 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// Upper bound for the blocking wait: NIC wait + 3×15s attempts + 2×5s delays + slack.
|
||||
private static readonly TimeSpan OverallBudget = TimeSpan.FromSeconds(75);
|
||||
|
||||
public static void TrySend(AgentConfiguration agentConfig, string agentVersion, AgentLogger logger)
|
||||
public static void TrySend(
|
||||
AgentConfiguration agentConfig,
|
||||
string agentVersion,
|
||||
double sessionAgeHours,
|
||||
RegistrationFailureSummary priorRegistrationFailure,
|
||||
AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
WaitForNetwork(logger);
|
||||
|
||||
var auth = BackendClientFactory.BuildAuthClients(agentConfig, agentVersion, logger);
|
||||
var message =
|
||||
$"Agent absolute session-age emergency break fired (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting.";
|
||||
var roundedAgeHours = Math.Round(sessionAgeHours, 1);
|
||||
var message = BuildMessage(roundedAgeHours, agentConfig.AbsoluteMaxSessionHours, priorRegistrationFailure);
|
||||
|
||||
// The process is about to exit, so block on the otherwise fire-and-forget send
|
||||
// rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP
|
||||
@@ -56,7 +61,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
message,
|
||||
attempts: SendAttempts,
|
||||
perAttemptTimeout: PerAttemptTimeout,
|
||||
retryDelay: RetryDelay)
|
||||
retryDelay: RetryDelay,
|
||||
sessionAgeHours: roundedAgeHours,
|
||||
priorRegistrationFailure: priorRegistrationFailure)
|
||||
.Wait(OverallBudget);
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -65,6 +72,25 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invariant culture: the text lands in the backend's ops feed and timeline, and a device
|
||||
/// locale would otherwise print "478,2h". The registration clause names what the ops feed
|
||||
/// cannot otherwise show for a session that never registered.
|
||||
/// </summary>
|
||||
internal static string BuildMessage(double roundedAgeHours, int capHours, RegistrationFailureSummary priorRegistrationFailure)
|
||||
{
|
||||
var message = FormattableString.Invariant(
|
||||
$"Agent absolute session-age emergency break fired at session age {roundedAgeHours:F1}h (cap {capHours}h) — cleaning up and exiting.");
|
||||
if (priorRegistrationFailure != null && priorRegistrationFailure.FailedRuns > 0)
|
||||
{
|
||||
message += FormattableString.Invariant(
|
||||
$" {priorRegistrationFailure.FailedRuns} earlier start(s) failed to register (last: {priorRegistrationFailure.Outcome}, network link at end: {FormatLink(priorRegistrationFailure.NetworkLinkUpAtEnd)}).");
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
private static string FormatLink(bool? up) => up == null ? "unknown" : (up.Value ? "up" : "down");
|
||||
|
||||
/// <summary>
|
||||
/// Boot-time NIC grace: the break fires during bootstrap, often seconds after boot while
|
||||
/// Wi-Fi is still associating. Polls the cheap link-level signal — it cannot prove backend
|
||||
|
||||
@@ -522,7 +522,8 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
.WithSkipUserEsp(value: true, sourceSignalOrdinal: 4)
|
||||
.WithEspSyncFailureTimeoutMinutes(value: 90, sourceSignalOrdinal: 5)
|
||||
.WithRegistrySelfDeployingProfile(value: false, sourceSignalOrdinal: 6)
|
||||
.WithCloudPc(value: true, sourceSignalOrdinal: 7))
|
||||
.WithCloudPc(value: true, sourceSignalOrdinal: 7)
|
||||
.WithAccountSetupProgressAtWhiteGloveSuccess(value: false, sourceSignalOrdinal: 8))
|
||||
.Build();
|
||||
|
||||
var json = StateSerializer.Serialize(state);
|
||||
@@ -538,6 +539,8 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
Assert.Equal(6, o.RegistrySelfDeployingProfile!.SourceSignalOrdinal);
|
||||
Assert.True(o.CloudPc!.Value);
|
||||
Assert.Equal(7, o.CloudPc!.SourceSignalOrdinal);
|
||||
Assert.False(o.AccountSetupProgressAtWhiteGloveSuccess!.Value);
|
||||
Assert.Equal(8, o.AccountSetupProgressAtWhiteGloveSuccess!.SourceSignalOrdinal);
|
||||
|
||||
// Never-observed facts must rehydrate as null (null ≠ veto — session 62e603c9).
|
||||
Assert.Null(o.WhiteGloveSealingPatternSeen);
|
||||
|
||||
@@ -31,7 +31,6 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
DecisionSignalKind.HelloResolved,
|
||||
DecisionSignalKind.ImeUserSessionCompleted,
|
||||
DecisionSignalKind.DeviceSetupProvisioningComplete,
|
||||
DecisionSignalKind.WhiteGloveShellCoreSuccess,
|
||||
DecisionSignalKind.WhiteGloveSealingPatternDetected,
|
||||
DecisionSignalKind.AadUserJoinedLate,
|
||||
DecisionSignalKind.DeviceInfoCollected,
|
||||
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.DecisionCore.Classifiers;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Serialization;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.DecisionCore.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// The sealing classifier's AccountSetup excluder (-40) applies only when the IME reported
|
||||
/// the AccountSetup phase AND the WhiteGlove_Success signal did not report an untouched
|
||||
/// AccountSetup registry. IME logs "EspPhase: AccountSetup" from the device session once
|
||||
/// the device apps are done — on pre-provisioning with the user ESP enabled that line
|
||||
/// routinely precedes the technician's success page and used to leave Part 1 unsealed.
|
||||
/// The gate may only ever lift the excluder, never add one.
|
||||
/// </summary>
|
||||
public sealed class WhiteGloveAccountSetupProgressGateTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 9, 29, 7, 25, 9, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public void Ime_account_setup_with_untouched_registry_seals_on_the_fast_path()
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
|
||||
|
||||
Assert.Equal(SessionStage.WhiteGloveSealed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.WhiteGlovePart1Sealed, step.NewState.Outcome);
|
||||
Assert.Equal(HypothesisLevel.Confirmed, step.NewState.ClassifierOutcomes.WhiteGloveSealing.Level);
|
||||
Assert.Equal(WhiteGloveSealingClassifier.WeightShellCoreWhiteGloveSuccess,
|
||||
step.NewState.ClassifierOutcomes.WhiteGloveSealing.Score);
|
||||
|
||||
var fact = step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess;
|
||||
Assert.NotNull(fact);
|
||||
Assert.False(fact!.Value);
|
||||
Assert.Equal(3, fact.SourceSignalOrdinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ime_account_setup_with_registry_progress_keeps_the_excluder()
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "true"));
|
||||
|
||||
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
|
||||
Assert.Contains(step.Effects, e => e.Kind == DecisionEffectKind.RunClassifier);
|
||||
Assert.True(DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState).HasAccountSetupActivity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Missing_progress_payload_keeps_the_excluder_for_older_agents()
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: null));
|
||||
|
||||
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
|
||||
Assert.Null(step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess);
|
||||
Assert.True(DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState).HasAccountSetupActivity);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData("yes")]
|
||||
public void Unparseable_progress_payload_is_treated_as_missing(string raw)
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: raw));
|
||||
|
||||
Assert.Null(step.NewState.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess);
|
||||
Assert.NotEqual(SessionStage.WhiteGloveSealed, step.NewState.Stage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Registry_progress_without_ime_account_setup_adds_no_excluder()
|
||||
{
|
||||
var state = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
|
||||
.WithStage(SessionStage.EspDeviceSetup)
|
||||
.WithStepIndex(1)
|
||||
.WithLastAppliedSignalOrdinal(0)
|
||||
.Build();
|
||||
|
||||
var step = new DecisionEngine().Reduce(state, MakeShellCoreSignal(ordinal: 1, accountSetupProgress: "true"));
|
||||
|
||||
Assert.Equal(SessionStage.WhiteGloveSealed, step.NewState.Stage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Untouched_registry_does_not_lift_a_hard_excluder()
|
||||
{
|
||||
var builder = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
|
||||
.WithStage(SessionStage.EspAccountSetup)
|
||||
.WithStepIndex(3)
|
||||
.WithLastAppliedSignalOrdinal(2);
|
||||
builder.AccountSetupEnteredUtc = new SignalFact<DateTime>(T0.AddMinutes(-3), 1);
|
||||
builder.DesktopArrivedUtc = new SignalFact<DateTime>(T0.AddMinutes(-1), 2);
|
||||
|
||||
var step = new DecisionEngine().Reduce(builder.Build(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
|
||||
|
||||
Assert.NotEqual(SessionStage.WhiteGloveSealed, step.NewState.Stage);
|
||||
var snapshot = DecisionEngine.BuildWhiteGloveSealingSnapshot(step.NewState);
|
||||
Assert.False(snapshot.HasAccountSetupActivity);
|
||||
Assert.True(snapshot.DesktopArrived);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Progress_fact_survives_the_state_serializer()
|
||||
{
|
||||
var step = new DecisionEngine().Reduce(
|
||||
StateWithImeAccountSetup(), MakeShellCoreSignal(ordinal: 3, accountSetupProgress: "false"));
|
||||
|
||||
var roundTripped = StateSerializer.Deserialize(StateSerializer.Serialize(step.NewState));
|
||||
|
||||
var fact = roundTripped.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess;
|
||||
Assert.NotNull(fact);
|
||||
Assert.False(fact!.Value);
|
||||
Assert.Equal(3, fact.SourceSignalOrdinal);
|
||||
}
|
||||
|
||||
private static DecisionState StateWithImeAccountSetup()
|
||||
{
|
||||
// The IME device-session line: AccountSetup entered before the success page.
|
||||
var builder = DecisionState.CreateInitial("s-gate", "t-gate").ToBuilder()
|
||||
.WithStage(SessionStage.EspAccountSetup)
|
||||
.WithStepIndex(3)
|
||||
.WithLastAppliedSignalOrdinal(2);
|
||||
builder.AccountSetupEnteredUtc = new SignalFact<DateTime>(T0.AddMinutes(-3), 2);
|
||||
return builder.Build();
|
||||
}
|
||||
|
||||
private static DecisionSignal MakeShellCoreSignal(long ordinal, string? accountSetupProgress)
|
||||
{
|
||||
var payload = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
if (accountSetupProgress != null)
|
||||
{
|
||||
payload[SignalPayloadKeys.AccountSetupProgress] = accountSetupProgress;
|
||||
}
|
||||
|
||||
return new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: DecisionSignalKind.WhiteGloveShellCoreSuccess,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: T0,
|
||||
sourceOrigin: "EspAndHelloTracker",
|
||||
evidence: new Evidence(
|
||||
kind: EvidenceKind.Raw,
|
||||
identifier: "ShellCore-62407",
|
||||
summary: "WhiteGlove_Success"),
|
||||
payload: payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,9 +4,11 @@ using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Ingest;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
@@ -94,14 +96,29 @@ public class ReportAgentErrorFunctionTests
|
||||
// (2026-07-23 hardening): timeline event + cross-session EventType index row +
|
||||
// AgentEmergencyBreak ops event, once per session. The index upsert is load-bearing:
|
||||
// StoreEventsBatchAsync alone leaves the event invisible to every search-by-eventType
|
||||
// surface (found in the 2026-07-22 incident analysis).
|
||||
// surface (found in the 2026-07-22 incident analysis). A session without a writable
|
||||
// row gets the ops event only.
|
||||
// =========================================================================
|
||||
|
||||
private static TableEntity Row(SessionStatus? status = SessionStatus.InProgress, string? deletionState = null)
|
||||
{
|
||||
var row = new TableEntity(TenantId, SessionId);
|
||||
if (status.HasValue) row["Status"] = status.Value.ToString();
|
||||
if (deletionState != null) row["DeletionState"] = deletionState;
|
||||
return row;
|
||||
}
|
||||
|
||||
private static ReportAgentErrorFunction.SessionRowRead Read(TableEntity? row) => new(row, Failed: false);
|
||||
|
||||
private static readonly ReportAgentErrorFunction.SessionRowRead ReadFailed = new(null, Failed: true);
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public readonly Mock<ISessionRepository> SessionRepo = new();
|
||||
public readonly List<OpsEventEntry> OpsEvents = new();
|
||||
public readonly OpsEventService OpsService;
|
||||
private readonly SessionDeletionGuard _deletionGuard = new(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(), NullLogger<SessionDeletionGuard>.Instance);
|
||||
|
||||
public Harness(List<EnrollmentEvent>? existingEvents = null)
|
||||
{
|
||||
@@ -125,9 +142,23 @@ public class ReportAgentErrorFunctionTests
|
||||
OpsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
}
|
||||
|
||||
/// <summary>Default read: an existing, open session row — the historical path.</summary>
|
||||
public Task RunAsync(AgentErrorReport report)
|
||||
=> RunAsync(report, Read(Row()));
|
||||
|
||||
public Task RunAsync(AgentErrorReport report, ReportAgentErrorFunction.SessionRowRead? sessionRead)
|
||||
=> ReportAgentErrorFunction.MaterializeEmergencyBreakArtifactsAsync(
|
||||
report, TenantId, SessionRepo.Object, OpsService, NullLogger<ReportAgentErrorFunction>.Instance);
|
||||
report, TenantId, sessionRead, SessionRepo.Object, _deletionGuard, OpsService,
|
||||
NullLogger<ReportAgentErrorFunction>.Instance);
|
||||
|
||||
public void VerifyNothingWrittenIntoTheSession()
|
||||
{
|
||||
SessionRepo.Verify(r => r.GetSessionEventsAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>()), Times.Never);
|
||||
SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Never);
|
||||
SessionRepo.Verify(r => r.UpsertEventTypeIndexBatchAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<IEnumerable<EnrollmentEvent>>()), Times.Never);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -165,7 +196,7 @@ public class ReportAgentErrorFunctionTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Non_break_error_types_and_missing_session_id_are_ignored()
|
||||
public async Task Non_break_error_types_and_ids_without_a_row_read_are_ignored()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
@@ -173,12 +204,10 @@ public class ReportAgentErrorFunctionTests
|
||||
other.ErrorType = AgentErrorType.ConfigFetchFailed;
|
||||
await h.RunAsync(other);
|
||||
|
||||
var noSession = Report(Now);
|
||||
noSession.SessionId = null!;
|
||||
await h.RunAsync(noSession);
|
||||
// No read = the ids were not GUIDs (ReadSessionRowAsync) — no session can exist to break.
|
||||
await h.RunAsync(Report(Now), sessionRead: null);
|
||||
|
||||
h.SessionRepo.Verify(r => r.GetSessionEventsAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>()), Times.Never);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
Assert.Empty(h.OpsEvents);
|
||||
}
|
||||
|
||||
@@ -192,10 +221,8 @@ public class ReportAgentErrorFunctionTests
|
||||
public async Task Break_on_already_terminal_session_is_info_late_cleanup(SessionStatus status)
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ReturnsAsync(new SessionSummary { SessionId = SessionId, TenantId = TenantId, Status = status });
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
await h.RunAsync(Report(Now), Read(Row(status)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
@@ -216,44 +243,220 @@ public class ReportAgentErrorFunctionTests
|
||||
public async Task Break_on_open_session_stays_warning(SessionStatus status)
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ReturnsAsync(new SessionSummary { SessionId = SessionId, TenantId = TenantId, Status = status });
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
await h.RunAsync(Report(Now), Read(Row(status)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains($"still {status}", ops.Message);
|
||||
Assert.Contains("\"lateCleanup\":false", ops.Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Once);
|
||||
}
|
||||
|
||||
// 2026-09-28 sweep: every "unknown" Warning named a session that never registered. The break is
|
||||
// the only thing the backend ever hears of it — nothing to close, and a timeline event would be
|
||||
// an orphan the maintenance sweep deletes a day later (with a Warning of its own).
|
||||
[Fact]
|
||||
public async Task Break_on_session_without_row_is_info_and_writes_nothing_into_the_session()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(null));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
Assert.Contains("no session row", ops.Message);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"missing\"", ops.Details);
|
||||
Assert.Contains("\"lateCleanup\":false", ops.Details);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Preparing)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Queued)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Running)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Poisoned)]
|
||||
public async Task Break_on_session_under_cascade_delete_writes_nothing_past_the_lock(string deletionState)
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(Row(SessionStatus.InProgress, deletionState)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"deleting\"", ops.Details);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Unknown_or_unreadable_session_status_keeps_warning()
|
||||
public async Task Unreadable_row_keeps_the_historical_warning_and_write()
|
||||
{
|
||||
// No row (Harness default: GetSessionAsync → null) — the historical Warning semantics.
|
||||
// Existence unknown — fail safe: the break may be the only signal for an open session.
|
||||
var h = new Harness();
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
await h.RunAsync(Report(Now), ReadFailed);
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"unknown\"", ops.Details);
|
||||
|
||||
// Row read throws — the break must still be recorded, still as Warning.
|
||||
var h2 = new Harness();
|
||||
h2.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ThrowsAsync(new InvalidOperationException("table down"));
|
||||
await h2.RunAsync(Report(Now));
|
||||
Assert.Equal(OpsEventSeverity.Warning, Assert.Single(h2.OpsEvents).Severity);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClassifyEmergencyBreak_is_terminal_set_only()
|
||||
public async Task Row_without_parseable_status_stays_warning()
|
||||
{
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Succeeded));
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Failed));
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Incomplete));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.AwaitingUser));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Unknown));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(null));
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(Row(status: null)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"unknown\"", ops.Details);
|
||||
}
|
||||
|
||||
// The agent-measured age separates a break just past the cap from a device that slept for
|
||||
// weeks; the backend cannot derive it (no row, or a row registered seconds before the break).
|
||||
[Fact]
|
||||
public async Task Reported_session_age_lands_in_ops_details_and_timeline_data()
|
||||
{
|
||||
var h = new Harness();
|
||||
var report = Report(Now);
|
||||
report.SessionAgeHours = 478.2;
|
||||
|
||||
await h.RunAsync(report);
|
||||
|
||||
Assert.Contains("\"sessionAgeHours\":478.2", Assert.Single(h.OpsEvents).Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => (double)e[0].Data["sessionAgeHours"] == 478.2)), Times.Once);
|
||||
}
|
||||
|
||||
// A session that never registered leaves the backend nothing but this report; the agent's
|
||||
// record of its failed starts is the only account of why.
|
||||
[Fact]
|
||||
public async Task Prior_registration_failure_lands_in_ops_details_even_without_a_row()
|
||||
{
|
||||
var h = new Harness();
|
||||
var report = Report(Now);
|
||||
report.PriorRegistrationFailure = ReportAgentErrorFunction.SanitizeRegistrationFailure(Failure());
|
||||
|
||||
await h.RunAsync(report, Read(null));
|
||||
|
||||
var details = Assert.Single(h.OpsEvents).Details;
|
||||
Assert.Contains("\"priorRegistrationFailure\":{\"failedRuns\":3", details);
|
||||
Assert.Contains("\"networkLinkUpAtEnd\":false", details);
|
||||
Assert.Contains("\"attemptWindowSeconds\":1204.5", details);
|
||||
Assert.Contains("\"configFetchOutcome\":\"UsedDefaults\"", details);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Prior_registration_failure_lands_in_the_timeline_data_of_a_registered_session()
|
||||
{
|
||||
var h = new Harness();
|
||||
var report = Report(Now);
|
||||
report.PriorRegistrationFailure = ReportAgentErrorFunction.SanitizeRegistrationFailure(Failure());
|
||||
|
||||
await h.RunAsync(report);
|
||||
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e =>
|
||||
((Dictionary<string, object?>)e[0].Data["priorRegistrationFailure"])["failedRuns"]!.Equals(3))), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_without_registration_record_leaves_it_null_and_out_of_the_timeline()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
Assert.Contains("\"priorRegistrationFailure\":null", Assert.Single(h.OpsEvents).Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => !e[0].Data.ContainsKey("priorRegistrationFailure"))), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SanitizeRegistrationFailure_bounds_device_input()
|
||||
{
|
||||
var raw = Failure();
|
||||
raw.LastError = new string('x', 5000);
|
||||
raw.Outcome = new string('o', 500);
|
||||
raw.ConfigFetchOutcome = new string('c', 500);
|
||||
raw.AttemptWindowSeconds = -3;
|
||||
|
||||
var clean = ReportAgentErrorFunction.SanitizeRegistrationFailure(raw)!;
|
||||
|
||||
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationErrorLength, clean.LastError!.Length);
|
||||
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationTokenLength, clean.Outcome.Length);
|
||||
Assert.Equal(ReportAgentErrorFunction.MaxRegistrationTokenLength, clean.ConfigFetchOutcome!.Length);
|
||||
Assert.Equal(0, clean.AttemptWindowSeconds);
|
||||
Assert.Equal(3, clean.FailedRuns);
|
||||
Assert.NotSame(raw, clean);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(-1)]
|
||||
public void SanitizeRegistrationFailure_drops_a_record_without_failed_runs(int failedRuns)
|
||||
{
|
||||
var raw = Failure();
|
||||
raw.FailedRuns = failedRuns;
|
||||
|
||||
Assert.Null(ReportAgentErrorFunction.SanitizeRegistrationFailure(raw));
|
||||
Assert.Null(ReportAgentErrorFunction.SanitizeRegistrationFailure(null));
|
||||
}
|
||||
|
||||
private static RegistrationFailureSummary Failure() => new()
|
||||
{
|
||||
FailedRuns = 3,
|
||||
FirstFailedAtUtc = Now.AddDays(-20),
|
||||
LastFailedAtUtc = Now.AddDays(-19),
|
||||
Outcome = "Failed",
|
||||
LastError = "No such host is known.",
|
||||
NetworkLinkUpAtStart = false,
|
||||
NetworkLinkUpAtEnd = false,
|
||||
AttemptWindowSeconds = 1204.5,
|
||||
ConfigFetchOutcome = "UsedDefaults",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_without_session_age_leaves_it_null_and_out_of_the_timeline()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
Assert.Contains("\"sessionAgeHours\":null", Assert.Single(h.OpsEvents).Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => !e[0].Data.ContainsKey("sessionAgeHours"))), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClassifyEmergencyBreak_follows_row_state()
|
||||
{
|
||||
static ReportAgentErrorFunction.EmergencyBreakVerdict Classify(TableEntity? row, bool failed = false, bool locked = false)
|
||||
=> ReportAgentErrorFunction.ClassifyEmergencyBreak(row, failed, locked);
|
||||
|
||||
foreach (var terminal in new[] { SessionStatus.Succeeded, SessionStatus.Failed, SessionStatus.Incomplete })
|
||||
{
|
||||
var v = Classify(Row(terminal));
|
||||
Assert.Equal((OpsEventSeverity.Info, true, true, terminal.ToString()), (v.Severity, v.LateCleanup, v.Materialize, v.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
foreach (var open in new[] { SessionStatus.InProgress, SessionStatus.Pending, SessionStatus.AwaitingUser, SessionStatus.Stalled, SessionStatus.Unknown })
|
||||
{
|
||||
var v = Classify(Row(open));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false, true, open.ToString()), (v.Severity, v.LateCleanup, v.Materialize, v.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
var missing = Classify(null);
|
||||
Assert.Equal((OpsEventSeverity.Info, false, false, "missing"), (missing.Severity, missing.LateCleanup, missing.Materialize, missing.SessionStatusAtBreak));
|
||||
|
||||
var deleting = Classify(Row(SessionStatus.Succeeded), locked: true);
|
||||
Assert.Equal((OpsEventSeverity.Info, false, false, "deleting"), (deleting.Severity, deleting.LateCleanup, deleting.Materialize, deleting.SessionStatusAtBreak));
|
||||
|
||||
var unreadable = Classify(null, failed: true);
|
||||
Assert.Equal((OpsEventSeverity.Warning, false, true, "unknown"), (unreadable.Severity, unreadable.LateCleanup, unreadable.Materialize, unreadable.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+216
-80
@@ -1,13 +1,16 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.ApplicationInsights;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
@@ -41,6 +44,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly OpsEventService _opsEventService;
|
||||
private readonly Services.Deletion.ISessionDeletionInventoryReader _sessionRowReader;
|
||||
private readonly SessionDeletionGuard _deletionGuard;
|
||||
private readonly SessionOwnerBindingObserver _ownerBinding;
|
||||
|
||||
public ReportAgentErrorFunction(
|
||||
@@ -58,6 +62,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
ISessionRepository sessionRepo,
|
||||
OpsEventService opsEventService,
|
||||
Services.Deletion.ISessionDeletionInventoryReader sessionRowReader,
|
||||
SessionDeletionGuard deletionGuard,
|
||||
SessionOwnerBindingObserver ownerBinding)
|
||||
{
|
||||
_logger = logger;
|
||||
@@ -74,6 +79,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
_sessionRepo = sessionRepo;
|
||||
_opsEventService = opsEventService;
|
||||
_sessionRowReader = sessionRowReader;
|
||||
_deletionGuard = deletionGuard;
|
||||
_ownerBinding = ownerBinding;
|
||||
}
|
||||
|
||||
@@ -161,6 +167,9 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
return req.CreateResponse(HttpStatusCode.OK);
|
||||
}
|
||||
|
||||
report.PriorRegistrationFailure = SanitizeRegistrationFailure(report.PriorRegistrationFailure);
|
||||
var priorRegistration = report.PriorRegistrationFailure;
|
||||
|
||||
// Emit a structured log entry (captured by App Insights as a trace)
|
||||
_logger.LogCritical(
|
||||
"AgentEmergencyError [{ErrorType}] tenant={TenantId} session={SessionId} http={HttpStatusCode} seq={SequenceNumber} ver={AgentVersion}: {Message}",
|
||||
@@ -184,16 +193,29 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
["AgentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["Message"] = report.Message ?? string.Empty,
|
||||
["AgentTimestamp"] = report.Timestamp.ToString("O"),
|
||||
["SessionAgeHours"] = report.SessionAgeHours?.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationFailedRuns"] = priorRegistration?.FailedRuns.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationOutcome"] = priorRegistration?.Outcome ?? string.Empty,
|
||||
["PriorRegistrationLastError"] = priorRegistration?.LastError ?? string.Empty,
|
||||
["PriorRegistrationLinkUpAtStart"] = FormatNullableBool(priorRegistration?.NetworkLinkUpAtStart),
|
||||
["PriorRegistrationLinkUpAtEnd"] = FormatNullableBool(priorRegistration?.NetworkLinkUpAtEnd),
|
||||
["PriorRegistrationWindowSeconds"] = priorRegistration?.AttemptWindowSeconds.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationConfigFetch"] = priorRegistration?.ConfigFetchOutcome ?? string.Empty,
|
||||
["PriorRegistrationFirstFailedAtUtc"] = priorRegistration?.FirstFailedAtUtc.ToString("O") ?? string.Empty,
|
||||
["PriorRegistrationLastFailedAtUtc"] = priorRegistration?.LastFailedAtUtc.ToString("O") ?? string.Empty,
|
||||
});
|
||||
|
||||
// One Sessions-row point-read serves the owner binding and the emergency-break verdict.
|
||||
var sessionRead = await ReadSessionRowAsync(tenantId, report.SessionId);
|
||||
|
||||
// SESSION-OWNER-BINDING: the emergency-break path below writes timeline events into
|
||||
// whatever session the report names (one point-read — this function has no guard row
|
||||
// in hand; no stamping from this path). A report naming a session bound to another
|
||||
// device identity keeps its device-scoped diagnostics above and the binary-integrity
|
||||
// check below, but must not write into that session. Still 200: the channel is never
|
||||
// retried, and the agent ignores its status.
|
||||
if (await IsSessionOwnerAllowedAsync(req, tenantId, report.SessionId, validation))
|
||||
await MaterializeEmergencyBreakArtifactsAsync(report, tenantId, _sessionRepo, _opsEventService, _logger);
|
||||
// whatever session the report names (no stamping from this path). A report naming a
|
||||
// session bound to another device identity keeps its device-scoped diagnostics above
|
||||
// and the binary-integrity check below, but must not write into that session. Still
|
||||
// 200: the channel is never retried, and the agent ignores its status.
|
||||
if (IsSessionOwnerAllowed(req, tenantId, report.SessionId, sessionRead, validation))
|
||||
await MaterializeEmergencyBreakArtifactsAsync(
|
||||
report, tenantId, sessionRead, _sessionRepo, _deletionGuard, _opsEventService, _logger);
|
||||
|
||||
var adminConfig = await _adminConfigService.GetConfigurationAsync();
|
||||
await MaterializeIntegrityMismatchAsync(
|
||||
@@ -204,18 +226,42 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// False only when the owner binding refuses the caller for the named session. Ids that are
|
||||
/// not GUIDs have no row to compare against; a failed row read fails open (our defect is
|
||||
/// not evidence of a foreign device).
|
||||
/// Outcome of the one Sessions-row point-read: <see cref="Row"/> is null when the row does
|
||||
/// not exist, <see cref="Failed"/> marks a read that threw (existence unknown).
|
||||
/// </summary>
|
||||
private async Task<bool> IsSessionOwnerAllowedAsync(HttpRequestData req, string tenantId, string? sessionId, SecurityValidationResult validation)
|
||||
internal readonly record struct SessionRowRead(TableEntity? Row, bool Failed);
|
||||
|
||||
/// <summary>
|
||||
/// Null when the ids are not GUIDs: no row can exist for them, so neither the owner binding
|
||||
/// nor the emergency break has anything to act on.
|
||||
/// </summary>
|
||||
private async Task<SessionRowRead?> ReadSessionRowAsync(string tenantId, string? sessionId)
|
||||
{
|
||||
if (!SecurityValidator.IsValidGuid(sessionId) || !SecurityValidator.IsValidGuid(tenantId))
|
||||
return null;
|
||||
try
|
||||
{
|
||||
return new SessionRowRead(await _sessionRowReader.GetSessionRowAsync(tenantId, sessionId!), Failed: false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "ReportAgentError: session row read failed for session {SessionId}", sessionId);
|
||||
return new SessionRowRead(null, Failed: true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// False only when the owner binding refuses the caller for the named session. Ids that are
|
||||
/// not GUIDs have no row to compare against; a failed row read or evaluation fails open (our
|
||||
/// defect is not evidence of a foreign device).
|
||||
/// </summary>
|
||||
private bool IsSessionOwnerAllowed(HttpRequestData req, string tenantId, string? sessionId, SessionRowRead? sessionRead, SecurityValidationResult validation)
|
||||
{
|
||||
if (sessionRead is not { Failed: false } read)
|
||||
return true;
|
||||
try
|
||||
{
|
||||
var row = await _sessionRowReader.GetSessionRowAsync(tenantId, sessionId!);
|
||||
return !_ownerBinding.Observe(req, tenantId, sessionId!, row, validation, "agent/error").Rejected;
|
||||
return !_ownerBinding.Observe(req, tenantId, sessionId!, read.Row, validation, "agent/error").Rejected;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -304,74 +350,65 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
/// Materializes the agent's silent 48h emergency break as (1) a timeline event so it shows
|
||||
/// in the session and the timeout classifier can see it, (2) the cross-session EventType
|
||||
/// index row, and (3) an <c>AgentEmergencyBreak</c> ops event for operator visibility
|
||||
/// (tasks/enrollment-status-reclassification.md). Static seam with explicit dependencies so
|
||||
/// tests can pin the artifact set without booting the Functions HTTP stack.
|
||||
/// Best-effort — a failure here must never turn the always-200 emergency channel into a
|
||||
/// retry loop.
|
||||
/// (tasks/enrollment-status-reclassification.md). (1) and (2) only when the session has a
|
||||
/// writable row (<see cref="ClassifyEmergencyBreak"/>); the ops event always, because the
|
||||
/// feed counts breaks. Static seam with explicit dependencies so tests can pin the artifact
|
||||
/// set without booting the Functions HTTP stack. Best-effort — a failure here must never
|
||||
/// turn the always-200 emergency channel into a retry loop.
|
||||
/// </summary>
|
||||
internal static async Task MaterializeEmergencyBreakArtifactsAsync(
|
||||
AgentErrorReport report,
|
||||
string tenantId,
|
||||
SessionRowRead? sessionRead,
|
||||
ISessionRepository sessionRepo,
|
||||
SessionDeletionGuard deletionGuard,
|
||||
OpsEventService opsEventService,
|
||||
ILogger logger)
|
||||
{
|
||||
if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak
|
||||
|| string.IsNullOrEmpty(report.SessionId))
|
||||
// No read means the ids are not GUIDs — no session can exist to break.
|
||||
if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak || sessionRead is not { } read)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000);
|
||||
// Idempotency: the agent's emergency channel can send up to a few reports per session;
|
||||
// only ever materialize one timeline event (and one ops event).
|
||||
var alreadyMaterialized = existing.Any(e =>
|
||||
string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase));
|
||||
if (alreadyMaterialized)
|
||||
var verdict = ClassifyEmergencyBreak(read.Row, read.Failed, IsDeletionLocked(read.Row, deletionGuard));
|
||||
var message = BreakMessage(report);
|
||||
|
||||
if (verdict.Materialize)
|
||||
{
|
||||
return;
|
||||
var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000);
|
||||
// Idempotency: the agent's emergency channel can send up to a few reports per
|
||||
// session; only ever materialize one timeline event (and one ops event).
|
||||
var alreadyMaterialized = existing.Any(e =>
|
||||
string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase));
|
||||
if (alreadyMaterialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow);
|
||||
await sessionRepo.StoreEventsBatchAsync(new List<EnrollmentEvent> { evt });
|
||||
|
||||
// StoreEventsBatchAsync writes the Events partition only — the cross-session
|
||||
// EventType index is normally written by EventIngestProcessor, which this
|
||||
// backend-materialized event never passes through. Without the upsert the event
|
||||
// exists on the session timeline but is invisible to every search-by-eventType
|
||||
// surface (portal cross-session search, MCP search_sessions_by_event /
|
||||
// query_raw_events) — found the hard way in the 2026-07-22 incident analysis.
|
||||
await sessionRepo.UpsertEventTypeIndexBatchAsync(
|
||||
tenantId, report.SessionId, new List<EnrollmentEvent> { evt });
|
||||
}
|
||||
|
||||
var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow);
|
||||
await sessionRepo.StoreEventsBatchAsync(new List<EnrollmentEvent> { evt });
|
||||
|
||||
// StoreEventsBatchAsync writes the Events partition only — the cross-session
|
||||
// EventType index is normally written by EventIngestProcessor, which this
|
||||
// backend-materialized event never passes through. Without the upsert the event
|
||||
// exists on the session timeline but is invisible to every search-by-eventType
|
||||
// surface (portal cross-session search, MCP search_sessions_by_event /
|
||||
// query_raw_events) — found the hard way in the 2026-07-22 incident analysis.
|
||||
await sessionRepo.UpsertEventTypeIndexBatchAsync(
|
||||
tenantId, report.SessionId, new List<EnrollmentEvent> { evt });
|
||||
|
||||
// Operator visibility: an emergency break means an agent silently gave up at its
|
||||
// absolute age cap — exactly the "are we losing agents?" signal. Emitted only on
|
||||
// first materialization so repeat reports for the same session cannot flood the
|
||||
// ops feed. OpsEventService never throws.
|
||||
//
|
||||
// Severity follows what the platform already knows about the session (ops audit
|
||||
// 2026-09-03): the break is wall-clock, so a device that was powered off for weeks
|
||||
// fires it on its first boot — long after the sweep reconciled the session. That
|
||||
// break is a late cleanup, not news; only a break on a still-open session is the
|
||||
// decision itself (classifier rule 3). The row read is best-effort: unknown status
|
||||
// keeps the historical Warning.
|
||||
SessionStatus? statusAtBreak = null;
|
||||
try
|
||||
{
|
||||
statusAtBreak = (await sessionRepo.GetSessionAsync(tenantId, report.SessionId))?.Status;
|
||||
}
|
||||
catch (Exception readEx)
|
||||
{
|
||||
logger.LogDebug(readEx,
|
||||
"ReportAgentError: session status read failed for emergency-break severity on {SessionId}", report.SessionId);
|
||||
}
|
||||
|
||||
var verdict = ClassifyEmergencyBreak(statusAtBreak);
|
||||
// absolute age cap — exactly the "are we losing agents?" signal. For a session with
|
||||
// a row it is emitted only on first materialization, so repeat reports cannot flood
|
||||
// the feed. OpsEventService never throws.
|
||||
await opsEventService.RecordAgentEmergencyBreakAsync(
|
||||
tenantId, report.SessionId, report.AgentVersion, evt.Message,
|
||||
verdict.Severity, statusAtBreak, verdict.LateCleanup);
|
||||
tenantId, report.SessionId, report.AgentVersion, message,
|
||||
verdict.Severity, verdict.SessionStatusAtBreak, verdict.LateCleanup, verdict.Context,
|
||||
report.SessionAgeHours, report.PriorRegistrationFailure);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -381,17 +418,99 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ops-event verdict for an emergency break, keyed on the session status the platform holds
|
||||
/// when the report arrives. Terminal (Succeeded / Failed / Incomplete) = the sweep already
|
||||
/// decided the enrollment and the agent merely cleaned up late (device powered on after a
|
||||
/// shelf period) — Info, <c>LateCleanup=true</c>. Open or unknown = the break is the first
|
||||
/// and only signal that nothing more will arrive — Warning, the historical semantics.
|
||||
/// The cascade-delete writer-block invariant, checked at its chokepoint on the row already in
|
||||
/// hand (no second read). A missing row needs no tombstone check: nothing is written for it.
|
||||
/// </summary>
|
||||
private static bool IsDeletionLocked(TableEntity? sessionRow, SessionDeletionGuard deletionGuard)
|
||||
{
|
||||
try
|
||||
{
|
||||
deletionGuard.ThrowIfLocked(sessionRow, "V2.ReportAgentError.EmergencyBreak");
|
||||
return false;
|
||||
}
|
||||
catch (SessionDeletionLockedException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>What the ops event says and whether the break is written into the session.</summary>
|
||||
internal readonly record struct EmergencyBreakVerdict(
|
||||
string Severity, bool LateCleanup, bool Materialize, string SessionStatusAtBreak, string Context);
|
||||
|
||||
/// <summary>
|
||||
/// Ops-event verdict for an emergency break, keyed on what the platform holds for the session
|
||||
/// when the report arrives. The break is wall-clock and checked at agent start, so it usually
|
||||
/// arrives long after the fact:
|
||||
/// <list type="bullet">
|
||||
/// <item>Terminal (Succeeded / Failed / Incomplete): the sweep already decided the enrollment
|
||||
/// and the agent merely cleaned up late after a shelf period — Info, <c>LateCleanup=true</c>
|
||||
/// (ops audit 2026-09-03).</item>
|
||||
/// <item>No row: in practice a session that never registered (the agent went dark before
|
||||
/// register-session) — nothing to close and nobody who could open it. Info, and no
|
||||
/// timeline event or index row: the orphan sweep would delete the event a day later with
|
||||
/// a Warning of its own, and the index row would point at a session that does not
|
||||
/// exist.</item>
|
||||
/// <item>Row under cascade delete: nothing may be written past the lock — Info.</item>
|
||||
/// <item>Open or unparseable status, or a failed read: the break is the first and only
|
||||
/// signal that nothing more will arrive (classifier rule 3) — Warning, and a failed read
|
||||
/// keeps the historical write.</item>
|
||||
/// </list>
|
||||
/// Pure so the rule is unit-testable.
|
||||
/// </summary>
|
||||
internal static (string Severity, bool LateCleanup) ClassifyEmergencyBreak(SessionStatus? statusAtBreak)
|
||||
=> statusAtBreak.HasValue && Helpers.DeviceJourneyCalculator.IsTerminal(statusAtBreak.Value)
|
||||
? (OpsEventSeverity.Info, true)
|
||||
: (OpsEventSeverity.Warning, false);
|
||||
internal static EmergencyBreakVerdict ClassifyEmergencyBreak(TableEntity? sessionRow, bool rowReadFailed, bool deletionLocked)
|
||||
{
|
||||
if (rowReadFailed)
|
||||
return new(OpsEventSeverity.Warning, false, true, "unknown", "session row unreadable when the break arrived");
|
||||
if (sessionRow == null)
|
||||
return new(OpsEventSeverity.Info, false, false, "missing", "no session row (never registered, or deleted long ago) — nothing to close");
|
||||
if (deletionLocked)
|
||||
return new(OpsEventSeverity.Info, false, false, "deleting", "session is being deleted — nothing written");
|
||||
|
||||
var status = IngestTelemetryFunction.TryReadSessionStatus(sessionRow);
|
||||
var statusText = status?.ToString() ?? "unknown";
|
||||
return status.HasValue && Helpers.DeviceJourneyCalculator.IsTerminal(status.Value)
|
||||
? new(OpsEventSeverity.Info, true, true, statusText, $"late cleanup — session already {statusText} when the break arrived")
|
||||
: new(OpsEventSeverity.Warning, false, true, statusText, $"session still {statusText} when the break arrived");
|
||||
}
|
||||
|
||||
private static string BreakMessage(AgentErrorReport report)
|
||||
=> string.IsNullOrWhiteSpace(report.Message)
|
||||
? "Agent absolute session-age emergency break fired — agent cleaned up and exited"
|
||||
: report.Message;
|
||||
|
||||
internal const int MaxRegistrationErrorLength = 256;
|
||||
internal const int MaxRegistrationTokenLength = 32;
|
||||
|
||||
/// <summary>
|
||||
/// Bounds the device-supplied failed-registration record before it reaches App Insights, the
|
||||
/// ops feed or a timeline: strings are truncated, a record without a positive run count is
|
||||
/// dropped, negative windows are clamped. Returns a copy; null in, null out.
|
||||
/// </summary>
|
||||
internal static RegistrationFailureSummary? SanitizeRegistrationFailure(RegistrationFailureSummary? source)
|
||||
{
|
||||
if (source == null || source.FailedRuns <= 0) return null;
|
||||
return new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = source.FailedRuns,
|
||||
FirstFailedAtUtc = source.FirstFailedAtUtc,
|
||||
LastFailedAtUtc = source.LastFailedAtUtc,
|
||||
Outcome = Truncate(source.Outcome, MaxRegistrationTokenLength) ?? string.Empty,
|
||||
LastError = Truncate(source.LastError, MaxRegistrationErrorLength),
|
||||
NetworkLinkUpAtStart = source.NetworkLinkUpAtStart,
|
||||
NetworkLinkUpAtEnd = source.NetworkLinkUpAtEnd,
|
||||
AttemptWindowSeconds = source.AttemptWindowSeconds < 0 || double.IsNaN(source.AttemptWindowSeconds)
|
||||
? 0
|
||||
: source.AttemptWindowSeconds,
|
||||
ConfigFetchOutcome = Truncate(source.ConfigFetchOutcome, MaxRegistrationTokenLength),
|
||||
};
|
||||
}
|
||||
|
||||
private static string? Truncate(string? value, int max) =>
|
||||
value == null || value.Length <= max ? value : value.Substring(0, max);
|
||||
|
||||
private static string FormatNullableBool(bool? value) =>
|
||||
value == null ? string.Empty : (value.Value ? "true" : "false");
|
||||
|
||||
/// <summary>
|
||||
/// Builds the backend-materialized <c>agent_emergency_break</c> timeline event from the agent's
|
||||
@@ -415,6 +534,30 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
? report.Timestamp.ToUniversalTime()
|
||||
: nowUtc;
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
["source"] = "emergency_channel",
|
||||
["agentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["reportedAtUtc"] = report.Timestamp.ToString("o"),
|
||||
};
|
||||
if (report.SessionAgeHours.HasValue)
|
||||
data["sessionAgeHours"] = report.SessionAgeHours.Value;
|
||||
if (report.PriorRegistrationFailure is { } prior)
|
||||
{
|
||||
data["priorRegistrationFailure"] = new Dictionary<string, object?>
|
||||
{
|
||||
["failedRuns"] = prior.FailedRuns,
|
||||
["firstFailedAtUtc"] = prior.FirstFailedAtUtc.ToString("o"),
|
||||
["lastFailedAtUtc"] = prior.LastFailedAtUtc.ToString("o"),
|
||||
["outcome"] = prior.Outcome,
|
||||
["lastError"] = prior.LastError,
|
||||
["networkLinkUpAtStart"] = prior.NetworkLinkUpAtStart,
|
||||
["networkLinkUpAtEnd"] = prior.NetworkLinkUpAtEnd,
|
||||
["attemptWindowSeconds"] = prior.AttemptWindowSeconds,
|
||||
["configFetchOutcome"] = prior.ConfigFetchOutcome,
|
||||
};
|
||||
}
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
TenantId = tenantId,
|
||||
@@ -425,15 +568,8 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Timestamp = breakAt,
|
||||
Sequence = maxSequence + 1,
|
||||
Message = string.IsNullOrWhiteSpace(report.Message)
|
||||
? "Agent absolute session-age emergency break fired — agent cleaned up and exited"
|
||||
: report.Message,
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
["source"] = "emergency_channel",
|
||||
["agentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["reportedAtUtc"] = report.Timestamp.ToString("o"),
|
||||
},
|
||||
Message = BreakMessage(report),
|
||||
Data = data,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -954,31 +954,50 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
/// <summary>
|
||||
/// An agent reported its absolute session-age emergency break (48h cap) over the
|
||||
/// emergency channel — it cleaned itself up and exited on a session that never reached
|
||||
/// a terminal state. This is the "are we silently losing agents?" signal; emitted by
|
||||
/// <see cref="Functions.Ingest.ReportAgentErrorFunction"/> once per session (guarded by
|
||||
/// the timeline-event idempotency check). Severity is decided by the caller from the
|
||||
/// session status at break time (<c>ReportAgentErrorFunction.ClassifyEmergencyBreak</c>):
|
||||
/// Warning when the session was still open (the break IS the verdict — Telegram-rule
|
||||
/// worthy), Info when the sweep had already terminalized it and the break is a late
|
||||
/// cleanup after the device was powered on again. The 2026-09-03 audit found 224 of 269
|
||||
/// breaks in three days were such late cleanups (kiosk devices stored for weeks); as
|
||||
/// Warnings they buried the real ones. <paramref name="statusAtBreak"/> lands in the
|
||||
/// details as <c>sessionStatusAtBreak</c> so the feed shows which case it was.
|
||||
/// emergency channel — it cleaned itself up and exited. This is the "are we silently
|
||||
/// losing agents?" signal; emitted by <see cref="Functions.Ingest.ReportAgentErrorFunction"/>
|
||||
/// once per break report (for a session with a row, guarded by the timeline-event
|
||||
/// idempotency check). Severity, <paramref name="sessionStatusAtBreak"/> and
|
||||
/// <paramref name="context"/> come from <c>ReportAgentErrorFunction.ClassifyEmergencyBreak</c>:
|
||||
/// Warning only when the session was still open (the break IS the verdict — Telegram-rule
|
||||
/// worthy) or its row could not be read. The 2026-09-03 audit found 224 of 269 breaks in
|
||||
/// three days were late cleanups on terminal sessions, the 2026-09-28 sweep that 115 of 160
|
||||
/// remaining Warnings named sessions that never registered; as Warnings both buried the real
|
||||
/// ones. <paramref name="sessionAgeHours"/> is the agent-measured age (null for agents that
|
||||
/// predate the field) — it separates a break just past the cap from a device that slept for
|
||||
/// weeks, which the backend cannot tell from its own rows. <paramref name="priorRegistrationFailure"/>
|
||||
/// is the agent's record of earlier starts that never registered (bounded by the caller;
|
||||
/// null when none or for older agents) — for a session without a row the only account of why.
|
||||
/// </summary>
|
||||
public Task RecordAgentEmergencyBreakAsync(
|
||||
string tenantId, string sessionId, string? agentVersion, string message,
|
||||
string severity, SessionStatus? statusAtBreak, bool lateCleanup)
|
||||
{
|
||||
var statusText = statusAtBreak?.ToString() ?? "unknown";
|
||||
var context = lateCleanup
|
||||
? $"late cleanup — session already {statusText} when the break arrived"
|
||||
: $"session still {statusText} when the break arrived";
|
||||
return WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
|
||||
string severity, string sessionStatusAtBreak, bool lateCleanup, string context, double? sessionAgeHours,
|
||||
RegistrationFailureSummary? priorRegistrationFailure = null)
|
||||
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
|
||||
$"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}, {context}): {message}",
|
||||
tenantId, "System.EmergencyChannel",
|
||||
new { sessionId, agentVersion, sessionStatusAtBreak = statusText, lateCleanup });
|
||||
}
|
||||
new
|
||||
{
|
||||
sessionId, agentVersion, sessionStatusAtBreak, lateCleanup, sessionAgeHours,
|
||||
priorRegistrationFailure = DescribeRegistrationFailure(priorRegistrationFailure),
|
||||
});
|
||||
|
||||
// camelCase like the rest of the details; the class itself would serialize in PascalCase.
|
||||
private static object? DescribeRegistrationFailure(RegistrationFailureSummary? failure) =>
|
||||
failure == null
|
||||
? null
|
||||
: new
|
||||
{
|
||||
failedRuns = failure.FailedRuns,
|
||||
firstFailedAtUtc = failure.FirstFailedAtUtc,
|
||||
lastFailedAtUtc = failure.LastFailedAtUtc,
|
||||
outcome = failure.Outcome,
|
||||
lastError = failure.LastError,
|
||||
networkLinkUpAtStart = failure.NetworkLinkUpAtStart,
|
||||
networkLinkUpAtEnd = failure.NetworkLinkUpAtEnd,
|
||||
attemptWindowSeconds = failure.AttemptWindowSeconds,
|
||||
configFetchOutcome = failure.ConfigFetchOutcome,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend
|
||||
|
||||
@@ -104,6 +104,15 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
/// <summary>On <c>AadUserJoinedLate</c>: user presence indicator ("true" / "false").</summary>
|
||||
public const string AadJoinedWithUser = "aadJoinedWithUser";
|
||||
|
||||
/// <summary>
|
||||
/// On <c>WhiteGloveShellCoreSuccess</c>: "true" / "false" — whether the ESP registry
|
||||
/// showed an AccountSetup subcategory past <c>notStarted</c> when the agent observed the
|
||||
/// Shell-Core WhiteGlove_Success event. Missing (registry not read yet, or agents
|
||||
/// predating the key) → the fact stays unset and the sealing classifier keeps the
|
||||
/// IME-phase AccountSetup excluder.
|
||||
/// </summary>
|
||||
public const string AccountSetupProgress = "accountSetupProgress";
|
||||
|
||||
/// <summary>On <c>ImeUserSessionCompleted</c>: matched pattern id.</summary>
|
||||
public const string ImePatternId = "patternId";
|
||||
|
||||
|
||||
@@ -53,6 +53,16 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
.WithLastAppliedSignalOrdinal(signal.SessionSignalOrdinal);
|
||||
builder.ScenarioObservations = builder.ScenarioObservations.WithShellCoreWhiteGloveSuccessSeen(signal.SessionSignalOrdinal);
|
||||
|
||||
// Recorded before the inline classifier runs: it decides whether the AccountSetup
|
||||
// excluder applies (see BuildWhiteGloveSealingSnapshot).
|
||||
if (signal.Payload != null
|
||||
&& signal.Payload.TryGetValue(SignalPayloadKeys.AccountSetupProgress, out var rawProgress)
|
||||
&& bool.TryParse(rawProgress, out var accountSetupProgress))
|
||||
{
|
||||
builder.ScenarioObservations = builder.ScenarioObservations
|
||||
.WithAccountSetupProgressAtWhiteGloveSuccess(accountSetupProgress, signal.SessionSignalOrdinal);
|
||||
}
|
||||
|
||||
// Option 3 fast-path: inline classifier evaluation on the strong WG signal.
|
||||
var afterObservation = builder.Build();
|
||||
var inlineSnapshot = BuildWhiteGloveSealingSnapshot(afterObservation);
|
||||
@@ -362,6 +372,15 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
/// <see cref="WhiteGloveSealingSnapshot.ComputeInputHash"/> is therefore byte-stable
|
||||
/// across the refactor, which preserves the anti-loop semantics.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <c>hasAccountSetupActivity</c> needs the IME AccountSetup phase AND no explicit
|
||||
/// "registry shows no AccountSetup progress" on the WhiteGlove_Success signal. IME
|
||||
/// reports AccountSetup from the device session as soon as the device apps are done, so
|
||||
/// on pre-provisioning with the user ESP enabled that line routinely precedes the
|
||||
/// technician's success page. The conjunction keeps the excluder a subset of the old
|
||||
/// one: a verdict can only rise, and a missing payload (older agent) scores exactly as
|
||||
/// before.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static WhiteGloveSealingSnapshot BuildWhiteGloveSealingSnapshot(DecisionState state) =>
|
||||
new WhiteGloveSealingSnapshot(
|
||||
@@ -370,7 +389,8 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
aadJoinedWithUser: state.ScenarioObservations.AadUserJoinWithUserObserved?.Value == true,
|
||||
desktopArrived: state.DesktopArrivedUtc != null,
|
||||
helloResolved: state.HelloResolvedUtc != null,
|
||||
hasAccountSetupActivity: state.AccountSetupEnteredUtc != null,
|
||||
hasAccountSetupActivity: state.AccountSetupEnteredUtc != null
|
||||
&& state.ScenarioObservations.AccountSetupProgressAtWhiteGloveSuccess?.Value != false,
|
||||
isDeviceOnlyDeploymentHypothesis:
|
||||
state.ClassifierOutcomes.DeviceOnlyDeployment.Level >= HypothesisLevel.Strong &&
|
||||
state.ClassifierOutcomes.DeviceOnlyDeployment.Reason == DeviceOnlyReasons.DeviceOnly,
|
||||
|
||||
@@ -144,6 +144,20 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
seen.Add("whiteglove_sealing_pattern");
|
||||
evidence["whiteGloveSealingPattern"] = OrdinalEvidence(obs.WhiteGloveSealingPatternSeen.SourceSignalOrdinal);
|
||||
}
|
||||
// Registry AccountSetup progress stamped on the WhiteGlove_Success signal — the gate of
|
||||
// the sealing classifier's AccountSetup excluder, so a Part-1 audit trail shows why
|
||||
// an IME AccountSetup phase did or did not count.
|
||||
if (obs.AccountSetupProgressAtWhiteGloveSuccess != null)
|
||||
{
|
||||
seen.Add(obs.AccountSetupProgressAtWhiteGloveSuccess.Value
|
||||
? "whiteglove_account_setup_progress"
|
||||
: "whiteglove_account_setup_untouched");
|
||||
evidence["accountSetupProgressAtWhiteGloveSuccess"] = new Dictionary<string, object>(capacity: 2, StringComparer.Ordinal)
|
||||
{
|
||||
["ordinal"] = obs.AccountSetupProgressAtWhiteGloveSuccess.SourceSignalOrdinal,
|
||||
["progress"] = obs.AccountSetupProgressAtWhiteGloveSuccess.Value,
|
||||
};
|
||||
}
|
||||
if (obs.AadUserJoinWithUserObserved != null)
|
||||
{
|
||||
seen.Add(obs.AadUserJoinWithUserObserved.Value
|
||||
|
||||
@@ -33,6 +33,17 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
/// <summary>True once <see cref="Signals.DecisionSignalKind.WhiteGloveSealingPatternDetected"/> has fired.</summary>
|
||||
public SignalFact<bool>? WhiteGloveSealingPatternSeen { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// ESP-registry AccountSetup progress as stamped on the
|
||||
/// <see cref="Signals.DecisionSignalKind.WhiteGloveShellCoreSuccess"/> payload:
|
||||
/// <c>true</c> = an AccountSetup subcategory had left <c>notStarted</c>, <c>false</c> =
|
||||
/// none had. <c>null</c> = not reported (registry not read yet, or an agent predating the
|
||||
/// payload key) and must be read as "unknown", never as "no progress". Gates the sealing classifier's
|
||||
/// AccountSetup excluder: IME logs "EspPhase: AccountSetup" from the device session as
|
||||
/// soon as the device apps are done, so only an explicit <c>false</c> here lifts it.
|
||||
/// </summary>
|
||||
public SignalFact<bool>? AccountSetupProgressAtWhiteGloveSuccess { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Payload-carrying observation from <see cref="Signals.DecisionSignalKind.AadUserJoinedLate"/>.
|
||||
/// <c>true</c> = late AADJ observed with a user-side principal (hard-excluder for
|
||||
@@ -113,6 +124,11 @@ namespace AutopilotMonitor.DecisionCore.State
|
||||
? this
|
||||
: this with { WhiteGloveSealingPatternSeen = new SignalFact<bool>(true, sourceSignalOrdinal) };
|
||||
|
||||
public EnrollmentScenarioObservations WithAccountSetupProgressAtWhiteGloveSuccess(bool value, long sourceSignalOrdinal) =>
|
||||
AccountSetupProgressAtWhiteGloveSuccess != null
|
||||
? this
|
||||
: this with { AccountSetupProgressAtWhiteGloveSuccess = new SignalFact<bool>(value, sourceSignalOrdinal) };
|
||||
|
||||
public EnrollmentScenarioObservations WithAadUserJoinWithUserObserved(bool value, long sourceSignalOrdinal) =>
|
||||
AadUserJoinWithUserObserved != null
|
||||
? this
|
||||
|
||||
@@ -81,6 +81,22 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public long? SequenceNumber { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Session age in hours the agent measured when its absolute session-age emergency break
|
||||
/// fired (<see cref="AgentErrorType.SessionAgeEmergencyBreak"/> only). The backend cannot
|
||||
/// derive it: a never-registered session has no row, and a registered one may have
|
||||
/// registered seconds before the break (hibernate resume). Null for other error types and
|
||||
/// for agents that predate the field.
|
||||
/// </summary>
|
||||
public double? SessionAgeHours { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Why earlier process starts of this session never registered
|
||||
/// (<see cref="AgentErrorType.SessionAgeEmergencyBreak"/> only). Null when every start
|
||||
/// registered, for other error types, and for agents that predate the field.
|
||||
/// </summary>
|
||||
public RegistrationFailureSummary? PriorRegistrationFailure { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Agent version string for diagnosing version-specific issues.
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// What the agent recorded locally each time a process start ended without a registered
|
||||
/// session (register-session failed after all retries, the agent exited). Carried on the
|
||||
/// emergency-break report (<see cref="AgentErrorReport.PriorRegistrationFailure"/>): for a
|
||||
/// session that never registered it is the only account of why that ever reaches the
|
||||
/// backend. Cleared by the next successful registration.
|
||||
/// </summary>
|
||||
public class RegistrationFailureSummary
|
||||
{
|
||||
/// <summary>Process starts whose registration failed since the last success.</summary>
|
||||
public int FailedRuns { get; set; }
|
||||
|
||||
/// <summary>UTC time the first of these runs gave up.</summary>
|
||||
public DateTime FirstFailedAtUtc { get; set; }
|
||||
|
||||
/// <summary>UTC time the latest of these runs gave up.</summary>
|
||||
public DateTime LastFailedAtUtc { get; set; }
|
||||
|
||||
/// <summary>Registration outcome of the latest failed run (e.g. <c>Failed</c>, <c>AuthFailed</c>).</summary>
|
||||
public string Outcome { get; set; } = default!;
|
||||
|
||||
/// <summary>Last error message of the latest failed run, truncated by the agent.</summary>
|
||||
public string? LastError { get; set; }
|
||||
|
||||
/// <summary>Network link up (any interface) when the latest failed run started registering.</summary>
|
||||
public bool? NetworkLinkUpAtStart { get; set; }
|
||||
|
||||
/// <summary>Network link up when the latest failed run gave up.</summary>
|
||||
public bool? NetworkLinkUpAtEnd { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Wall-clock seconds the latest failed run spent registering (link wait plus all attempts).
|
||||
/// Far above the retry schedule means the device slept or was suspended meanwhile.
|
||||
/// </summary>
|
||||
public double AttemptWindowSeconds { get; set; }
|
||||
|
||||
/// <summary>Outcome of the remote-config fetch in the latest failed run (e.g. <c>Succeeded</c>, <c>UsedDefaults</c>).</summary>
|
||||
public string? ConfigFetchOutcome { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# WhiteGloveShellCoreSuccess v1 — Shell-Core 62407 "CommercialOOBE_ESPProgress_WhiteGlove_Success".
|
||||
# Posted by EspAndHelloTrackerAdapter (fire-once) when the ShellCoreTracker observes the
|
||||
# technician-flow success page. Payload `accountSetupProgress` (added 2026-09-29) carries the ESP
|
||||
# registry's AccountSetup progress at that moment ("true" = a subcategory had left notStarted);
|
||||
# signals from older agents carry no payload.
|
||||
# Reducer: HandleWhiteGloveShellCoreSuccessV1 → records ShellCoreWhiteGloveSuccessSeen and, when
|
||||
# the payload parses, AccountSetupProgressAtWhiteGloveSuccess; runs the sealing classifier inline.
|
||||
# The IME AccountSetup excluder (-40) is lifted only by an explicit "false".
|
||||
{"SessionSignalOrdinal":1,"SessionTraceOrdinal":1,"Kind":"WhiteGloveShellCoreSuccess","KindSchemaVersion":1,"OccurredAtUtc":"2026-09-14T07:25:09.7628703Z","SourceOrigin":"EspAndHelloTracker","Evidence":{"Kind":"Derived","Identifier":"esp-and-hello-tracker-v1","Summary":"WhiteGlove sealing success (coordinator-forwarded)","DerivationInputs":{"subSource":"ShellCoreTracker","accountSetupProgress":"false"}},"Payload":{"accountSetupProgress":"false"}}
|
||||
Reference in New Issue
Block a user