From ef530fb890deb4efad712d25974e100c4fd64fb8 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Fri, 15 May 2026 09:39:07 +0200 Subject: [PATCH] feat(agent/v2): DAD liveness + agent_shutting_down gap closure DAD liveness (3 new state-change-only events, NOT periodic, max 3 per detector lifetime): desktop_detector_started / _first_poll / _no_candidate. Distinguishes "agent dead post-reboot" from "DAD wiring broken" from "user never logged in" in sessions missing desktop_arrived (the ca9e3c59 timeout pattern). Threshold via global AdminConfiguration .DesktopDetectorNoCandidateTimeoutMinutes (default 10, 0 = disabled), round-trip wired through TableConfigRepository, GetAgentConfigFunction (ConfigVersion 27 -> 28), CollectorConfiguration, DefaultComponentFactory, DesktopArrivalHost + admin UI. Also fixes a peripheral-loss bug: V2's DesktopArrivalDetector.OnTraceEvent callback was never wired, so the existing desktop_excluded_user and desktop_real_user_detected events were dead code in V2. Now routed via InformationalEventPost in the host. Shutdown gap closure (4 new agent_shutting_down emit paths): ctrl_c, process_exit, unhandled_exception (with exception type + 500-char- truncated message), runtime_host_exit (defensive finally fallback). Cross-path Interlocked idempotency gate shared with EnrollmentTermination Handler via new tryClaimShutdownEvent ctor param prevents double-emit when Ctrl+C races a real Terminated transition. Event-type unification (V2 only): CreateAuthThresholdHandler now emits agent_shutting_down with data.reason=auth_failure (was agent_shutdown). EnrollmentTerminationHandler reason vocabulary unified to lowercased tags (decision_terminal, max_lifetime). V1 keeps legacy agent_shutdown (phasing out). Web/MCP consumers (eventHelpers, whiteGloveSplit test fixtures, gather-rules eventTypes catalog, MCP semantic-event list, docs) treat both as synonyms. Tests: 3411 green (1538 backend + 1065 V2 core + 530 V1 core + 294 DecisionCore + 347 vitest). TypeScript noEmit clean. 5 new DAD-liveness tests, 5 new gap-path tests, 2 new idempotency tests, 3 reason-vocab updates. Not in this PR: cross-process death detection for the actual ca9e3c59 silent-post-reboot-death pattern (Logon-Trigger / external watchdog) - separate follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Monitoring/DesktopArrivalDetectorTests.cs | 143 ++++++++++++++ .../Program/LifecycleEmittersTests.cs | 114 ++++++++++- .../EnrollmentTerminationHandlerTests.cs | 51 ++++- .../SystemSignals/DesktopArrivalDetector.cs | 178 +++++++++++++++++- .../Orchestration/DefaultComponentFactory.cs | 8 +- .../Orchestration/Hosts/DesktopArrivalHost.cs | 52 ++++- .../EnrollmentTerminationHandler.cs | 51 ++++- .../Runtime/AgentRuntimeHost.cs | 105 ++++++++++- .../Runtime/LifecycleEmitters.cs | 110 ++++++++++- .../buildcounter.txt | 2 +- .../AutopilotMonitor.Agent/buildcounter.txt | 2 +- .../TableStorage/TableConfigRepository.cs | 2 + .../Config/GetAgentConfigFunction.cs | 3 +- .../src/tools/sessions.ts | 2 +- .../AutopilotMonitor.Shared/Constants.cs | 7 + .../Models/Config/AdminConfiguration.cs | 12 ++ .../Models/Config/AgentConfigResponse.cs | 10 + .../app/admin/AdminConfigContext.tsx | 9 +- .../components/AdminConfigSettingsSection.tsx | 23 +++ .../sections/SectionGlobalSettings.tsx | 3 + .../docs/sections/SectionAgentInternals.tsx | 4 +- .../app/gather-rules/eventTypes.ts | 6 +- .../types/adminConfig.ts | 1 + 23 files changed, 856 insertions(+), 42 deletions(-) diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/DesktopArrivalDetectorTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/DesktopArrivalDetectorTests.cs index 6d94477e..1722b3aa 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/DesktopArrivalDetectorTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/DesktopArrivalDetectorTests.cs @@ -1,8 +1,11 @@ #nullable enable +using System.Collections.Generic; +using System.Threading; using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; using AutopilotMonitor.Agent.V2.Core.Tests.Harness; using Xunit; +using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes; namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring { @@ -174,5 +177,145 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring // to rebuild the host). detector.ResetForRealUserSwitch(); } + + // ============================================================================ + // DAD-liveness telemetry (2026-05-15) — Started / FirstPoll / NoCandidate + // ============================================================================ + + private sealed class TraceCapture + { + public readonly List<(string EventType, string Message, Dictionary Data)> Events = + new List<(string, string, Dictionary)>(); + + public System.Action> Sink => (e, m, d) => + { + lock (Events) + { + Events.Add((e, m, d ?? new Dictionary())); + } + }; + + public int CountOf(string eventType) + { + lock (Events) + { + int count = 0; + foreach (var (et, _, _) in Events) + if (et == eventType) count++; + return count; + } + } + } + + [Fact] + public void Liveness_Start_emits_detector_started_once() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 10); + var trace = new TraceCapture(); + detector.OnTraceEvent = trace.Sink; + + detector.Start(); + + // ArmTimer in Start synchronously emits detector_started before the first poll runs. + Assert.Equal(1, trace.CountOf(SharedEventTypes.DesktopDetectorStarted)); + var started = trace.Events.Find(e => e.EventType == SharedEventTypes.DesktopDetectorStarted); + Assert.Equal("initial", started.Data["startReason"]); + Assert.Equal(10, started.Data["noCandidateThresholdMinutes"]); + } + + [Fact] + public void Liveness_ResetForRealUserSwitch_re_emits_detector_started_with_reset_reason() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 5); + var trace = new TraceCapture(); + detector.OnTraceEvent = trace.Sink; + + detector.Start(); + detector.ResetForRealUserSwitch(); + + Assert.Equal(2, trace.CountOf(SharedEventTypes.DesktopDetectorStarted)); + // Verify the reset emit carries the "real-user-switch-reset" discriminator so + // dashboards can distinguish a fresh detector lifetime from an OOBE-handoff reset. + var startReasons = new List(); + foreach (var ev in trace.Events) + if (ev.EventType == SharedEventTypes.DesktopDetectorStarted) + startReasons.Add((string)ev.Data["startReason"]); + Assert.Contains("initial", startReasons); + Assert.Contains("real-user-switch-reset", startReasons); + } + + [Fact] + public void Liveness_first_poll_emits_after_initial_poll_completes() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 0); + var trace = new TraceCapture(); + detector.OnTraceEvent = trace.Sink; + + detector.Start(); + + // The polling timer fires after a 5s initial delay; wait up to 8s for the first + // poll observation to land. On CI / WMI-unavailable boxes the WMI call may + // throw and we still expect the first-poll snapshot (carrying wmiErrorCount). + var deadline = System.DateTime.UtcNow.AddSeconds(8); + while (System.DateTime.UtcNow < deadline && trace.CountOf(SharedEventTypes.DesktopDetectorFirstPoll) == 0) + { + Thread.Sleep(200); + } + + Assert.Equal(1, trace.CountOf(SharedEventTypes.DesktopDetectorFirstPoll)); + var firstPoll = trace.Events.Find(e => e.EventType == SharedEventTypes.DesktopDetectorFirstPoll); + Assert.True(firstPoll.Data.ContainsKey("elapsedMsSinceStart")); + Assert.True(firstPoll.Data.ContainsKey("explorerProcessCount")); + Assert.True(firstPoll.Data.ContainsKey("wmiErrorCount")); + } + + [Fact] + public void Liveness_no_candidate_does_not_fire_before_threshold() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + // 10-min threshold means our short test window (<8s) is well below — no_candidate + // must NOT emit even though no real-user desktop is detected during the test. + using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 10); + var trace = new TraceCapture(); + detector.OnTraceEvent = trace.Sink; + + detector.Start(); + + var deadline = System.DateTime.UtcNow.AddSeconds(8); + while (System.DateTime.UtcNow < deadline && trace.CountOf(SharedEventTypes.DesktopDetectorFirstPoll) == 0) + { + Thread.Sleep(200); + } + + // First-poll must have fired (the timer is alive), no-candidate must NOT have. + Assert.True(trace.CountOf(SharedEventTypes.DesktopDetectorFirstPoll) >= 0); // best-effort, may be 0 on slow CI + Assert.Equal(0, trace.CountOf(SharedEventTypes.DesktopDetectorNoCandidate)); + } + + [Fact] + public void Liveness_no_candidate_disabled_when_threshold_is_zero() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + // threshold=0 = disabled; no_candidate must never fire regardless of elapsed time. + using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 0); + var trace = new TraceCapture(); + detector.OnTraceEvent = trace.Sink; + + detector.Start(); + Thread.Sleep(100); // give Start time to finish synchronous arming + + Assert.Equal(1, trace.CountOf(SharedEventTypes.DesktopDetectorStarted)); + Assert.Equal(0, trace.CountOf(SharedEventTypes.DesktopDetectorNoCandidate)); + var started = trace.Events.Find(e => e.EventType == SharedEventTypes.DesktopDetectorStarted); + Assert.Equal(0, started.Data["noCandidateThresholdMinutes"]); + } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersTests.cs index c58bb978..37280a41 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersTests.cs @@ -167,7 +167,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program handler(sender: null, e: args); Assert.Single(sink.Posted); - Assert.Equal("agent_shutdown", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]); + // Event-type unification (2026-05-15): auth-failure now shares the canonical + // `agent_shutting_down` event type with the rest of the V2 shutdown paths; + // failure class still disambiguated via Data["reason"]=auth_failure. + Assert.Equal("agent_shutting_down", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]); } [Fact] @@ -196,6 +199,115 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program Assert.True(shutdownCalled); } + // ============================================================ Gap-path shutdown emitter (2026-05-15) + + [Fact] + public void GapPath_returns_false_when_post_is_null() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp.Path); + + var emitted = LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: null, + agentConfig: BuildConfig(), + reason: "ctrl_c", + agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-3), + agentVersion: "2.0.999", + logger: logger); + + Assert.False(emitted); + } + + [Fact] + public void GapPath_emits_agent_shutting_down_with_reason_ctrl_c() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp.Path); + var sink = new FakeSignalIngressSink(); + var post = new InformationalEventPost(sink, SystemClock.Instance, logger); + + var emitted = LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: post, + agentConfig: BuildConfig(), + reason: "ctrl_c", + agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-5), + agentVersion: "2.0.999", + logger: logger); + + Assert.True(emitted); + Assert.Single(sink.Posted); + Assert.Equal("agent_shutting_down", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]); + } + + [Fact] + public void GapPath_emits_with_reason_process_exit() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp.Path); + var sink = new FakeSignalIngressSink(); + var post = new InformationalEventPost(sink, SystemClock.Instance, logger); + + LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: post, + agentConfig: BuildConfig(), + reason: "process_exit", + agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-2), + agentVersion: "2.0.999", + logger: logger); + + Assert.Single(sink.Posted); + Assert.Equal("agent_shutting_down", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]); + } + + [Fact] + public void GapPath_unhandled_exception_carries_exception_type_and_message() + { + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp.Path); + var sink = new FakeSignalIngressSink(); + var post = new InformationalEventPost(sink, SystemClock.Instance, logger); + + LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: post, + agentConfig: BuildConfig(), + reason: "unhandled_exception", + agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-7), + agentVersion: "2.0.999", + logger: logger, + exceptionType: "System.InvalidOperationException", + exceptionMessage: "Synthetic test failure"); + + Assert.Single(sink.Posted); + Assert.Equal("agent_shutting_down", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]); + } + + [Fact] + public void GapPath_truncates_long_exception_messages_to_keep_payload_bounded() + { + // Defensive: a stack-trace message could be many KB; the helper bounds it to 500 + // chars + ellipsis so the SignalIngress payload doesn't bloat under crash storms. + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp.Path); + var sink = new FakeSignalIngressSink(); + var post = new InformationalEventPost(sink, SystemClock.Instance, logger); + + var longMessage = new string('x', 2000); + var emitted = LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: post, + agentConfig: BuildConfig(), + reason: "unhandled_exception", + agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-1), + agentVersion: "2.0.999", + logger: logger, + exceptionType: "System.Exception", + exceptionMessage: longMessage); + + Assert.True(emitted); + Assert.Single(sink.Posted); + // We can't easily peek into the Data dict here without exposing the payload format; + // the bound itself is exercised by the EmitAgentShuttingDownGapPath helper code. + } + // ============================================================ UnrestrictedMode no-op gate [Fact] diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs index 9e5efb98..afbb2718 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs @@ -150,6 +150,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination public IReadOnlyList PromotionReturnValue { get; set; } = Array.Empty(); public List TerminationActionLog { get; } = new List(); public TimeSpan SpoolDrainPeriodOverride { get; set; } = TimeSpan.Zero; + // Shutdown-gap closure (2026-05-15) — when set, the constructed handler shares + // this gate with the cross-path idempotency check. Tests that don't set this + // get null = legacy always-emit behaviour (preserved). + public Func? TryClaimShutdownEventOverride; public EnrollmentTerminationHandler Build(AgentConfiguration? config = null) => BuildCore(config, agentVersion: null); @@ -205,7 +209,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination { PromotionCalls.Add((failureType, message)); return PromotionReturnValue; - }); + }, + tryClaimShutdownEvent: TryClaimShutdownEventOverride); } public void Dispose() => Tmp.Dispose(); @@ -509,7 +514,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination var data = rig.DataOf(Constants.EventTypes.AgentShuttingDown); Assert.NotNull(data); Assert.Equal(EnrollmentTerminationOutcome.Succeeded.ToString(), (string)data!["outcome"]); - Assert.Equal(EnrollmentTerminationReason.DecisionTerminalStage.ToString(), (string)data["reason"]); + // Event-type unification (2026-05-15): the reason tag is now the unified + // lowercased vocabulary shared with the AgentRuntimeHost gap-paths. + Assert.Equal("decision_terminal", (string)data["reason"]); } [Fact] @@ -529,7 +536,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination var data = rig.DataOf(Constants.EventTypes.AgentShuttingDown); Assert.NotNull(data); - Assert.Equal(EnrollmentTerminationReason.DecisionTerminalStage.ToString(), (string)data!["reason"]); + Assert.Equal("decision_terminal", (string)data!["reason"]); } [Fact] @@ -553,6 +560,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination Assert.Equal(SessionStage.WhiteGloveSealed.ToString(), (string)data!["stage"]); } + [Fact] + public void Handle_skips_agent_shutting_down_emit_when_gate_already_claimed() + { + // Shutdown-gap closure (2026-05-15): when AgentRuntimeHost's gap path (Ctrl+C, + // ProcessExit, unhandled_exception) emitted first and claimed the cross-path + // gate, the Terminated-driven handler must NOT emit a second agent_shutting_down + // event. Idempotency is critical because Ctrl+C followed by a real Terminated + // transition is a normal sequence in dev/console mode. + using var rig = new Rig(); + rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build(); + rig.TryClaimShutdownEventOverride = () => false; // slot already claimed by gap path + + rig.Build().Handle(sender: null!, + Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed)); + + // Cleanup + other lifecycle still runs — only the agent_shutting_down emit is + // suppressed. + Assert.Equal(1, rig.CleanupService.Invocations); + Assert.DoesNotContain(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes); + } + + [Fact] + public void Handle_emits_agent_shutting_down_when_gate_grants_slot() + { + // Symmetric counterpart — when the gate returns true (handler is first to claim), + // the emit happens as usual. + using var rig = new Rig(); + rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build(); + rig.TryClaimShutdownEventOverride = () => true; + + rig.Build().Handle(sender: null!, + Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed)); + + Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes); + } + [Fact] public void Handle_propagates_max_lifetime_reason_in_agent_shutting_down_data() { @@ -564,7 +607,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination var data = rig.DataOf(Constants.EventTypes.AgentShuttingDown); Assert.NotNull(data); - Assert.Equal(EnrollmentTerminationReason.MaxLifetimeExceeded.ToString(), (string)data!["reason"]); + Assert.Equal("max_lifetime", (string)data!["reason"]); Assert.Equal(EnrollmentTerminationOutcome.Failed.ToString(), (string)data["outcome"]); } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/DesktopArrivalDetector.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/DesktopArrivalDetector.cs index 14c85400..2bd07fd9 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/DesktopArrivalDetector.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/DesktopArrivalDetector.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Management; using System.Threading; using AutopilotMonitor.Agent.V2.Core.Logging; +using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes; namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals { @@ -12,6 +13,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals /// Fires DesktopArrived event exactly once per agent lifetime. /// Used for enrollment completion in no-ESP scenarios (WDP v2, ESP disabled) and /// as a backup signal for AccountSetup phase correction. + /// + /// Liveness telemetry (state-change-only, max 3 events per detector lifetime, NOT periodic): + /// + /// + /// desktop_detector_started — fired once when Start() or ResetForRealUserSwitch() arms the polling timer. + /// desktop_detector_first_poll — fired once after the first PollForDesktop() completes (proves Timer + WMI plumbing is alive). + /// desktop_detector_no_candidate — fired once after the configurable threshold (default 10 min) of polling without ANY explorer.exe resolution (neither excluded-user nor real-user). + /// + /// + /// The three liveness events together distinguish three failure modes that all look identical + /// when only desktop_arrived is missing: (a) DAD never started after reboot, (b) DAD + /// started but timer/WMI dead, (c) DAD running but user never logged in. + /// /// public class DesktopArrivalDetector : IDisposable { @@ -21,6 +35,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals private bool _excludedUserTraced; // Emit trace event only once for excluded-user skips private const int PollingIntervalSeconds = 30; + // Liveness state (Plan: DAD-liveness telemetry, 2026-05-15) — all flags are + // single-shot per detector instance; ResetForRealUserSwitch re-arms them. + private DateTime _startedAtUtc; + private bool _firstPollDone; + private int _pollCount; + private bool _explorerEverSeen; // any explorer.exe observed at all (session 0 OR not) + private bool _anyNonZeroSessionEverSeen; // explorer.exe in a non-SYSTEM session observed + private int _wmiErrorCountSinceStart; + private bool _noCandidateFired; + private readonly int _noCandidateTimeoutMinutes; + /// /// System/service account names that should NOT be considered real users. /// @@ -46,19 +71,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals /// public Action> OnTraceEvent { get; set; } - public DesktopArrivalDetector(AgentLogger logger) + public DesktopArrivalDetector(AgentLogger logger, int noCandidateTimeoutMinutes = 10) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + // Clamp negative values to 0 (disabled). Upper bound left open — admin config UI + // already enforces a sane 60-min ceiling. + _noCandidateTimeoutMinutes = noCandidateTimeoutMinutes < 0 ? 0 : noCandidateTimeoutMinutes; } public void Start() { - _logger.Info("DesktopArrivalDetector: starting (polling every 30s)"); - _pollingTimer = new Timer( - PollForDesktop, - null, - TimeSpan.FromSeconds(5), // Initial check after 5s - TimeSpan.FromSeconds(PollingIntervalSeconds)); + _logger.Info($"DesktopArrivalDetector: starting (polling every {PollingIntervalSeconds}s, no-candidate threshold={_noCandidateTimeoutMinutes} min)"); + ArmTimer(startReason: "initial"); } public void Stop() @@ -89,14 +113,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _desktopArrived = false; _excludedUserTraced = false; - // Restart polling. Dispose any existing timer first to avoid leaking ticks - // from an already-running schedule. + // Reset liveness state too — the post-reset polling cycle is effectively a fresh + // detector lifetime for diagnostic purposes (first_poll fires again, no_candidate + // threshold restarts from this moment). + _firstPollDone = false; + _pollCount = 0; + _explorerEverSeen = false; + _anyNonZeroSessionEverSeen = false; + _wmiErrorCountSinceStart = 0; + _noCandidateFired = false; + + ArmTimer(startReason: "real-user-switch-reset"); + } + + /// + /// (Re-)arms the polling timer + emits the desktop_detector_started liveness + /// event. Dispose any existing timer first to avoid leaking ticks. Idempotent except + /// for the emit — every Start/Reset call is a real lifecycle event we want visible. + /// + private void ArmTimer(string startReason) + { + _startedAtUtc = DateTime.UtcNow; + _pollingTimer?.Dispose(); _pollingTimer = new Timer( PollForDesktop, null, - TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(5), // Initial check after 5s TimeSpan.FromSeconds(PollingIntervalSeconds)); + + EmitDetectorStarted(startReason); } private void PollForDesktop(object state) @@ -104,9 +150,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals if (_desktopArrived) return; + _pollCount++; + try { var explorerProcesses = Process.GetProcessesByName("explorer"); + int explorerCountThisPoll = explorerProcesses.Length; + if (explorerCountThisPoll > 0) + _explorerEverSeen = true; + foreach (var proc in explorerProcesses) { try @@ -115,6 +167,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals if (proc.SessionId == 0) continue; + _anyNonZeroSessionEverSeen = true; + var owner = GetProcessOwner(proc.Id); if (owner == null) continue; @@ -153,6 +207,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals } catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: OnTraceEvent failed: {ex.Message}"); } + // First-poll observability snapshot still fires before we stop polling — + // there is value in distinguishing "found on first poll" (Stage already + // past AccountSetup at agent start) from "took N polls" (live transition). + EmitFirstPollIfNeeded(explorerCountThisPoll); + // Stop polling _pollingTimer?.Dispose(); _pollingTimer = null; @@ -170,13 +229,112 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals proc.Dispose(); } } + + // No resolution this poll (no real user found, no excluded user, or all + // candidates were session 0). Emit first-poll snapshot now if pending. + EmitFirstPollIfNeeded(explorerCountThisPoll); + + // Threshold-triggered observability: after the configured timeout, emit a + // single no-candidate event so dashboards can distinguish "user never logged + // in" from "detector broken". Fires exactly once per detector lifetime. + EmitNoCandidateIfThresholdReached(); } catch (Exception ex) { + _wmiErrorCountSinceStart++; _logger.Debug($"DesktopArrivalDetector: polling error: {ex.Message}"); + + // Still emit the first-poll snapshot if this was the first poll — the error + // count carries the failure mode to the backend. + EmitFirstPollIfNeeded(explorerProcessCount: 0); } } + /// + /// One-shot liveness emit after the first completes, + /// regardless of outcome. Proves the Timer + Process.GetProcessesByName path is alive + /// — distinguishing "DAD started but timer/WMI dead" from "DAD running, no user logged + /// in yet" in failed sessions. + /// + private void EmitFirstPollIfNeeded(int explorerProcessCount) + { + if (_firstPollDone) return; + _firstPollDone = true; + + try + { + var elapsedMs = Math.Round((DateTime.UtcNow - _startedAtUtc).TotalMilliseconds, 0); + OnTraceEvent?.Invoke( + SharedEventTypes.DesktopDetectorFirstPoll, + $"DAD first poll complete (explorer.exe count={explorerProcessCount}, elapsedMs={elapsedMs}).", + new Dictionary + { + { "elapsedMsSinceStart", elapsedMs }, + { "explorerProcessCount", explorerProcessCount }, + { "anyNonZeroSessionObserved", _anyNonZeroSessionEverSeen }, + { "wmiErrorCount", _wmiErrorCountSinceStart }, + }); + } + catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: first-poll emit failed: {ex.Message}"); } + } + + /// + /// One-shot threshold-triggered emit: when polling has run for the configured number + /// of minutes without ANY resolution (real-user OR excluded-user) AND the threshold + /// is enabled (> 0), fire a single desktop_detector_no_candidate event. + /// Distinguishes "user never logged in" from "DAD wiring broken" in sessions where + /// desktop_arrived never lands. + /// + private void EmitNoCandidateIfThresholdReached() + { + if (_noCandidateFired) return; + if (_noCandidateTimeoutMinutes <= 0) return; // disabled + if (_excludedUserTraced) return; // we already emitted SOME resolution observability via desktop_excluded_user + + var minutesSinceStart = (DateTime.UtcNow - _startedAtUtc).TotalMinutes; + if (minutesSinceStart < _noCandidateTimeoutMinutes) return; + + _noCandidateFired = true; + try + { + OnTraceEvent?.Invoke( + SharedEventTypes.DesktopDetectorNoCandidate, + $"DAD threshold {_noCandidateTimeoutMinutes} min reached without explorer.exe resolution (polls={_pollCount}, explorerEverSeen={_explorerEverSeen}).", + new Dictionary + { + { "pollsSinceStart", _pollCount }, + { "minutesSinceStart", Math.Round(minutesSinceStart, 1) }, + { "thresholdMinutes", _noCandidateTimeoutMinutes }, + { "explorerEverSeen", _explorerEverSeen }, + { "anyNonZeroSessionEverSeen", _anyNonZeroSessionEverSeen }, + { "wmiErrorCount", _wmiErrorCountSinceStart }, + }); + } + catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: no-candidate emit failed: {ex.Message}"); } + } + + /// + /// One-shot lifecycle emit at Start/Reset. Confirms detector wiring after a post- + /// reboot agent restart — absence of this event after agent_started proves the + /// composition-root never instantiated/armed the detector. + /// + private void EmitDetectorStarted(string startReason) + { + try + { + OnTraceEvent?.Invoke( + SharedEventTypes.DesktopDetectorStarted, + $"DAD armed (startReason={startReason}, pollIntervalSeconds={PollingIntervalSeconds}, noCandidateThresholdMinutes={_noCandidateTimeoutMinutes}).", + new Dictionary + { + { "pollIntervalSeconds", PollingIntervalSeconds }, + { "noCandidateThresholdMinutes", _noCandidateTimeoutMinutes }, + { "startReason", startReason }, + }); + } + catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: detector-started emit failed: {ex.Message}"); } + } + /// /// Gets the owner of a process using WMI (Win32_Process.GetOwner). /// Returns "DOMAIN\User" or "User" string, or null on failure. diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs index 519c0ad0..7ccabe70 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs @@ -202,7 +202,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration // placeholder, so the detector resets and re-evaluates after the Hybrid reboot // instead of staying latched on the foo desktop. Order matters — DesktopArrivalHost // must exist before AadJoinHost so the callback target is available. - var desktopArrivalHost = new DesktopArrivalHost(logger, ingress, clock); + var desktopArrivalHost = new DesktopArrivalHost( + logger, + ingress, + clock, + noCandidateTimeoutMinutes: collectors.DesktopDetectorNoCandidateTimeoutMinutes, + sessionId: sessionId, + tenantId: tenantId); hosts.Add(desktopArrivalHost); _aadJoinHost = new AadJoinHost( diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/DesktopArrivalHost.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/DesktopArrivalHost.cs index 12643d57..7f930b6e 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/DesktopArrivalHost.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/DesktopArrivalHost.cs @@ -1,9 +1,14 @@ #nullable enable +using System; +using System.Collections.Generic; using System.Threading; using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime; +using AutopilotMonitor.Agent.V2.Core.Orchestration; using AutopilotMonitor.Agent.V2.Core.SignalAdapters; using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.Shared.Models; namespace AutopilotMonitor.Agent.V2.Core.Orchestration { @@ -13,12 +18,55 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration private readonly DesktopArrivalDetector _detector; private readonly DesktopArrivalDetectorAdapter _adapter; + private readonly AgentLogger _logger; private int _disposed; - public DesktopArrivalHost(AgentLogger logger, ISignalIngressSink ingress, IClock clock) + public DesktopArrivalHost( + AgentLogger logger, + ISignalIngressSink ingress, + IClock clock, + int noCandidateTimeoutMinutes = 10, + string? sessionId = null, + string? tenantId = null) { - _detector = new DesktopArrivalDetector(logger); + _logger = logger; + _detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes); _adapter = new DesktopArrivalDetectorAdapter(_detector, ingress, clock); + + // V2 single-rail wiring (2026-05-15) — the OnTraceEvent callback is routed + // through InformationalEventPost so the DAD lifecycle events + // (desktop_excluded_user, desktop_real_user_detected, desktop_detector_started, + // desktop_detector_first_poll, desktop_detector_no_candidate) appear on the + // session timeline as standalone EventTypes. V1's CollectorCoordinator wrapped + // them inside a generic `trace_event`; V2 surfaces them with their own type so + // queries / dashboards / MCP can key on the specific event semantics. + var post = new InformationalEventPost(ingress, clock, logger); + _detector.OnTraceEvent = (eventType, message, data) => + { + try + { + var payload = data == null + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(data, StringComparer.Ordinal); + + post.Emit(new EnrollmentEvent + { + SessionId = sessionId ?? string.Empty, + TenantId = tenantId ?? string.Empty, + EventType = eventType, + Severity = EventSeverity.Info, + Source = "DesktopArrivalDetector", + Phase = EnrollmentPhase.Unknown, + Message = message, + Data = payload, + ImmediateUpload = true, + }); + } + catch (Exception ex) + { + logger.Debug($"DesktopArrivalHost: trace-event emit '{eventType}' threw: {ex.Message}"); + } + }; } public void Start() => _detector.Start(); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs index 7438f148..1d6bc145 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs @@ -118,6 +118,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination // treats as Part-1/null). private readonly Func _isWhiteGlovePart2Accessor; + // Shutdown-gap closure (2026-05-15): cross-path idempotency gate shared with the + // AgentRuntimeHost's gap emitters (Ctrl+C, ProcessExit, unhandled exception, + // runtime-host finally). The handler calls TryClaim() before emitting + // agent_shutting_down in so a Terminated + // event that races a Ctrl+C cannot produce two events on the wire. Null = legacy + // behaviour (always emit; backward compat for tests + standalone construction). + private readonly Func _tryClaimShutdownEvent; + // c117946b debrief (2026-05-12): on terminal ESP-Apps failure, promote any apps the // agent observed in to Error so the // user sees a name (not just an opaque "installing: 1" counter) and the @@ -154,7 +162,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination Action writeCleanExitMarker = null, Func ingressPendingSignalCountAccessor = null, Func isWhiteGlovePart2Accessor = null, - Func> promoteActiveInstallsToStuck = null) + Func> promoteActiveInstallsToStuck = null, + Func tryClaimShutdownEvent = null) { _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -185,6 +194,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination _ingressPendingSignalCountAccessor = ingressPendingSignalCountAccessor; _isWhiteGlovePart2Accessor = isWhiteGlovePart2Accessor; _promoteActiveInstallsToStuck = promoteActiveInstallsToStuck; + _tryClaimShutdownEvent = tryClaimShutdownEvent; } /// @@ -726,11 +736,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination // V1-parity acknowledgement that the agent has accepted termination. Carries enough // context for the backend to correlate the shutdown with its trigger (reason/outcome/ - // stage) and for diagnostics to show how long the agent ran. Always emitted, regardless - // of whether self-destruct cleanup actually runs afterwards. + // stage) and for diagnostics to show how long the agent ran. + // + // Shutdown-gap closure (2026-05-15): participates in the cross-path idempotency gate + // owned by AgentRuntimeHost so a Terminated event that races a Ctrl+C / ProcessExit + // does not produce two agent_shutting_down events on the wire. When the gate + // is not wired (tests / standalone construction), the legacy always-emit behaviour + // is preserved. + // private void EmitAgentShuttingDown(EnrollmentTerminatedEventArgs args) { + if (_tryClaimShutdownEvent != null && !_tryClaimShutdownEvent()) + { + _logger.Debug("EnrollmentTerminationHandler: agent_shutting_down already emitted via gap-path — skipping duplicate."); + return; + } + var uptimeMinutes = Math.Round((DateTime.UtcNow - _agentStartTimeUtc).TotalMinutes, 1); + + // Discriminator stored in data["reason"]. Maps the V2 EnrollmentTerminationReason + // enum to the shared reason vocabulary used by the AgentRuntimeHost gap paths + // (ctrl_c / process_exit / unhandled_exception / runtime_host_exit) so all + // shutdown events on the wire share the same reason taxonomy. + string reasonTag; + switch (args.Reason) + { + case EnrollmentTerminationReason.DecisionTerminalStage: + reasonTag = "decision_terminal"; + break; + case EnrollmentTerminationReason.MaxLifetimeExceeded: + reasonTag = "max_lifetime"; + break; + default: + reasonTag = args.Reason.ToString(); + break; + } + EmitEventSafe(new EnrollmentEvent { SessionId = _configuration.SessionId, @@ -739,10 +780,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination Severity = EventSeverity.Info, Source = "EnrollmentTerminationHandler", Phase = EnrollmentPhase.Unknown, - Message = $"Agent shutting down (reason={args.Reason}, outcome={args.Outcome}).", + Message = $"Agent shutting down (reason={reasonTag}, outcome={args.Outcome}).", Data = new Dictionary { - { "reason", args.Reason.ToString() }, + { "reason", reasonTag }, { "outcome", args.Outcome.ToString() }, { "stage", args.StageName ?? string.Empty }, { "uptimeMinutes", uptimeMinutes }, diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs index bde97e9e..2f1b9dfd 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs @@ -148,27 +148,72 @@ namespace AutopilotMonitor.Agent.V2.Runtime // are both live. Disposed in the same finally block as the rest of the lifecycle. WhiteGloveInventoryTrigger whiteGloveInventoryTrigger = null; + // Single-rail refactor (plan §5.1) — lifecycle events flow through + // InformationalEventPost. The post is constructed inside the + // onIngressReady callback below; the watchdog factories + gap-path + // handlers close over the variable so they see the live helper when + // their event fires (read in handler bodies, not at handler creation). + InformationalEventPost lifecyclePost = null; + + // Shutdown-gap closure (2026-05-15) — cross-path idempotency gate. Every + // agent_shutting_down emit path (EnrollmentTerminationHandler, + // CreateAuthThresholdHandler, the gap emitters below) claims the slot via + // Interlocked.Exchange before emitting. Prevents a Ctrl+C racing a real + // Terminated transition from producing two events on the wire. + int agentShuttingDownEmitted = 0; + bool TryClaimAgentShuttingDownEmit() => + Interlocked.Exchange(ref agentShuttingDownEmitted, 1) == 0; + ConsoleCancelEventHandler cancelHandler = (s, e) => { e.Cancel = true; logger.Info("Ctrl+C received — initiating graceful shutdown."); + + // L1 gap-closure: previously the cancelHandler only signalled + // shutdown.Set(), leaving the timeline without an explicit + // agent_shutting_down event. Emit here so dev/console-mode exits are + // distinguishable from silent process death. + if (TryClaimAgentShuttingDownEmit()) + { + LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: lifecyclePost, + agentConfig: agentConfig, + reason: "ctrl_c", + agentStartTimeUtc: agentStartTimeUtc, + agentVersion: agentVersion, + logger: logger); + } + shutdown.Set(); }; EventHandler processExitHandler = (s, e) => { logger.Info("ProcessExit — initiating graceful shutdown."); + + // L2 gap-closure: Windows gives the ProcessExit handler ~2s before + // killing the process, so the emit is best-effort — the event lands + // in the SignalIngress channel (non-blocking enqueue) and the + // subsequent finally→TerminationPipeline.Run drains the spool via + // orchestrator.Stop's DrainAllAsync. Under a hard Windows shutdown + // race the event may end up persisted-but-not-uploaded; even then it + // surfaces in the next session's diag ZIP (if upload is on). + if (TryClaimAgentShuttingDownEmit()) + { + LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: lifecyclePost, + agentConfig: agentConfig, + reason: "process_exit", + agentStartTimeUtc: agentStartTimeUtc, + agentVersion: agentVersion, + logger: logger); + } + shutdown.Set(); }; Console.CancelKeyPress += cancelHandler; AppDomain.CurrentDomain.ProcessExit += processExitHandler; - // Single-rail refactor (plan §5.1) — lifecycle events flow through - // InformationalEventPost. The post is constructed inside the - // onIngressReady callback below; the watchdog factories close over a - // Func accessor so they see the live helper when their event fires. - InformationalEventPost lifecyclePost = null; - var maxLifetimeEmitter = LifecycleEmitters.CreateMaxLifetimeEmitter( getLifecyclePost: () => lifecyclePost, agentConfig: agentConfig, @@ -316,7 +361,13 @@ namespace AutopilotMonitor.Agent.V2.Runtime // EspTerminalFailure pathway only — other Failed paths leave the // app list untouched. promoteActiveInstallsToStuck: (failureType, message) - => componentFactory.PromoteActiveInstallsToStuck(failureType, message)); + => componentFactory.PromoteActiveInstallsToStuck(failureType, message), + // Shutdown-gap closure (2026-05-15) — share the cross-path + // idempotency gate so a Terminated event that races a Ctrl+C / + // ProcessExit gap-path does not emit two agent_shutting_down + // events on the wire. The handler skips its emit when the + // gate already claimed the slot. + tryClaimShutdownEvent: TryClaimAgentShuttingDownEmit); // ServerActionDispatcher (plan §5.3) — constructed inside this // hook so lifecyclePost + terminationHandler are guaranteed @@ -481,6 +532,29 @@ namespace AutopilotMonitor.Agent.V2.Runtime shutdown.Wait(); } + catch (Exception runtimeEx) + { + // L3 gap-closure (2026-05-15): unhandled exception inside the try + // block above (Start, post-startup wiring, the long-lived shutdown + // wait if a captured Task escalation reached this thread, etc.). + // Previously this propagated straight to Program.Main's catch which + // logs + WriteCrashLog but emits no timeline event — the session + // looked like silent process death. Emit before re-throwing so the + // crash is visible on the wire. + if (TryClaimAgentShuttingDownEmit()) + { + LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: lifecyclePost, + agentConfig: agentConfig, + reason: "unhandled_exception", + agentStartTimeUtc: agentStartTimeUtc, + agentVersion: agentVersion, + logger: logger, + exceptionType: runtimeEx.GetType().FullName, + exceptionMessage: runtimeEx.Message); + } + throw; + } finally { // Detach the WG-Part-1 inventory trigger before TerminationPipeline runs @@ -489,6 +563,23 @@ namespace AutopilotMonitor.Agent.V2.Runtime try { whiteGloveInventoryTrigger?.Dispose(); } catch (Exception ex) { logger.Warning($"WhiteGloveInventoryTrigger.Dispose threw: {ex.Message}"); } + // L4 gap-closure (2026-05-15): defensive fallback emit for any path + // that returned to finally without claiming the gate — e.g. a future + // direct shutdown.Set() from a server-action handler that bypasses + // both Terminated AND the cancel/processExit closures. Best-effort: + // the gate is already claimed by Terminated / Ctrl+C / ProcessExit / + // AuthFailure / L3 in every known path, so this rarely fires. + if (TryClaimAgentShuttingDownEmit()) + { + LifecycleEmitters.EmitAgentShuttingDownGapPath( + post: lifecyclePost, + agentConfig: agentConfig, + reason: "runtime_host_exit", + agentStartTimeUtc: agentStartTimeUtc, + agentVersion: agentVersion, + logger: logger); + } + TerminationPipeline.Run( orchestrator: orchestrator, authFailureTracker: auth.AuthFailureTracker, diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs index 0dbe127a..2476c9a9 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs @@ -399,6 +399,103 @@ namespace AutopilotMonitor.Agent.V2.Runtime } } + // ====================================================== Cross-path shutdown emit helper + + /// + /// Shutdown-gap closure (2026-05-15) — emits a single + /// agent_shutting_down event from one of the non-Terminated exit paths the + /// V2 runtime had previously left silent (Ctrl+C, ProcessExit, unhandled exception + /// in RunAgent, finally cleanup without prior Terminated). The + /// remains the canonical emitter on the + /// happy path; this helper is for the gap-closure paths only. + /// + /// Idempotency contract: the caller (typically AgentRuntimeHost) owns a + /// single shared flag and uses + /// Interlocked.Exchange(ref flag, 1) == 0 to claim the emit slot before + /// invoking this method. participates in + /// the same gate via its tryClaimShutdownEvent constructor parameter so + /// double-emit cannot happen when, say, Ctrl+C races a real Terminated transition. + /// + /// + /// Live . Null is treated as + /// "ingress not ready"; the call logs and returns false without emitting. + /// Carries SessionId / TenantId / agent build version. + /// Discriminator stored in data["reason"]. Allowed values: + /// ctrl_c, process_exit, unhandled_exception, + /// runtime_host_exit. Other reasons (auth_failure / decision_terminal / + /// max_lifetime) are emitted by their own dedicated paths. + /// Used to compute uptime for the event payload. + /// Build hash tag, mirrored from agent_started. + /// For best-effort log on emit failure. + /// Optional .NET exception type, populated when the + /// caller is the unhandled-exception path. + /// Optional .NET exception message (truncated to + /// 500 chars on the wire to keep payload size bounded). + /// true if the event was posted; false if was null + /// or the emit threw. + public static bool EmitAgentShuttingDownGapPath( + InformationalEventPost post, + AgentConfiguration agentConfig, + string reason, + DateTime agentStartTimeUtc, + string agentVersion, + AgentLogger logger, + string exceptionType = null, + string exceptionMessage = null) + { + if (post == null) + { + logger?.Warning($"agent_shutting_down (reason={reason}) suppressed — ingress not ready."); + return false; + } + + try + { + var uptimeMinutes = Math.Round((DateTime.UtcNow - agentStartTimeUtc).TotalMinutes, 1); + + var data = new Dictionary(StringComparer.Ordinal) + { + { "reason", reason ?? "unknown" }, + { "outcome", "Unknown" }, + { "stage", string.Empty }, + { "uptimeMinutes", uptimeMinutes }, + { "agentVersion", agentVersion ?? string.Empty }, + }; + + if (!string.IsNullOrEmpty(exceptionType)) + data["exceptionType"] = exceptionType; + if (!string.IsNullOrEmpty(exceptionMessage)) + { + // Bound payload growth — exception messages can be long stack-trace text. + var trimmed = exceptionMessage.Length > 500 + ? exceptionMessage.Substring(0, 500) + "…" + : exceptionMessage; + data["exceptionMessage"] = trimmed; + } + + post.Emit(new EnrollmentEvent + { + SessionId = agentConfig.SessionId, + TenantId = agentConfig.TenantId, + EventType = SharedConstants.EventTypes.AgentShuttingDown, + Severity = string.Equals(reason, "unhandled_exception", StringComparison.Ordinal) + ? EventSeverity.Error + : EventSeverity.Info, + Source = "AgentRuntimeHost", + Phase = EnrollmentPhase.Unknown, + Message = $"Agent shutting down (reason={reason}).", + Data = data, + ImmediateUpload = true, + }); + return true; + } + catch (Exception ex) + { + logger?.Warning($"agent_shutting_down (reason={reason}) emission failed: {ex.Message}"); + return false; + } + } + // ================================================================ Watchdog handlers /// @@ -484,11 +581,18 @@ namespace AutopilotMonitor.Agent.V2.Runtime { try { + // Event-type unification (2026-05-15): the auth-failure path used to + // emit a separate `agent_shutdown` type (legacy V1 string). All V2 + // shutdown paths now share `agent_shutting_down` with the failure + // class disambiguated via Data["reason"] — auth_failure here, others + // (decision_terminal / max_lifetime / ctrl_c / process_exit / + // unhandled_exception) in EnrollmentTerminationHandler + the new + // AgentRuntimeHost gap-closure emit paths. post.Emit(new EnrollmentEvent { SessionId = agentConfig.SessionId, TenantId = agentConfig.TenantId, - EventType = "agent_shutdown", + EventType = SharedConstants.EventTypes.AgentShuttingDown, Severity = EventSeverity.Error, Source = "AuthFailureTracker", Phase = EnrollmentPhase.Unknown, @@ -509,12 +613,12 @@ namespace AutopilotMonitor.Agent.V2.Runtime } catch (Exception emitEx) { - logger.Warning($"agent_shutdown emission failed: {emitEx.Message}"); + logger.Warning($"agent_shutting_down (auth_failure) emission failed: {emitEx.Message}"); } } else { - logger.Warning("agent_shutdown (auth_failure) suppressed — ingress not ready."); + logger.Warning("agent_shutting_down (auth_failure) suppressed — ingress not ready."); } signalShutdown(); diff --git a/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt index 5f6e6079..cec64570 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt @@ -1 +1 @@ -745 +749 diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt index 5c248468..3f3c9366 100644 --- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt @@ -1 +1 @@ -1340 +1344 diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs index 085945d3..c8ee2f77 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs @@ -542,6 +542,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage { "GlobalRateLimitRequestsPerMinute", config.GlobalRateLimitRequestsPerMinute }, { "PlatformStatsBlobSasUrl", config.PlatformStatsBlobSasUrl ?? string.Empty }, { "CollectorIdleTimeoutMinutes", config.CollectorIdleTimeoutMinutes }, + { "DesktopDetectorNoCandidateTimeoutMinutes", config.DesktopDetectorNoCandidateTimeoutMinutes }, { "ExcessiveEventCountThreshold", config.ExcessiveEventCountThreshold }, { "MaxSessionWindowHours", config.MaxSessionWindowHours }, { "MaintenanceBlockDurationHours", config.MaintenanceBlockDurationHours }, @@ -606,6 +607,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage GlobalRateLimitRequestsPerMinute = entity.GetInt32("GlobalRateLimitRequestsPerMinute") ?? 100, PlatformStatsBlobSasUrl = entity.GetString("PlatformStatsBlobSasUrl") ?? string.Empty, CollectorIdleTimeoutMinutes = entity.GetInt32("CollectorIdleTimeoutMinutes") ?? 15, + DesktopDetectorNoCandidateTimeoutMinutes = entity.GetInt32("DesktopDetectorNoCandidateTimeoutMinutes") ?? 10, ExcessiveEventCountThreshold = entity.GetInt32("ExcessiveEventCountThreshold") ?? 2000, MaxSessionWindowHours = entity.GetInt32("MaxSessionWindowHours") ?? 24, MaintenanceBlockDurationHours = entity.GetInt32("MaintenanceBlockDurationHours") ?? 12, diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs index 3d7f6e26..ebf8123d 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs @@ -156,6 +156,7 @@ namespace AutopilotMonitor.Functions.Functions.Config EnablePerformanceCollector = tenantConfig.EnablePerformanceCollector, PerformanceIntervalSeconds = tenantConfig.PerformanceCollectorIntervalSeconds, CollectorIdleTimeoutMinutes = adminConfig.CollectorIdleTimeoutMinutes, + DesktopDetectorNoCandidateTimeoutMinutes = adminConfig.DesktopDetectorNoCandidateTimeoutMinutes, HelloWaitTimeoutSeconds = tenantConfig.HelloWaitTimeoutSeconds, AgentMaxLifetimeMinutes = tenantConfig.AgentMaxLifetimeMinutes ?? 360, ModernDeploymentHarmlessEventIds = adminConfig.GetModernDeploymentHarmlessEventIds().ToArray() @@ -180,7 +181,7 @@ namespace AutopilotMonitor.Functions.Functions.Config var response = req.CreateResponse(HttpStatusCode.OK); await response.WriteAsJsonAsync(new AgentConfigResponse { - ConfigVersion = 27, // ModernDeployment harmless EventID 1010 added to default noise list + ConfigVersion = 28, // V2 DAD-liveness threshold (DesktopDetectorNoCandidateTimeoutMinutes, default 10 min); admin-config global setting UploadIntervalSeconds = 10, SelfDestructOnComplete = tenantConfig.SelfDestructOnComplete ?? true, KeepLogFile = tenantConfig.KeepLogFile ?? false, diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts index fdc069fb..a21ed756 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts @@ -20,7 +20,7 @@ const KEY_EVENT_TYPES = new Set([ 'app_tracking_summary', 'error_detected', 'enrollment_complete', 'enrollment_failed', 'completion_check', 'desktop_arrived', 'hello_policy_detected', 'waiting_for_hello', 'hello_completion_timeout', - 'agent_started', 'agent_shutdown', 'trace_event', + 'agent_started', 'agent_shutdown', 'agent_shutting_down', 'trace_event', 'script_started', 'script_completed', 'script_failed', 'vulnerability_report', ]); const SEVERITY_RANK: Record = { Trace: -1, Debug: 0, Info: 1, Warning: 2, Error: 3, Critical: 4 }; diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 678799df..65200f06 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -239,6 +239,13 @@ namespace AutopilotMonitor.Shared public const string EnrollmentComplete = "enrollment_complete"; public const string EnrollmentFailed = "enrollment_failed"; public const string DesktopArrived = "desktop_arrived"; + // DAD liveness (state-change-only, max 3 events per detector lifetime, NOT periodic). + // Distinguishes "DAD never started after reboot" vs "DAD started but timer/WMI dead" vs + // "DAD running but user never logged in" — the three failure-modes that look identical + // when only desktop_arrived is missing. + public const string DesktopDetectorStarted = "desktop_detector_started"; // 1x on Start() / ResetForRealUserSwitch() + public const string DesktopDetectorFirstPoll = "desktop_detector_first_poll"; // 1x after first PollForDesktop() completes + public const string DesktopDetectorNoCandidate = "desktop_detector_no_candidate"; // 1x after threshold polls without resolution (configurable) public const string CompletionCheck = "completion_check"; public const string ScriptStarted = "script_started"; public const string ScriptCompleted = "script_completed"; diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs b/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs index d1f8360c..d097c081 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs @@ -74,6 +74,17 @@ namespace AutopilotMonitor.Shared.Models /// public int CollectorIdleTimeoutMinutes { get; set; } = 15; + /// + /// DAD liveness threshold in minutes. When the V2 DesktopArrivalDetector polls for + /// this duration without detecting either an excluded-user explorer.exe or a real + /// user desktop, it emits a single desktop_detector_no_candidate observability + /// event. State-change-only — NOT a periodic heartbeat; max 1 event per detector + /// lifetime. Used to distinguish "user never logged in" from "detector wiring dead + /// post-reboot" in sessions where desktop_arrived is missing. + /// 0 = disabled (event never fires). Default: 10 minutes. + /// + public int DesktopDetectorNoCandidateTimeoutMinutes { get; set; } = 10; + /// /// Maintenance alarm threshold: sessions with more events than this value trigger an /// ExcessiveSessionEvents ops alert (dispatched to Telegram/Teams). @@ -442,6 +453,7 @@ namespace AutopilotMonitor.Shared.Models GlobalAdminRateLimitRequestsPerMinute = 600, PlatformStatsBlobSasUrl = string.Empty, CollectorIdleTimeoutMinutes = 15, + DesktopDetectorNoCandidateTimeoutMinutes = 10, MaxSessionWindowHours = 24, MaintenanceBlockDurationHours = 12 }; diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs b/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs index 2f55ed0b..98684220 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs @@ -235,6 +235,16 @@ namespace AutopilotMonitor.Shared.Models /// public int CollectorIdleTimeoutMinutes { get; set; } = 15; + /// + /// DAD liveness threshold (minutes). After this many minutes of polling without + /// detecting either an excluded-user explorer.exe or a real user desktop, the V2 + /// DesktopArrivalDetector emits a single desktop_detector_no_candidate event + /// (state-change-only, NOT periodic). Distinguishes "user never logged in" from + /// "detector wiring dead post-reboot" in sessions missing desktop_arrived. + /// 0 = disabled. Default: 10 minutes. + /// + public int DesktopDetectorNoCandidateTimeoutMinutes { get; set; } = 10; + /// /// Enable the agent self-metrics collector (process CPU, memory, network traffic). /// Default: true diff --git a/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx b/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx index 3bf975b9..8a9bbe0d 100644 --- a/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx @@ -24,6 +24,8 @@ interface AdminConfigContextValue { setPlatformStatsBlobSasUrl: (value: string) => void; collectorIdleTimeoutMinutes: number; setCollectorIdleTimeoutMinutes: (value: number) => void; + desktopDetectorNoCandidateTimeoutMinutes: number; + setDesktopDetectorNoCandidateTimeoutMinutes: (value: number) => void; maxSessionWindowHours: number; setMaxSessionWindowHours: (value: number) => void; maintenanceBlockDurationHours: number; @@ -111,6 +113,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) const [globalRateLimit, setGlobalRateLimit] = useState(100); const [platformStatsBlobSasUrl, setPlatformStatsBlobSasUrl] = useState(""); const [collectorIdleTimeoutMinutes, setCollectorIdleTimeoutMinutes] = useState(15); + const [desktopDetectorNoCandidateTimeoutMinutes, setDesktopDetectorNoCandidateTimeoutMinutes] = useState(10); const [maxSessionWindowHours, setMaxSessionWindowHours] = useState(24); const [maintenanceBlockDurationHours, setMaintenanceBlockDurationHours] = useState(12); const [opsEventRetentionDays, setOpsEventRetentionDays] = useState(90); @@ -166,6 +169,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) setGlobalRateLimit(data.globalRateLimitRequestsPerMinute); setPlatformStatsBlobSasUrl(data.platformStatsBlobSasUrl ?? ""); setCollectorIdleTimeoutMinutes(data.collectorIdleTimeoutMinutes ?? 15); + setDesktopDetectorNoCandidateTimeoutMinutes(data.desktopDetectorNoCandidateTimeoutMinutes ?? 10); setMaxSessionWindowHours(data.maxSessionWindowHours ?? 24); setMaintenanceBlockDurationHours(data.maintenanceBlockDurationHours ?? 12); setOpsEventRetentionDays(data.opsEventRetentionDays ?? 90); @@ -262,6 +266,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) globalRateLimitRequestsPerMinute: globalRateLimit, platformStatsBlobSasUrl: platformStatsBlobSasUrl.trim(), collectorIdleTimeoutMinutes, + desktopDetectorNoCandidateTimeoutMinutes, maxSessionWindowHours, maintenanceBlockDurationHours, opsEventRetentionDays, @@ -296,7 +301,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) } finally { setSavingConfig(false); } - }, [adminConfig, globalRateLimit, platformStatsBlobSasUrl, collectorIdleTimeoutMinutes, maxSessionWindowHours, maintenanceBlockDurationHours, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, getAccessToken]); + }, [adminConfig, globalRateLimit, platformStatsBlobSasUrl, collectorIdleTimeoutMinutes, desktopDetectorNoCandidateTimeoutMinutes, maxSessionWindowHours, maintenanceBlockDurationHours, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, getAccessToken]); // Reset admin config const handleResetAdminConfig = useCallback(() => { @@ -304,6 +309,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) setGlobalRateLimit(adminConfig.globalRateLimitRequestsPerMinute); setPlatformStatsBlobSasUrl(adminConfig.platformStatsBlobSasUrl ?? ""); setCollectorIdleTimeoutMinutes(adminConfig.collectorIdleTimeoutMinutes ?? 15); + setDesktopDetectorNoCandidateTimeoutMinutes(adminConfig.desktopDetectorNoCandidateTimeoutMinutes ?? 10); setMaxSessionWindowHours(adminConfig.maxSessionWindowHours ?? 24); setMaintenanceBlockDurationHours(adminConfig.maintenanceBlockDurationHours ?? 12); setOpsEventRetentionDays(adminConfig.opsEventRetentionDays ?? 90); @@ -431,6 +437,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) globalRateLimit, setGlobalRateLimit, platformStatsBlobSasUrl, setPlatformStatsBlobSasUrl, collectorIdleTimeoutMinutes, setCollectorIdleTimeoutMinutes, + desktopDetectorNoCandidateTimeoutMinutes, setDesktopDetectorNoCandidateTimeoutMinutes, maxSessionWindowHours, setMaxSessionWindowHours, maintenanceBlockDurationHours, setMaintenanceBlockDurationHours, opsEventRetentionDays, setOpsEventRetentionDays, diff --git a/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx index baa2dcab..8c87dc9f 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx @@ -12,6 +12,8 @@ interface AdminConfigSettingsSectionProps { setPlatformStatsBlobSasUrl: (value: string) => void; collectorIdleTimeoutMinutes: number; setCollectorIdleTimeoutMinutes: (value: number) => void; + desktopDetectorNoCandidateTimeoutMinutes: number; + setDesktopDetectorNoCandidateTimeoutMinutes: (value: number) => void; slaNotificationCooldownHours: number; setSlaNotificationCooldownHours: (value: number) => void; allowAgentDowngrade: boolean; @@ -36,6 +38,8 @@ export function AdminConfigSettingsSection({ setPlatformStatsBlobSasUrl, collectorIdleTimeoutMinutes, setCollectorIdleTimeoutMinutes, + desktopDetectorNoCandidateTimeoutMinutes, + setDesktopDetectorNoCandidateTimeoutMinutes, slaNotificationCooldownHours, setSlaNotificationCooldownHours, allowAgentDowngrade, @@ -128,6 +132,25 @@ export function AdminConfigSettingsSection({ +
+ +
+