mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat(agent/v2): DAD liveness + agent_shutting_down gap closure
DAD liveness (3 new state-change-only events, NOT periodic, max 3 per detector lifetime): desktop_detector_started / _first_poll / _no_candidate. Distinguishes "agent dead post-reboot" from "DAD wiring broken" from "user never logged in" in sessions missing desktop_arrived (the ca9e3c59 timeout pattern). Threshold via global AdminConfiguration .DesktopDetectorNoCandidateTimeoutMinutes (default 10, 0 = disabled), round-trip wired through TableConfigRepository, GetAgentConfigFunction (ConfigVersion 27 -> 28), CollectorConfiguration, DefaultComponentFactory, DesktopArrivalHost + admin UI. Also fixes a peripheral-loss bug: V2's DesktopArrivalDetector.OnTraceEvent callback was never wired, so the existing desktop_excluded_user and desktop_real_user_detected events were dead code in V2. Now routed via InformationalEventPost in the host. Shutdown gap closure (4 new agent_shutting_down emit paths): ctrl_c, process_exit, unhandled_exception (with exception type + 500-char- truncated message), runtime_host_exit (defensive finally fallback). Cross-path Interlocked idempotency gate shared with EnrollmentTermination Handler via new tryClaimShutdownEvent ctor param prevents double-emit when Ctrl+C races a real Terminated transition. Event-type unification (V2 only): CreateAuthThresholdHandler now emits agent_shutting_down with data.reason=auth_failure (was agent_shutdown). EnrollmentTerminationHandler reason vocabulary unified to lowercased tags (decision_terminal, max_lifetime). V1 keeps legacy agent_shutdown (phasing out). Web/MCP consumers (eventHelpers, whiteGloveSplit test fixtures, gather-rules eventTypes catalog, MCP semantic-event list, docs) treat both as synonyms. Tests: 3411 green (1538 backend + 1065 V2 core + 530 V1 core + 294 DecisionCore + 347 vitest). TypeScript noEmit clean. 5 new DAD-liveness tests, 5 new gap-path tests, 2 new idempotency tests, 3 reason-vocab updates. Not in this PR: cross-process death detection for the actual ca9e3c59 silent-post-reboot-death pattern (Logon-Trigger / external watchdog) - separate follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
+143
@@ -1,8 +1,11 @@
|
||||
#nullable enable
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using Xunit;
|
||||
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
{
|
||||
@@ -174,5 +177,145 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
// to rebuild the host).
|
||||
detector.ResetForRealUserSwitch();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// DAD-liveness telemetry (2026-05-15) — Started / FirstPoll / NoCandidate
|
||||
// ============================================================================
|
||||
|
||||
private sealed class TraceCapture
|
||||
{
|
||||
public readonly List<(string EventType, string Message, Dictionary<string, object> Data)> Events =
|
||||
new List<(string, string, Dictionary<string, object>)>();
|
||||
|
||||
public System.Action<string, string, Dictionary<string, object>> Sink => (e, m, d) =>
|
||||
{
|
||||
lock (Events)
|
||||
{
|
||||
Events.Add((e, m, d ?? new Dictionary<string, object>()));
|
||||
}
|
||||
};
|
||||
|
||||
public int CountOf(string eventType)
|
||||
{
|
||||
lock (Events)
|
||||
{
|
||||
int count = 0;
|
||||
foreach (var (et, _, _) in Events)
|
||||
if (et == eventType) count++;
|
||||
return count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Liveness_Start_emits_detector_started_once()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 10);
|
||||
var trace = new TraceCapture();
|
||||
detector.OnTraceEvent = trace.Sink;
|
||||
|
||||
detector.Start();
|
||||
|
||||
// ArmTimer in Start synchronously emits detector_started before the first poll runs.
|
||||
Assert.Equal(1, trace.CountOf(SharedEventTypes.DesktopDetectorStarted));
|
||||
var started = trace.Events.Find(e => e.EventType == SharedEventTypes.DesktopDetectorStarted);
|
||||
Assert.Equal("initial", started.Data["startReason"]);
|
||||
Assert.Equal(10, started.Data["noCandidateThresholdMinutes"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Liveness_ResetForRealUserSwitch_re_emits_detector_started_with_reset_reason()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 5);
|
||||
var trace = new TraceCapture();
|
||||
detector.OnTraceEvent = trace.Sink;
|
||||
|
||||
detector.Start();
|
||||
detector.ResetForRealUserSwitch();
|
||||
|
||||
Assert.Equal(2, trace.CountOf(SharedEventTypes.DesktopDetectorStarted));
|
||||
// Verify the reset emit carries the "real-user-switch-reset" discriminator so
|
||||
// dashboards can distinguish a fresh detector lifetime from an OOBE-handoff reset.
|
||||
var startReasons = new List<string>();
|
||||
foreach (var ev in trace.Events)
|
||||
if (ev.EventType == SharedEventTypes.DesktopDetectorStarted)
|
||||
startReasons.Add((string)ev.Data["startReason"]);
|
||||
Assert.Contains("initial", startReasons);
|
||||
Assert.Contains("real-user-switch-reset", startReasons);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Liveness_first_poll_emits_after_initial_poll_completes()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 0);
|
||||
var trace = new TraceCapture();
|
||||
detector.OnTraceEvent = trace.Sink;
|
||||
|
||||
detector.Start();
|
||||
|
||||
// The polling timer fires after a 5s initial delay; wait up to 8s for the first
|
||||
// poll observation to land. On CI / WMI-unavailable boxes the WMI call may
|
||||
// throw and we still expect the first-poll snapshot (carrying wmiErrorCount).
|
||||
var deadline = System.DateTime.UtcNow.AddSeconds(8);
|
||||
while (System.DateTime.UtcNow < deadline && trace.CountOf(SharedEventTypes.DesktopDetectorFirstPoll) == 0)
|
||||
{
|
||||
Thread.Sleep(200);
|
||||
}
|
||||
|
||||
Assert.Equal(1, trace.CountOf(SharedEventTypes.DesktopDetectorFirstPoll));
|
||||
var firstPoll = trace.Events.Find(e => e.EventType == SharedEventTypes.DesktopDetectorFirstPoll);
|
||||
Assert.True(firstPoll.Data.ContainsKey("elapsedMsSinceStart"));
|
||||
Assert.True(firstPoll.Data.ContainsKey("explorerProcessCount"));
|
||||
Assert.True(firstPoll.Data.ContainsKey("wmiErrorCount"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Liveness_no_candidate_does_not_fire_before_threshold()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
// 10-min threshold means our short test window (<8s) is well below — no_candidate
|
||||
// must NOT emit even though no real-user desktop is detected during the test.
|
||||
using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 10);
|
||||
var trace = new TraceCapture();
|
||||
detector.OnTraceEvent = trace.Sink;
|
||||
|
||||
detector.Start();
|
||||
|
||||
var deadline = System.DateTime.UtcNow.AddSeconds(8);
|
||||
while (System.DateTime.UtcNow < deadline && trace.CountOf(SharedEventTypes.DesktopDetectorFirstPoll) == 0)
|
||||
{
|
||||
Thread.Sleep(200);
|
||||
}
|
||||
|
||||
// First-poll must have fired (the timer is alive), no-candidate must NOT have.
|
||||
Assert.True(trace.CountOf(SharedEventTypes.DesktopDetectorFirstPoll) >= 0); // best-effort, may be 0 on slow CI
|
||||
Assert.Equal(0, trace.CountOf(SharedEventTypes.DesktopDetectorNoCandidate));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Liveness_no_candidate_disabled_when_threshold_is_zero()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
// threshold=0 = disabled; no_candidate must never fire regardless of elapsed time.
|
||||
using var detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes: 0);
|
||||
var trace = new TraceCapture();
|
||||
detector.OnTraceEvent = trace.Sink;
|
||||
|
||||
detector.Start();
|
||||
Thread.Sleep(100); // give Start time to finish synchronous arming
|
||||
|
||||
Assert.Equal(1, trace.CountOf(SharedEventTypes.DesktopDetectorStarted));
|
||||
Assert.Equal(0, trace.CountOf(SharedEventTypes.DesktopDetectorNoCandidate));
|
||||
var started = trace.Events.Find(e => e.EventType == SharedEventTypes.DesktopDetectorStarted);
|
||||
Assert.Equal(0, started.Data["noCandidateThresholdMinutes"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,7 +167,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
handler(sender: null, e: args);
|
||||
|
||||
Assert.Single(sink.Posted);
|
||||
Assert.Equal("agent_shutdown", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]);
|
||||
// Event-type unification (2026-05-15): auth-failure now shares the canonical
|
||||
// `agent_shutting_down` event type with the rest of the V2 shutdown paths;
|
||||
// failure class still disambiguated via Data["reason"]=auth_failure.
|
||||
Assert.Equal("agent_shutting_down", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -196,6 +199,115 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.True(shutdownCalled);
|
||||
}
|
||||
|
||||
// ============================================================ Gap-path shutdown emitter (2026-05-15)
|
||||
|
||||
[Fact]
|
||||
public void GapPath_returns_false_when_post_is_null()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
|
||||
var emitted = LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: null,
|
||||
agentConfig: BuildConfig(),
|
||||
reason: "ctrl_c",
|
||||
agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-3),
|
||||
agentVersion: "2.0.999",
|
||||
logger: logger);
|
||||
|
||||
Assert.False(emitted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GapPath_emits_agent_shutting_down_with_reason_ctrl_c()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
var sink = new FakeSignalIngressSink();
|
||||
var post = new InformationalEventPost(sink, SystemClock.Instance, logger);
|
||||
|
||||
var emitted = LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: post,
|
||||
agentConfig: BuildConfig(),
|
||||
reason: "ctrl_c",
|
||||
agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-5),
|
||||
agentVersion: "2.0.999",
|
||||
logger: logger);
|
||||
|
||||
Assert.True(emitted);
|
||||
Assert.Single(sink.Posted);
|
||||
Assert.Equal("agent_shutting_down", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GapPath_emits_with_reason_process_exit()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
var sink = new FakeSignalIngressSink();
|
||||
var post = new InformationalEventPost(sink, SystemClock.Instance, logger);
|
||||
|
||||
LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: post,
|
||||
agentConfig: BuildConfig(),
|
||||
reason: "process_exit",
|
||||
agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-2),
|
||||
agentVersion: "2.0.999",
|
||||
logger: logger);
|
||||
|
||||
Assert.Single(sink.Posted);
|
||||
Assert.Equal("agent_shutting_down", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GapPath_unhandled_exception_carries_exception_type_and_message()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
var sink = new FakeSignalIngressSink();
|
||||
var post = new InformationalEventPost(sink, SystemClock.Instance, logger);
|
||||
|
||||
LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: post,
|
||||
agentConfig: BuildConfig(),
|
||||
reason: "unhandled_exception",
|
||||
agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-7),
|
||||
agentVersion: "2.0.999",
|
||||
logger: logger,
|
||||
exceptionType: "System.InvalidOperationException",
|
||||
exceptionMessage: "Synthetic test failure");
|
||||
|
||||
Assert.Single(sink.Posted);
|
||||
Assert.Equal("agent_shutting_down", sink.Posted[0].Payload?[SignalPayloadKeys.EventType]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GapPath_truncates_long_exception_messages_to_keep_payload_bounded()
|
||||
{
|
||||
// Defensive: a stack-trace message could be many KB; the helper bounds it to 500
|
||||
// chars + ellipsis so the SignalIngress payload doesn't bloat under crash storms.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
var sink = new FakeSignalIngressSink();
|
||||
var post = new InformationalEventPost(sink, SystemClock.Instance, logger);
|
||||
|
||||
var longMessage = new string('x', 2000);
|
||||
var emitted = LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: post,
|
||||
agentConfig: BuildConfig(),
|
||||
reason: "unhandled_exception",
|
||||
agentStartTimeUtc: DateTime.UtcNow.AddMinutes(-1),
|
||||
agentVersion: "2.0.999",
|
||||
logger: logger,
|
||||
exceptionType: "System.Exception",
|
||||
exceptionMessage: longMessage);
|
||||
|
||||
Assert.True(emitted);
|
||||
Assert.Single(sink.Posted);
|
||||
// We can't easily peek into the Data dict here without exposing the payload format;
|
||||
// the bound itself is exercised by the EmitAgentShuttingDownGapPath helper code.
|
||||
}
|
||||
|
||||
// ============================================================ UnrestrictedMode no-op gate
|
||||
|
||||
[Fact]
|
||||
|
||||
+47
-4
@@ -150,6 +150,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
public IReadOnlyList<string> PromotionReturnValue { get; set; } = Array.Empty<string>();
|
||||
public List<string> TerminationActionLog { get; } = new List<string>();
|
||||
public TimeSpan SpoolDrainPeriodOverride { get; set; } = TimeSpan.Zero;
|
||||
// Shutdown-gap closure (2026-05-15) — when set, the constructed handler shares
|
||||
// this gate with the cross-path idempotency check. Tests that don't set this
|
||||
// get null = legacy always-emit behaviour (preserved).
|
||||
public Func<bool>? TryClaimShutdownEventOverride;
|
||||
|
||||
public EnrollmentTerminationHandler Build(AgentConfiguration? config = null) =>
|
||||
BuildCore(config, agentVersion: null);
|
||||
@@ -205,7 +209,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
{
|
||||
PromotionCalls.Add((failureType, message));
|
||||
return PromotionReturnValue;
|
||||
});
|
||||
},
|
||||
tryClaimShutdownEvent: TryClaimShutdownEventOverride);
|
||||
}
|
||||
|
||||
public void Dispose() => Tmp.Dispose();
|
||||
@@ -509,7 +514,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
var data = rig.DataOf(Constants.EventTypes.AgentShuttingDown);
|
||||
Assert.NotNull(data);
|
||||
Assert.Equal(EnrollmentTerminationOutcome.Succeeded.ToString(), (string)data!["outcome"]);
|
||||
Assert.Equal(EnrollmentTerminationReason.DecisionTerminalStage.ToString(), (string)data["reason"]);
|
||||
// Event-type unification (2026-05-15): the reason tag is now the unified
|
||||
// lowercased vocabulary shared with the AgentRuntimeHost gap-paths.
|
||||
Assert.Equal("decision_terminal", (string)data["reason"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -529,7 +536,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
|
||||
var data = rig.DataOf(Constants.EventTypes.AgentShuttingDown);
|
||||
Assert.NotNull(data);
|
||||
Assert.Equal(EnrollmentTerminationReason.DecisionTerminalStage.ToString(), (string)data!["reason"]);
|
||||
Assert.Equal("decision_terminal", (string)data!["reason"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -553,6 +560,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.Equal(SessionStage.WhiteGloveSealed.ToString(), (string)data!["stage"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_skips_agent_shutting_down_emit_when_gate_already_claimed()
|
||||
{
|
||||
// Shutdown-gap closure (2026-05-15): when AgentRuntimeHost's gap path (Ctrl+C,
|
||||
// ProcessExit, unhandled_exception) emitted first and claimed the cross-path
|
||||
// gate, the Terminated-driven handler must NOT emit a second agent_shutting_down
|
||||
// event. Idempotency is critical because Ctrl+C followed by a real Terminated
|
||||
// transition is a normal sequence in dev/console mode.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build();
|
||||
rig.TryClaimShutdownEventOverride = () => false; // slot already claimed by gap path
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed));
|
||||
|
||||
// Cleanup + other lifecycle still runs — only the agent_shutting_down emit is
|
||||
// suppressed.
|
||||
Assert.Equal(1, rig.CleanupService.Invocations);
|
||||
Assert.DoesNotContain(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_emits_agent_shutting_down_when_gate_grants_slot()
|
||||
{
|
||||
// Symmetric counterpart — when the gate returns true (handler is first to claim),
|
||||
// the emit happens as usual.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build();
|
||||
rig.TryClaimShutdownEventOverride = () => true;
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed));
|
||||
|
||||
Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_propagates_max_lifetime_reason_in_agent_shutting_down_data()
|
||||
{
|
||||
@@ -564,7 +607,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
|
||||
var data = rig.DataOf(Constants.EventTypes.AgentShuttingDown);
|
||||
Assert.NotNull(data);
|
||||
Assert.Equal(EnrollmentTerminationReason.MaxLifetimeExceeded.ToString(), (string)data!["reason"]);
|
||||
Assert.Equal("max_lifetime", (string)data!["reason"]);
|
||||
Assert.Equal(EnrollmentTerminationOutcome.Failed.ToString(), (string)data["outcome"]);
|
||||
}
|
||||
|
||||
|
||||
+168
-10
@@ -4,6 +4,7 @@ using System.Diagnostics;
|
||||
using System.Management;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
@@ -12,6 +13,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// Fires DesktopArrived event exactly once per agent lifetime.
|
||||
/// Used for enrollment completion in no-ESP scenarios (WDP v2, ESP disabled) and
|
||||
/// as a backup signal for AccountSetup phase correction.
|
||||
/// <para>
|
||||
/// <b>Liveness telemetry</b> (state-change-only, max 3 events per detector lifetime, NOT periodic):
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item><c>desktop_detector_started</c> — fired once when Start() or ResetForRealUserSwitch() arms the polling timer.</item>
|
||||
/// <item><c>desktop_detector_first_poll</c> — fired once after the first PollForDesktop() completes (proves Timer + WMI plumbing is alive).</item>
|
||||
/// <item><c>desktop_detector_no_candidate</c> — fired once after the configurable threshold (default 10 min) of polling without ANY explorer.exe resolution (neither excluded-user nor real-user).</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// The three liveness events together distinguish three failure modes that all look identical
|
||||
/// when only <c>desktop_arrived</c> is missing: (a) DAD never started after reboot, (b) DAD
|
||||
/// started but timer/WMI dead, (c) DAD running but user never logged in.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class DesktopArrivalDetector : IDisposable
|
||||
{
|
||||
@@ -21,6 +35,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
private bool _excludedUserTraced; // Emit trace event only once for excluded-user skips
|
||||
private const int PollingIntervalSeconds = 30;
|
||||
|
||||
// Liveness state (Plan: DAD-liveness telemetry, 2026-05-15) — all flags are
|
||||
// single-shot per detector instance; ResetForRealUserSwitch re-arms them.
|
||||
private DateTime _startedAtUtc;
|
||||
private bool _firstPollDone;
|
||||
private int _pollCount;
|
||||
private bool _explorerEverSeen; // any explorer.exe observed at all (session 0 OR not)
|
||||
private bool _anyNonZeroSessionEverSeen; // explorer.exe in a non-SYSTEM session observed
|
||||
private int _wmiErrorCountSinceStart;
|
||||
private bool _noCandidateFired;
|
||||
private readonly int _noCandidateTimeoutMinutes;
|
||||
|
||||
/// <summary>
|
||||
/// System/service account names that should NOT be considered real users.
|
||||
/// </summary>
|
||||
@@ -46,19 +71,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// </summary>
|
||||
public Action<string, string, Dictionary<string, object>> OnTraceEvent { get; set; }
|
||||
|
||||
public DesktopArrivalDetector(AgentLogger logger)
|
||||
public DesktopArrivalDetector(AgentLogger logger, int noCandidateTimeoutMinutes = 10)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
// Clamp negative values to 0 (disabled). Upper bound left open — admin config UI
|
||||
// already enforces a sane 60-min ceiling.
|
||||
_noCandidateTimeoutMinutes = noCandidateTimeoutMinutes < 0 ? 0 : noCandidateTimeoutMinutes;
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
_logger.Info("DesktopArrivalDetector: starting (polling every 30s)");
|
||||
_pollingTimer = new Timer(
|
||||
PollForDesktop,
|
||||
null,
|
||||
TimeSpan.FromSeconds(5), // Initial check after 5s
|
||||
TimeSpan.FromSeconds(PollingIntervalSeconds));
|
||||
_logger.Info($"DesktopArrivalDetector: starting (polling every {PollingIntervalSeconds}s, no-candidate threshold={_noCandidateTimeoutMinutes} min)");
|
||||
ArmTimer(startReason: "initial");
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
@@ -89,14 +113,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_desktopArrived = false;
|
||||
_excludedUserTraced = false;
|
||||
|
||||
// Restart polling. Dispose any existing timer first to avoid leaking ticks
|
||||
// from an already-running schedule.
|
||||
// Reset liveness state too — the post-reset polling cycle is effectively a fresh
|
||||
// detector lifetime for diagnostic purposes (first_poll fires again, no_candidate
|
||||
// threshold restarts from this moment).
|
||||
_firstPollDone = false;
|
||||
_pollCount = 0;
|
||||
_explorerEverSeen = false;
|
||||
_anyNonZeroSessionEverSeen = false;
|
||||
_wmiErrorCountSinceStart = 0;
|
||||
_noCandidateFired = false;
|
||||
|
||||
ArmTimer(startReason: "real-user-switch-reset");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// (Re-)arms the polling timer + emits the <c>desktop_detector_started</c> liveness
|
||||
/// event. Dispose any existing timer first to avoid leaking ticks. Idempotent except
|
||||
/// for the emit — every Start/Reset call is a real lifecycle event we want visible.
|
||||
/// </summary>
|
||||
private void ArmTimer(string startReason)
|
||||
{
|
||||
_startedAtUtc = DateTime.UtcNow;
|
||||
|
||||
_pollingTimer?.Dispose();
|
||||
_pollingTimer = new Timer(
|
||||
PollForDesktop,
|
||||
null,
|
||||
TimeSpan.FromSeconds(5),
|
||||
TimeSpan.FromSeconds(5), // Initial check after 5s
|
||||
TimeSpan.FromSeconds(PollingIntervalSeconds));
|
||||
|
||||
EmitDetectorStarted(startReason);
|
||||
}
|
||||
|
||||
private void PollForDesktop(object state)
|
||||
@@ -104,9 +150,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
if (_desktopArrived)
|
||||
return;
|
||||
|
||||
_pollCount++;
|
||||
|
||||
try
|
||||
{
|
||||
var explorerProcesses = Process.GetProcessesByName("explorer");
|
||||
int explorerCountThisPoll = explorerProcesses.Length;
|
||||
if (explorerCountThisPoll > 0)
|
||||
_explorerEverSeen = true;
|
||||
|
||||
foreach (var proc in explorerProcesses)
|
||||
{
|
||||
try
|
||||
@@ -115,6 +167,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
if (proc.SessionId == 0)
|
||||
continue;
|
||||
|
||||
_anyNonZeroSessionEverSeen = true;
|
||||
|
||||
var owner = GetProcessOwner(proc.Id);
|
||||
if (owner == null)
|
||||
continue;
|
||||
@@ -153,6 +207,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
}
|
||||
catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: OnTraceEvent failed: {ex.Message}"); }
|
||||
|
||||
// First-poll observability snapshot still fires before we stop polling —
|
||||
// there is value in distinguishing "found on first poll" (Stage already
|
||||
// past AccountSetup at agent start) from "took N polls" (live transition).
|
||||
EmitFirstPollIfNeeded(explorerCountThisPoll);
|
||||
|
||||
// Stop polling
|
||||
_pollingTimer?.Dispose();
|
||||
_pollingTimer = null;
|
||||
@@ -170,13 +229,112 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
proc.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
// No resolution this poll (no real user found, no excluded user, or all
|
||||
// candidates were session 0). Emit first-poll snapshot now if pending.
|
||||
EmitFirstPollIfNeeded(explorerCountThisPoll);
|
||||
|
||||
// Threshold-triggered observability: after the configured timeout, emit a
|
||||
// single no-candidate event so dashboards can distinguish "user never logged
|
||||
// in" from "detector broken". Fires exactly once per detector lifetime.
|
||||
EmitNoCandidateIfThresholdReached();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_wmiErrorCountSinceStart++;
|
||||
_logger.Debug($"DesktopArrivalDetector: polling error: {ex.Message}");
|
||||
|
||||
// Still emit the first-poll snapshot if this was the first poll — the error
|
||||
// count carries the failure mode to the backend.
|
||||
EmitFirstPollIfNeeded(explorerProcessCount: 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One-shot liveness emit after the first <see cref="PollForDesktop"/> completes,
|
||||
/// regardless of outcome. Proves the Timer + Process.GetProcessesByName path is alive
|
||||
/// — distinguishing "DAD started but timer/WMI dead" from "DAD running, no user logged
|
||||
/// in yet" in failed sessions.
|
||||
/// </summary>
|
||||
private void EmitFirstPollIfNeeded(int explorerProcessCount)
|
||||
{
|
||||
if (_firstPollDone) return;
|
||||
_firstPollDone = true;
|
||||
|
||||
try
|
||||
{
|
||||
var elapsedMs = Math.Round((DateTime.UtcNow - _startedAtUtc).TotalMilliseconds, 0);
|
||||
OnTraceEvent?.Invoke(
|
||||
SharedEventTypes.DesktopDetectorFirstPoll,
|
||||
$"DAD first poll complete (explorer.exe count={explorerProcessCount}, elapsedMs={elapsedMs}).",
|
||||
new Dictionary<string, object>
|
||||
{
|
||||
{ "elapsedMsSinceStart", elapsedMs },
|
||||
{ "explorerProcessCount", explorerProcessCount },
|
||||
{ "anyNonZeroSessionObserved", _anyNonZeroSessionEverSeen },
|
||||
{ "wmiErrorCount", _wmiErrorCountSinceStart },
|
||||
});
|
||||
}
|
||||
catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: first-poll emit failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One-shot threshold-triggered emit: when polling has run for the configured number
|
||||
/// of minutes without ANY resolution (real-user OR excluded-user) AND the threshold
|
||||
/// is enabled (> 0), fire a single <c>desktop_detector_no_candidate</c> event.
|
||||
/// Distinguishes "user never logged in" from "DAD wiring broken" in sessions where
|
||||
/// <c>desktop_arrived</c> never lands.
|
||||
/// </summary>
|
||||
private void EmitNoCandidateIfThresholdReached()
|
||||
{
|
||||
if (_noCandidateFired) return;
|
||||
if (_noCandidateTimeoutMinutes <= 0) return; // disabled
|
||||
if (_excludedUserTraced) return; // we already emitted SOME resolution observability via desktop_excluded_user
|
||||
|
||||
var minutesSinceStart = (DateTime.UtcNow - _startedAtUtc).TotalMinutes;
|
||||
if (minutesSinceStart < _noCandidateTimeoutMinutes) return;
|
||||
|
||||
_noCandidateFired = true;
|
||||
try
|
||||
{
|
||||
OnTraceEvent?.Invoke(
|
||||
SharedEventTypes.DesktopDetectorNoCandidate,
|
||||
$"DAD threshold {_noCandidateTimeoutMinutes} min reached without explorer.exe resolution (polls={_pollCount}, explorerEverSeen={_explorerEverSeen}).",
|
||||
new Dictionary<string, object>
|
||||
{
|
||||
{ "pollsSinceStart", _pollCount },
|
||||
{ "minutesSinceStart", Math.Round(minutesSinceStart, 1) },
|
||||
{ "thresholdMinutes", _noCandidateTimeoutMinutes },
|
||||
{ "explorerEverSeen", _explorerEverSeen },
|
||||
{ "anyNonZeroSessionEverSeen", _anyNonZeroSessionEverSeen },
|
||||
{ "wmiErrorCount", _wmiErrorCountSinceStart },
|
||||
});
|
||||
}
|
||||
catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: no-candidate emit failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One-shot lifecycle emit at Start/Reset. Confirms detector wiring after a post-
|
||||
/// reboot agent restart — absence of this event after <c>agent_started</c> proves the
|
||||
/// composition-root never instantiated/armed the detector.
|
||||
/// </summary>
|
||||
private void EmitDetectorStarted(string startReason)
|
||||
{
|
||||
try
|
||||
{
|
||||
OnTraceEvent?.Invoke(
|
||||
SharedEventTypes.DesktopDetectorStarted,
|
||||
$"DAD armed (startReason={startReason}, pollIntervalSeconds={PollingIntervalSeconds}, noCandidateThresholdMinutes={_noCandidateTimeoutMinutes}).",
|
||||
new Dictionary<string, object>
|
||||
{
|
||||
{ "pollIntervalSeconds", PollingIntervalSeconds },
|
||||
{ "noCandidateThresholdMinutes", _noCandidateTimeoutMinutes },
|
||||
{ "startReason", startReason },
|
||||
});
|
||||
}
|
||||
catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: detector-started emit failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the owner of a process using WMI (Win32_Process.GetOwner).
|
||||
/// Returns "DOMAIN\User" or "User" string, or null on failure.
|
||||
|
||||
@@ -202,7 +202,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
// placeholder, so the detector resets and re-evaluates after the Hybrid reboot
|
||||
// instead of staying latched on the foo desktop. Order matters — DesktopArrivalHost
|
||||
// must exist before AadJoinHost so the callback target is available.
|
||||
var desktopArrivalHost = new DesktopArrivalHost(logger, ingress, clock);
|
||||
var desktopArrivalHost = new DesktopArrivalHost(
|
||||
logger,
|
||||
ingress,
|
||||
clock,
|
||||
noCandidateTimeoutMinutes: collectors.DesktopDetectorNoCandidateTimeoutMinutes,
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId);
|
||||
hosts.Add(desktopArrivalHost);
|
||||
|
||||
_aadJoinHost = new AadJoinHost(
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
@@ -13,12 +18,55 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
|
||||
private readonly DesktopArrivalDetector _detector;
|
||||
private readonly DesktopArrivalDetectorAdapter _adapter;
|
||||
private readonly AgentLogger _logger;
|
||||
private int _disposed;
|
||||
|
||||
public DesktopArrivalHost(AgentLogger logger, ISignalIngressSink ingress, IClock clock)
|
||||
public DesktopArrivalHost(
|
||||
AgentLogger logger,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
int noCandidateTimeoutMinutes = 10,
|
||||
string? sessionId = null,
|
||||
string? tenantId = null)
|
||||
{
|
||||
_detector = new DesktopArrivalDetector(logger);
|
||||
_logger = logger;
|
||||
_detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes);
|
||||
_adapter = new DesktopArrivalDetectorAdapter(_detector, ingress, clock);
|
||||
|
||||
// V2 single-rail wiring (2026-05-15) — the OnTraceEvent callback is routed
|
||||
// through InformationalEventPost so the DAD lifecycle events
|
||||
// (desktop_excluded_user, desktop_real_user_detected, desktop_detector_started,
|
||||
// desktop_detector_first_poll, desktop_detector_no_candidate) appear on the
|
||||
// session timeline as standalone EventTypes. V1's CollectorCoordinator wrapped
|
||||
// them inside a generic `trace_event`; V2 surfaces them with their own type so
|
||||
// queries / dashboards / MCP can key on the specific event semantics.
|
||||
var post = new InformationalEventPost(ingress, clock, logger);
|
||||
_detector.OnTraceEvent = (eventType, message, data) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var payload = data == null
|
||||
? new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
: new Dictionary<string, object>(data, StringComparer.Ordinal);
|
||||
|
||||
post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = sessionId ?? string.Empty,
|
||||
TenantId = tenantId ?? string.Empty,
|
||||
EventType = eventType,
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "DesktopArrivalDetector",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = message,
|
||||
Data = payload,
|
||||
ImmediateUpload = true,
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Debug($"DesktopArrivalHost: trace-event emit '{eventType}' threw: {ex.Message}");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public void Start() => _detector.Start();
|
||||
|
||||
+46
-5
@@ -118,6 +118,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
// treats as Part-1/null).
|
||||
private readonly Func<bool> _isWhiteGlovePart2Accessor;
|
||||
|
||||
// Shutdown-gap closure (2026-05-15): cross-path idempotency gate shared with the
|
||||
// AgentRuntimeHost's gap emitters (Ctrl+C, ProcessExit, unhandled exception,
|
||||
// runtime-host finally). The handler calls TryClaim() before emitting
|
||||
// <c>agent_shutting_down</c> in <see cref="EmitAgentShuttingDown"/> so a Terminated
|
||||
// event that races a Ctrl+C cannot produce two events on the wire. Null = legacy
|
||||
// behaviour (always emit; backward compat for tests + standalone construction).
|
||||
private readonly Func<bool> _tryClaimShutdownEvent;
|
||||
|
||||
// c117946b debrief (2026-05-12): on terminal ESP-Apps failure, promote any apps the
|
||||
// agent observed in <see cref="AppInstallationState.Installing"/> to Error so the
|
||||
// user sees a name (not just an opaque "installing: 1" counter) and the
|
||||
@@ -154,7 +162,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
Action writeCleanExitMarker = null,
|
||||
Func<long> ingressPendingSignalCountAccessor = null,
|
||||
Func<bool> isWhiteGlovePart2Accessor = null,
|
||||
Func<string, string, IReadOnlyList<string>> promoteActiveInstallsToStuck = null)
|
||||
Func<string, string, IReadOnlyList<string>> promoteActiveInstallsToStuck = null,
|
||||
Func<bool> tryClaimShutdownEvent = null)
|
||||
{
|
||||
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
@@ -185,6 +194,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
_ingressPendingSignalCountAccessor = ingressPendingSignalCountAccessor;
|
||||
_isWhiteGlovePart2Accessor = isWhiteGlovePart2Accessor;
|
||||
_promoteActiveInstallsToStuck = promoteActiveInstallsToStuck;
|
||||
_tryClaimShutdownEvent = tryClaimShutdownEvent;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -726,11 +736,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
|
||||
// V1-parity acknowledgement that the agent has accepted termination. Carries enough
|
||||
// context for the backend to correlate the shutdown with its trigger (reason/outcome/
|
||||
// stage) and for diagnostics to show how long the agent ran. Always emitted, regardless
|
||||
// of whether self-destruct cleanup actually runs afterwards.
|
||||
// stage) and for diagnostics to show how long the agent ran.
|
||||
// <para>
|
||||
// Shutdown-gap closure (2026-05-15): participates in the cross-path idempotency gate
|
||||
// owned by AgentRuntimeHost so a Terminated event that races a Ctrl+C / ProcessExit
|
||||
// does not produce two <c>agent_shutting_down</c> events on the wire. When the gate
|
||||
// is not wired (tests / standalone construction), the legacy always-emit behaviour
|
||||
// is preserved.
|
||||
// </para>
|
||||
private void EmitAgentShuttingDown(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
if (_tryClaimShutdownEvent != null && !_tryClaimShutdownEvent())
|
||||
{
|
||||
_logger.Debug("EnrollmentTerminationHandler: agent_shutting_down already emitted via gap-path — skipping duplicate.");
|
||||
return;
|
||||
}
|
||||
|
||||
var uptimeMinutes = Math.Round((DateTime.UtcNow - _agentStartTimeUtc).TotalMinutes, 1);
|
||||
|
||||
// Discriminator stored in data["reason"]. Maps the V2 EnrollmentTerminationReason
|
||||
// enum to the shared reason vocabulary used by the AgentRuntimeHost gap paths
|
||||
// (ctrl_c / process_exit / unhandled_exception / runtime_host_exit) so all
|
||||
// shutdown events on the wire share the same reason taxonomy.
|
||||
string reasonTag;
|
||||
switch (args.Reason)
|
||||
{
|
||||
case EnrollmentTerminationReason.DecisionTerminalStage:
|
||||
reasonTag = "decision_terminal";
|
||||
break;
|
||||
case EnrollmentTerminationReason.MaxLifetimeExceeded:
|
||||
reasonTag = "max_lifetime";
|
||||
break;
|
||||
default:
|
||||
reasonTag = args.Reason.ToString();
|
||||
break;
|
||||
}
|
||||
|
||||
EmitEventSafe(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _configuration.SessionId,
|
||||
@@ -739,10 +780,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "EnrollmentTerminationHandler",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"Agent shutting down (reason={args.Reason}, outcome={args.Outcome}).",
|
||||
Message = $"Agent shutting down (reason={reasonTag}, outcome={args.Outcome}).",
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
{ "reason", args.Reason.ToString() },
|
||||
{ "reason", reasonTag },
|
||||
{ "outcome", args.Outcome.ToString() },
|
||||
{ "stage", args.StageName ?? string.Empty },
|
||||
{ "uptimeMinutes", uptimeMinutes },
|
||||
|
||||
@@ -148,27 +148,72 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// are both live. Disposed in the same finally block as the rest of the lifecycle.
|
||||
WhiteGloveInventoryTrigger whiteGloveInventoryTrigger = null;
|
||||
|
||||
// Single-rail refactor (plan §5.1) — lifecycle events flow through
|
||||
// InformationalEventPost. The post is constructed inside the
|
||||
// onIngressReady callback below; the watchdog factories + gap-path
|
||||
// handlers close over the variable so they see the live helper when
|
||||
// their event fires (read in handler bodies, not at handler creation).
|
||||
InformationalEventPost lifecyclePost = null;
|
||||
|
||||
// Shutdown-gap closure (2026-05-15) — cross-path idempotency gate. Every
|
||||
// <c>agent_shutting_down</c> emit path (EnrollmentTerminationHandler,
|
||||
// CreateAuthThresholdHandler, the gap emitters below) claims the slot via
|
||||
// Interlocked.Exchange before emitting. Prevents a Ctrl+C racing a real
|
||||
// Terminated transition from producing two events on the wire.
|
||||
int agentShuttingDownEmitted = 0;
|
||||
bool TryClaimAgentShuttingDownEmit() =>
|
||||
Interlocked.Exchange(ref agentShuttingDownEmitted, 1) == 0;
|
||||
|
||||
ConsoleCancelEventHandler cancelHandler = (s, e) =>
|
||||
{
|
||||
e.Cancel = true;
|
||||
logger.Info("Ctrl+C received — initiating graceful shutdown.");
|
||||
|
||||
// L1 gap-closure: previously the cancelHandler only signalled
|
||||
// shutdown.Set(), leaving the timeline without an explicit
|
||||
// agent_shutting_down event. Emit here so dev/console-mode exits are
|
||||
// distinguishable from silent process death.
|
||||
if (TryClaimAgentShuttingDownEmit())
|
||||
{
|
||||
LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: lifecyclePost,
|
||||
agentConfig: agentConfig,
|
||||
reason: "ctrl_c",
|
||||
agentStartTimeUtc: agentStartTimeUtc,
|
||||
agentVersion: agentVersion,
|
||||
logger: logger);
|
||||
}
|
||||
|
||||
shutdown.Set();
|
||||
};
|
||||
EventHandler processExitHandler = (s, e) =>
|
||||
{
|
||||
logger.Info("ProcessExit — initiating graceful shutdown.");
|
||||
|
||||
// L2 gap-closure: Windows gives the ProcessExit handler ~2s before
|
||||
// killing the process, so the emit is best-effort — the event lands
|
||||
// in the SignalIngress channel (non-blocking enqueue) and the
|
||||
// subsequent finally→TerminationPipeline.Run drains the spool via
|
||||
// orchestrator.Stop's DrainAllAsync. Under a hard Windows shutdown
|
||||
// race the event may end up persisted-but-not-uploaded; even then it
|
||||
// surfaces in the next session's diag ZIP (if upload is on).
|
||||
if (TryClaimAgentShuttingDownEmit())
|
||||
{
|
||||
LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: lifecyclePost,
|
||||
agentConfig: agentConfig,
|
||||
reason: "process_exit",
|
||||
agentStartTimeUtc: agentStartTimeUtc,
|
||||
agentVersion: agentVersion,
|
||||
logger: logger);
|
||||
}
|
||||
|
||||
shutdown.Set();
|
||||
};
|
||||
|
||||
Console.CancelKeyPress += cancelHandler;
|
||||
AppDomain.CurrentDomain.ProcessExit += processExitHandler;
|
||||
|
||||
// Single-rail refactor (plan §5.1) — lifecycle events flow through
|
||||
// InformationalEventPost. The post is constructed inside the
|
||||
// onIngressReady callback below; the watchdog factories close over a
|
||||
// Func accessor so they see the live helper when their event fires.
|
||||
InformationalEventPost lifecyclePost = null;
|
||||
|
||||
var maxLifetimeEmitter = LifecycleEmitters.CreateMaxLifetimeEmitter(
|
||||
getLifecyclePost: () => lifecyclePost,
|
||||
agentConfig: agentConfig,
|
||||
@@ -316,7 +361,13 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// EspTerminalFailure pathway only — other Failed paths leave the
|
||||
// app list untouched.
|
||||
promoteActiveInstallsToStuck: (failureType, message)
|
||||
=> componentFactory.PromoteActiveInstallsToStuck(failureType, message));
|
||||
=> componentFactory.PromoteActiveInstallsToStuck(failureType, message),
|
||||
// Shutdown-gap closure (2026-05-15) — share the cross-path
|
||||
// idempotency gate so a Terminated event that races a Ctrl+C /
|
||||
// ProcessExit gap-path does not emit two agent_shutting_down
|
||||
// events on the wire. The handler skips its emit when the
|
||||
// gate already claimed the slot.
|
||||
tryClaimShutdownEvent: TryClaimAgentShuttingDownEmit);
|
||||
|
||||
// ServerActionDispatcher (plan §5.3) — constructed inside this
|
||||
// hook so lifecyclePost + terminationHandler are guaranteed
|
||||
@@ -481,6 +532,29 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
|
||||
shutdown.Wait();
|
||||
}
|
||||
catch (Exception runtimeEx)
|
||||
{
|
||||
// L3 gap-closure (2026-05-15): unhandled exception inside the try
|
||||
// block above (Start, post-startup wiring, the long-lived shutdown
|
||||
// wait if a captured Task escalation reached this thread, etc.).
|
||||
// Previously this propagated straight to Program.Main's catch which
|
||||
// logs + WriteCrashLog but emits no timeline event — the session
|
||||
// looked like silent process death. Emit before re-throwing so the
|
||||
// crash is visible on the wire.
|
||||
if (TryClaimAgentShuttingDownEmit())
|
||||
{
|
||||
LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: lifecyclePost,
|
||||
agentConfig: agentConfig,
|
||||
reason: "unhandled_exception",
|
||||
agentStartTimeUtc: agentStartTimeUtc,
|
||||
agentVersion: agentVersion,
|
||||
logger: logger,
|
||||
exceptionType: runtimeEx.GetType().FullName,
|
||||
exceptionMessage: runtimeEx.Message);
|
||||
}
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Detach the WG-Part-1 inventory trigger before TerminationPipeline runs
|
||||
@@ -489,6 +563,23 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
try { whiteGloveInventoryTrigger?.Dispose(); }
|
||||
catch (Exception ex) { logger.Warning($"WhiteGloveInventoryTrigger.Dispose threw: {ex.Message}"); }
|
||||
|
||||
// L4 gap-closure (2026-05-15): defensive fallback emit for any path
|
||||
// that returned to finally without claiming the gate — e.g. a future
|
||||
// direct shutdown.Set() from a server-action handler that bypasses
|
||||
// both Terminated AND the cancel/processExit closures. Best-effort:
|
||||
// the gate is already claimed by Terminated / Ctrl+C / ProcessExit /
|
||||
// AuthFailure / L3 in every known path, so this rarely fires.
|
||||
if (TryClaimAgentShuttingDownEmit())
|
||||
{
|
||||
LifecycleEmitters.EmitAgentShuttingDownGapPath(
|
||||
post: lifecyclePost,
|
||||
agentConfig: agentConfig,
|
||||
reason: "runtime_host_exit",
|
||||
agentStartTimeUtc: agentStartTimeUtc,
|
||||
agentVersion: agentVersion,
|
||||
logger: logger);
|
||||
}
|
||||
|
||||
TerminationPipeline.Run(
|
||||
orchestrator: orchestrator,
|
||||
authFailureTracker: auth.AuthFailureTracker,
|
||||
|
||||
@@ -399,6 +399,103 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
// ====================================================== Cross-path shutdown emit helper
|
||||
|
||||
/// <summary>
|
||||
/// Shutdown-gap closure (2026-05-15) — emits a single
|
||||
/// <c>agent_shutting_down</c> event from one of the non-Terminated exit paths the
|
||||
/// V2 runtime had previously left silent (Ctrl+C, ProcessExit, unhandled exception
|
||||
/// in <c>RunAgent</c>, finally cleanup without prior Terminated). The
|
||||
/// <see cref="EnrollmentTerminationHandler"/> remains the canonical emitter on the
|
||||
/// happy path; this helper is for the gap-closure paths only.
|
||||
/// <para>
|
||||
/// <b>Idempotency contract:</b> the caller (typically <c>AgentRuntimeHost</c>) owns a
|
||||
/// single shared <see cref="int"/> flag and uses
|
||||
/// <c>Interlocked.Exchange(ref flag, 1) == 0</c> to claim the emit slot before
|
||||
/// invoking this method. <see cref="EnrollmentTerminationHandler"/> participates in
|
||||
/// the same gate via its <c>tryClaimShutdownEvent</c> constructor parameter so
|
||||
/// double-emit cannot happen when, say, Ctrl+C races a real Terminated transition.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <param name="post">Live <see cref="InformationalEventPost"/>. Null is treated as
|
||||
/// "ingress not ready"; the call logs and returns false without emitting.</param>
|
||||
/// <param name="agentConfig">Carries SessionId / TenantId / agent build version.</param>
|
||||
/// <param name="reason">Discriminator stored in <c>data["reason"]</c>. Allowed values:
|
||||
/// <c>ctrl_c</c>, <c>process_exit</c>, <c>unhandled_exception</c>,
|
||||
/// <c>runtime_host_exit</c>. Other reasons (auth_failure / decision_terminal /
|
||||
/// max_lifetime) are emitted by their own dedicated paths.</param>
|
||||
/// <param name="agentStartTimeUtc">Used to compute uptime for the event payload.</param>
|
||||
/// <param name="agentVersion">Build hash tag, mirrored from <c>agent_started</c>.</param>
|
||||
/// <param name="logger">For best-effort log on emit failure.</param>
|
||||
/// <param name="exceptionType">Optional .NET exception type, populated when the
|
||||
/// caller is the unhandled-exception path.</param>
|
||||
/// <param name="exceptionMessage">Optional .NET exception message (truncated to
|
||||
/// 500 chars on the wire to keep payload size bounded).</param>
|
||||
/// <returns>true if the event was posted; false if <paramref name="post"/> was null
|
||||
/// or the emit threw.</returns>
|
||||
public static bool EmitAgentShuttingDownGapPath(
|
||||
InformationalEventPost post,
|
||||
AgentConfiguration agentConfig,
|
||||
string reason,
|
||||
DateTime agentStartTimeUtc,
|
||||
string agentVersion,
|
||||
AgentLogger logger,
|
||||
string exceptionType = null,
|
||||
string exceptionMessage = null)
|
||||
{
|
||||
if (post == null)
|
||||
{
|
||||
logger?.Warning($"agent_shutting_down (reason={reason}) suppressed — ingress not ready.");
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var uptimeMinutes = Math.Round((DateTime.UtcNow - agentStartTimeUtc).TotalMinutes, 1);
|
||||
|
||||
var data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
{ "reason", reason ?? "unknown" },
|
||||
{ "outcome", "Unknown" },
|
||||
{ "stage", string.Empty },
|
||||
{ "uptimeMinutes", uptimeMinutes },
|
||||
{ "agentVersion", agentVersion ?? string.Empty },
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(exceptionType))
|
||||
data["exceptionType"] = exceptionType;
|
||||
if (!string.IsNullOrEmpty(exceptionMessage))
|
||||
{
|
||||
// Bound payload growth — exception messages can be long stack-trace text.
|
||||
var trimmed = exceptionMessage.Length > 500
|
||||
? exceptionMessage.Substring(0, 500) + "…"
|
||||
: exceptionMessage;
|
||||
data["exceptionMessage"] = trimmed;
|
||||
}
|
||||
|
||||
post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = agentConfig.SessionId,
|
||||
TenantId = agentConfig.TenantId,
|
||||
EventType = SharedConstants.EventTypes.AgentShuttingDown,
|
||||
Severity = string.Equals(reason, "unhandled_exception", StringComparison.Ordinal)
|
||||
? EventSeverity.Error
|
||||
: EventSeverity.Info,
|
||||
Source = "AgentRuntimeHost",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"Agent shutting down (reason={reason}).",
|
||||
Data = data,
|
||||
ImmediateUpload = true,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"agent_shutting_down (reason={reason}) emission failed: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================ Watchdog handlers
|
||||
|
||||
/// <summary>
|
||||
@@ -484,11 +581,18 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{
|
||||
try
|
||||
{
|
||||
// Event-type unification (2026-05-15): the auth-failure path used to
|
||||
// emit a separate `agent_shutdown` type (legacy V1 string). All V2
|
||||
// shutdown paths now share `agent_shutting_down` with the failure
|
||||
// class disambiguated via Data["reason"] — auth_failure here, others
|
||||
// (decision_terminal / max_lifetime / ctrl_c / process_exit /
|
||||
// unhandled_exception) in EnrollmentTerminationHandler + the new
|
||||
// AgentRuntimeHost gap-closure emit paths.
|
||||
post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = agentConfig.SessionId,
|
||||
TenantId = agentConfig.TenantId,
|
||||
EventType = "agent_shutdown",
|
||||
EventType = SharedConstants.EventTypes.AgentShuttingDown,
|
||||
Severity = EventSeverity.Error,
|
||||
Source = "AuthFailureTracker",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
@@ -509,12 +613,12 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
}
|
||||
catch (Exception emitEx)
|
||||
{
|
||||
logger.Warning($"agent_shutdown emission failed: {emitEx.Message}");
|
||||
logger.Warning($"agent_shutting_down (auth_failure) emission failed: {emitEx.Message}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.Warning("agent_shutdown (auth_failure) suppressed — ingress not ready.");
|
||||
logger.Warning("agent_shutting_down (auth_failure) suppressed — ingress not ready.");
|
||||
}
|
||||
|
||||
signalShutdown();
|
||||
|
||||
@@ -1 +1 @@
|
||||
745
|
||||
749
|
||||
|
||||
@@ -1 +1 @@
|
||||
1340
|
||||
1344
|
||||
|
||||
+2
@@ -542,6 +542,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{ "GlobalRateLimitRequestsPerMinute", config.GlobalRateLimitRequestsPerMinute },
|
||||
{ "PlatformStatsBlobSasUrl", config.PlatformStatsBlobSasUrl ?? string.Empty },
|
||||
{ "CollectorIdleTimeoutMinutes", config.CollectorIdleTimeoutMinutes },
|
||||
{ "DesktopDetectorNoCandidateTimeoutMinutes", config.DesktopDetectorNoCandidateTimeoutMinutes },
|
||||
{ "ExcessiveEventCountThreshold", config.ExcessiveEventCountThreshold },
|
||||
{ "MaxSessionWindowHours", config.MaxSessionWindowHours },
|
||||
{ "MaintenanceBlockDurationHours", config.MaintenanceBlockDurationHours },
|
||||
@@ -606,6 +607,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
GlobalRateLimitRequestsPerMinute = entity.GetInt32("GlobalRateLimitRequestsPerMinute") ?? 100,
|
||||
PlatformStatsBlobSasUrl = entity.GetString("PlatformStatsBlobSasUrl") ?? string.Empty,
|
||||
CollectorIdleTimeoutMinutes = entity.GetInt32("CollectorIdleTimeoutMinutes") ?? 15,
|
||||
DesktopDetectorNoCandidateTimeoutMinutes = entity.GetInt32("DesktopDetectorNoCandidateTimeoutMinutes") ?? 10,
|
||||
ExcessiveEventCountThreshold = entity.GetInt32("ExcessiveEventCountThreshold") ?? 2000,
|
||||
MaxSessionWindowHours = entity.GetInt32("MaxSessionWindowHours") ?? 24,
|
||||
MaintenanceBlockDurationHours = entity.GetInt32("MaintenanceBlockDurationHours") ?? 12,
|
||||
|
||||
@@ -156,6 +156,7 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
EnablePerformanceCollector = tenantConfig.EnablePerformanceCollector,
|
||||
PerformanceIntervalSeconds = tenantConfig.PerformanceCollectorIntervalSeconds,
|
||||
CollectorIdleTimeoutMinutes = adminConfig.CollectorIdleTimeoutMinutes,
|
||||
DesktopDetectorNoCandidateTimeoutMinutes = adminConfig.DesktopDetectorNoCandidateTimeoutMinutes,
|
||||
HelloWaitTimeoutSeconds = tenantConfig.HelloWaitTimeoutSeconds,
|
||||
AgentMaxLifetimeMinutes = tenantConfig.AgentMaxLifetimeMinutes ?? 360,
|
||||
ModernDeploymentHarmlessEventIds = adminConfig.GetModernDeploymentHarmlessEventIds().ToArray()
|
||||
@@ -180,7 +181,7 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new AgentConfigResponse
|
||||
{
|
||||
ConfigVersion = 27, // ModernDeployment harmless EventID 1010 added to default noise list
|
||||
ConfigVersion = 28, // V2 DAD-liveness threshold (DesktopDetectorNoCandidateTimeoutMinutes, default 10 min); admin-config global setting
|
||||
UploadIntervalSeconds = 10,
|
||||
SelfDestructOnComplete = tenantConfig.SelfDestructOnComplete ?? true,
|
||||
KeepLogFile = tenantConfig.KeepLogFile ?? false,
|
||||
|
||||
@@ -20,7 +20,7 @@ const KEY_EVENT_TYPES = new Set([
|
||||
'app_tracking_summary', 'error_detected',
|
||||
'enrollment_complete', 'enrollment_failed', 'completion_check',
|
||||
'desktop_arrived', 'hello_policy_detected', 'waiting_for_hello', 'hello_completion_timeout',
|
||||
'agent_started', 'agent_shutdown', 'trace_event',
|
||||
'agent_started', 'agent_shutdown', 'agent_shutting_down', 'trace_event',
|
||||
'script_started', 'script_completed', 'script_failed', 'vulnerability_report',
|
||||
]);
|
||||
const SEVERITY_RANK: Record<string, number> = { Trace: -1, Debug: 0, Info: 1, Warning: 2, Error: 3, Critical: 4 };
|
||||
|
||||
@@ -239,6 +239,13 @@ namespace AutopilotMonitor.Shared
|
||||
public const string EnrollmentComplete = "enrollment_complete";
|
||||
public const string EnrollmentFailed = "enrollment_failed";
|
||||
public const string DesktopArrived = "desktop_arrived";
|
||||
// DAD liveness (state-change-only, max 3 events per detector lifetime, NOT periodic).
|
||||
// Distinguishes "DAD never started after reboot" vs "DAD started but timer/WMI dead" vs
|
||||
// "DAD running but user never logged in" — the three failure-modes that look identical
|
||||
// when only desktop_arrived is missing.
|
||||
public const string DesktopDetectorStarted = "desktop_detector_started"; // 1x on Start() / ResetForRealUserSwitch()
|
||||
public const string DesktopDetectorFirstPoll = "desktop_detector_first_poll"; // 1x after first PollForDesktop() completes
|
||||
public const string DesktopDetectorNoCandidate = "desktop_detector_no_candidate"; // 1x after threshold polls without resolution (configurable)
|
||||
public const string CompletionCheck = "completion_check";
|
||||
public const string ScriptStarted = "script_started";
|
||||
public const string ScriptCompleted = "script_completed";
|
||||
|
||||
@@ -74,6 +74,17 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public int CollectorIdleTimeoutMinutes { get; set; } = 15;
|
||||
|
||||
/// <summary>
|
||||
/// DAD liveness threshold in minutes. When the V2 DesktopArrivalDetector polls for
|
||||
/// this duration without detecting either an excluded-user explorer.exe or a real
|
||||
/// user desktop, it emits a single <c>desktop_detector_no_candidate</c> observability
|
||||
/// event. State-change-only — NOT a periodic heartbeat; max 1 event per detector
|
||||
/// lifetime. Used to distinguish "user never logged in" from "detector wiring dead
|
||||
/// post-reboot" in sessions where <c>desktop_arrived</c> is missing.
|
||||
/// 0 = disabled (event never fires). Default: 10 minutes.
|
||||
/// </summary>
|
||||
public int DesktopDetectorNoCandidateTimeoutMinutes { get; set; } = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Maintenance alarm threshold: sessions with more events than this value trigger an
|
||||
/// ExcessiveSessionEvents ops alert (dispatched to Telegram/Teams).
|
||||
@@ -442,6 +453,7 @@ namespace AutopilotMonitor.Shared.Models
|
||||
GlobalAdminRateLimitRequestsPerMinute = 600,
|
||||
PlatformStatsBlobSasUrl = string.Empty,
|
||||
CollectorIdleTimeoutMinutes = 15,
|
||||
DesktopDetectorNoCandidateTimeoutMinutes = 10,
|
||||
MaxSessionWindowHours = 24,
|
||||
MaintenanceBlockDurationHours = 12
|
||||
};
|
||||
|
||||
@@ -235,6 +235,16 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public int CollectorIdleTimeoutMinutes { get; set; } = 15;
|
||||
|
||||
/// <summary>
|
||||
/// DAD liveness threshold (minutes). After this many minutes of polling without
|
||||
/// detecting either an excluded-user explorer.exe or a real user desktop, the V2
|
||||
/// DesktopArrivalDetector emits a single <c>desktop_detector_no_candidate</c> event
|
||||
/// (state-change-only, NOT periodic). Distinguishes "user never logged in" from
|
||||
/// "detector wiring dead post-reboot" in sessions missing <c>desktop_arrived</c>.
|
||||
/// 0 = disabled. Default: 10 minutes.
|
||||
/// </summary>
|
||||
public int DesktopDetectorNoCandidateTimeoutMinutes { get; set; } = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Enable the agent self-metrics collector (process CPU, memory, network traffic).
|
||||
/// Default: true
|
||||
|
||||
@@ -24,6 +24,8 @@ interface AdminConfigContextValue {
|
||||
setPlatformStatsBlobSasUrl: (value: string) => void;
|
||||
collectorIdleTimeoutMinutes: number;
|
||||
setCollectorIdleTimeoutMinutes: (value: number) => void;
|
||||
desktopDetectorNoCandidateTimeoutMinutes: number;
|
||||
setDesktopDetectorNoCandidateTimeoutMinutes: (value: number) => void;
|
||||
maxSessionWindowHours: number;
|
||||
setMaxSessionWindowHours: (value: number) => void;
|
||||
maintenanceBlockDurationHours: number;
|
||||
@@ -111,6 +113,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
const [globalRateLimit, setGlobalRateLimit] = useState(100);
|
||||
const [platformStatsBlobSasUrl, setPlatformStatsBlobSasUrl] = useState("");
|
||||
const [collectorIdleTimeoutMinutes, setCollectorIdleTimeoutMinutes] = useState(15);
|
||||
const [desktopDetectorNoCandidateTimeoutMinutes, setDesktopDetectorNoCandidateTimeoutMinutes] = useState(10);
|
||||
const [maxSessionWindowHours, setMaxSessionWindowHours] = useState(24);
|
||||
const [maintenanceBlockDurationHours, setMaintenanceBlockDurationHours] = useState(12);
|
||||
const [opsEventRetentionDays, setOpsEventRetentionDays] = useState(90);
|
||||
@@ -166,6 +169,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
setGlobalRateLimit(data.globalRateLimitRequestsPerMinute);
|
||||
setPlatformStatsBlobSasUrl(data.platformStatsBlobSasUrl ?? "");
|
||||
setCollectorIdleTimeoutMinutes(data.collectorIdleTimeoutMinutes ?? 15);
|
||||
setDesktopDetectorNoCandidateTimeoutMinutes(data.desktopDetectorNoCandidateTimeoutMinutes ?? 10);
|
||||
setMaxSessionWindowHours(data.maxSessionWindowHours ?? 24);
|
||||
setMaintenanceBlockDurationHours(data.maintenanceBlockDurationHours ?? 12);
|
||||
setOpsEventRetentionDays(data.opsEventRetentionDays ?? 90);
|
||||
@@ -262,6 +266,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
globalRateLimitRequestsPerMinute: globalRateLimit,
|
||||
platformStatsBlobSasUrl: platformStatsBlobSasUrl.trim(),
|
||||
collectorIdleTimeoutMinutes,
|
||||
desktopDetectorNoCandidateTimeoutMinutes,
|
||||
maxSessionWindowHours,
|
||||
maintenanceBlockDurationHours,
|
||||
opsEventRetentionDays,
|
||||
@@ -296,7 +301,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
} finally {
|
||||
setSavingConfig(false);
|
||||
}
|
||||
}, [adminConfig, globalRateLimit, platformStatsBlobSasUrl, collectorIdleTimeoutMinutes, maxSessionWindowHours, maintenanceBlockDurationHours, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, getAccessToken]);
|
||||
}, [adminConfig, globalRateLimit, platformStatsBlobSasUrl, collectorIdleTimeoutMinutes, desktopDetectorNoCandidateTimeoutMinutes, maxSessionWindowHours, maintenanceBlockDurationHours, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, getAccessToken]);
|
||||
|
||||
// Reset admin config
|
||||
const handleResetAdminConfig = useCallback(() => {
|
||||
@@ -304,6 +309,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
setGlobalRateLimit(adminConfig.globalRateLimitRequestsPerMinute);
|
||||
setPlatformStatsBlobSasUrl(adminConfig.platformStatsBlobSasUrl ?? "");
|
||||
setCollectorIdleTimeoutMinutes(adminConfig.collectorIdleTimeoutMinutes ?? 15);
|
||||
setDesktopDetectorNoCandidateTimeoutMinutes(adminConfig.desktopDetectorNoCandidateTimeoutMinutes ?? 10);
|
||||
setMaxSessionWindowHours(adminConfig.maxSessionWindowHours ?? 24);
|
||||
setMaintenanceBlockDurationHours(adminConfig.maintenanceBlockDurationHours ?? 12);
|
||||
setOpsEventRetentionDays(adminConfig.opsEventRetentionDays ?? 90);
|
||||
@@ -431,6 +437,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
globalRateLimit, setGlobalRateLimit,
|
||||
platformStatsBlobSasUrl, setPlatformStatsBlobSasUrl,
|
||||
collectorIdleTimeoutMinutes, setCollectorIdleTimeoutMinutes,
|
||||
desktopDetectorNoCandidateTimeoutMinutes, setDesktopDetectorNoCandidateTimeoutMinutes,
|
||||
maxSessionWindowHours, setMaxSessionWindowHours,
|
||||
maintenanceBlockDurationHours, setMaintenanceBlockDurationHours,
|
||||
opsEventRetentionDays, setOpsEventRetentionDays,
|
||||
|
||||
@@ -12,6 +12,8 @@ interface AdminConfigSettingsSectionProps {
|
||||
setPlatformStatsBlobSasUrl: (value: string) => void;
|
||||
collectorIdleTimeoutMinutes: number;
|
||||
setCollectorIdleTimeoutMinutes: (value: number) => void;
|
||||
desktopDetectorNoCandidateTimeoutMinutes: number;
|
||||
setDesktopDetectorNoCandidateTimeoutMinutes: (value: number) => void;
|
||||
slaNotificationCooldownHours: number;
|
||||
setSlaNotificationCooldownHours: (value: number) => void;
|
||||
allowAgentDowngrade: boolean;
|
||||
@@ -36,6 +38,8 @@ export function AdminConfigSettingsSection({
|
||||
setPlatformStatsBlobSasUrl,
|
||||
collectorIdleTimeoutMinutes,
|
||||
setCollectorIdleTimeoutMinutes,
|
||||
desktopDetectorNoCandidateTimeoutMinutes,
|
||||
setDesktopDetectorNoCandidateTimeoutMinutes,
|
||||
slaNotificationCooldownHours,
|
||||
setSlaNotificationCooldownHours,
|
||||
allowAgentDowngrade,
|
||||
@@ -128,6 +132,25 @@ export function AdminConfigSettingsSection({
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block">
|
||||
<span className="text-indigo-900 dark:text-indigo-100 font-medium">Desktop Detector No-Candidate Timeout (Minutes)</span>
|
||||
<p className="text-sm text-indigo-800 dark:text-gray-300 mb-2">
|
||||
The V2 agent's Desktop Arrival Detector emits a single <code className="text-xs bg-indigo-100 dark:bg-indigo-900 dark:text-indigo-200 px-1 rounded">desktop_detector_no_candidate</code> observability event after this many minutes of polling without detecting either an excluded-user explorer.exe or a real user desktop.
|
||||
State-change-only (NOT a periodic heartbeat) — fires at most once per agent run. Used to distinguish "user never logged in" from "detector wiring dead post-reboot" in sessions missing <code className="text-xs bg-indigo-100 dark:bg-indigo-900 dark:text-indigo-200 px-1 rounded">desktop_arrived</code>.
|
||||
Set to <strong>0</strong> to disable. Default: <strong>10</strong>. Range: 0–60.
|
||||
</p>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="60"
|
||||
value={desktopDetectorNoCandidateTimeoutMinutes}
|
||||
onChange={(e) => setDesktopDetectorNoCandidateTimeoutMinutes(parseInt(e.target.value) || 0)}
|
||||
className="mt-1 block w-full px-4 py-2 border border-indigo-300 dark:border-indigo-600 rounded-lg bg-white dark:bg-gray-700 text-gray-900 dark:text-gray-100 placeholder-gray-400 dark:placeholder-gray-500 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:border-indigo-500 transition-colors"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block">
|
||||
<span className="text-indigo-900 dark:text-indigo-100 font-medium">SLA Notification Cooldown (Hours)</span>
|
||||
|
||||
@@ -10,6 +10,7 @@ export function SectionGlobalSettings() {
|
||||
globalRateLimit, setGlobalRateLimit,
|
||||
platformStatsBlobSasUrl, setPlatformStatsBlobSasUrl,
|
||||
collectorIdleTimeoutMinutes, setCollectorIdleTimeoutMinutes,
|
||||
desktopDetectorNoCandidateTimeoutMinutes, setDesktopDetectorNoCandidateTimeoutMinutes,
|
||||
slaNotificationCooldownHours, setSlaNotificationCooldownHours,
|
||||
allowAgentDowngrade, setAllowAgentDowngrade,
|
||||
modernDeploymentHarmlessEventIds, setModernDeploymentHarmlessEventIds,
|
||||
@@ -31,6 +32,8 @@ export function SectionGlobalSettings() {
|
||||
setPlatformStatsBlobSasUrl={setPlatformStatsBlobSasUrl}
|
||||
collectorIdleTimeoutMinutes={collectorIdleTimeoutMinutes}
|
||||
setCollectorIdleTimeoutMinutes={setCollectorIdleTimeoutMinutes}
|
||||
desktopDetectorNoCandidateTimeoutMinutes={desktopDetectorNoCandidateTimeoutMinutes}
|
||||
setDesktopDetectorNoCandidateTimeoutMinutes={setDesktopDetectorNoCandidateTimeoutMinutes}
|
||||
slaNotificationCooldownHours={slaNotificationCooldownHours}
|
||||
setSlaNotificationCooldownHours={setSlaNotificationCooldownHours}
|
||||
allowAgentDowngrade={allowAgentDowngrade}
|
||||
|
||||
@@ -238,7 +238,7 @@ export function SectionAgentInternals() {
|
||||
["Upload", "UploadIntervalSeconds, MaxBatchSize, MaxRetryAttempts", "30s, 100, 5"],
|
||||
["Collector Toggles", "EnablePerformanceCollector, EnableAgentSelfMetrics, EnableDeliveryOptimization", "varies"],
|
||||
["Collector Timing", "PerformanceIntervalSeconds, AgentSelfMetricsIntervalSeconds, DOIntervalSeconds", "30s, 60s, 5s"],
|
||||
["Idle Management", "CollectorIdleTimeoutMinutes", "15 min"],
|
||||
["Idle Management", "CollectorIdleTimeoutMinutes, DesktopDetectorNoCandidateTimeoutMinutes", "15 min, 10 min"],
|
||||
["Logging", "LogLevel, SendTraceEvents", "Info, true"],
|
||||
["Geo / Time", "EnableGeoLocation, EnableTimezoneAutoSet, NtpServer", "true, false, time.windows.com"],
|
||||
["Diagnostics", "DiagnosticsUploadEnabled, DiagnosticsUploadMode, DiagnosticsLogPaths", "false, Off, []"],
|
||||
@@ -883,7 +883,7 @@ export function SectionAgentInternals() {
|
||||
headers={["Event Type", "Source", "Description"]}
|
||||
rows={[
|
||||
["agent_started", "MonitoringService", "Agent started (version, config, previous exit type)"],
|
||||
["agent_shutdown", "MonitoringService", "Graceful shutdown"],
|
||||
["agent_shutdown / agent_shutting_down", "MonitoringService / Agent", "Graceful shutdown. V1 emits agent_shutdown; V2 emits agent_shutting_down with reason= decision_terminal | max_lifetime | auth_failure | ctrl_c | process_exit | unhandled_exception"],
|
||||
["agent_trace", "Various", "Decision tracing (verbose diagnostics)"],
|
||||
["agent_version_check", "MonitoringService", "Version check outcome (up_to_date / updated / skipped / check_failed) - emitted every startup, session-scoped dedup for up_to_date"],
|
||||
["system_reboot_detected", "MonitoringService", "Reboot detected between sessions"],
|
||||
|
||||
@@ -36,8 +36,10 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [
|
||||
description: "Throttled state-machine snapshot during completion evaluation." },
|
||||
{ value: "agent_started", label: "agent_started", category: "enrollment",
|
||||
description: "Agent process started." },
|
||||
{ value: "agent_shutdown", label: "agent_shutdown", category: "enrollment",
|
||||
description: "Agent process is shutting down." },
|
||||
{ value: "agent_shutdown", label: "agent_shutdown (V1)", category: "enrollment",
|
||||
description: "Agent process is shutting down (legacy V1 event type — V2 emits agent_shutting_down)." },
|
||||
{ value: "agent_shutting_down", label: "agent_shutting_down (V2)", category: "enrollment",
|
||||
description: "Agent process is shutting down. V2 canonical event; reason field disambiguates (decision_terminal / max_lifetime / auth_failure / ctrl_c / process_exit / unhandled_exception)." },
|
||||
{ value: "phase_transition", label: "phase_transition", category: "enrollment",
|
||||
description: "Enrollment phase transition detected." },
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ export interface AdminConfiguration {
|
||||
globalRateLimitRequestsPerMinute: number;
|
||||
platformStatsBlobSasUrl?: string;
|
||||
collectorIdleTimeoutMinutes?: number;
|
||||
desktopDetectorNoCandidateTimeoutMinutes?: number;
|
||||
excessiveEventCountThreshold?: number;
|
||||
maxSessionWindowHours?: number;
|
||||
maintenanceBlockDurationHours?: number;
|
||||
|
||||
Reference in New Issue
Block a user