diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IngestMaxLifetimeShutdownTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IngestMaxLifetimeShutdownTests.cs new file mode 100644 index 00000000..01f78f9a --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/IngestMaxLifetimeShutdownTests.cs @@ -0,0 +1,81 @@ +using AutopilotMonitor.Functions.Functions.Ingest; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Session 8bc1180f (2026-06-12): the V2 max-lifetime watchdog stops the agent permanently +/// WITHOUT emitting enrollment_failed ("notbremse, not a session verdict"), so the +/// agent_shutting_down(reason=max_lifetime) event is the last one the session ever +/// sends — without a backend mapping the session stays InProgress forever. These tests pin +/// the shared detection predicate used by both classification copies (legacy ingest + +/// EventIngestProcessor); the status write itself rides the existing +/// UpdateSessionStatusAsync(Failed) path as the lowest-priority status writer. +/// +public class IngestMaxLifetimeShutdownTests +{ + private static EnrollmentEvent Shutdown(string? reason, string eventType = "agent_shutting_down") + { + var evt = new EnrollmentEvent + { + EventType = eventType, + Message = "Agent shutting down (reason=max_lifetime, outcome=TimedOut).", + }; + if (reason != null) + { + evt.Data = new Dictionary + { + ["reason"] = reason, + ["outcome"] = "TimedOut", + ["uptimeMinutes"] = 360.2, + }; + } + return evt; + } + + [Fact] + public void MaxLifetimeReason_IsDetected() + { + Assert.True(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown("max_lifetime"))); + } + + [Fact] + public void MaxLifetimeReason_IsDetected_CaseInsensitively() + { + Assert.True(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown("Max_Lifetime"))); + } + + [Theory] + // The full V2 shutdown-reason taxonomy: decision_terminal follows a real terminal event + // (enrollment_complete / enrollment_failed already drove the status); the gap paths say + // nothing terminal about the enrollment. + [InlineData("decision_terminal")] + [InlineData("ctrl_c")] + [InlineData("process_exit")] + [InlineData("unhandled_exception")] + [InlineData("runtime_host_exit")] + public void OtherShutdownReasons_AreNotDetected(string reason) + { + Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown(reason))); + } + + [Fact] + public void MissingReasonData_IsNotDetected() + { + Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown(reason: null))); + } + + [Fact] + public void OtherEventTypes_AreNotDetected_EvenWithMaxLifetimeReason() + { + Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown( + Shutdown("max_lifetime", eventType: "agent_shutdown"))); + } + + [Fact] + public void NullEvent_IsNotDetected() + { + Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(null)); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs index 5179816c..1c874557 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs @@ -613,6 +613,10 @@ namespace AutopilotMonitor.Functions.Functions.Ingest case "session_stalled": classification.SessionStalledEvent = evt; break; + case "agent_shutting_down": + if (IsMaxLifetimeAgentShutdown(evt)) + classification.AgentMaxLifetimeShutdownEvent = evt; + break; case "system_reboot_detected": // Per-batch incremental reboot count (live value during enrollment). // Overwritten with an authoritative distinct count at the terminal @@ -642,6 +646,26 @@ namespace AutopilotMonitor.Functions.Functions.Ingest return classification; } + /// + /// True for an agent_shutting_down event whose Data.reason is + /// max_lifetime — the V2 watchdog shutdown (session 8bc1180f). By design that + /// path is a "notbremse, not a session verdict": the agent stops permanently WITHOUT + /// emitting enrollment_failed, so this is the last event the session ever + /// sends and the backend must map it to a terminal status itself. Other shutdown + /// reasons (decision_terminal, ctrl_c, process_exit, unhandled_exception, ...) either + /// follow a real terminal event or imply nothing terminal. Shared by both + /// classification copies (legacy ingest + ). + /// + internal static bool IsMaxLifetimeAgentShutdown(EnrollmentEvent? evt) + { + if (evt == null || !string.Equals(evt.EventType, "agent_shutting_down", StringComparison.Ordinal)) + return false; + var reason = evt.Data != null && evt.Data.ContainsKey("reason") + ? evt.Data["reason"]?.ToString() + : null; + return string.Equals(reason, "max_lifetime", StringComparison.OrdinalIgnoreCase); + } + /// /// Classifies whether an event type should count as "real enrollment activity" for /// Stalled→InProgress healing. Periodic telemetry and stall-probe self-reports are @@ -754,6 +778,39 @@ namespace AutopilotMonitor.Functions.Functions.Ingest _logger.LogInformation("{SessionPrefix} WhiteGlove resumed skipped, session already {Status}", sessionPrefix, currentSession?.Status); } } + else if (c.AgentMaxLifetimeShutdownEvent != null) + { + // Lowest-priority status writer (session 8bc1180f): the V2 max-lifetime + // watchdog stops the agent permanently without a session verdict — this + // shutdown event is the last one the session ever sends, so without this + // mapping the session stays InProgress forever. Any genuine terminal event + // in the same batch wins via the else-if chain above; an already-terminal + // session is protected by the repository's idempotency guard. Pending + // (WhiteGlove) sessions are skipped — they are deliberately long-lived and + // resume via re-registration. + var currentSession = await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId); + if (currentSession?.Status == SessionStatus.Pending) + { + _logger.LogInformation( + "{SessionPrefix} max_lifetime shutdown ignored — session is Pending (WhiteGlove)", sessionPrefix); + } + else + { + var uptimeMinutes = c.AgentMaxLifetimeShutdownEvent.Data?.ContainsKey("uptimeMinutes") == true + ? c.AgentMaxLifetimeShutdownEvent.Data["uptimeMinutes"]?.ToString() : null; + failureReason = string.IsNullOrEmpty(uptimeMinutes) + ? "Agent reached its maximum lifetime without a terminal enrollment verdict (max_lifetime watchdog shutdown)." + : $"Agent reached its maximum lifetime ({uptimeMinutes} min) without a terminal enrollment verdict (max_lifetime watchdog shutdown)."; + + statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync( + request.TenantId, request.SessionId, SessionStatus.Failed, c.AgentMaxLifetimeShutdownEvent.Phase, failureReason, + completedAt: c.AgentMaxLifetimeShutdownEvent.Timestamp, + earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp, + failureSource: "max_lifetime_watchdog"); + _logger.LogWarning("{SessionPrefix} Status: Failed via max_lifetime shutdown mapping (transitioned={Transitioned})", + sessionPrefix, statusTransitioned); + } + } // whiteglove_started (EventID 509) is a soft signal only — it fires on hybrid-join // devices too. Do NOT set IsPreProvisioned here. Only whiteglove_complete (confirmed by @@ -1114,6 +1171,14 @@ namespace AutopilotMonitor.Functions.Functions.Ingest public EnrollmentEvent? WhiteGloveResumedEvent { get; set; } public EnrollmentEvent? EspFailureEvent { get; set; } public EnrollmentEvent? SessionStalledEvent { get; set; } + + /// + /// agent_shutting_down with Data.reason == "max_lifetime" — the V2 + /// watchdog shutdown that deliberately carries no enrollment verdict. Mapped to a + /// terminal Failed status as the lowest-priority status writer so the session does + /// not stay InProgress forever (session 8bc1180f). + /// + public EnrollmentEvent? AgentMaxLifetimeShutdownEvent { get; set; } public bool HasNonPeriodicRealEvent { get; set; } public EnrollmentEvent? DeviceLocationEvent { get; set; } public DateTime? EarliestEventTimestamp { get; set; } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs index 270969fa..ac017019 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs @@ -65,6 +65,10 @@ namespace AutopilotMonitor.Functions.Services case "session_stalled": classification.SessionStalledEvent = evt; break; + case "agent_shutting_down": + if (IngestEventsFunction.IsMaxLifetimeAgentShutdown(evt)) + classification.AgentMaxLifetimeShutdownEvent = evt; + break; case "system_reboot_detected": // Per-batch incremental reboot count (live value during enrollment). // Overwritten with an authoritative distinct count at the terminal @@ -190,6 +194,40 @@ namespace AutopilotMonitor.Functions.Services _logger.LogInformation("{SessionPrefix} WhiteGlove resumed skipped, session already {Status}", sessionPrefix, currentSession?.Status); } } + else if (c.AgentMaxLifetimeShutdownEvent != null) + { + // Lowest-priority status writer (session 8bc1180f): the V2 max-lifetime + // watchdog stops the agent permanently without a session verdict — this + // shutdown event is the last one the session ever sends, so without this + // mapping the session stays InProgress forever. Any genuine terminal event + // in the same batch wins via the else-if chain above; an already-terminal + // session is protected by the repository's idempotency guard. Pending + // (WhiteGlove) sessions are skipped — they are deliberately long-lived and + // resume via re-registration. + var currentStatus = preFetchedStatus + ?? (await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId))?.Status; + if (currentStatus == SessionStatus.Pending) + { + _logger.LogInformation( + "{SessionPrefix} max_lifetime shutdown ignored — session is Pending (WhiteGlove)", sessionPrefix); + } + else + { + var uptimeMinutes = c.AgentMaxLifetimeShutdownEvent.Data?.ContainsKey("uptimeMinutes") == true + ? c.AgentMaxLifetimeShutdownEvent.Data["uptimeMinutes"]?.ToString() : null; + failureReason = string.IsNullOrEmpty(uptimeMinutes) + ? "Agent reached its maximum lifetime without a terminal enrollment verdict (max_lifetime watchdog shutdown)." + : $"Agent reached its maximum lifetime ({uptimeMinutes} min) without a terminal enrollment verdict (max_lifetime watchdog shutdown)."; + + statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync( + request.TenantId, request.SessionId, SessionStatus.Failed, c.AgentMaxLifetimeShutdownEvent.Phase, failureReason, + completedAt: c.AgentMaxLifetimeShutdownEvent.Timestamp, + earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp, + failureSource: "max_lifetime_watchdog"); + _logger.LogWarning("{SessionPrefix} Status: Failed via max_lifetime shutdown mapping (transitioned={Transitioned})", + sessionPrefix, statusTransitioned); + } + } if (c.WhiteGloveStartedEvent != null) { @@ -215,7 +253,8 @@ namespace AutopilotMonitor.Functions.Services // falsely heal the session back to InProgress over the just-written status. var batchWroteStatus = c.CompletionEvent != null || c.FailureEvent != null || c.EspFailureEvent != null || c.GatherCompletionEvent != null - || c.WhiteGloveEvent != null || c.WhiteGloveResumedEvent != null; + || c.WhiteGloveEvent != null || c.WhiteGloveResumedEvent != null + || c.AgentMaxLifetimeShutdownEvent != null; var currentStatus = !batchWroteStatus && preFetchedStatus.HasValue ? preFetchedStatus : (await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId))?.Status;