mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
fix(backend): map agent_shutting_down(max_lifetime) to a terminal Failed session status
The V2 max-lifetime watchdog is deliberately "notbremse, not a session verdict": the agent stops permanently and emits only agent_shutting_down(reason=max_lifetime) - no enrollment_failed. Since the backend flips session status exclusively on enrollment_complete / enrollment_failed / esp_failure, such sessions stayed InProgress forever (session 8bc1180f). V1 agents already emit enrollment_failed at max lifetime; the gap is V2-only. Both classification copies (legacy ingest + EventIngestProcessor, kept in sync per the documented copy contract) now classify the event via a shared IsMaxLifetimeAgentShutdown predicate and map it as the LOWEST-priority status writer: any genuine terminal event in the same batch wins via the else-if chain, already-terminal sessions are protected by the repository idempotency guard, Pending (WhiteGlove) sessions are skipped, and Stalled sessions finally resolve to Failed. failureReason carries the uptime; failureSource=max_lifetime_watchdog. This is the defense-in-depth half of the 8bc1180f fix - it works for every field agent immediately on backend deploy, no agent redeploy needed. The root-cause half (AdvisoryCompletion deadline in DecisionCore) resolves the dead-end before max-lifetime is ever reached. Backend tests 2284/2284 green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
using AutopilotMonitor.Functions.Functions.Ingest;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Session 8bc1180f (2026-06-12): the V2 max-lifetime watchdog stops the agent permanently
|
||||
/// WITHOUT emitting <c>enrollment_failed</c> ("notbremse, not a session verdict"), so the
|
||||
/// <c>agent_shutting_down(reason=max_lifetime)</c> event is the last one the session ever
|
||||
/// sends — without a backend mapping the session stays InProgress forever. These tests pin
|
||||
/// the shared detection predicate used by both classification copies (legacy ingest +
|
||||
/// EventIngestProcessor); the status write itself rides the existing
|
||||
/// <c>UpdateSessionStatusAsync(Failed)</c> path as the lowest-priority status writer.
|
||||
/// </summary>
|
||||
public class IngestMaxLifetimeShutdownTests
|
||||
{
|
||||
private static EnrollmentEvent Shutdown(string? reason, string eventType = "agent_shutting_down")
|
||||
{
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = eventType,
|
||||
Message = "Agent shutting down (reason=max_lifetime, outcome=TimedOut).",
|
||||
};
|
||||
if (reason != null)
|
||||
{
|
||||
evt.Data = new Dictionary<string, object>
|
||||
{
|
||||
["reason"] = reason,
|
||||
["outcome"] = "TimedOut",
|
||||
["uptimeMinutes"] = 360.2,
|
||||
};
|
||||
}
|
||||
return evt;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MaxLifetimeReason_IsDetected()
|
||||
{
|
||||
Assert.True(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown("max_lifetime")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MaxLifetimeReason_IsDetected_CaseInsensitively()
|
||||
{
|
||||
Assert.True(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown("Max_Lifetime")));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// The full V2 shutdown-reason taxonomy: decision_terminal follows a real terminal event
|
||||
// (enrollment_complete / enrollment_failed already drove the status); the gap paths say
|
||||
// nothing terminal about the enrollment.
|
||||
[InlineData("decision_terminal")]
|
||||
[InlineData("ctrl_c")]
|
||||
[InlineData("process_exit")]
|
||||
[InlineData("unhandled_exception")]
|
||||
[InlineData("runtime_host_exit")]
|
||||
public void OtherShutdownReasons_AreNotDetected(string reason)
|
||||
{
|
||||
Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown(reason)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingReasonData_IsNotDetected()
|
||||
{
|
||||
Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown(reason: null)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OtherEventTypes_AreNotDetected_EvenWithMaxLifetimeReason()
|
||||
{
|
||||
Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(
|
||||
Shutdown("max_lifetime", eventType: "agent_shutdown")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullEvent_IsNotDetected()
|
||||
{
|
||||
Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(null));
|
||||
}
|
||||
}
|
||||
+65
@@ -613,6 +613,10 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
case "session_stalled":
|
||||
classification.SessionStalledEvent = evt;
|
||||
break;
|
||||
case "agent_shutting_down":
|
||||
if (IsMaxLifetimeAgentShutdown(evt))
|
||||
classification.AgentMaxLifetimeShutdownEvent = evt;
|
||||
break;
|
||||
case "system_reboot_detected":
|
||||
// Per-batch incremental reboot count (live value during enrollment).
|
||||
// Overwritten with an authoritative distinct count at the terminal
|
||||
@@ -642,6 +646,26 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
return classification;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True for an <c>agent_shutting_down</c> event whose <c>Data.reason</c> is
|
||||
/// <c>max_lifetime</c> — the V2 watchdog shutdown (session 8bc1180f). By design that
|
||||
/// path is a "notbremse, not a session verdict": the agent stops permanently WITHOUT
|
||||
/// emitting <c>enrollment_failed</c>, so this is the last event the session ever
|
||||
/// sends and the backend must map it to a terminal status itself. Other shutdown
|
||||
/// reasons (decision_terminal, ctrl_c, process_exit, unhandled_exception, ...) either
|
||||
/// follow a real terminal event or imply nothing terminal. Shared by both
|
||||
/// classification copies (legacy ingest + <see cref="Services.EventIngestProcessor"/>).
|
||||
/// </summary>
|
||||
internal static bool IsMaxLifetimeAgentShutdown(EnrollmentEvent? evt)
|
||||
{
|
||||
if (evt == null || !string.Equals(evt.EventType, "agent_shutting_down", StringComparison.Ordinal))
|
||||
return false;
|
||||
var reason = evt.Data != null && evt.Data.ContainsKey("reason")
|
||||
? evt.Data["reason"]?.ToString()
|
||||
: null;
|
||||
return string.Equals(reason, "max_lifetime", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Classifies whether an event type should count as "real enrollment activity" for
|
||||
/// Stalled→InProgress healing. Periodic telemetry and stall-probe self-reports are
|
||||
@@ -754,6 +778,39 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
_logger.LogInformation("{SessionPrefix} WhiteGlove resumed skipped, session already {Status}", sessionPrefix, currentSession?.Status);
|
||||
}
|
||||
}
|
||||
else if (c.AgentMaxLifetimeShutdownEvent != null)
|
||||
{
|
||||
// Lowest-priority status writer (session 8bc1180f): the V2 max-lifetime
|
||||
// watchdog stops the agent permanently without a session verdict — this
|
||||
// shutdown event is the last one the session ever sends, so without this
|
||||
// mapping the session stays InProgress forever. Any genuine terminal event
|
||||
// in the same batch wins via the else-if chain above; an already-terminal
|
||||
// session is protected by the repository's idempotency guard. Pending
|
||||
// (WhiteGlove) sessions are skipped — they are deliberately long-lived and
|
||||
// resume via re-registration.
|
||||
var currentSession = await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId);
|
||||
if (currentSession?.Status == SessionStatus.Pending)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"{SessionPrefix} max_lifetime shutdown ignored — session is Pending (WhiteGlove)", sessionPrefix);
|
||||
}
|
||||
else
|
||||
{
|
||||
var uptimeMinutes = c.AgentMaxLifetimeShutdownEvent.Data?.ContainsKey("uptimeMinutes") == true
|
||||
? c.AgentMaxLifetimeShutdownEvent.Data["uptimeMinutes"]?.ToString() : null;
|
||||
failureReason = string.IsNullOrEmpty(uptimeMinutes)
|
||||
? "Agent reached its maximum lifetime without a terminal enrollment verdict (max_lifetime watchdog shutdown)."
|
||||
: $"Agent reached its maximum lifetime ({uptimeMinutes} min) without a terminal enrollment verdict (max_lifetime watchdog shutdown).";
|
||||
|
||||
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, c.AgentMaxLifetimeShutdownEvent.Phase, failureReason,
|
||||
completedAt: c.AgentMaxLifetimeShutdownEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
failureSource: "max_lifetime_watchdog");
|
||||
_logger.LogWarning("{SessionPrefix} Status: Failed via max_lifetime shutdown mapping (transitioned={Transitioned})",
|
||||
sessionPrefix, statusTransitioned);
|
||||
}
|
||||
}
|
||||
|
||||
// whiteglove_started (EventID 509) is a soft signal only — it fires on hybrid-join
|
||||
// devices too. Do NOT set IsPreProvisioned here. Only whiteglove_complete (confirmed by
|
||||
@@ -1114,6 +1171,14 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
public EnrollmentEvent? WhiteGloveResumedEvent { get; set; }
|
||||
public EnrollmentEvent? EspFailureEvent { get; set; }
|
||||
public EnrollmentEvent? SessionStalledEvent { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// <c>agent_shutting_down</c> with <c>Data.reason == "max_lifetime"</c> — the V2
|
||||
/// watchdog shutdown that deliberately carries no enrollment verdict. Mapped to a
|
||||
/// terminal Failed status as the lowest-priority status writer so the session does
|
||||
/// not stay InProgress forever (session 8bc1180f).
|
||||
/// </summary>
|
||||
public EnrollmentEvent? AgentMaxLifetimeShutdownEvent { get; set; }
|
||||
public bool HasNonPeriodicRealEvent { get; set; }
|
||||
public EnrollmentEvent? DeviceLocationEvent { get; set; }
|
||||
public DateTime? EarliestEventTimestamp { get; set; }
|
||||
|
||||
+40
-1
@@ -65,6 +65,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
case "session_stalled":
|
||||
classification.SessionStalledEvent = evt;
|
||||
break;
|
||||
case "agent_shutting_down":
|
||||
if (IngestEventsFunction.IsMaxLifetimeAgentShutdown(evt))
|
||||
classification.AgentMaxLifetimeShutdownEvent = evt;
|
||||
break;
|
||||
case "system_reboot_detected":
|
||||
// Per-batch incremental reboot count (live value during enrollment).
|
||||
// Overwritten with an authoritative distinct count at the terminal
|
||||
@@ -190,6 +194,40 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_logger.LogInformation("{SessionPrefix} WhiteGlove resumed skipped, session already {Status}", sessionPrefix, currentSession?.Status);
|
||||
}
|
||||
}
|
||||
else if (c.AgentMaxLifetimeShutdownEvent != null)
|
||||
{
|
||||
// Lowest-priority status writer (session 8bc1180f): the V2 max-lifetime
|
||||
// watchdog stops the agent permanently without a session verdict — this
|
||||
// shutdown event is the last one the session ever sends, so without this
|
||||
// mapping the session stays InProgress forever. Any genuine terminal event
|
||||
// in the same batch wins via the else-if chain above; an already-terminal
|
||||
// session is protected by the repository's idempotency guard. Pending
|
||||
// (WhiteGlove) sessions are skipped — they are deliberately long-lived and
|
||||
// resume via re-registration.
|
||||
var currentStatus = preFetchedStatus
|
||||
?? (await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId))?.Status;
|
||||
if (currentStatus == SessionStatus.Pending)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"{SessionPrefix} max_lifetime shutdown ignored — session is Pending (WhiteGlove)", sessionPrefix);
|
||||
}
|
||||
else
|
||||
{
|
||||
var uptimeMinutes = c.AgentMaxLifetimeShutdownEvent.Data?.ContainsKey("uptimeMinutes") == true
|
||||
? c.AgentMaxLifetimeShutdownEvent.Data["uptimeMinutes"]?.ToString() : null;
|
||||
failureReason = string.IsNullOrEmpty(uptimeMinutes)
|
||||
? "Agent reached its maximum lifetime without a terminal enrollment verdict (max_lifetime watchdog shutdown)."
|
||||
: $"Agent reached its maximum lifetime ({uptimeMinutes} min) without a terminal enrollment verdict (max_lifetime watchdog shutdown).";
|
||||
|
||||
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, c.AgentMaxLifetimeShutdownEvent.Phase, failureReason,
|
||||
completedAt: c.AgentMaxLifetimeShutdownEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
failureSource: "max_lifetime_watchdog");
|
||||
_logger.LogWarning("{SessionPrefix} Status: Failed via max_lifetime shutdown mapping (transitioned={Transitioned})",
|
||||
sessionPrefix, statusTransitioned);
|
||||
}
|
||||
}
|
||||
|
||||
if (c.WhiteGloveStartedEvent != null)
|
||||
{
|
||||
@@ -215,7 +253,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// falsely heal the session back to InProgress over the just-written status.
|
||||
var batchWroteStatus = c.CompletionEvent != null || c.FailureEvent != null
|
||||
|| c.EspFailureEvent != null || c.GatherCompletionEvent != null
|
||||
|| c.WhiteGloveEvent != null || c.WhiteGloveResumedEvent != null;
|
||||
|| c.WhiteGloveEvent != null || c.WhiteGloveResumedEvent != null
|
||||
|| c.AgentMaxLifetimeShutdownEvent != null;
|
||||
var currentStatus = !batchWroteStatus && preFetchedStatus.HasValue
|
||||
? preFetchedStatus
|
||||
: (await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId))?.Status;
|
||||
|
||||
Reference in New Issue
Block a user