fix(backend): map agent_shutting_down(max_lifetime) to a terminal Failed session status

The V2 max-lifetime watchdog is deliberately "notbremse, not a session
verdict": the agent stops permanently and emits only
agent_shutting_down(reason=max_lifetime) - no enrollment_failed. Since the
backend flips session status exclusively on enrollment_complete /
enrollment_failed / esp_failure, such sessions stayed InProgress forever
(session 8bc1180f). V1 agents already emit enrollment_failed at max lifetime;
the gap is V2-only.

Both classification copies (legacy ingest + EventIngestProcessor, kept in
sync per the documented copy contract) now classify the event via a shared
IsMaxLifetimeAgentShutdown predicate and map it as the LOWEST-priority status
writer: any genuine terminal event in the same batch wins via the else-if
chain, already-terminal sessions are protected by the repository idempotency
guard, Pending (WhiteGlove) sessions are skipped, and Stalled sessions
finally resolve to Failed. failureReason carries the uptime;
failureSource=max_lifetime_watchdog.

This is the defense-in-depth half of the 8bc1180f fix - it works for every
field agent immediately on backend deploy, no agent redeploy needed. The
root-cause half (AdvisoryCompletion deadline in DecisionCore) resolves the
dead-end before max-lifetime is ever reached.

Backend tests 2284/2284 green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-06-12 13:09:17 +02:00
parent 30f71e1b54
commit e5bebf4bcb
3 changed files with 186 additions and 1 deletions
@@ -0,0 +1,81 @@
using AutopilotMonitor.Functions.Functions.Ingest;
using AutopilotMonitor.Shared.Models;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Session 8bc1180f (2026-06-12): the V2 max-lifetime watchdog stops the agent permanently
/// WITHOUT emitting <c>enrollment_failed</c> ("notbremse, not a session verdict"), so the
/// <c>agent_shutting_down(reason=max_lifetime)</c> event is the last one the session ever
/// sends — without a backend mapping the session stays InProgress forever. These tests pin
/// the shared detection predicate used by both classification copies (legacy ingest +
/// EventIngestProcessor); the status write itself rides the existing
/// <c>UpdateSessionStatusAsync(Failed)</c> path as the lowest-priority status writer.
/// </summary>
public class IngestMaxLifetimeShutdownTests
{
private static EnrollmentEvent Shutdown(string? reason, string eventType = "agent_shutting_down")
{
var evt = new EnrollmentEvent
{
EventType = eventType,
Message = "Agent shutting down (reason=max_lifetime, outcome=TimedOut).",
};
if (reason != null)
{
evt.Data = new Dictionary<string, object>
{
["reason"] = reason,
["outcome"] = "TimedOut",
["uptimeMinutes"] = 360.2,
};
}
return evt;
}
[Fact]
public void MaxLifetimeReason_IsDetected()
{
Assert.True(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown("max_lifetime")));
}
[Fact]
public void MaxLifetimeReason_IsDetected_CaseInsensitively()
{
Assert.True(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown("Max_Lifetime")));
}
[Theory]
// The full V2 shutdown-reason taxonomy: decision_terminal follows a real terminal event
// (enrollment_complete / enrollment_failed already drove the status); the gap paths say
// nothing terminal about the enrollment.
[InlineData("decision_terminal")]
[InlineData("ctrl_c")]
[InlineData("process_exit")]
[InlineData("unhandled_exception")]
[InlineData("runtime_host_exit")]
public void OtherShutdownReasons_AreNotDetected(string reason)
{
Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown(reason)));
}
[Fact]
public void MissingReasonData_IsNotDetected()
{
Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(Shutdown(reason: null)));
}
[Fact]
public void OtherEventTypes_AreNotDetected_EvenWithMaxLifetimeReason()
{
Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(
Shutdown("max_lifetime", eventType: "agent_shutdown")));
}
[Fact]
public void NullEvent_IsNotDetected()
{
Assert.False(IngestEventsFunction.IsMaxLifetimeAgentShutdown(null));
}
}
@@ -613,6 +613,10 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
case "session_stalled":
classification.SessionStalledEvent = evt;
break;
case "agent_shutting_down":
if (IsMaxLifetimeAgentShutdown(evt))
classification.AgentMaxLifetimeShutdownEvent = evt;
break;
case "system_reboot_detected":
// Per-batch incremental reboot count (live value during enrollment).
// Overwritten with an authoritative distinct count at the terminal
@@ -642,6 +646,26 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
return classification;
}
/// <summary>
/// True for an <c>agent_shutting_down</c> event whose <c>Data.reason</c> is
/// <c>max_lifetime</c> — the V2 watchdog shutdown (session 8bc1180f). By design that
/// path is a "notbremse, not a session verdict": the agent stops permanently WITHOUT
/// emitting <c>enrollment_failed</c>, so this is the last event the session ever
/// sends and the backend must map it to a terminal status itself. Other shutdown
/// reasons (decision_terminal, ctrl_c, process_exit, unhandled_exception, ...) either
/// follow a real terminal event or imply nothing terminal. Shared by both
/// classification copies (legacy ingest + <see cref="Services.EventIngestProcessor"/>).
/// </summary>
internal static bool IsMaxLifetimeAgentShutdown(EnrollmentEvent? evt)
{
if (evt == null || !string.Equals(evt.EventType, "agent_shutting_down", StringComparison.Ordinal))
return false;
var reason = evt.Data != null && evt.Data.ContainsKey("reason")
? evt.Data["reason"]?.ToString()
: null;
return string.Equals(reason, "max_lifetime", StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// Classifies whether an event type should count as "real enrollment activity" for
/// Stalled→InProgress healing. Periodic telemetry and stall-probe self-reports are
@@ -754,6 +778,39 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
_logger.LogInformation("{SessionPrefix} WhiteGlove resumed skipped, session already {Status}", sessionPrefix, currentSession?.Status);
}
}
else if (c.AgentMaxLifetimeShutdownEvent != null)
{
// Lowest-priority status writer (session 8bc1180f): the V2 max-lifetime
// watchdog stops the agent permanently without a session verdict — this
// shutdown event is the last one the session ever sends, so without this
// mapping the session stays InProgress forever. Any genuine terminal event
// in the same batch wins via the else-if chain above; an already-terminal
// session is protected by the repository's idempotency guard. Pending
// (WhiteGlove) sessions are skipped — they are deliberately long-lived and
// resume via re-registration.
var currentSession = await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId);
if (currentSession?.Status == SessionStatus.Pending)
{
_logger.LogInformation(
"{SessionPrefix} max_lifetime shutdown ignored — session is Pending (WhiteGlove)", sessionPrefix);
}
else
{
var uptimeMinutes = c.AgentMaxLifetimeShutdownEvent.Data?.ContainsKey("uptimeMinutes") == true
? c.AgentMaxLifetimeShutdownEvent.Data["uptimeMinutes"]?.ToString() : null;
failureReason = string.IsNullOrEmpty(uptimeMinutes)
? "Agent reached its maximum lifetime without a terminal enrollment verdict (max_lifetime watchdog shutdown)."
: $"Agent reached its maximum lifetime ({uptimeMinutes} min) without a terminal enrollment verdict (max_lifetime watchdog shutdown).";
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
request.TenantId, request.SessionId, SessionStatus.Failed, c.AgentMaxLifetimeShutdownEvent.Phase, failureReason,
completedAt: c.AgentMaxLifetimeShutdownEvent.Timestamp,
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
failureSource: "max_lifetime_watchdog");
_logger.LogWarning("{SessionPrefix} Status: Failed via max_lifetime shutdown mapping (transitioned={Transitioned})",
sessionPrefix, statusTransitioned);
}
}
// whiteglove_started (EventID 509) is a soft signal only — it fires on hybrid-join
// devices too. Do NOT set IsPreProvisioned here. Only whiteglove_complete (confirmed by
@@ -1114,6 +1171,14 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
public EnrollmentEvent? WhiteGloveResumedEvent { get; set; }
public EnrollmentEvent? EspFailureEvent { get; set; }
public EnrollmentEvent? SessionStalledEvent { get; set; }
/// <summary>
/// <c>agent_shutting_down</c> with <c>Data.reason == "max_lifetime"</c> — the V2
/// watchdog shutdown that deliberately carries no enrollment verdict. Mapped to a
/// terminal Failed status as the lowest-priority status writer so the session does
/// not stay InProgress forever (session 8bc1180f).
/// </summary>
public EnrollmentEvent? AgentMaxLifetimeShutdownEvent { get; set; }
public bool HasNonPeriodicRealEvent { get; set; }
public EnrollmentEvent? DeviceLocationEvent { get; set; }
public DateTime? EarliestEventTimestamp { get; set; }
@@ -65,6 +65,10 @@ namespace AutopilotMonitor.Functions.Services
case "session_stalled":
classification.SessionStalledEvent = evt;
break;
case "agent_shutting_down":
if (IngestEventsFunction.IsMaxLifetimeAgentShutdown(evt))
classification.AgentMaxLifetimeShutdownEvent = evt;
break;
case "system_reboot_detected":
// Per-batch incremental reboot count (live value during enrollment).
// Overwritten with an authoritative distinct count at the terminal
@@ -190,6 +194,40 @@ namespace AutopilotMonitor.Functions.Services
_logger.LogInformation("{SessionPrefix} WhiteGlove resumed skipped, session already {Status}", sessionPrefix, currentSession?.Status);
}
}
else if (c.AgentMaxLifetimeShutdownEvent != null)
{
// Lowest-priority status writer (session 8bc1180f): the V2 max-lifetime
// watchdog stops the agent permanently without a session verdict — this
// shutdown event is the last one the session ever sends, so without this
// mapping the session stays InProgress forever. Any genuine terminal event
// in the same batch wins via the else-if chain above; an already-terminal
// session is protected by the repository's idempotency guard. Pending
// (WhiteGlove) sessions are skipped — they are deliberately long-lived and
// resume via re-registration.
var currentStatus = preFetchedStatus
?? (await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId))?.Status;
if (currentStatus == SessionStatus.Pending)
{
_logger.LogInformation(
"{SessionPrefix} max_lifetime shutdown ignored — session is Pending (WhiteGlove)", sessionPrefix);
}
else
{
var uptimeMinutes = c.AgentMaxLifetimeShutdownEvent.Data?.ContainsKey("uptimeMinutes") == true
? c.AgentMaxLifetimeShutdownEvent.Data["uptimeMinutes"]?.ToString() : null;
failureReason = string.IsNullOrEmpty(uptimeMinutes)
? "Agent reached its maximum lifetime without a terminal enrollment verdict (max_lifetime watchdog shutdown)."
: $"Agent reached its maximum lifetime ({uptimeMinutes} min) without a terminal enrollment verdict (max_lifetime watchdog shutdown).";
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
request.TenantId, request.SessionId, SessionStatus.Failed, c.AgentMaxLifetimeShutdownEvent.Phase, failureReason,
completedAt: c.AgentMaxLifetimeShutdownEvent.Timestamp,
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
failureSource: "max_lifetime_watchdog");
_logger.LogWarning("{SessionPrefix} Status: Failed via max_lifetime shutdown mapping (transitioned={Transitioned})",
sessionPrefix, statusTransitioned);
}
}
if (c.WhiteGloveStartedEvent != null)
{
@@ -215,7 +253,8 @@ namespace AutopilotMonitor.Functions.Services
// falsely heal the session back to InProgress over the just-written status.
var batchWroteStatus = c.CompletionEvent != null || c.FailureEvent != null
|| c.EspFailureEvent != null || c.GatherCompletionEvent != null
|| c.WhiteGloveEvent != null || c.WhiteGloveResumedEvent != null;
|| c.WhiteGloveEvent != null || c.WhiteGloveResumedEvent != null
|| c.AgentMaxLifetimeShutdownEvent != null;
var currentStatus = !batchWroteStatus && preFetchedStatus.HasValue
? preFetchedStatus
: (await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId))?.Status;