diff --git a/.claude/commands b/.claude/commands index 313fe61e..88fa1b02 160000 --- a/.claude/commands +++ b/.claude/commands @@ -1 +1 @@ -Subproject commit 313fe61e980899ee02edcdbcced1508e20e44385 +Subproject commit 88fa1b02749d729fddefdf4b78443fe58abc891b diff --git a/internal b/internal index 8f1f02e6..efec9897 160000 --- a/internal +++ b/internal @@ -1 +1 @@ -Subproject commit 8f1f02e6818d97d56003c4393d80b9a646415b82 +Subproject commit efec98979a7f7e2774c9f84ba4b9733c8cb97187 diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs index b76ba4e1..3fc55f73 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs @@ -20,6 +20,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport private sealed class FakeApiClient : BackendApiClient { public readonly List Timeouts = new List(); + public readonly List Reports = new List(); public readonly Queue Results = new Queue(); public FakeApiClient() : base( @@ -40,6 +41,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport public override Task ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null) { Timeouts.Add(timeout); + Reports.Add(report); return Task.FromResult(Results.Count > 0 && Results.Dequeue()); } } @@ -139,5 +141,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport Assert.Equal(2, client.Calls); } + + [Fact] + public async Task Session_age_is_carried_in_the_report_and_absent_by_default() + { + // One reporter per report: a delivered report starts the 10-minute cooldown. + var client = new FakeApiClient(); + client.Results.Enqueue(true); + client.Results.Enqueue(true); + + await Reporter(client).TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m", sessionAgeHours: 478.2); + await Reporter(client).TrySendAsync(AgentErrorType.ConfigFetchFailed, "m"); + + Assert.Equal(2, client.Reports.Count); + Assert.Equal(478.2, client.Reports[0].SessionAgeHours); + Assert.Null(client.Reports[1].SessionAgeHours); + } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/ProgramGuardsTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/ProgramGuardsTests.cs index 73eaeee0..31203ab0 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/ProgramGuardsTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/ProgramGuardsTests.cs @@ -149,6 +149,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program persistence.SaveSessionCreatedAt(DateTime.UtcNow.AddHours(-100)); var stateDir = Path.Combine(tmp.Path, "State"); + double? reportedAgeHours = null; var tripped = AutopilotMonitor.Agent.V2.Program.CheckSessionAgeEmergencyBreak( dataDirectory: tmp.Path, @@ -157,9 +158,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program selfDestructOnComplete: false, cleanupServiceFactory: null, logger: logger, - consoleMode: false); + consoleMode: false, + onBreakFired: ageHours => reportedAgeHours = ageHours); Assert.True(tripped); + // The measured age reaches the break report (the backend cannot derive it). + Assert.NotNull(reportedAgeHours); + Assert.InRange(reportedAgeHours.Value, 99.9, 100.1); // Marker must have been written so the next restart exits cleanly. Assert.True(File.Exists(Path.Combine(stateDir, "enrollment-complete.marker"))); // Session must have been cleared. diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs index 0333d421..984a2e94 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs @@ -79,6 +79,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// exist for callers whose process is about to exit and cannot come back later /// (emergency break): retries stay INSIDE the one reservation, so they never /// multiply against the per-session budget. + /// + /// is the emergency break's measured session age + /// (); null for every other error type. /// public virtual async Task TrySendAsync( AgentErrorType errorType, @@ -87,7 +90,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport long? sequenceNumber = null, int attempts = 1, TimeSpan? perAttemptTimeout = null, - TimeSpan? retryDelay = null) + TimeSpan? retryDelay = null, + double? sessionAgeHours = null) { // Deduplicate by error type + status code: same failure category sent only once per session. // All anti-flood checks are protected by a lock to prevent race conditions when @@ -135,6 +139,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport Message = message, HttpStatusCode = httpStatusCode, SequenceNumber = sequenceNumber, + SessionAgeHours = sessionAgeHours, AgentVersion = _agentVersion, Timestamp = DateTime.UtcNow }; diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Program.Guards.cs b/src/Agent/AutopilotMonitor.Agent.V2/Program.Guards.cs index 70162f9c..9f33125a 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Program.Guards.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Program.Guards.cs @@ -308,7 +308,7 @@ namespace AutopilotMonitor.Agent.V2 Func cleanupServiceFactory, AgentLogger logger, bool consoleMode, - Action onBreakFired = null) + Action onBreakFired = null) { try { @@ -347,7 +347,7 @@ namespace AutopilotMonitor.Agent.V2 // (tasks/enrollment-status-reclassification.md). Fired BEFORE cleanup, while the // session state and network are still intact. Never throws — a send failure (e.g. no // network) must not block the cleanup/exit that is the whole point of this guard. - try { onBreakFired?.Invoke(); } + try { onBreakFired?.Invoke(sessionAgeHours); } catch (Exception cbEx) { logger.Debug($"Emergency-break notify callback failed: {cbEx.Message}"); } // Write enrollment-complete.marker so the next start exits cleanly even if diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentBootstrap.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentBootstrap.cs index d7cb2778..f0d899e5 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentBootstrap.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentBootstrap.cs @@ -87,7 +87,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime cleanupServiceFactory, logger, consoleMode, // Best-effort: surface the otherwise-silent 48h break to the backend before cleanup // (tasks/enrollment-status-reclassification.md). Never blocks the exit. - onBreakFired: () => EmergencyBreakReporter.TrySend(agentConfig, Program.GetAgentVersion(), logger))) + onBreakFired: sessionAgeHours => EmergencyBreakReporter.TrySend( + agentConfig, Program.GetAgentVersion(), sessionAgeHours, logger))) { logger.Info("Emergency break fired — agent exiting."); return BootstrapResult.Exit(0); diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/EmergencyBreakReporter.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/EmergencyBreakReporter.cs index 0374f8c8..6006e50a 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/EmergencyBreakReporter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/EmergencyBreakReporter.cs @@ -36,15 +36,18 @@ namespace AutopilotMonitor.Agent.V2.Runtime // Upper bound for the blocking wait: NIC wait + 3×15s attempts + 2×5s delays + slack. private static readonly TimeSpan OverallBudget = TimeSpan.FromSeconds(75); - public static void TrySend(AgentConfiguration agentConfig, string agentVersion, AgentLogger logger) + public static void TrySend(AgentConfiguration agentConfig, string agentVersion, double sessionAgeHours, AgentLogger logger) { try { WaitForNetwork(logger); var auth = BackendClientFactory.BuildAuthClients(agentConfig, agentVersion, logger); - var message = - $"Agent absolute session-age emergency break fired (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting."; + // Invariant: the text lands in the backend's ops feed and timeline; a device locale + // would otherwise print "478,2h". + var roundedAgeHours = Math.Round(sessionAgeHours, 1); + var message = FormattableString.Invariant( + $"Agent absolute session-age emergency break fired at session age {roundedAgeHours:F1}h (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting."); // The process is about to exit, so block on the otherwise fire-and-forget send // rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP @@ -56,7 +59,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime message, attempts: SendAttempts, perAttemptTimeout: PerAttemptTimeout, - retryDelay: RetryDelay) + retryDelay: RetryDelay, + sessionAgeHours: roundedAgeHours) .Wait(OverallBudget); } catch (Exception ex) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs index 449ea47d..2a3018c4 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs @@ -4,9 +4,11 @@ using System.Net.Http; using System.Threading.Tasks; using AutopilotMonitor.Functions.Functions.Ingest; using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; using AutopilotMonitor.Functions.Services.Notifications; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; +using Azure.Data.Tables; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -94,14 +96,29 @@ public class ReportAgentErrorFunctionTests // (2026-07-23 hardening): timeline event + cross-session EventType index row + // AgentEmergencyBreak ops event, once per session. The index upsert is load-bearing: // StoreEventsBatchAsync alone leaves the event invisible to every search-by-eventType - // surface (found in the 2026-07-22 incident analysis). + // surface (found in the 2026-07-22 incident analysis). A session without a writable + // row gets the ops event only. // ========================================================================= + private static TableEntity Row(SessionStatus? status = SessionStatus.InProgress, string? deletionState = null) + { + var row = new TableEntity(TenantId, SessionId); + if (status.HasValue) row["Status"] = status.Value.ToString(); + if (deletionState != null) row["DeletionState"] = deletionState; + return row; + } + + private static ReportAgentErrorFunction.SessionRowRead Read(TableEntity? row) => new(row, Failed: false); + + private static readonly ReportAgentErrorFunction.SessionRowRead ReadFailed = new(null, Failed: true); + private sealed class Harness { public readonly Mock SessionRepo = new(); public readonly List OpsEvents = new(); public readonly OpsEventService OpsService; + private readonly SessionDeletionGuard _deletionGuard = new( + Mock.Of(), NullLogger.Instance); public Harness(List? existingEvents = null) { @@ -125,9 +142,23 @@ public class ReportAgentErrorFunctionTests OpsService = new OpsEventService(opsRepo.Object, NullLogger.Instance, alertDispatch); } + /// Default read: an existing, open session row — the historical path. public Task RunAsync(AgentErrorReport report) + => RunAsync(report, Read(Row())); + + public Task RunAsync(AgentErrorReport report, ReportAgentErrorFunction.SessionRowRead? sessionRead) => ReportAgentErrorFunction.MaterializeEmergencyBreakArtifactsAsync( - report, TenantId, SessionRepo.Object, OpsService, NullLogger.Instance); + report, TenantId, sessionRead, SessionRepo.Object, _deletionGuard, OpsService, + NullLogger.Instance); + + public void VerifyNothingWrittenIntoTheSession() + { + SessionRepo.Verify(r => r.GetSessionEventsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny>()), Times.Never); + SessionRepo.Verify(r => r.UpsertEventTypeIndexBatchAsync( + It.IsAny(), It.IsAny(), It.IsAny>()), Times.Never); + } } [Fact] @@ -165,7 +196,7 @@ public class ReportAgentErrorFunctionTests } [Fact] - public async Task Non_break_error_types_and_missing_session_id_are_ignored() + public async Task Non_break_error_types_and_ids_without_a_row_read_are_ignored() { var h = new Harness(); @@ -173,12 +204,10 @@ public class ReportAgentErrorFunctionTests other.ErrorType = AgentErrorType.ConfigFetchFailed; await h.RunAsync(other); - var noSession = Report(Now); - noSession.SessionId = null!; - await h.RunAsync(noSession); + // No read = the ids were not GUIDs (ReadSessionRowAsync) — no session can exist to break. + await h.RunAsync(Report(Now), sessionRead: null); - h.SessionRepo.Verify(r => r.GetSessionEventsAsync( - It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + h.VerifyNothingWrittenIntoTheSession(); Assert.Empty(h.OpsEvents); } @@ -192,10 +221,8 @@ public class ReportAgentErrorFunctionTests public async Task Break_on_already_terminal_session_is_info_late_cleanup(SessionStatus status) { var h = new Harness(); - h.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId)) - .ReturnsAsync(new SessionSummary { SessionId = SessionId, TenantId = TenantId, Status = status }); - await h.RunAsync(Report(Now)); + await h.RunAsync(Report(Now), Read(Row(status))); var ops = Assert.Single(h.OpsEvents); Assert.Equal(OpsEventSeverity.Info, ops.Severity); @@ -216,44 +243,131 @@ public class ReportAgentErrorFunctionTests public async Task Break_on_open_session_stays_warning(SessionStatus status) { var h = new Harness(); - h.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId)) - .ReturnsAsync(new SessionSummary { SessionId = SessionId, TenantId = TenantId, Status = status }); - await h.RunAsync(Report(Now)); + await h.RunAsync(Report(Now), Read(Row(status))); var ops = Assert.Single(h.OpsEvents); Assert.Equal(OpsEventSeverity.Warning, ops.Severity); Assert.Contains($"still {status}", ops.Message); Assert.Contains("\"lateCleanup\":false", ops.Details); + h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny>()), Times.Once); + } + + // 2026-09-28 sweep: every "unknown" Warning named a session that never registered. The break is + // the only thing the backend ever hears of it — nothing to close, and a timeline event would be + // an orphan the maintenance sweep deletes a day later (with a Warning of its own). + [Fact] + public async Task Break_on_session_without_row_is_info_and_writes_nothing_into_the_session() + { + var h = new Harness(); + + await h.RunAsync(Report(Now), Read(null)); + + var ops = Assert.Single(h.OpsEvents); + Assert.Equal(OpsEventSeverity.Info, ops.Severity); + Assert.Contains("no session row", ops.Message); + Assert.Contains("\"sessionStatusAtBreak\":\"missing\"", ops.Details); + Assert.Contains("\"lateCleanup\":false", ops.Details); + h.VerifyNothingWrittenIntoTheSession(); + } + + [Theory] + [InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Preparing)] + [InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Queued)] + [InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Running)] + [InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Poisoned)] + public async Task Break_on_session_under_cascade_delete_writes_nothing_past_the_lock(string deletionState) + { + var h = new Harness(); + + await h.RunAsync(Report(Now), Read(Row(SessionStatus.InProgress, deletionState))); + + var ops = Assert.Single(h.OpsEvents); + Assert.Equal(OpsEventSeverity.Info, ops.Severity); + Assert.Contains("\"sessionStatusAtBreak\":\"deleting\"", ops.Details); + h.VerifyNothingWrittenIntoTheSession(); } [Fact] - public async Task Unknown_or_unreadable_session_status_keeps_warning() + public async Task Unreadable_row_keeps_the_historical_warning_and_write() { - // No row (Harness default: GetSessionAsync → null) — the historical Warning semantics. + // Existence unknown — fail safe: the break may be the only signal for an open session. var h = new Harness(); - await h.RunAsync(Report(Now)); + + await h.RunAsync(Report(Now), ReadFailed); + var ops = Assert.Single(h.OpsEvents); Assert.Equal(OpsEventSeverity.Warning, ops.Severity); Assert.Contains("\"sessionStatusAtBreak\":\"unknown\"", ops.Details); - - // Row read throws — the break must still be recorded, still as Warning. - var h2 = new Harness(); - h2.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId)) - .ThrowsAsync(new InvalidOperationException("table down")); - await h2.RunAsync(Report(Now)); - Assert.Equal(OpsEventSeverity.Warning, Assert.Single(h2.OpsEvents).Severity); + h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny>()), Times.Once); } [Fact] - public void ClassifyEmergencyBreak_is_terminal_set_only() + public async Task Row_without_parseable_status_stays_warning() { - Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Succeeded)); - Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Failed)); - Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Incomplete)); - Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.AwaitingUser)); - Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Unknown)); - Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(null)); + var h = new Harness(); + + await h.RunAsync(Report(Now), Read(Row(status: null))); + + var ops = Assert.Single(h.OpsEvents); + Assert.Equal(OpsEventSeverity.Warning, ops.Severity); + Assert.Contains("\"sessionStatusAtBreak\":\"unknown\"", ops.Details); + } + + // The agent-measured age separates a break just past the cap from a device that slept for + // weeks; the backend cannot derive it (no row, or a row registered seconds before the break). + [Fact] + public async Task Reported_session_age_lands_in_ops_details_and_timeline_data() + { + var h = new Harness(); + var report = Report(Now); + report.SessionAgeHours = 478.2; + + await h.RunAsync(report); + + Assert.Contains("\"sessionAgeHours\":478.2", Assert.Single(h.OpsEvents).Details); + h.SessionRepo.Verify(r => r.StoreEventsBatchAsync( + It.Is>(e => (double)e[0].Data["sessionAgeHours"] == 478.2)), Times.Once); + } + + [Fact] + public async Task Agent_without_session_age_leaves_it_null_and_out_of_the_timeline() + { + var h = new Harness(); + + await h.RunAsync(Report(Now)); + + Assert.Contains("\"sessionAgeHours\":null", Assert.Single(h.OpsEvents).Details); + h.SessionRepo.Verify(r => r.StoreEventsBatchAsync( + It.Is>(e => !e[0].Data.ContainsKey("sessionAgeHours"))), Times.Once); + } + + [Fact] + public void ClassifyEmergencyBreak_follows_row_state() + { + static ReportAgentErrorFunction.EmergencyBreakVerdict Classify(TableEntity? row, bool failed = false, bool locked = false) + => ReportAgentErrorFunction.ClassifyEmergencyBreak(row, failed, locked); + + foreach (var terminal in new[] { SessionStatus.Succeeded, SessionStatus.Failed, SessionStatus.Incomplete }) + { + var v = Classify(Row(terminal)); + Assert.Equal((OpsEventSeverity.Info, true, true, terminal.ToString()), (v.Severity, v.LateCleanup, v.Materialize, v.SessionStatusAtBreak)); + } + + foreach (var open in new[] { SessionStatus.InProgress, SessionStatus.Pending, SessionStatus.AwaitingUser, SessionStatus.Stalled, SessionStatus.Unknown }) + { + var v = Classify(Row(open)); + Assert.Equal((OpsEventSeverity.Warning, false, true, open.ToString()), (v.Severity, v.LateCleanup, v.Materialize, v.SessionStatusAtBreak)); + } + + var missing = Classify(null); + Assert.Equal((OpsEventSeverity.Info, false, false, "missing"), (missing.Severity, missing.LateCleanup, missing.Materialize, missing.SessionStatusAtBreak)); + + var deleting = Classify(Row(SessionStatus.Succeeded), locked: true); + Assert.Equal((OpsEventSeverity.Info, false, false, "deleting"), (deleting.Severity, deleting.LateCleanup, deleting.Materialize, deleting.SessionStatusAtBreak)); + + var unreadable = Classify(null, failed: true); + Assert.Equal((OpsEventSeverity.Warning, false, true, "unknown"), (unreadable.Severity, unreadable.LateCleanup, unreadable.Materialize, unreadable.SessionStatusAtBreak)); } [Fact] diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs index b1656932..4840e260 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs @@ -1,13 +1,16 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Net; using System.Text.Json; using AutopilotMonitor.Functions.Security; using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; +using Azure.Data.Tables; using Microsoft.ApplicationInsights; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; @@ -41,6 +44,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest private readonly ISessionRepository _sessionRepo; private readonly OpsEventService _opsEventService; private readonly Services.Deletion.ISessionDeletionInventoryReader _sessionRowReader; + private readonly SessionDeletionGuard _deletionGuard; private readonly SessionOwnerBindingObserver _ownerBinding; public ReportAgentErrorFunction( @@ -58,6 +62,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest ISessionRepository sessionRepo, OpsEventService opsEventService, Services.Deletion.ISessionDeletionInventoryReader sessionRowReader, + SessionDeletionGuard deletionGuard, SessionOwnerBindingObserver ownerBinding) { _logger = logger; @@ -74,6 +79,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest _sessionRepo = sessionRepo; _opsEventService = opsEventService; _sessionRowReader = sessionRowReader; + _deletionGuard = deletionGuard; _ownerBinding = ownerBinding; } @@ -184,16 +190,20 @@ namespace AutopilotMonitor.Functions.Functions.Ingest ["AgentVersion"] = report.AgentVersion ?? string.Empty, ["Message"] = report.Message ?? string.Empty, ["AgentTimestamp"] = report.Timestamp.ToString("O"), + ["SessionAgeHours"] = report.SessionAgeHours?.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty, }); + // One Sessions-row point-read serves the owner binding and the emergency-break verdict. + var sessionRead = await ReadSessionRowAsync(tenantId, report.SessionId); + // SESSION-OWNER-BINDING: the emergency-break path below writes timeline events into - // whatever session the report names (one point-read — this function has no guard row - // in hand; no stamping from this path). A report naming a session bound to another - // device identity keeps its device-scoped diagnostics above and the binary-integrity - // check below, but must not write into that session. Still 200: the channel is never - // retried, and the agent ignores its status. - if (await IsSessionOwnerAllowedAsync(req, tenantId, report.SessionId, validation)) - await MaterializeEmergencyBreakArtifactsAsync(report, tenantId, _sessionRepo, _opsEventService, _logger); + // whatever session the report names (no stamping from this path). A report naming a + // session bound to another device identity keeps its device-scoped diagnostics above + // and the binary-integrity check below, but must not write into that session. Still + // 200: the channel is never retried, and the agent ignores its status. + if (IsSessionOwnerAllowed(req, tenantId, report.SessionId, sessionRead, validation)) + await MaterializeEmergencyBreakArtifactsAsync( + report, tenantId, sessionRead, _sessionRepo, _deletionGuard, _opsEventService, _logger); var adminConfig = await _adminConfigService.GetConfigurationAsync(); await MaterializeIntegrityMismatchAsync( @@ -204,18 +214,42 @@ namespace AutopilotMonitor.Functions.Functions.Ingest } /// - /// False only when the owner binding refuses the caller for the named session. Ids that are - /// not GUIDs have no row to compare against; a failed row read fails open (our defect is - /// not evidence of a foreign device). + /// Outcome of the one Sessions-row point-read: is null when the row does + /// not exist, marks a read that threw (existence unknown). /// - private async Task IsSessionOwnerAllowedAsync(HttpRequestData req, string tenantId, string? sessionId, SecurityValidationResult validation) + internal readonly record struct SessionRowRead(TableEntity? Row, bool Failed); + + /// + /// Null when the ids are not GUIDs: no row can exist for them, so neither the owner binding + /// nor the emergency break has anything to act on. + /// + private async Task ReadSessionRowAsync(string tenantId, string? sessionId) { if (!SecurityValidator.IsValidGuid(sessionId) || !SecurityValidator.IsValidGuid(tenantId)) + return null; + try + { + return new SessionRowRead(await _sessionRowReader.GetSessionRowAsync(tenantId, sessionId!), Failed: false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "ReportAgentError: session row read failed for session {SessionId}", sessionId); + return new SessionRowRead(null, Failed: true); + } + } + + /// + /// False only when the owner binding refuses the caller for the named session. Ids that are + /// not GUIDs have no row to compare against; a failed row read or evaluation fails open (our + /// defect is not evidence of a foreign device). + /// + private bool IsSessionOwnerAllowed(HttpRequestData req, string tenantId, string? sessionId, SessionRowRead? sessionRead, SecurityValidationResult validation) + { + if (sessionRead is not { Failed: false } read) return true; try { - var row = await _sessionRowReader.GetSessionRowAsync(tenantId, sessionId!); - return !_ownerBinding.Observe(req, tenantId, sessionId!, row, validation, "agent/error").Rejected; + return !_ownerBinding.Observe(req, tenantId, sessionId!, read.Row, validation, "agent/error").Rejected; } catch (Exception ex) { @@ -304,74 +338,65 @@ namespace AutopilotMonitor.Functions.Functions.Ingest /// Materializes the agent's silent 48h emergency break as (1) a timeline event so it shows /// in the session and the timeout classifier can see it, (2) the cross-session EventType /// index row, and (3) an AgentEmergencyBreak ops event for operator visibility - /// (tasks/enrollment-status-reclassification.md). Static seam with explicit dependencies so - /// tests can pin the artifact set without booting the Functions HTTP stack. - /// Best-effort — a failure here must never turn the always-200 emergency channel into a - /// retry loop. + /// (tasks/enrollment-status-reclassification.md). (1) and (2) only when the session has a + /// writable row (); the ops event always, because the + /// feed counts breaks. Static seam with explicit dependencies so tests can pin the artifact + /// set without booting the Functions HTTP stack. Best-effort — a failure here must never + /// turn the always-200 emergency channel into a retry loop. /// internal static async Task MaterializeEmergencyBreakArtifactsAsync( AgentErrorReport report, string tenantId, + SessionRowRead? sessionRead, ISessionRepository sessionRepo, + SessionDeletionGuard deletionGuard, OpsEventService opsEventService, ILogger logger) { - if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak - || string.IsNullOrEmpty(report.SessionId)) + // No read means the ids are not GUIDs — no session can exist to break. + if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak || sessionRead is not { } read) { return; } try { - var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000); - // Idempotency: the agent's emergency channel can send up to a few reports per session; - // only ever materialize one timeline event (and one ops event). - var alreadyMaterialized = existing.Any(e => - string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase)); - if (alreadyMaterialized) + var verdict = ClassifyEmergencyBreak(read.Row, read.Failed, IsDeletionLocked(read.Row, deletionGuard)); + var message = BreakMessage(report); + + if (verdict.Materialize) { - return; + var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000); + // Idempotency: the agent's emergency channel can send up to a few reports per + // session; only ever materialize one timeline event (and one ops event). + var alreadyMaterialized = existing.Any(e => + string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase)); + if (alreadyMaterialized) + { + return; + } + + var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow); + await sessionRepo.StoreEventsBatchAsync(new List { evt }); + + // StoreEventsBatchAsync writes the Events partition only — the cross-session + // EventType index is normally written by EventIngestProcessor, which this + // backend-materialized event never passes through. Without the upsert the event + // exists on the session timeline but is invisible to every search-by-eventType + // surface (portal cross-session search, MCP search_sessions_by_event / + // query_raw_events) — found the hard way in the 2026-07-22 incident analysis. + await sessionRepo.UpsertEventTypeIndexBatchAsync( + tenantId, report.SessionId, new List { evt }); } - var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow); - await sessionRepo.StoreEventsBatchAsync(new List { evt }); - - // StoreEventsBatchAsync writes the Events partition only — the cross-session - // EventType index is normally written by EventIngestProcessor, which this - // backend-materialized event never passes through. Without the upsert the event - // exists on the session timeline but is invisible to every search-by-eventType - // surface (portal cross-session search, MCP search_sessions_by_event / - // query_raw_events) — found the hard way in the 2026-07-22 incident analysis. - await sessionRepo.UpsertEventTypeIndexBatchAsync( - tenantId, report.SessionId, new List { evt }); - // Operator visibility: an emergency break means an agent silently gave up at its - // absolute age cap — exactly the "are we losing agents?" signal. Emitted only on - // first materialization so repeat reports for the same session cannot flood the - // ops feed. OpsEventService never throws. - // - // Severity follows what the platform already knows about the session (ops audit - // 2026-09-03): the break is wall-clock, so a device that was powered off for weeks - // fires it on its first boot — long after the sweep reconciled the session. That - // break is a late cleanup, not news; only a break on a still-open session is the - // decision itself (classifier rule 3). The row read is best-effort: unknown status - // keeps the historical Warning. - SessionStatus? statusAtBreak = null; - try - { - statusAtBreak = (await sessionRepo.GetSessionAsync(tenantId, report.SessionId))?.Status; - } - catch (Exception readEx) - { - logger.LogDebug(readEx, - "ReportAgentError: session status read failed for emergency-break severity on {SessionId}", report.SessionId); - } - - var verdict = ClassifyEmergencyBreak(statusAtBreak); + // absolute age cap — exactly the "are we losing agents?" signal. For a session with + // a row it is emitted only on first materialization, so repeat reports cannot flood + // the feed. OpsEventService never throws. await opsEventService.RecordAgentEmergencyBreakAsync( - tenantId, report.SessionId, report.AgentVersion, evt.Message, - verdict.Severity, statusAtBreak, verdict.LateCleanup); + tenantId, report.SessionId, report.AgentVersion, message, + verdict.Severity, verdict.SessionStatusAtBreak, verdict.LateCleanup, verdict.Context, + report.SessionAgeHours); } catch (Exception ex) { @@ -381,17 +406,66 @@ namespace AutopilotMonitor.Functions.Functions.Ingest } /// - /// Ops-event verdict for an emergency break, keyed on the session status the platform holds - /// when the report arrives. Terminal (Succeeded / Failed / Incomplete) = the sweep already - /// decided the enrollment and the agent merely cleaned up late (device powered on after a - /// shelf period) — Info, LateCleanup=true. Open or unknown = the break is the first - /// and only signal that nothing more will arrive — Warning, the historical semantics. + /// The cascade-delete writer-block invariant, checked at its chokepoint on the row already in + /// hand (no second read). A missing row needs no tombstone check: nothing is written for it. + /// + private static bool IsDeletionLocked(TableEntity? sessionRow, SessionDeletionGuard deletionGuard) + { + try + { + deletionGuard.ThrowIfLocked(sessionRow, "V2.ReportAgentError.EmergencyBreak"); + return false; + } + catch (SessionDeletionLockedException) + { + return true; + } + } + + /// What the ops event says and whether the break is written into the session. + internal readonly record struct EmergencyBreakVerdict( + string Severity, bool LateCleanup, bool Materialize, string SessionStatusAtBreak, string Context); + + /// + /// Ops-event verdict for an emergency break, keyed on what the platform holds for the session + /// when the report arrives. The break is wall-clock and checked at agent start, so it usually + /// arrives long after the fact: + /// + /// Terminal (Succeeded / Failed / Incomplete): the sweep already decided the enrollment + /// and the agent merely cleaned up late after a shelf period — Info, LateCleanup=true + /// (ops audit 2026-09-03). + /// No row: in practice a session that never registered (the agent went dark before + /// register-session) — nothing to close and nobody who could open it. Info, and no + /// timeline event or index row: the orphan sweep would delete the event a day later with + /// a Warning of its own, and the index row would point at a session that does not + /// exist. + /// Row under cascade delete: nothing may be written past the lock — Info. + /// Open or unparseable status, or a failed read: the break is the first and only + /// signal that nothing more will arrive (classifier rule 3) — Warning, and a failed read + /// keeps the historical write. + /// /// Pure so the rule is unit-testable. /// - internal static (string Severity, bool LateCleanup) ClassifyEmergencyBreak(SessionStatus? statusAtBreak) - => statusAtBreak.HasValue && Helpers.DeviceJourneyCalculator.IsTerminal(statusAtBreak.Value) - ? (OpsEventSeverity.Info, true) - : (OpsEventSeverity.Warning, false); + internal static EmergencyBreakVerdict ClassifyEmergencyBreak(TableEntity? sessionRow, bool rowReadFailed, bool deletionLocked) + { + if (rowReadFailed) + return new(OpsEventSeverity.Warning, false, true, "unknown", "session row unreadable when the break arrived"); + if (sessionRow == null) + return new(OpsEventSeverity.Info, false, false, "missing", "no session row (never registered, or deleted long ago) — nothing to close"); + if (deletionLocked) + return new(OpsEventSeverity.Info, false, false, "deleting", "session is being deleted — nothing written"); + + var status = IngestTelemetryFunction.TryReadSessionStatus(sessionRow); + var statusText = status?.ToString() ?? "unknown"; + return status.HasValue && Helpers.DeviceJourneyCalculator.IsTerminal(status.Value) + ? new(OpsEventSeverity.Info, true, true, statusText, $"late cleanup — session already {statusText} when the break arrived") + : new(OpsEventSeverity.Warning, false, true, statusText, $"session still {statusText} when the break arrived"); + } + + private static string BreakMessage(AgentErrorReport report) + => string.IsNullOrWhiteSpace(report.Message) + ? "Agent absolute session-age emergency break fired — agent cleaned up and exited" + : report.Message; /// /// Builds the backend-materialized agent_emergency_break timeline event from the agent's @@ -415,6 +489,15 @@ namespace AutopilotMonitor.Functions.Functions.Ingest ? report.Timestamp.ToUniversalTime() : nowUtc; + var data = new Dictionary + { + ["source"] = "emergency_channel", + ["agentVersion"] = report.AgentVersion ?? string.Empty, + ["reportedAtUtc"] = report.Timestamp.ToString("o"), + }; + if (report.SessionAgeHours.HasValue) + data["sessionAgeHours"] = report.SessionAgeHours.Value; + return new EnrollmentEvent { TenantId = tenantId, @@ -425,15 +508,8 @@ namespace AutopilotMonitor.Functions.Functions.Ingest Phase = EnrollmentPhase.Unknown, Timestamp = breakAt, Sequence = maxSequence + 1, - Message = string.IsNullOrWhiteSpace(report.Message) - ? "Agent absolute session-age emergency break fired — agent cleaned up and exited" - : report.Message, - Data = new Dictionary - { - ["source"] = "emergency_channel", - ["agentVersion"] = report.AgentVersion ?? string.Empty, - ["reportedAtUtc"] = report.Timestamp.ToString("o"), - }, + Message = BreakMessage(report), + Data = data, }; } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index ba3cd0f2..11115be4 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -954,31 +954,26 @@ namespace AutopilotMonitor.Functions.Services /// /// An agent reported its absolute session-age emergency break (48h cap) over the - /// emergency channel — it cleaned itself up and exited on a session that never reached - /// a terminal state. This is the "are we silently losing agents?" signal; emitted by - /// once per session (guarded by - /// the timeline-event idempotency check). Severity is decided by the caller from the - /// session status at break time (ReportAgentErrorFunction.ClassifyEmergencyBreak): - /// Warning when the session was still open (the break IS the verdict — Telegram-rule - /// worthy), Info when the sweep had already terminalized it and the break is a late - /// cleanup after the device was powered on again. The 2026-09-03 audit found 224 of 269 - /// breaks in three days were such late cleanups (kiosk devices stored for weeks); as - /// Warnings they buried the real ones. lands in the - /// details as sessionStatusAtBreak so the feed shows which case it was. + /// emergency channel — it cleaned itself up and exited. This is the "are we silently + /// losing agents?" signal; emitted by + /// once per break report (for a session with a row, guarded by the timeline-event + /// idempotency check). Severity, and + /// come from ReportAgentErrorFunction.ClassifyEmergencyBreak: + /// Warning only when the session was still open (the break IS the verdict — Telegram-rule + /// worthy) or its row could not be read. The 2026-09-03 audit found 224 of 269 breaks in + /// three days were late cleanups on terminal sessions, the 2026-09-28 sweep that 115 of 160 + /// remaining Warnings named sessions that never registered; as Warnings both buried the real + /// ones. is the agent-measured age (null for agents that + /// predate the field) — it separates a break just past the cap from a device that slept for + /// weeks, which the backend cannot tell from its own rows. /// public Task RecordAgentEmergencyBreakAsync( string tenantId, string sessionId, string? agentVersion, string message, - string severity, SessionStatus? statusAtBreak, bool lateCleanup) - { - var statusText = statusAtBreak?.ToString() ?? "unknown"; - var context = lateCleanup - ? $"late cleanup — session already {statusText} when the break arrived" - : $"session still {statusText} when the break arrived"; - return WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity, + string severity, string sessionStatusAtBreak, bool lateCleanup, string context, double? sessionAgeHours) + => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity, $"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}, {context}): {message}", tenantId, "System.EmergencyChannel", - new { sessionId, agentVersion, sessionStatusAtBreak = statusText, lateCleanup }); - } + new { sessionId, agentVersion, sessionStatusAtBreak, lateCleanup, sessionAgeHours }); /// /// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/AgentErrorReport.cs b/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/AgentErrorReport.cs index e668ef70..aa84f36d 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/AgentErrorReport.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/AgentErrorReport.cs @@ -81,6 +81,15 @@ namespace AutopilotMonitor.Shared.Models /// public long? SequenceNumber { get; set; } + /// + /// Session age in hours the agent measured when its absolute session-age emergency break + /// fired ( only). The backend cannot + /// derive it: a never-registered session has no row, and a registered one may have + /// registered seconds before the break (hibernate resume). Null for other error types and + /// for agents that predate the field. + /// + public double? SessionAgeHours { get; set; } + /// /// Agent version string for diagnosing version-specific issues. ///