From ad3632e9fb3cfa68c45174d4f26f075d7c65d301 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 23 Jul 2026 09:56:46 +0200 Subject: [PATCH] fix(emergency-break): survivable self-destruct, reliable break report, ops visibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incident 2026-07-22: the 48h emergency break left exe + loaded DLLs orphaned forever — a second agent instance held the images while the cleanup script (waiting on the launcher PID only) force-deleted marker + state around the locks AFTER removing the Scheduled Task, destroying the retry path. Agent — cleanup script (LIFE-F7): - wait for ALL agent processes by name (PID-reuse-proof), then force-kill stragglers; single-flight lock file serializes concurrent cleanup scripts - probe-rename (GUID target) BEFORE task removal; any failure renames back and aborts with task + marker + files intact for a next-boot retry; the destructive Remove-Item-on-original fallback is gone - script generation extracted to BuildCleanupScript, pinned by 13 tests; all 4 variants parser-validated + E2E-smoked (happy path, locked tree) Agent — break report delivery: - EmergencyReporter counts DELIVERED reports: reservation rolls back on failure (mark-before-send permanently suppressed a key after its first transport failure); optional in-reservation retries - break path: NIC wait <=15s, 3 attempts x 15s timeout (survives Functions cold starts observed as >5s 503s), 75s overall bound Backend: - backend-materialized events (agent_emergency_break, session_timeout) now upsert the cross-session EventType index — they were invisible to every search-by-eventType surface (portal + MCP) - new AgentEmergencyBreak ops event (Agent/Warning, once per session) so operators see silently-lost agents; dual-registered in OPS_EVENT_TYPES - materialization extracted to a static seam, covered by 4 new tests Agent 1963/1963, Backend 3198/3198, tsc clean. Co-Authored-By: Claude Fable 5 --- .../Runtime/CleanupServiceScriptTests.cs | 135 ++++++++++ .../Transport/EmergencyReporterTests.cs | 143 +++++++++++ .../Monitoring/Runtime/CleanupService.cs | 237 ++++++++++++------ .../Monitoring/Transport/BackendApiClient.cs | 15 +- .../Monitoring/Transport/EmergencyReporter.cs | 46 +++- .../Runtime/EmergencyBreakReporter.cs | 62 ++++- .../ReportAgentErrorFunctionTests.cs | 120 +++++++++ .../Ingest/ReportAgentErrorFunction.cs | 91 +++++-- .../Services/MaintenanceService.cs | 7 + .../Services/OpsEventService.cs | 14 ++ .../admin/components/OpsAlertRulesSection.tsx | 12 +- 11 files changed, 771 insertions(+), 111 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Runtime/CleanupServiceScriptTests.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Runtime/CleanupServiceScriptTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Runtime/CleanupServiceScriptTests.cs new file mode 100644 index 00000000..3f7e08a6 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Runtime/CleanupServiceScriptTests.cs @@ -0,0 +1,135 @@ +using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime +{ + /// + /// Pins the LIFE-F7 hardening invariants of the self-destruct cleanup script + /// (incident 2026-07-22: a second agent instance held exe + loaded DLLs while the + /// old script — waiting on the launcher PID only — force-deleted around the locks + /// AFTER removing the Scheduled Task, orphaning the binaries with no retry path). + /// These are string-level pins on : + /// weakening any of them silently re-introduces the orphaned-binaries failure mode. + /// + public class CleanupServiceScriptTests + { + private const string ProcessName = "AutopilotMonitor.Agent"; + private const string BasePath = @"C:\ProgramData\AutopilotMonitor"; + private const string TaskName = "AutopilotMonitorAgent"; + + private static string Script(bool keepLogs = false, bool reboot = false) + => CleanupService.BuildCleanupScript(ProcessName, BasePath, keepLogs, TaskName, reboot); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Waits_by_process_name_not_launcher_pid(bool keepLogs) + { + var script = Script(keepLogs); + + // LIFE-F7 #2: name-based wait covers marker-retry / self-update / manual instances + // and is immune to PID reuse. Wait-Process -Id saw exactly one PID and nothing else. + Assert.Contains($"Get-Process -Name '{ProcessName}'", script); + Assert.DoesNotContain("Wait-Process", script); + + // Backstop: stragglers are force-killed — their images are deleted right after. + Assert.Contains("Stop-Process -Force", script); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Never_deletes_the_original_path(bool keepLogs) + { + var script = Script(keepLogs); + + // LIFE-F7 #3: deletion happens exclusively on successfully renamed targets + // ($entry.Renamed). A Remove-Item against the original tree is what deleted + // marker + state around locked binaries in the incident. + Assert.DoesNotContain($"Remove-Item -Path '{BasePath}'", script); + Assert.Contains("Remove-Item -Path $entry.Renamed", script); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Task_removal_is_gated_on_probe_and_deletion_on_task_gone(bool keepLogs) + { + var script = Script(keepLogs); + + // Probe-rename first; only a successful probe reaches task removal, and only a + // removed task reaches deletion. Both failure branches rename BACK so the next + // boot retries with task + marker + files intact. + Assert.Contains("if ($probeOk)", script); + Assert.Contains("if ($taskGone)", script); + Assert.Contains("Rename-Item -Path $entry.Renamed -NewName $entry.Original", script); + + // LIFE-F6 task-removal loop + module-independent fallback stay intact. + Assert.Contains("Unregister-ScheduledTask -TaskName $taskName", script); + Assert.Contains("schtasks.exe /Delete /TN $taskName /F", script); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Concurrent_cleanup_scripts_are_serialized_by_lock_file(bool keepLogs) + { + var script = Script(keepLogs); + + // LIFE-F7 #1: CreateNew is the atomic single-flight gate; the loser exits without + // touching the tree. Stale locks (>10 min) are taken over. + Assert.Contains("AutopilotMonitor-Cleanup.lock", script); + Assert.Contains("[System.IO.FileMode]::CreateNew", script); + Assert.Contains("$lockAge.TotalMinutes -gt 10", script); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Rename_targets_carry_a_guid_suffix(bool keepLogs) + { + // A fixed '.del' target collides with leftovers of earlier attempts and with a + // concurrent script's rename — the GUID makes every attempt's target unique. + Assert.Contains(".del-' + [Guid]::NewGuid().ToString('N')", Script(keepLogs)); + } + + [Fact] + public void KeepLogs_variant_excludes_logs_directory() + { + var script = Script(keepLogs: true); + Assert.Contains("-Exclude 'Logs'", script); + Assert.DoesNotContain($"Rename-Item -Path '{BasePath}' -NewName", script); + } + + [Fact] + public void Full_variant_renames_the_whole_tree() + { + var script = Script(keepLogs: false); + Assert.Contains($"Rename-Item -Path '{BasePath}' -NewName $renamedPath", script); + Assert.DoesNotContain("-Exclude 'Logs'", script); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Reboot_flag_controls_shutdown_invocation(bool reboot) + { + var script = Script(keepLogs: false, reboot: reboot); + if (reboot) + Assert.Contains("shutdown.exe /r /t 10", script); + else + Assert.DoesNotContain("shutdown.exe", script); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Script_and_lock_are_released_in_finally(bool keepLogs) + { + var script = Script(keepLogs); + Assert.Contains("finally", script); + Assert.Contains("$lockStream.Dispose()", script); + Assert.Contains("Remove-Item -Path $scriptPath -Force", script); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs new file mode 100644 index 00000000..b76ba4e1 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport +{ + /// + /// Pins the anti-flood semantics of after the + /// delivery-hardening change (2026-07-23): the once-per-session dedup and the per-session + /// budget count DELIVERED reports. The previous mark-before-send permanently suppressed an + /// error key after its first transport failure — fatal for single-shot callers like the 48h + /// emergency break, whose process exits right after the attempt. + /// + public class EmergencyReporterTests + { + private sealed class FakeApiClient : BackendApiClient + { + public readonly List Timeouts = new List(); + public readonly Queue Results = new Queue(); + + public FakeApiClient() : base( + httpClient: new HttpClient(), + baseUrl: "http://localhost", + manufacturer: string.Empty, + model: string.Empty, + serialNumber: string.Empty, + useBootstrapTokenAuth: false, + bootstrapToken: null, + agentVersion: "0.0.0", + logger: null) + { + } + + public int Calls => Timeouts.Count; + + public override Task ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null) + { + Timeouts.Add(timeout); + return Task.FromResult(Results.Count > 0 && Results.Dequeue()); + } + } + + private static EmergencyReporter Reporter(FakeApiClient client) + => new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null); + + [Fact] + public async Task Failed_send_rolls_back_reservation_so_a_later_call_retries() + { + var client = new FakeApiClient(); + client.Results.Enqueue(false); // first call fails + client.Results.Enqueue(true); // second call succeeds + var reporter = Reporter(client); + + await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m"); + await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m"); + + // Old behaviour: second call suppressed as "already reported" ⇒ 1 call. + Assert.Equal(2, client.Calls); + } + + [Fact] + public async Task Delivered_send_is_deduplicated_per_key() + { + var client = new FakeApiClient(); + client.Results.Enqueue(true); + var reporter = Reporter(client); + + await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m"); + await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m"); + + Assert.Equal(1, client.Calls); + } + + [Fact] + public async Task Retries_stay_inside_one_reservation_and_stop_on_success() + { + var client = new FakeApiClient(); + client.Results.Enqueue(false); + client.Results.Enqueue(true); // second attempt delivers — third must not fire + var reporter = Reporter(client); + + await reporter.TrySendAsync( + AgentErrorType.SessionAgeEmergencyBreak, "m", + attempts: 3, perAttemptTimeout: TimeSpan.FromMilliseconds(50), retryDelay: TimeSpan.Zero); + + Assert.Equal(2, client.Calls); + + // Delivered ⇒ dedup holds for the session. + await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m"); + Assert.Equal(2, client.Calls); + } + + [Fact] + public async Task Per_attempt_timeout_is_forwarded_to_the_api_client() + { + var client = new FakeApiClient(); + client.Results.Enqueue(true); + var reporter = Reporter(client); + + var timeout = TimeSpan.FromSeconds(15); + await reporter.TrySendAsync( + AgentErrorType.SessionAgeEmergencyBreak, "m", + attempts: 1, perAttemptTimeout: timeout); + + Assert.Equal(timeout, Assert.Single(client.Timeouts)); + } + + [Fact] + public async Task All_attempts_failing_rolls_back_after_exhausting_attempts() + { + var client = new FakeApiClient(); // queue empty ⇒ every attempt fails + var reporter = Reporter(client); + + await reporter.TrySendAsync( + AgentErrorType.SessionAgeEmergencyBreak, "m", + attempts: 3, retryDelay: TimeSpan.Zero); + Assert.Equal(3, client.Calls); + + // Rolled back ⇒ a later call may try again (budget + dedup + cooldown restored). + client.Results.Enqueue(true); + await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m"); + Assert.Equal(4, client.Calls); + } + + [Fact] + public async Task Different_keys_do_not_suppress_each_other_after_rollback() + { + var client = new FakeApiClient(); + client.Results.Enqueue(false); // break report fails → rollback (incl. cooldown restore) + client.Results.Enqueue(true); // unrelated report must still go out + var reporter = Reporter(client); + + await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m"); + await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "m"); + + Assert.Equal(2, client.Calls); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/CleanupService.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/CleanupService.cs index 2cd957c1..44b911ea 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/CleanupService.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/CleanupService.cs @@ -31,84 +31,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { _logger.Info($"Executing FULL SELF-DESTRUCT (Scheduled Task + Files{(_configuration.RebootOnComplete ? " + Reboot" : "")})"); - var currentProcessId = Process.GetCurrentProcess().Id; + var currentProcess = Process.GetCurrentProcess(); var agentBasePath = Path.GetFullPath(Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "AutopilotMonitor" )); - var logDir = Path.GetFullPath(Path.Combine(agentBasePath, "Logs")); - var keepLogs = _configuration.KeepLogFile; - - // Create a self-deleting PowerShell script - var cleanupScript = $@" -$scriptPath = $MyInvocation.MyCommand.Path - -# Wait for agent process to exit -Start-Sleep -Seconds 2 -Wait-Process -Id {currentProcessId} -ErrorAction SilentlyContinue -Timeout 30 -Start-Sleep -Seconds 1 - -# Remove Scheduled Task — verify removal BEFORE deleting files (LIFE-F6). An orphan task firing -# against a deleted exe spams Task Scheduler errors every boot forever and, once the files (incl. -# the enrollment-complete marker) are gone, has no cleanup-retry path. So confirm the task is gone -# first; if it cannot be removed, SKIP the file deletion and leave everything intact for a clean -# cleanup retry on the next boot (the marker survives, so bootstrap re-runs cleanup). -$taskName = '{_configuration.ScheduledTaskName}' -$taskGone = $false -for ($i = 1; $i -le 5; $i++) {{ - try {{ Stop-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue }} catch {{ }} - try {{ Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue }} catch {{ }} - $still = $null - try {{ $still = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue }} catch {{ }} - if (-not $still) {{ $taskGone = $true; break }} - Start-Sleep -Seconds 1 -}} -if (-not $taskGone) {{ - # Fallback: schtasks.exe /Delete does not depend on the ScheduledTasks PowerShell module. - try {{ & schtasks.exe /Delete /TN $taskName /F 2>$null | Out-Null }} catch {{ }} - try {{ $taskGone = -not (Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue) }} catch {{ }} -}} - -{(keepLogs ? $@" -if ($taskGone) {{ - # Delete everything except Logs directory - Get-ChildItem -Path '{agentBasePath}' -Exclude 'Logs' | ForEach-Object {{ - $dest = $_.FullName + '.del' - try {{ Rename-Item -Path $_.FullName -NewName $dest -Force -ErrorAction SilentlyContinue }} catch {{ }} - Remove-Item -Path $dest -Recurse -Force -ErrorAction SilentlyContinue - }} -}} -" : $@" -if ($taskGone) {{ - # Rename folder then delete. Retry a few times to let the OS release handles. - $renamedPath = '{agentBasePath}.del' - $renamed = $false - for ($i = 1; $i -le 10; $i++) {{ - try {{ - Rename-Item -Path '{agentBasePath}' -NewName $renamedPath -Force -ErrorAction Stop - $renamed = $true - break - }} catch {{ - Start-Sleep -Seconds 2 - }} - }} - if ($renamed) {{ - Remove-Item -Path $renamedPath -Recurse -Force -ErrorAction SilentlyContinue - }} else {{ - Remove-Item -Path '{agentBasePath}' -Recurse -Force -ErrorAction SilentlyContinue - }} -}} -")} -{(_configuration.RebootOnComplete ? @" -# shutdown.exe (NOT Restart-Computer): Restart-Computer has no -Comment parameter and -Delay is -# only honoured with -Wait, so the previous invocation failed parameter binding and the device -# silently never rebooted. shutdown.exe /r /t 10 mirrors the standalone reboot path and reboots -# after a 10 s delay (lets this script finish + the agent exit). Review LIFE-F2. -shutdown.exe /r /t 10 /c 'Autopilot enrollment completed - Autopilot Monitor is rebooting' -" : "")} -Remove-Item -Path $scriptPath -Force -ErrorAction SilentlyContinue -"; + var cleanupScript = BuildCleanupScript( + agentProcessName: currentProcess.ProcessName, + agentBasePath: agentBasePath, + keepLogs: _configuration.KeepLogFile, + scheduledTaskName: _configuration.ScheduledTaskName, + rebootOnComplete: _configuration.RebootOnComplete); // Write cleanup script to temp location (outside of agent folder) var tempScriptPath = Path.Combine(Path.GetTempPath(), $"AutopilotMonitor-Cleanup-{Guid.NewGuid()}.ps1"); @@ -143,5 +77,162 @@ Remove-Item -Path $scriptPath -Force -ErrorAction SilentlyContinue } } + /// + /// Builds the self-deleting cleanup PowerShell script. Extracted (internal static, pure) + /// so tests can pin the hardened invariants without launching processes. + /// + /// LIFE-F7 hardening (incident 2026-07-22 — exe + loaded DLLs orphaned forever): + /// + /// + /// Single-flight lock — the enrollment-complete marker is written BEFORE + /// cleanup launches, so any agent start during an in-flight cleanup spawns a SECOND + /// cleanup script. Two unsynchronized scripts interleave their rename/delete phases + /// and leave partial trees behind. A CreateNew lock file in the SYSTEM temp dir + /// serializes them; a stale lock (>10 min) is taken over. + /// Wait by process NAME, not launcher PID — marker-retry instances, + /// self-update restarts and manual runs all load the same images but have PIDs the + /// old Wait-Process -Id never saw (and an already-freed PID can be reused by + /// an unrelated boot-time process). After the deadline any straggler is force-killed: + /// its images are about to be deleted anyway. + /// Probe-rename BEFORE task removal, never delete the original path — the + /// rename is the atomic lock probe. If it fails, something still holds the tree: abort + /// with task + files + marker intact so the next boot retries (file-side symmetry to + /// the LIFE-F6 task-side rule). If the task cannot be removed afterwards, rename BACK + /// and abort for the same reason. Only a fully renamed tree with the task gone is + /// deleted — the old Remove-Item-on-original fallback deleted marker + state + /// around locked binaries AFTER the task was gone, converting a transient lock into a + /// permanently orphaned exe. + /// + /// + internal static string BuildCleanupScript( + string agentProcessName, + string agentBasePath, + bool keepLogs, + string scheduledTaskName, + bool rebootOnComplete) + { + return $@" +$scriptPath = $MyInvocation.MyCommand.Path + +# Single-flight lock (LIFE-F7 #1): serialize concurrent cleanup scripts. CreateNew is +# atomic; the loser backs off and lets the winner own the deletion. +$lockPath = Join-Path ([System.IO.Path]::GetTempPath()) 'AutopilotMonitor-Cleanup.lock' +$lockStream = $null +try {{ + if (Test-Path $lockPath) {{ + $lockAge = (Get-Date) - (Get-Item $lockPath -ErrorAction Stop).LastWriteTime + if ($lockAge.TotalMinutes -gt 10) {{ Remove-Item -Path $lockPath -Force -ErrorAction SilentlyContinue }} + }} + $lockStream = [System.IO.File]::Open($lockPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) +}} catch {{ + Remove-Item -Path $scriptPath -Force -ErrorAction SilentlyContinue + exit 0 +}} + +try {{ + # Wait for EVERY agent process to exit, by NAME (LIFE-F7 #2) — covers marker-retry + # instances, self-update restarts and manual runs, and is immune to PID reuse. + Start-Sleep -Seconds 2 + $deadline = (Get-Date).AddSeconds(60) + while ((Get-Date) -lt $deadline) {{ + if (-not (Get-Process -Name '{agentProcessName}' -ErrorAction SilentlyContinue)) {{ break }} + Start-Sleep -Seconds 2 + }} + # Backstop: force-kill stragglers — their images are about to be deleted anyway. + Get-Process -Name '{agentProcessName}' -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue + Start-Sleep -Seconds 1 + +{(keepLogs ? $@" + # Probe-rename all children except Logs (LIFE-F7 #3). Any failure ⇒ rename back and + # abort with the Scheduled Task intact so the next boot retries cleanly. + $renamed = @() + $probeOk = $true + foreach ($item in @(Get-ChildItem -Path '{agentBasePath}' -Exclude 'Logs' -ErrorAction SilentlyContinue)) {{ + $dest = $item.FullName + '.del-' + [Guid]::NewGuid().ToString('N') + $itemOk = $false + for ($i = 1; $i -le 10; $i++) {{ + try {{ + Rename-Item -Path $item.FullName -NewName $dest -Force -ErrorAction Stop + $itemOk = $true + break + }} catch {{ + Start-Sleep -Seconds 2 + }} + }} + if ($itemOk) {{ $renamed += @{{ Original = $item.FullName; Renamed = $dest }} }} + else {{ $probeOk = $false; break }} + }} +" : $@" + # Probe-rename the whole tree (LIFE-F7 #3). Failure ⇒ abort with task + files + marker + # intact so the next boot retries cleanly. GUID suffix avoids collisions with leftovers + # of earlier attempts. + $renamedPath = '{agentBasePath}.del-' + [Guid]::NewGuid().ToString('N') + $probeOk = $false + for ($i = 1; $i -le 10; $i++) {{ + try {{ + Rename-Item -Path '{agentBasePath}' -NewName $renamedPath -Force -ErrorAction Stop + $probeOk = $true + break + }} catch {{ + Start-Sleep -Seconds 2 + }} + }} + $renamed = @(@{{ Original = '{agentBasePath}'; Renamed = $renamedPath }}) +")} + if ($probeOk) {{ + # Remove Scheduled Task — verify removal BEFORE deleting files (LIFE-F6). An orphan + # task firing against a deleted exe spams Task Scheduler errors every boot forever + # and, once the files (incl. the enrollment-complete marker) are gone, has no + # cleanup-retry path. + $taskName = '{scheduledTaskName}' + $taskGone = $false + for ($i = 1; $i -le 5; $i++) {{ + try {{ Stop-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue }} catch {{ }} + try {{ Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue }} catch {{ }} + $still = $null + try {{ $still = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue }} catch {{ }} + if (-not $still) {{ $taskGone = $true; break }} + Start-Sleep -Seconds 1 + }} + if (-not $taskGone) {{ + # Fallback: schtasks.exe /Delete does not depend on the ScheduledTasks PowerShell module. + try {{ & schtasks.exe /Delete /TN $taskName /F 2>$null | Out-Null }} catch {{ }} + try {{ $taskGone = -not (Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue) }} catch {{ }} + }} + + if ($taskGone) {{ + # Task gone + tree renamed ⇒ safe to delete. Never touches the original path. + foreach ($entry in $renamed) {{ + Remove-Item -Path $entry.Renamed -Recurse -Force -ErrorAction SilentlyContinue + }} + }} else {{ + # Task survived ⇒ rename back and leave everything for the next-boot retry + # (the marker survives, so bootstrap re-runs cleanup). + foreach ($entry in $renamed) {{ + try {{ Rename-Item -Path $entry.Renamed -NewName $entry.Original -Force -ErrorAction SilentlyContinue }} catch {{ }} + }} + }} + }} else {{ + # Probe failed ⇒ something still holds the tree. Rename back whatever succeeded and + # abort WITHOUT deleting anything — task + marker stay, next boot retries. + foreach ($entry in $renamed) {{ + try {{ Rename-Item -Path $entry.Renamed -NewName $entry.Original -Force -ErrorAction SilentlyContinue }} catch {{ }} + }} + }} +{(rebootOnComplete ? @" + # shutdown.exe (NOT Restart-Computer): Restart-Computer has no -Comment parameter and -Delay is + # only honoured with -Wait, so the previous invocation failed parameter binding and the device + # silently never rebooted. shutdown.exe /r /t 10 mirrors the standalone reboot path and reboots + # after a 10 s delay (lets this script finish + the agent exit). Review LIFE-F2. + shutdown.exe /r /t 10 /c 'Autopilot enrollment completed - Autopilot Monitor is rebooting' +" : "")} +}} finally {{ + if ($lockStream) {{ $lockStream.Dispose() }} + Remove-Item -Path $lockPath -Force -ErrorAction SilentlyContinue + Remove-Item -Path $scriptPath -Force -ErrorAction SilentlyContinue +}} +"; + } + } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs index ca9c30a1..2494e1ed 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs @@ -169,9 +169,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// /// Sends a critical error report to the emergency channel endpoint. /// Fire-and-forget: swallows all exceptions so a failure here never cascades. - /// Uses a 5-second per-request timeout so it cannot block the upload loop. + /// Default 5-second per-request timeout so it cannot block the upload loop; the + /// emergency-break path passes a longer because its first + /// request of a freshly booted process pays cold TLS + DNS + a possible backend + /// cold start (observed 503s take well over 5 s). + /// Returns true when the HTTP round-trip completed (the endpoint always + /// answers 200), false on timeout/transport failure — the caller-side + /// anti-flood layer uses this to decide whether the report counts as delivered. + /// Virtual for tests. /// - public async Task ReportAgentErrorAsync(AgentErrorReport report) + public virtual async Task ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null) { try { @@ -190,16 +197,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport httpRequest.Headers.Add("X-Tenant-Id", report.TenantId); AddSecurityHeaders(httpRequest); - using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + using (var cts = new CancellationTokenSource(timeout ?? TimeSpan.FromSeconds(5))) using (var response = await _httpClient.SendAsync(httpRequest, cts.Token).ConfigureAwait(false)) { // Response status is deliberately ignored — endpoint always returns 200. + return true; } } } catch (Exception ex) { _logger?.Debug($"ReportAgentErrorAsync: emergency channel failed: {ex.Message}"); + return false; } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs index 2cd57209..51f5d2f3 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs @@ -57,19 +57,33 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// (intended to be called as fire-and-forget via _ = reporter.TrySendAsync(...)). /// /// All anti-flood checks are applied synchronously before the async HTTP call, - /// so repeated calls are cheap when suppressed. + /// so repeated calls are cheap when suppressed. The anti-flood reservation is + /// taken up front (so concurrent same-key calls cannot double-send) but ROLLED + /// BACK when every attempt fails: the budget and the once-per-session dedup are + /// meant to count DELIVERED reports — the previous mark-before-send made the very + /// first transport failure permanently suppress that error key for the session + /// (found via the 48h emergency-break report, which is a single shot at boot). + /// + /// // + /// exist for callers whose process is about to exit and cannot come back later + /// (emergency break): retries stay INSIDE the one reservation, so they never + /// multiply against the per-session budget. /// public virtual async Task TrySendAsync( AgentErrorType errorType, string message, int? httpStatusCode = null, - long? sequenceNumber = null) + long? sequenceNumber = null, + int attempts = 1, + TimeSpan? perAttemptTimeout = null, + TimeSpan? retryDelay = null) { // Deduplicate by error type + status code: same failure category sent only once per session. // All anti-flood checks are protected by a lock to prevent race conditions when // TrySendAsync is called concurrently (fire-and-forget from multiple threads). var key = $"{errorType}:{httpStatusCode}"; int currentReportCount; + DateTime? previousLastReportTime; lock (_antiFloodLock) { @@ -92,6 +106,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport return; } + previousLastReportTime = _lastReportTime; _reportedKeys.Add(key); _reportCount++; _lastReportTime = DateTime.UtcNow; @@ -113,15 +128,40 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport Timestamp = DateTime.UtcNow }; + var delivered = false; + var totalAttempts = attempts < 1 ? 1 : attempts; try { - await _apiClient.ReportAgentErrorAsync(report); + for (var attempt = 1; attempt <= totalAttempts && !delivered; attempt++) + { + if (attempt > 1 && retryDelay.HasValue) + { + await Task.Delay(retryDelay.Value).ConfigureAwait(false); + } + + delivered = await _apiClient.ReportAgentErrorAsync(report, perAttemptTimeout).ConfigureAwait(false); + if (!delivered && totalAttempts > 1) + { + _logger?.Debug($"[EmergencyChannel] Attempt {attempt}/{totalAttempts} failed: {key}"); + } + } } catch (Exception ex) { // Swallow — emergency channel must never cascade failures into the caller _logger?.Debug($"[EmergencyChannel] Failed to send report: {ex.Message}"); } + + if (!delivered) + { + lock (_antiFloodLock) + { + _reportedKeys.Remove(key); + _reportCount--; + _lastReportTime = previousLastReportTime; + } + _logger?.Debug($"[EmergencyChannel] Report not delivered — reservation rolled back: {key}"); + } } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/EmergencyBreakReporter.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/EmergencyBreakReporter.cs index 9db297dc..0374f8c8 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/EmergencyBreakReporter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/EmergencyBreakReporter.cs @@ -1,4 +1,6 @@ using System; +using System.Net.NetworkInformation; +using System.Threading; using AutopilotMonitor.Agent.V2.Core.Configuration; using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Shared.Models; @@ -15,28 +17,78 @@ namespace AutopilotMonitor.Agent.V2.Runtime /// classifier terminalize the session instead of waiting out the grace. Fully swallowed — a send /// failure (no network, no cert) must never delay the cleanup/exit the break exists to perform. /// See tasks/enrollment-status-reclassification.md. + /// + /// Delivery hardening (2026-07-23): this is the FIRST request of a freshly booted process and + /// the process never comes back — a miss is gone forever. The old single shot with a 5 s cap + /// died on boot-time realities (NIC still associating, cold TLS/DNS, backend cold starts + /// observed as >5 s 503s). Now: wait briefly for the NIC, then up to + /// attempts with a per-attempt timeout that survives a Functions + /// cold start. The session is 48 h dead — spending up to ~75 s here is irrelevant, losing the + /// only signal that the agent gave up is not. + /// /// internal static class EmergencyBreakReporter { + private static readonly TimeSpan NetworkWaitMax = TimeSpan.FromSeconds(15); + private const int SendAttempts = 3; + private static readonly TimeSpan PerAttemptTimeout = TimeSpan.FromSeconds(15); + private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(5); + // Upper bound for the blocking wait: NIC wait + 3×15s attempts + 2×5s delays + slack. + private static readonly TimeSpan OverallBudget = TimeSpan.FromSeconds(75); + public static void TrySend(AgentConfiguration agentConfig, string agentVersion, AgentLogger logger) { try { + WaitForNetwork(logger); + var auth = BackendClientFactory.BuildAuthClients(agentConfig, agentVersion, logger); var message = $"Agent absolute session-age emergency break fired (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting."; - // The process is about to exit, so block briefly on the otherwise fire-and-forget send - // rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP errors; - // the auth bundle's HttpClients are intentionally left for process teardown. + // The process is about to exit, so block on the otherwise fire-and-forget send + // rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP + // errors and keeps the retries inside one anti-flood reservation; the auth + // bundle's HttpClients are intentionally left for process teardown. auth.EmergencyReporter - .TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, message) - .Wait(TimeSpan.FromSeconds(5)); + .TrySendAsync( + AgentErrorType.SessionAgeEmergencyBreak, + message, + attempts: SendAttempts, + perAttemptTimeout: PerAttemptTimeout, + retryDelay: RetryDelay) + .Wait(OverallBudget); } catch (Exception ex) { logger?.Debug($"Emergency-break report send failed (best-effort): {ex.Message}"); } } + + /// + /// Boot-time NIC grace: the break fires during bootstrap, often seconds after boot while + /// Wi-Fi is still associating. Polls the cheap link-level signal — it cannot prove backend + /// reachability (the send attempts handle that), it only avoids burning the first attempts + /// into a known-dead link. Best-effort: probe errors abort the wait, never the send. + /// + private static void WaitForNetwork(AgentLogger logger) + { + try + { + if (NetworkInterface.GetIsNetworkAvailable()) return; + + logger?.Info($"Emergency-break report: no network link yet — waiting up to {NetworkWaitMax.TotalSeconds:F0}s."); + var deadline = DateTime.UtcNow + NetworkWaitMax; + while (DateTime.UtcNow < deadline) + { + Thread.Sleep(TimeSpan.FromSeconds(1)); + if (NetworkInterface.GetIsNetworkAvailable()) return; + } + } + catch (Exception ex) + { + logger?.Debug($"Emergency-break report: network probe failed ({ex.Message}) — sending anyway."); + } + } } } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs index 3cf2d963..46d19b6a 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs @@ -1,7 +1,15 @@ using System; using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; using AutopilotMonitor.Functions.Functions.Ingest; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Notifications; +using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; namespace AutopilotMonitor.Functions.Tests; @@ -80,4 +88,116 @@ public class ReportAgentErrorFunctionTests var evt = ReportAgentErrorFunction.BuildAgentEmergencyBreakEvent(Report(Now, message: ""), TenantId, null!, Now); Assert.Contains("emergency break", evt.Message, StringComparison.OrdinalIgnoreCase); } + + // ========================================================================= + // MaterializeEmergencyBreakArtifactsAsync — the artifact set per break report + // (2026-07-23 hardening): timeline event + cross-session EventType index row + + // AgentEmergencyBreak ops event, once per session. The index upsert is load-bearing: + // StoreEventsBatchAsync alone leaves the event invisible to every search-by-eventType + // surface (found in the 2026-07-22 incident analysis). + // ========================================================================= + + private sealed class Harness + { + public readonly Mock SessionRepo = new(); + public readonly List OpsEvents = new(); + public readonly OpsEventService OpsService; + + public Harness(List? existingEvents = null) + { + SessionRepo.Setup(r => r.GetSessionEventsAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync(existingEvents ?? new List()); + SessionRepo.Setup(r => r.StoreEventsBatchAsync(It.IsAny>())) + .ReturnsAsync((List e) => e); + SessionRepo.Setup(r => r.UpsertEventTypeIndexBatchAsync( + It.IsAny(), It.IsAny(), It.IsAny>())) + .Returns(Task.CompletedTask); + + var opsRepo = new Mock(); + opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny())) + .Callback(e => { lock (OpsEvents) OpsEvents.Add(e); }) + .Returns(Task.CompletedTask); + + var adminConfig = new Mock( + Mock.Of(), NullLogger.Instance, + new MemoryCache(new MemoryCacheOptions())); + var alertDispatch = new OpsAlertDispatchService( + adminConfig.Object, + new TelegramNotificationService(new HttpClient(), Mock.Of(), + NullLogger.Instance), + new WebhookNotificationService(new HttpClient(), + NullLogger.Instance), + NullLogger.Instance); + OpsService = new OpsEventService(opsRepo.Object, NullLogger.Instance, alertDispatch); + } + + public Task RunAsync(AgentErrorReport report) + => ReportAgentErrorFunction.MaterializeEmergencyBreakArtifactsAsync( + report, TenantId, SessionRepo.Object, OpsService, NullLogger.Instance); + } + + [Fact] + public async Task First_break_report_materializes_event_index_row_and_ops_event() + { + var h = new Harness(); + + await h.RunAsync(Report(Now)); + + h.SessionRepo.Verify(r => r.StoreEventsBatchAsync( + It.Is>(e => e.Count == 1 + && e[0].EventType == AutopilotMonitor.Shared.Constants.EventTypes.AgentEmergencyBreak)), Times.Once); + h.SessionRepo.Verify(r => r.UpsertEventTypeIndexBatchAsync( + TenantId, SessionId, It.IsAny>()), Times.Once); + + var ops = Assert.Single(h.OpsEvents); + Assert.Equal("AgentEmergencyBreak", ops.EventType); + Assert.Equal(TenantId, ops.TenantId); + } + + [Fact] + public async Task Already_materialized_session_produces_no_second_artifact_set() + { + var h = new Harness(new List + { + new() { EventType = AutopilotMonitor.Shared.Constants.EventTypes.AgentEmergencyBreak, Sequence = 45 }, + }); + + await h.RunAsync(Report(Now)); + + h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny>()), Times.Never); + h.SessionRepo.Verify(r => r.UpsertEventTypeIndexBatchAsync( + It.IsAny(), It.IsAny(), It.IsAny>()), Times.Never); + Assert.Empty(h.OpsEvents); + } + + [Fact] + public async Task Non_break_error_types_and_missing_session_id_are_ignored() + { + var h = new Harness(); + + var other = Report(Now); + other.ErrorType = AgentErrorType.ConfigFetchFailed; + await h.RunAsync(other); + + var noSession = Report(Now); + noSession.SessionId = null!; + await h.RunAsync(noSession); + + h.SessionRepo.Verify(r => r.GetSessionEventsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + Assert.Empty(h.OpsEvents); + } + + [Fact] + public async Task Storage_failure_is_swallowed_and_never_throws() + { + var h = new Harness(); + h.SessionRepo.Setup(r => r.StoreEventsBatchAsync(It.IsAny>())) + .ThrowsAsync(new InvalidOperationException("storage down")); + + // Must not throw — the always-200 emergency channel must never turn into a retry loop. + await h.RunAsync(Report(Now)); + + Assert.Empty(h.OpsEvents); + } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs index 20141749..220245a8 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs @@ -37,6 +37,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest private readonly TelemetryClient _telemetryClient; private readonly BootstrapSessionService _bootstrapSessionService; private readonly ISessionRepository _sessionRepo; + private readonly OpsEventService _opsEventService; public ReportAgentErrorFunction( ILogger logger, @@ -48,7 +49,8 @@ namespace AutopilotMonitor.Functions.Functions.Ingest DeviceAssociationValidator deviceAssociationValidator, TelemetryClient telemetryClient, BootstrapSessionService bootstrapSessionService, - ISessionRepository sessionRepo) + ISessionRepository sessionRepo, + OpsEventService opsEventService) { _logger = logger; _configService = configService; @@ -60,6 +62,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest _telemetryClient = telemetryClient; _bootstrapSessionService = bootstrapSessionService; _sessionRepo = sessionRepo; + _opsEventService = opsEventService; } [Function("ReportAgentError")] @@ -169,35 +172,71 @@ namespace AutopilotMonitor.Functions.Functions.Ingest ["AgentTimestamp"] = report.Timestamp.ToString("O"), }); - // Materialize the agent's silent 48h emergency break as a timeline event so it shows in the - // session and the timeout classifier can see it (tasks/enrollment-status-reclassification.md). - // Best-effort — a failure here must never turn the always-200 emergency channel into a retry loop. - if (report.ErrorType == AgentErrorType.SessionAgeEmergencyBreak - && !string.IsNullOrEmpty(report.SessionId)) - { - try - { - var existing = await _sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000); - // Idempotency: the agent's emergency channel can send up to a few reports per session; - // only ever materialize one timeline event. - var alreadyMaterialized = existing.Any(e => - string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase)); - if (!alreadyMaterialized) - { - var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow); - await _sessionRepo.StoreEventsBatchAsync(new List { evt }); - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, - "ReportAgentError: failed to materialize agent_emergency_break event for session {SessionId}", report.SessionId); - } - } + await MaterializeEmergencyBreakArtifactsAsync(report, tenantId, _sessionRepo, _opsEventService, _logger); return req.CreateResponse(HttpStatusCode.OK); } + /// + /// Materializes the agent's silent 48h emergency break as (1) a timeline event so it shows + /// in the session and the timeout classifier can see it, (2) the cross-session EventType + /// index row, and (3) an AgentEmergencyBreak ops event for operator visibility + /// (tasks/enrollment-status-reclassification.md). Static seam with explicit dependencies so + /// tests can pin the artifact set without booting the Functions HTTP stack. + /// Best-effort — a failure here must never turn the always-200 emergency channel into a + /// retry loop. + /// + internal static async Task MaterializeEmergencyBreakArtifactsAsync( + AgentErrorReport report, + string tenantId, + ISessionRepository sessionRepo, + OpsEventService opsEventService, + ILogger logger) + { + if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak + || string.IsNullOrEmpty(report.SessionId)) + { + return; + } + + try + { + var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000); + // Idempotency: the agent's emergency channel can send up to a few reports per session; + // only ever materialize one timeline event (and one ops event). + var alreadyMaterialized = existing.Any(e => + string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase)); + if (alreadyMaterialized) + { + return; + } + + var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow); + await sessionRepo.StoreEventsBatchAsync(new List { evt }); + + // StoreEventsBatchAsync writes the Events partition only — the cross-session + // EventType index is normally written by EventIngestProcessor, which this + // backend-materialized event never passes through. Without the upsert the event + // exists on the session timeline but is invisible to every search-by-eventType + // surface (portal cross-session search, MCP search_sessions_by_event / + // query_raw_events) — found the hard way in the 2026-07-22 incident analysis. + await sessionRepo.UpsertEventTypeIndexBatchAsync( + tenantId, report.SessionId, new List { evt }); + + // Operator visibility: an emergency break means an agent silently gave up at its + // absolute age cap — exactly the "are we losing agents?" signal. Emitted only on + // first materialization so repeat reports for the same session cannot flood the + // ops feed. OpsEventService never throws. + await opsEventService.RecordAgentEmergencyBreakAsync( + tenantId, report.SessionId, report.AgentVersion, evt.Message); + } + catch (Exception ex) + { + logger.LogWarning(ex, + "ReportAgentError: failed to materialize agent_emergency_break event for session {SessionId}", report.SessionId); + } + } + /// /// Builds the backend-materialized agent_emergency_break timeline event from the agent's /// best-effort emergency report. Static + pure (analog to diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 491c370c..aa7f887d 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -439,6 +439,13 @@ namespace AutopilotMonitor.Functions.Services var timeoutEvent = BuildSessionTimeoutEvent(session, timeoutHours, sessionEvents, now); await _sessionRepo.StoreEventsBatchAsync(new List { timeoutEvent }); + // Backend-materialized events bypass EventIngestProcessor and thus the + // cross-session EventType index — without this upsert session_timeout is + // invisible to every search-by-eventType surface (portal cross-session + // search, MCP search_sessions_by_event / query_raw_events). + await _sessionRepo.UpsertEventTypeIndexBatchAsync( + session.TenantId, session.SessionId, new List { timeoutEvent }); + // StoreEventsBatchAsync only bumps the orphan side-index, not the // session's EventCount, and the terminal reconcile already ran inside // UpdateSessionStatusAsync *before* this synthetic event existed. Recount diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 987eb27b..bf6a1e96 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -493,6 +493,20 @@ namespace AutopilotMonitor.Functions.Services $"{sessionCount} session(s) timed out after {timeoutHours}h", tenantId, "System.Maintenance", new { sessionCount, timeoutHours }); + /// + /// An agent reported its absolute session-age emergency break (48h cap) over the + /// emergency channel — it cleaned itself up and exited on a session that never reached + /// a terminal state. This is the "are we silently losing agents?" signal; emitted by + /// once per session (guarded by + /// the timeline-event idempotency check). Warning-tier so operators can wire a Telegram + /// rule; if real-world volume turns out noisy, downgrade or remove — the timeline event + /// on the session is the durable record. + /// + public Task RecordAgentEmergencyBreakAsync(string tenantId, string sessionId, string? agentVersion, string message) + => WriteAsync(OpsEventCategory.Agent, "AgentEmergencyBreak", OpsEventSeverity.Warning, + $"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}): {message}", + tenantId, "System.EmergencyChannel", new { sessionId, agentVersion }); + public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold) => WriteAsync(OpsEventCategory.Agent, "ExcessiveSessionEvents", OpsEventSeverity.Warning, $"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug", diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index 80a10d13..83799d7b 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -95,7 +95,17 @@ const OPS_EVENT_TYPES: Record = { "TenantTrialExpiring", "TenantTrialExpired", ], - Agent: ["BlobStorageMissing", "BlobStorageUnreachable", "NewImeVersionDetected", "ExcessiveSessionEvents"], + Agent: [ + "BlobStorageMissing", + "BlobStorageUnreachable", + "NewImeVersionDetected", + "ExcessiveSessionEvents", + // 48h session-age emergency break reported over the agent's emergency channel — the + // "are we silently losing agents?" signal. Emitted once per session by + // ReportAgentErrorFunction (RecordAgentEmergencyBreakAsync). Dual-register per memory + // feedback_ops_event_types_dual_register. + "AgentEmergencyBreak", + ], }; const SEVERITIES = ["Info", "Warning", "Error", "Critical"];