mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 04:32:32 +00:00
Emergency break reports the session age; a break without a session row is Info
The agent measured the session age at its 48 h break but never sent it. AgentErrorReport now carries SessionAgeHours (optional, null for older agents), and the backend puts it into the ops details, the timeline data and the App Insights dimension. It is information only: the backend cannot derive it (no row, or a row registered seconds before the break after a hibernate resume), and it never feeds the severity. The backend now reads the Sessions row once for the owner binding and the break verdict. A report for a session without a row (in practice one that never registered) is Info and writes no timeline event and no EventType index row: nothing can close or open that session, and the orphan sweep deleted the event a day later with a Warning of its own. A row under cascade delete gets nothing written past the lock. Terminal stays Info with lateCleanup; an open status or a failed read stays Warning with the historical write. Break materialization now requires GUID ids. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+1
-1
Submodule .claude/commands updated: 313fe61e98...88fa1b0274
+1
-1
Submodule internal updated: 8f1f02e681...efec98979a
+18
@@ -20,6 +20,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
private sealed class FakeApiClient : BackendApiClient
|
||||
{
|
||||
public readonly List<TimeSpan?> Timeouts = new List<TimeSpan?>();
|
||||
public readonly List<AgentErrorReport> Reports = new List<AgentErrorReport>();
|
||||
public readonly Queue<bool> Results = new Queue<bool>();
|
||||
|
||||
public FakeApiClient() : base(
|
||||
@@ -40,6 +41,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
public override Task<bool> ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null)
|
||||
{
|
||||
Timeouts.Add(timeout);
|
||||
Reports.Add(report);
|
||||
return Task.FromResult(Results.Count > 0 && Results.Dequeue());
|
||||
}
|
||||
}
|
||||
@@ -139,5 +141,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_age_is_carried_in_the_report_and_absent_by_default()
|
||||
{
|
||||
// One reporter per report: a delivered report starts the 10-minute cooldown.
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m", sessionAgeHours: 478.2);
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
|
||||
Assert.Equal(2, client.Reports.Count);
|
||||
Assert.Equal(478.2, client.Reports[0].SessionAgeHours);
|
||||
Assert.Null(client.Reports[1].SessionAgeHours);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
persistence.SaveSessionCreatedAt(DateTime.UtcNow.AddHours(-100));
|
||||
|
||||
var stateDir = Path.Combine(tmp.Path, "State");
|
||||
double? reportedAgeHours = null;
|
||||
|
||||
var tripped = AutopilotMonitor.Agent.V2.Program.CheckSessionAgeEmergencyBreak(
|
||||
dataDirectory: tmp.Path,
|
||||
@@ -157,9 +158,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
selfDestructOnComplete: false,
|
||||
cleanupServiceFactory: null,
|
||||
logger: logger,
|
||||
consoleMode: false);
|
||||
consoleMode: false,
|
||||
onBreakFired: ageHours => reportedAgeHours = ageHours);
|
||||
|
||||
Assert.True(tripped);
|
||||
// The measured age reaches the break report (the backend cannot derive it).
|
||||
Assert.NotNull(reportedAgeHours);
|
||||
Assert.InRange(reportedAgeHours.Value, 99.9, 100.1);
|
||||
// Marker must have been written so the next restart exits cleanly.
|
||||
Assert.True(File.Exists(Path.Combine(stateDir, "enrollment-complete.marker")));
|
||||
// Session must have been cleared.
|
||||
|
||||
@@ -79,6 +79,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// exist for callers whose process is about to exit and cannot come back later
|
||||
/// (emergency break): retries stay INSIDE the one reservation, so they never
|
||||
/// multiply against the per-session budget.
|
||||
///
|
||||
/// <paramref name="sessionAgeHours"/> is the emergency break's measured session age
|
||||
/// (<see cref="AgentErrorReport.SessionAgeHours"/>); null for every other error type.
|
||||
/// </summary>
|
||||
public virtual async Task TrySendAsync(
|
||||
AgentErrorType errorType,
|
||||
@@ -87,7 +90,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
long? sequenceNumber = null,
|
||||
int attempts = 1,
|
||||
TimeSpan? perAttemptTimeout = null,
|
||||
TimeSpan? retryDelay = null)
|
||||
TimeSpan? retryDelay = null,
|
||||
double? sessionAgeHours = null)
|
||||
{
|
||||
// Deduplicate by error type + status code: same failure category sent only once per session.
|
||||
// All anti-flood checks are protected by a lock to prevent race conditions when
|
||||
@@ -135,6 +139,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
Message = message,
|
||||
HttpStatusCode = httpStatusCode,
|
||||
SequenceNumber = sequenceNumber,
|
||||
SessionAgeHours = sessionAgeHours,
|
||||
AgentVersion = _agentVersion,
|
||||
Timestamp = DateTime.UtcNow
|
||||
};
|
||||
|
||||
@@ -308,7 +308,7 @@ namespace AutopilotMonitor.Agent.V2
|
||||
Func<CleanupService> cleanupServiceFactory,
|
||||
AgentLogger logger,
|
||||
bool consoleMode,
|
||||
Action onBreakFired = null)
|
||||
Action<double> onBreakFired = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -347,7 +347,7 @@ namespace AutopilotMonitor.Agent.V2
|
||||
// (tasks/enrollment-status-reclassification.md). Fired BEFORE cleanup, while the
|
||||
// session state and network are still intact. Never throws — a send failure (e.g. no
|
||||
// network) must not block the cleanup/exit that is the whole point of this guard.
|
||||
try { onBreakFired?.Invoke(); }
|
||||
try { onBreakFired?.Invoke(sessionAgeHours); }
|
||||
catch (Exception cbEx) { logger.Debug($"Emergency-break notify callback failed: {cbEx.Message}"); }
|
||||
|
||||
// Write enrollment-complete.marker so the next start exits cleanly even if
|
||||
|
||||
@@ -87,7 +87,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
cleanupServiceFactory, logger, consoleMode,
|
||||
// Best-effort: surface the otherwise-silent 48h break to the backend before cleanup
|
||||
// (tasks/enrollment-status-reclassification.md). Never blocks the exit.
|
||||
onBreakFired: () => EmergencyBreakReporter.TrySend(agentConfig, Program.GetAgentVersion(), logger)))
|
||||
onBreakFired: sessionAgeHours => EmergencyBreakReporter.TrySend(
|
||||
agentConfig, Program.GetAgentVersion(), sessionAgeHours, logger)))
|
||||
{
|
||||
logger.Info("Emergency break fired — agent exiting.");
|
||||
return BootstrapResult.Exit(0);
|
||||
|
||||
@@ -36,15 +36,18 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// Upper bound for the blocking wait: NIC wait + 3×15s attempts + 2×5s delays + slack.
|
||||
private static readonly TimeSpan OverallBudget = TimeSpan.FromSeconds(75);
|
||||
|
||||
public static void TrySend(AgentConfiguration agentConfig, string agentVersion, AgentLogger logger)
|
||||
public static void TrySend(AgentConfiguration agentConfig, string agentVersion, double sessionAgeHours, AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
WaitForNetwork(logger);
|
||||
|
||||
var auth = BackendClientFactory.BuildAuthClients(agentConfig, agentVersion, logger);
|
||||
var message =
|
||||
$"Agent absolute session-age emergency break fired (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting.";
|
||||
// Invariant: the text lands in the backend's ops feed and timeline; a device locale
|
||||
// would otherwise print "478,2h".
|
||||
var roundedAgeHours = Math.Round(sessionAgeHours, 1);
|
||||
var message = FormattableString.Invariant(
|
||||
$"Agent absolute session-age emergency break fired at session age {roundedAgeHours:F1}h (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting.");
|
||||
|
||||
// The process is about to exit, so block on the otherwise fire-and-forget send
|
||||
// rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP
|
||||
@@ -56,7 +59,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
message,
|
||||
attempts: SendAttempts,
|
||||
perAttemptTimeout: PerAttemptTimeout,
|
||||
retryDelay: RetryDelay)
|
||||
retryDelay: RetryDelay,
|
||||
sessionAgeHours: roundedAgeHours)
|
||||
.Wait(OverallBudget);
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -4,9 +4,11 @@ using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Ingest;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
@@ -94,14 +96,29 @@ public class ReportAgentErrorFunctionTests
|
||||
// (2026-07-23 hardening): timeline event + cross-session EventType index row +
|
||||
// AgentEmergencyBreak ops event, once per session. The index upsert is load-bearing:
|
||||
// StoreEventsBatchAsync alone leaves the event invisible to every search-by-eventType
|
||||
// surface (found in the 2026-07-22 incident analysis).
|
||||
// surface (found in the 2026-07-22 incident analysis). A session without a writable
|
||||
// row gets the ops event only.
|
||||
// =========================================================================
|
||||
|
||||
private static TableEntity Row(SessionStatus? status = SessionStatus.InProgress, string? deletionState = null)
|
||||
{
|
||||
var row = new TableEntity(TenantId, SessionId);
|
||||
if (status.HasValue) row["Status"] = status.Value.ToString();
|
||||
if (deletionState != null) row["DeletionState"] = deletionState;
|
||||
return row;
|
||||
}
|
||||
|
||||
private static ReportAgentErrorFunction.SessionRowRead Read(TableEntity? row) => new(row, Failed: false);
|
||||
|
||||
private static readonly ReportAgentErrorFunction.SessionRowRead ReadFailed = new(null, Failed: true);
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public readonly Mock<ISessionRepository> SessionRepo = new();
|
||||
public readonly List<OpsEventEntry> OpsEvents = new();
|
||||
public readonly OpsEventService OpsService;
|
||||
private readonly SessionDeletionGuard _deletionGuard = new(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(), NullLogger<SessionDeletionGuard>.Instance);
|
||||
|
||||
public Harness(List<EnrollmentEvent>? existingEvents = null)
|
||||
{
|
||||
@@ -125,9 +142,23 @@ public class ReportAgentErrorFunctionTests
|
||||
OpsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
}
|
||||
|
||||
/// <summary>Default read: an existing, open session row — the historical path.</summary>
|
||||
public Task RunAsync(AgentErrorReport report)
|
||||
=> RunAsync(report, Read(Row()));
|
||||
|
||||
public Task RunAsync(AgentErrorReport report, ReportAgentErrorFunction.SessionRowRead? sessionRead)
|
||||
=> ReportAgentErrorFunction.MaterializeEmergencyBreakArtifactsAsync(
|
||||
report, TenantId, SessionRepo.Object, OpsService, NullLogger<ReportAgentErrorFunction>.Instance);
|
||||
report, TenantId, sessionRead, SessionRepo.Object, _deletionGuard, OpsService,
|
||||
NullLogger<ReportAgentErrorFunction>.Instance);
|
||||
|
||||
public void VerifyNothingWrittenIntoTheSession()
|
||||
{
|
||||
SessionRepo.Verify(r => r.GetSessionEventsAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>()), Times.Never);
|
||||
SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Never);
|
||||
SessionRepo.Verify(r => r.UpsertEventTypeIndexBatchAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<IEnumerable<EnrollmentEvent>>()), Times.Never);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -165,7 +196,7 @@ public class ReportAgentErrorFunctionTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Non_break_error_types_and_missing_session_id_are_ignored()
|
||||
public async Task Non_break_error_types_and_ids_without_a_row_read_are_ignored()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
@@ -173,12 +204,10 @@ public class ReportAgentErrorFunctionTests
|
||||
other.ErrorType = AgentErrorType.ConfigFetchFailed;
|
||||
await h.RunAsync(other);
|
||||
|
||||
var noSession = Report(Now);
|
||||
noSession.SessionId = null!;
|
||||
await h.RunAsync(noSession);
|
||||
// No read = the ids were not GUIDs (ReadSessionRowAsync) — no session can exist to break.
|
||||
await h.RunAsync(Report(Now), sessionRead: null);
|
||||
|
||||
h.SessionRepo.Verify(r => r.GetSessionEventsAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>()), Times.Never);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
Assert.Empty(h.OpsEvents);
|
||||
}
|
||||
|
||||
@@ -192,10 +221,8 @@ public class ReportAgentErrorFunctionTests
|
||||
public async Task Break_on_already_terminal_session_is_info_late_cleanup(SessionStatus status)
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ReturnsAsync(new SessionSummary { SessionId = SessionId, TenantId = TenantId, Status = status });
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
await h.RunAsync(Report(Now), Read(Row(status)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
@@ -216,44 +243,131 @@ public class ReportAgentErrorFunctionTests
|
||||
public async Task Break_on_open_session_stays_warning(SessionStatus status)
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ReturnsAsync(new SessionSummary { SessionId = SessionId, TenantId = TenantId, Status = status });
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
await h.RunAsync(Report(Now), Read(Row(status)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains($"still {status}", ops.Message);
|
||||
Assert.Contains("\"lateCleanup\":false", ops.Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Once);
|
||||
}
|
||||
|
||||
// 2026-09-28 sweep: every "unknown" Warning named a session that never registered. The break is
|
||||
// the only thing the backend ever hears of it — nothing to close, and a timeline event would be
|
||||
// an orphan the maintenance sweep deletes a day later (with a Warning of its own).
|
||||
[Fact]
|
||||
public async Task Break_on_session_without_row_is_info_and_writes_nothing_into_the_session()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(null));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
Assert.Contains("no session row", ops.Message);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"missing\"", ops.Details);
|
||||
Assert.Contains("\"lateCleanup\":false", ops.Details);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Preparing)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Queued)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Running)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Poisoned)]
|
||||
public async Task Break_on_session_under_cascade_delete_writes_nothing_past_the_lock(string deletionState)
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(Row(SessionStatus.InProgress, deletionState)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"deleting\"", ops.Details);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Unknown_or_unreadable_session_status_keeps_warning()
|
||||
public async Task Unreadable_row_keeps_the_historical_warning_and_write()
|
||||
{
|
||||
// No row (Harness default: GetSessionAsync → null) — the historical Warning semantics.
|
||||
// Existence unknown — fail safe: the break may be the only signal for an open session.
|
||||
var h = new Harness();
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
await h.RunAsync(Report(Now), ReadFailed);
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"unknown\"", ops.Details);
|
||||
|
||||
// Row read throws — the break must still be recorded, still as Warning.
|
||||
var h2 = new Harness();
|
||||
h2.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ThrowsAsync(new InvalidOperationException("table down"));
|
||||
await h2.RunAsync(Report(Now));
|
||||
Assert.Equal(OpsEventSeverity.Warning, Assert.Single(h2.OpsEvents).Severity);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClassifyEmergencyBreak_is_terminal_set_only()
|
||||
public async Task Row_without_parseable_status_stays_warning()
|
||||
{
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Succeeded));
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Failed));
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Incomplete));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.AwaitingUser));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Unknown));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(null));
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(Row(status: null)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"unknown\"", ops.Details);
|
||||
}
|
||||
|
||||
// The agent-measured age separates a break just past the cap from a device that slept for
|
||||
// weeks; the backend cannot derive it (no row, or a row registered seconds before the break).
|
||||
[Fact]
|
||||
public async Task Reported_session_age_lands_in_ops_details_and_timeline_data()
|
||||
{
|
||||
var h = new Harness();
|
||||
var report = Report(Now);
|
||||
report.SessionAgeHours = 478.2;
|
||||
|
||||
await h.RunAsync(report);
|
||||
|
||||
Assert.Contains("\"sessionAgeHours\":478.2", Assert.Single(h.OpsEvents).Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => (double)e[0].Data["sessionAgeHours"] == 478.2)), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_without_session_age_leaves_it_null_and_out_of_the_timeline()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
Assert.Contains("\"sessionAgeHours\":null", Assert.Single(h.OpsEvents).Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => !e[0].Data.ContainsKey("sessionAgeHours"))), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClassifyEmergencyBreak_follows_row_state()
|
||||
{
|
||||
static ReportAgentErrorFunction.EmergencyBreakVerdict Classify(TableEntity? row, bool failed = false, bool locked = false)
|
||||
=> ReportAgentErrorFunction.ClassifyEmergencyBreak(row, failed, locked);
|
||||
|
||||
foreach (var terminal in new[] { SessionStatus.Succeeded, SessionStatus.Failed, SessionStatus.Incomplete })
|
||||
{
|
||||
var v = Classify(Row(terminal));
|
||||
Assert.Equal((OpsEventSeverity.Info, true, true, terminal.ToString()), (v.Severity, v.LateCleanup, v.Materialize, v.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
foreach (var open in new[] { SessionStatus.InProgress, SessionStatus.Pending, SessionStatus.AwaitingUser, SessionStatus.Stalled, SessionStatus.Unknown })
|
||||
{
|
||||
var v = Classify(Row(open));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false, true, open.ToString()), (v.Severity, v.LateCleanup, v.Materialize, v.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
var missing = Classify(null);
|
||||
Assert.Equal((OpsEventSeverity.Info, false, false, "missing"), (missing.Severity, missing.LateCleanup, missing.Materialize, missing.SessionStatusAtBreak));
|
||||
|
||||
var deleting = Classify(Row(SessionStatus.Succeeded), locked: true);
|
||||
Assert.Equal((OpsEventSeverity.Info, false, false, "deleting"), (deleting.Severity, deleting.LateCleanup, deleting.Materialize, deleting.SessionStatusAtBreak));
|
||||
|
||||
var unreadable = Classify(null, failed: true);
|
||||
Assert.Equal((OpsEventSeverity.Warning, false, true, "unknown"), (unreadable.Severity, unreadable.LateCleanup, unreadable.Materialize, unreadable.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+156
-80
@@ -1,13 +1,16 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.ApplicationInsights;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
@@ -41,6 +44,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly OpsEventService _opsEventService;
|
||||
private readonly Services.Deletion.ISessionDeletionInventoryReader _sessionRowReader;
|
||||
private readonly SessionDeletionGuard _deletionGuard;
|
||||
private readonly SessionOwnerBindingObserver _ownerBinding;
|
||||
|
||||
public ReportAgentErrorFunction(
|
||||
@@ -58,6 +62,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
ISessionRepository sessionRepo,
|
||||
OpsEventService opsEventService,
|
||||
Services.Deletion.ISessionDeletionInventoryReader sessionRowReader,
|
||||
SessionDeletionGuard deletionGuard,
|
||||
SessionOwnerBindingObserver ownerBinding)
|
||||
{
|
||||
_logger = logger;
|
||||
@@ -74,6 +79,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
_sessionRepo = sessionRepo;
|
||||
_opsEventService = opsEventService;
|
||||
_sessionRowReader = sessionRowReader;
|
||||
_deletionGuard = deletionGuard;
|
||||
_ownerBinding = ownerBinding;
|
||||
}
|
||||
|
||||
@@ -184,16 +190,20 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
["AgentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["Message"] = report.Message ?? string.Empty,
|
||||
["AgentTimestamp"] = report.Timestamp.ToString("O"),
|
||||
["SessionAgeHours"] = report.SessionAgeHours?.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
});
|
||||
|
||||
// One Sessions-row point-read serves the owner binding and the emergency-break verdict.
|
||||
var sessionRead = await ReadSessionRowAsync(tenantId, report.SessionId);
|
||||
|
||||
// SESSION-OWNER-BINDING: the emergency-break path below writes timeline events into
|
||||
// whatever session the report names (one point-read — this function has no guard row
|
||||
// in hand; no stamping from this path). A report naming a session bound to another
|
||||
// device identity keeps its device-scoped diagnostics above and the binary-integrity
|
||||
// check below, but must not write into that session. Still 200: the channel is never
|
||||
// retried, and the agent ignores its status.
|
||||
if (await IsSessionOwnerAllowedAsync(req, tenantId, report.SessionId, validation))
|
||||
await MaterializeEmergencyBreakArtifactsAsync(report, tenantId, _sessionRepo, _opsEventService, _logger);
|
||||
// whatever session the report names (no stamping from this path). A report naming a
|
||||
// session bound to another device identity keeps its device-scoped diagnostics above
|
||||
// and the binary-integrity check below, but must not write into that session. Still
|
||||
// 200: the channel is never retried, and the agent ignores its status.
|
||||
if (IsSessionOwnerAllowed(req, tenantId, report.SessionId, sessionRead, validation))
|
||||
await MaterializeEmergencyBreakArtifactsAsync(
|
||||
report, tenantId, sessionRead, _sessionRepo, _deletionGuard, _opsEventService, _logger);
|
||||
|
||||
var adminConfig = await _adminConfigService.GetConfigurationAsync();
|
||||
await MaterializeIntegrityMismatchAsync(
|
||||
@@ -204,18 +214,42 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// False only when the owner binding refuses the caller for the named session. Ids that are
|
||||
/// not GUIDs have no row to compare against; a failed row read fails open (our defect is
|
||||
/// not evidence of a foreign device).
|
||||
/// Outcome of the one Sessions-row point-read: <see cref="Row"/> is null when the row does
|
||||
/// not exist, <see cref="Failed"/> marks a read that threw (existence unknown).
|
||||
/// </summary>
|
||||
private async Task<bool> IsSessionOwnerAllowedAsync(HttpRequestData req, string tenantId, string? sessionId, SecurityValidationResult validation)
|
||||
internal readonly record struct SessionRowRead(TableEntity? Row, bool Failed);
|
||||
|
||||
/// <summary>
|
||||
/// Null when the ids are not GUIDs: no row can exist for them, so neither the owner binding
|
||||
/// nor the emergency break has anything to act on.
|
||||
/// </summary>
|
||||
private async Task<SessionRowRead?> ReadSessionRowAsync(string tenantId, string? sessionId)
|
||||
{
|
||||
if (!SecurityValidator.IsValidGuid(sessionId) || !SecurityValidator.IsValidGuid(tenantId))
|
||||
return null;
|
||||
try
|
||||
{
|
||||
return new SessionRowRead(await _sessionRowReader.GetSessionRowAsync(tenantId, sessionId!), Failed: false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "ReportAgentError: session row read failed for session {SessionId}", sessionId);
|
||||
return new SessionRowRead(null, Failed: true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// False only when the owner binding refuses the caller for the named session. Ids that are
|
||||
/// not GUIDs have no row to compare against; a failed row read or evaluation fails open (our
|
||||
/// defect is not evidence of a foreign device).
|
||||
/// </summary>
|
||||
private bool IsSessionOwnerAllowed(HttpRequestData req, string tenantId, string? sessionId, SessionRowRead? sessionRead, SecurityValidationResult validation)
|
||||
{
|
||||
if (sessionRead is not { Failed: false } read)
|
||||
return true;
|
||||
try
|
||||
{
|
||||
var row = await _sessionRowReader.GetSessionRowAsync(tenantId, sessionId!);
|
||||
return !_ownerBinding.Observe(req, tenantId, sessionId!, row, validation, "agent/error").Rejected;
|
||||
return !_ownerBinding.Observe(req, tenantId, sessionId!, read.Row, validation, "agent/error").Rejected;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -304,74 +338,65 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
/// Materializes the agent's silent 48h emergency break as (1) a timeline event so it shows
|
||||
/// in the session and the timeout classifier can see it, (2) the cross-session EventType
|
||||
/// index row, and (3) an <c>AgentEmergencyBreak</c> ops event for operator visibility
|
||||
/// (tasks/enrollment-status-reclassification.md). Static seam with explicit dependencies so
|
||||
/// tests can pin the artifact set without booting the Functions HTTP stack.
|
||||
/// Best-effort — a failure here must never turn the always-200 emergency channel into a
|
||||
/// retry loop.
|
||||
/// (tasks/enrollment-status-reclassification.md). (1) and (2) only when the session has a
|
||||
/// writable row (<see cref="ClassifyEmergencyBreak"/>); the ops event always, because the
|
||||
/// feed counts breaks. Static seam with explicit dependencies so tests can pin the artifact
|
||||
/// set without booting the Functions HTTP stack. Best-effort — a failure here must never
|
||||
/// turn the always-200 emergency channel into a retry loop.
|
||||
/// </summary>
|
||||
internal static async Task MaterializeEmergencyBreakArtifactsAsync(
|
||||
AgentErrorReport report,
|
||||
string tenantId,
|
||||
SessionRowRead? sessionRead,
|
||||
ISessionRepository sessionRepo,
|
||||
SessionDeletionGuard deletionGuard,
|
||||
OpsEventService opsEventService,
|
||||
ILogger logger)
|
||||
{
|
||||
if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak
|
||||
|| string.IsNullOrEmpty(report.SessionId))
|
||||
// No read means the ids are not GUIDs — no session can exist to break.
|
||||
if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak || sessionRead is not { } read)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000);
|
||||
// Idempotency: the agent's emergency channel can send up to a few reports per session;
|
||||
// only ever materialize one timeline event (and one ops event).
|
||||
var alreadyMaterialized = existing.Any(e =>
|
||||
string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase));
|
||||
if (alreadyMaterialized)
|
||||
var verdict = ClassifyEmergencyBreak(read.Row, read.Failed, IsDeletionLocked(read.Row, deletionGuard));
|
||||
var message = BreakMessage(report);
|
||||
|
||||
if (verdict.Materialize)
|
||||
{
|
||||
return;
|
||||
var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000);
|
||||
// Idempotency: the agent's emergency channel can send up to a few reports per
|
||||
// session; only ever materialize one timeline event (and one ops event).
|
||||
var alreadyMaterialized = existing.Any(e =>
|
||||
string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase));
|
||||
if (alreadyMaterialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow);
|
||||
await sessionRepo.StoreEventsBatchAsync(new List<EnrollmentEvent> { evt });
|
||||
|
||||
// StoreEventsBatchAsync writes the Events partition only — the cross-session
|
||||
// EventType index is normally written by EventIngestProcessor, which this
|
||||
// backend-materialized event never passes through. Without the upsert the event
|
||||
// exists on the session timeline but is invisible to every search-by-eventType
|
||||
// surface (portal cross-session search, MCP search_sessions_by_event /
|
||||
// query_raw_events) — found the hard way in the 2026-07-22 incident analysis.
|
||||
await sessionRepo.UpsertEventTypeIndexBatchAsync(
|
||||
tenantId, report.SessionId, new List<EnrollmentEvent> { evt });
|
||||
}
|
||||
|
||||
var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow);
|
||||
await sessionRepo.StoreEventsBatchAsync(new List<EnrollmentEvent> { evt });
|
||||
|
||||
// StoreEventsBatchAsync writes the Events partition only — the cross-session
|
||||
// EventType index is normally written by EventIngestProcessor, which this
|
||||
// backend-materialized event never passes through. Without the upsert the event
|
||||
// exists on the session timeline but is invisible to every search-by-eventType
|
||||
// surface (portal cross-session search, MCP search_sessions_by_event /
|
||||
// query_raw_events) — found the hard way in the 2026-07-22 incident analysis.
|
||||
await sessionRepo.UpsertEventTypeIndexBatchAsync(
|
||||
tenantId, report.SessionId, new List<EnrollmentEvent> { evt });
|
||||
|
||||
// Operator visibility: an emergency break means an agent silently gave up at its
|
||||
// absolute age cap — exactly the "are we losing agents?" signal. Emitted only on
|
||||
// first materialization so repeat reports for the same session cannot flood the
|
||||
// ops feed. OpsEventService never throws.
|
||||
//
|
||||
// Severity follows what the platform already knows about the session (ops audit
|
||||
// 2026-09-03): the break is wall-clock, so a device that was powered off for weeks
|
||||
// fires it on its first boot — long after the sweep reconciled the session. That
|
||||
// break is a late cleanup, not news; only a break on a still-open session is the
|
||||
// decision itself (classifier rule 3). The row read is best-effort: unknown status
|
||||
// keeps the historical Warning.
|
||||
SessionStatus? statusAtBreak = null;
|
||||
try
|
||||
{
|
||||
statusAtBreak = (await sessionRepo.GetSessionAsync(tenantId, report.SessionId))?.Status;
|
||||
}
|
||||
catch (Exception readEx)
|
||||
{
|
||||
logger.LogDebug(readEx,
|
||||
"ReportAgentError: session status read failed for emergency-break severity on {SessionId}", report.SessionId);
|
||||
}
|
||||
|
||||
var verdict = ClassifyEmergencyBreak(statusAtBreak);
|
||||
// absolute age cap — exactly the "are we losing agents?" signal. For a session with
|
||||
// a row it is emitted only on first materialization, so repeat reports cannot flood
|
||||
// the feed. OpsEventService never throws.
|
||||
await opsEventService.RecordAgentEmergencyBreakAsync(
|
||||
tenantId, report.SessionId, report.AgentVersion, evt.Message,
|
||||
verdict.Severity, statusAtBreak, verdict.LateCleanup);
|
||||
tenantId, report.SessionId, report.AgentVersion, message,
|
||||
verdict.Severity, verdict.SessionStatusAtBreak, verdict.LateCleanup, verdict.Context,
|
||||
report.SessionAgeHours);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -381,17 +406,66 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ops-event verdict for an emergency break, keyed on the session status the platform holds
|
||||
/// when the report arrives. Terminal (Succeeded / Failed / Incomplete) = the sweep already
|
||||
/// decided the enrollment and the agent merely cleaned up late (device powered on after a
|
||||
/// shelf period) — Info, <c>LateCleanup=true</c>. Open or unknown = the break is the first
|
||||
/// and only signal that nothing more will arrive — Warning, the historical semantics.
|
||||
/// The cascade-delete writer-block invariant, checked at its chokepoint on the row already in
|
||||
/// hand (no second read). A missing row needs no tombstone check: nothing is written for it.
|
||||
/// </summary>
|
||||
private static bool IsDeletionLocked(TableEntity? sessionRow, SessionDeletionGuard deletionGuard)
|
||||
{
|
||||
try
|
||||
{
|
||||
deletionGuard.ThrowIfLocked(sessionRow, "V2.ReportAgentError.EmergencyBreak");
|
||||
return false;
|
||||
}
|
||||
catch (SessionDeletionLockedException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>What the ops event says and whether the break is written into the session.</summary>
|
||||
internal readonly record struct EmergencyBreakVerdict(
|
||||
string Severity, bool LateCleanup, bool Materialize, string SessionStatusAtBreak, string Context);
|
||||
|
||||
/// <summary>
|
||||
/// Ops-event verdict for an emergency break, keyed on what the platform holds for the session
|
||||
/// when the report arrives. The break is wall-clock and checked at agent start, so it usually
|
||||
/// arrives long after the fact:
|
||||
/// <list type="bullet">
|
||||
/// <item>Terminal (Succeeded / Failed / Incomplete): the sweep already decided the enrollment
|
||||
/// and the agent merely cleaned up late after a shelf period — Info, <c>LateCleanup=true</c>
|
||||
/// (ops audit 2026-09-03).</item>
|
||||
/// <item>No row: in practice a session that never registered (the agent went dark before
|
||||
/// register-session) — nothing to close and nobody who could open it. Info, and no
|
||||
/// timeline event or index row: the orphan sweep would delete the event a day later with
|
||||
/// a Warning of its own, and the index row would point at a session that does not
|
||||
/// exist.</item>
|
||||
/// <item>Row under cascade delete: nothing may be written past the lock — Info.</item>
|
||||
/// <item>Open or unparseable status, or a failed read: the break is the first and only
|
||||
/// signal that nothing more will arrive (classifier rule 3) — Warning, and a failed read
|
||||
/// keeps the historical write.</item>
|
||||
/// </list>
|
||||
/// Pure so the rule is unit-testable.
|
||||
/// </summary>
|
||||
internal static (string Severity, bool LateCleanup) ClassifyEmergencyBreak(SessionStatus? statusAtBreak)
|
||||
=> statusAtBreak.HasValue && Helpers.DeviceJourneyCalculator.IsTerminal(statusAtBreak.Value)
|
||||
? (OpsEventSeverity.Info, true)
|
||||
: (OpsEventSeverity.Warning, false);
|
||||
internal static EmergencyBreakVerdict ClassifyEmergencyBreak(TableEntity? sessionRow, bool rowReadFailed, bool deletionLocked)
|
||||
{
|
||||
if (rowReadFailed)
|
||||
return new(OpsEventSeverity.Warning, false, true, "unknown", "session row unreadable when the break arrived");
|
||||
if (sessionRow == null)
|
||||
return new(OpsEventSeverity.Info, false, false, "missing", "no session row (never registered, or deleted long ago) — nothing to close");
|
||||
if (deletionLocked)
|
||||
return new(OpsEventSeverity.Info, false, false, "deleting", "session is being deleted — nothing written");
|
||||
|
||||
var status = IngestTelemetryFunction.TryReadSessionStatus(sessionRow);
|
||||
var statusText = status?.ToString() ?? "unknown";
|
||||
return status.HasValue && Helpers.DeviceJourneyCalculator.IsTerminal(status.Value)
|
||||
? new(OpsEventSeverity.Info, true, true, statusText, $"late cleanup — session already {statusText} when the break arrived")
|
||||
: new(OpsEventSeverity.Warning, false, true, statusText, $"session still {statusText} when the break arrived");
|
||||
}
|
||||
|
||||
private static string BreakMessage(AgentErrorReport report)
|
||||
=> string.IsNullOrWhiteSpace(report.Message)
|
||||
? "Agent absolute session-age emergency break fired — agent cleaned up and exited"
|
||||
: report.Message;
|
||||
|
||||
/// <summary>
|
||||
/// Builds the backend-materialized <c>agent_emergency_break</c> timeline event from the agent's
|
||||
@@ -415,6 +489,15 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
? report.Timestamp.ToUniversalTime()
|
||||
: nowUtc;
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
["source"] = "emergency_channel",
|
||||
["agentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["reportedAtUtc"] = report.Timestamp.ToString("o"),
|
||||
};
|
||||
if (report.SessionAgeHours.HasValue)
|
||||
data["sessionAgeHours"] = report.SessionAgeHours.Value;
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
TenantId = tenantId,
|
||||
@@ -425,15 +508,8 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Timestamp = breakAt,
|
||||
Sequence = maxSequence + 1,
|
||||
Message = string.IsNullOrWhiteSpace(report.Message)
|
||||
? "Agent absolute session-age emergency break fired — agent cleaned up and exited"
|
||||
: report.Message,
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
["source"] = "emergency_channel",
|
||||
["agentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["reportedAtUtc"] = report.Timestamp.ToString("o"),
|
||||
},
|
||||
Message = BreakMessage(report),
|
||||
Data = data,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -954,31 +954,26 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
/// <summary>
|
||||
/// An agent reported its absolute session-age emergency break (48h cap) over the
|
||||
/// emergency channel — it cleaned itself up and exited on a session that never reached
|
||||
/// a terminal state. This is the "are we silently losing agents?" signal; emitted by
|
||||
/// <see cref="Functions.Ingest.ReportAgentErrorFunction"/> once per session (guarded by
|
||||
/// the timeline-event idempotency check). Severity is decided by the caller from the
|
||||
/// session status at break time (<c>ReportAgentErrorFunction.ClassifyEmergencyBreak</c>):
|
||||
/// Warning when the session was still open (the break IS the verdict — Telegram-rule
|
||||
/// worthy), Info when the sweep had already terminalized it and the break is a late
|
||||
/// cleanup after the device was powered on again. The 2026-09-03 audit found 224 of 269
|
||||
/// breaks in three days were such late cleanups (kiosk devices stored for weeks); as
|
||||
/// Warnings they buried the real ones. <paramref name="statusAtBreak"/> lands in the
|
||||
/// details as <c>sessionStatusAtBreak</c> so the feed shows which case it was.
|
||||
/// emergency channel — it cleaned itself up and exited. This is the "are we silently
|
||||
/// losing agents?" signal; emitted by <see cref="Functions.Ingest.ReportAgentErrorFunction"/>
|
||||
/// once per break report (for a session with a row, guarded by the timeline-event
|
||||
/// idempotency check). Severity, <paramref name="sessionStatusAtBreak"/> and
|
||||
/// <paramref name="context"/> come from <c>ReportAgentErrorFunction.ClassifyEmergencyBreak</c>:
|
||||
/// Warning only when the session was still open (the break IS the verdict — Telegram-rule
|
||||
/// worthy) or its row could not be read. The 2026-09-03 audit found 224 of 269 breaks in
|
||||
/// three days were late cleanups on terminal sessions, the 2026-09-28 sweep that 115 of 160
|
||||
/// remaining Warnings named sessions that never registered; as Warnings both buried the real
|
||||
/// ones. <paramref name="sessionAgeHours"/> is the agent-measured age (null for agents that
|
||||
/// predate the field) — it separates a break just past the cap from a device that slept for
|
||||
/// weeks, which the backend cannot tell from its own rows.
|
||||
/// </summary>
|
||||
public Task RecordAgentEmergencyBreakAsync(
|
||||
string tenantId, string sessionId, string? agentVersion, string message,
|
||||
string severity, SessionStatus? statusAtBreak, bool lateCleanup)
|
||||
{
|
||||
var statusText = statusAtBreak?.ToString() ?? "unknown";
|
||||
var context = lateCleanup
|
||||
? $"late cleanup — session already {statusText} when the break arrived"
|
||||
: $"session still {statusText} when the break arrived";
|
||||
return WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
|
||||
string severity, string sessionStatusAtBreak, bool lateCleanup, string context, double? sessionAgeHours)
|
||||
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
|
||||
$"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}, {context}): {message}",
|
||||
tenantId, "System.EmergencyChannel",
|
||||
new { sessionId, agentVersion, sessionStatusAtBreak = statusText, lateCleanup });
|
||||
}
|
||||
new { sessionId, agentVersion, sessionStatusAtBreak, lateCleanup, sessionAgeHours });
|
||||
|
||||
/// <summary>
|
||||
/// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend
|
||||
|
||||
@@ -81,6 +81,15 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public long? SequenceNumber { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Session age in hours the agent measured when its absolute session-age emergency break
|
||||
/// fired (<see cref="AgentErrorType.SessionAgeEmergencyBreak"/> only). The backend cannot
|
||||
/// derive it: a never-registered session has no row, and a registered one may have
|
||||
/// registered seconds before the break (hibernate resume). Null for other error types and
|
||||
/// for agents that predate the field.
|
||||
/// </summary>
|
||||
public double? SessionAgeHours { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Agent version string for diagnosing version-specific issues.
|
||||
/// </summary>
|
||||
|
||||
Reference in New Issue
Block a user