mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 13:10:13 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| edf30d35e0 | |||
| 3b4228e3a1 |
+1
-1
Submodule .claude/commands updated: 313fe61e98...88fa1b0274
+1
-1
Submodule internal updated: df3667d672...efec98979a
+18
@@ -20,6 +20,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
private sealed class FakeApiClient : BackendApiClient
|
||||
{
|
||||
public readonly List<TimeSpan?> Timeouts = new List<TimeSpan?>();
|
||||
public readonly List<AgentErrorReport> Reports = new List<AgentErrorReport>();
|
||||
public readonly Queue<bool> Results = new Queue<bool>();
|
||||
|
||||
public FakeApiClient() : base(
|
||||
@@ -40,6 +41,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
public override Task<bool> ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null)
|
||||
{
|
||||
Timeouts.Add(timeout);
|
||||
Reports.Add(report);
|
||||
return Task.FromResult(Results.Count > 0 && Results.Dequeue());
|
||||
}
|
||||
}
|
||||
@@ -139,5 +141,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_age_is_carried_in_the_report_and_absent_by_default()
|
||||
{
|
||||
// One reporter per report: a delivered report starts the 10-minute cooldown.
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m", sessionAgeHours: 478.2);
|
||||
await Reporter(client).TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
|
||||
Assert.Equal(2, client.Reports.Count);
|
||||
Assert.Equal(478.2, client.Reports[0].SessionAgeHours);
|
||||
Assert.Null(client.Reports[1].SessionAgeHours);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
persistence.SaveSessionCreatedAt(DateTime.UtcNow.AddHours(-100));
|
||||
|
||||
var stateDir = Path.Combine(tmp.Path, "State");
|
||||
double? reportedAgeHours = null;
|
||||
|
||||
var tripped = AutopilotMonitor.Agent.V2.Program.CheckSessionAgeEmergencyBreak(
|
||||
dataDirectory: tmp.Path,
|
||||
@@ -157,9 +158,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
selfDestructOnComplete: false,
|
||||
cleanupServiceFactory: null,
|
||||
logger: logger,
|
||||
consoleMode: false);
|
||||
consoleMode: false,
|
||||
onBreakFired: ageHours => reportedAgeHours = ageHours);
|
||||
|
||||
Assert.True(tripped);
|
||||
// The measured age reaches the break report (the backend cannot derive it).
|
||||
Assert.NotNull(reportedAgeHours);
|
||||
Assert.InRange(reportedAgeHours.Value, 99.9, 100.1);
|
||||
// Marker must have been written so the next restart exits cleanly.
|
||||
Assert.True(File.Exists(Path.Combine(stateDir, "enrollment-complete.marker")));
|
||||
// Session must have been cleared.
|
||||
|
||||
@@ -79,6 +79,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// exist for callers whose process is about to exit and cannot come back later
|
||||
/// (emergency break): retries stay INSIDE the one reservation, so they never
|
||||
/// multiply against the per-session budget.
|
||||
///
|
||||
/// <paramref name="sessionAgeHours"/> is the emergency break's measured session age
|
||||
/// (<see cref="AgentErrorReport.SessionAgeHours"/>); null for every other error type.
|
||||
/// </summary>
|
||||
public virtual async Task TrySendAsync(
|
||||
AgentErrorType errorType,
|
||||
@@ -87,7 +90,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
long? sequenceNumber = null,
|
||||
int attempts = 1,
|
||||
TimeSpan? perAttemptTimeout = null,
|
||||
TimeSpan? retryDelay = null)
|
||||
TimeSpan? retryDelay = null,
|
||||
double? sessionAgeHours = null)
|
||||
{
|
||||
// Deduplicate by error type + status code: same failure category sent only once per session.
|
||||
// All anti-flood checks are protected by a lock to prevent race conditions when
|
||||
@@ -135,6 +139,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
Message = message,
|
||||
HttpStatusCode = httpStatusCode,
|
||||
SequenceNumber = sequenceNumber,
|
||||
SessionAgeHours = sessionAgeHours,
|
||||
AgentVersion = _agentVersion,
|
||||
Timestamp = DateTime.UtcNow
|
||||
};
|
||||
|
||||
@@ -308,7 +308,7 @@ namespace AutopilotMonitor.Agent.V2
|
||||
Func<CleanupService> cleanupServiceFactory,
|
||||
AgentLogger logger,
|
||||
bool consoleMode,
|
||||
Action onBreakFired = null)
|
||||
Action<double> onBreakFired = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -347,7 +347,7 @@ namespace AutopilotMonitor.Agent.V2
|
||||
// (tasks/enrollment-status-reclassification.md). Fired BEFORE cleanup, while the
|
||||
// session state and network are still intact. Never throws — a send failure (e.g. no
|
||||
// network) must not block the cleanup/exit that is the whole point of this guard.
|
||||
try { onBreakFired?.Invoke(); }
|
||||
try { onBreakFired?.Invoke(sessionAgeHours); }
|
||||
catch (Exception cbEx) { logger.Debug($"Emergency-break notify callback failed: {cbEx.Message}"); }
|
||||
|
||||
// Write enrollment-complete.marker so the next start exits cleanly even if
|
||||
|
||||
@@ -87,7 +87,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
cleanupServiceFactory, logger, consoleMode,
|
||||
// Best-effort: surface the otherwise-silent 48h break to the backend before cleanup
|
||||
// (tasks/enrollment-status-reclassification.md). Never blocks the exit.
|
||||
onBreakFired: () => EmergencyBreakReporter.TrySend(agentConfig, Program.GetAgentVersion(), logger)))
|
||||
onBreakFired: sessionAgeHours => EmergencyBreakReporter.TrySend(
|
||||
agentConfig, Program.GetAgentVersion(), sessionAgeHours, logger)))
|
||||
{
|
||||
logger.Info("Emergency break fired — agent exiting.");
|
||||
return BootstrapResult.Exit(0);
|
||||
|
||||
@@ -36,15 +36,18 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// Upper bound for the blocking wait: NIC wait + 3×15s attempts + 2×5s delays + slack.
|
||||
private static readonly TimeSpan OverallBudget = TimeSpan.FromSeconds(75);
|
||||
|
||||
public static void TrySend(AgentConfiguration agentConfig, string agentVersion, AgentLogger logger)
|
||||
public static void TrySend(AgentConfiguration agentConfig, string agentVersion, double sessionAgeHours, AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
WaitForNetwork(logger);
|
||||
|
||||
var auth = BackendClientFactory.BuildAuthClients(agentConfig, agentVersion, logger);
|
||||
var message =
|
||||
$"Agent absolute session-age emergency break fired (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting.";
|
||||
// Invariant: the text lands in the backend's ops feed and timeline; a device locale
|
||||
// would otherwise print "478,2h".
|
||||
var roundedAgeHours = Math.Round(sessionAgeHours, 1);
|
||||
var message = FormattableString.Invariant(
|
||||
$"Agent absolute session-age emergency break fired at session age {roundedAgeHours:F1}h (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting.");
|
||||
|
||||
// The process is about to exit, so block on the otherwise fire-and-forget send
|
||||
// rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP
|
||||
@@ -56,7 +59,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
message,
|
||||
attempts: SendAttempts,
|
||||
perAttemptTimeout: PerAttemptTimeout,
|
||||
retryDelay: RetryDelay)
|
||||
retryDelay: RetryDelay,
|
||||
sessionAgeHours: roundedAgeHours)
|
||||
.Wait(OverallBudget);
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -4,9 +4,11 @@ using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Ingest;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
@@ -94,14 +96,29 @@ public class ReportAgentErrorFunctionTests
|
||||
// (2026-07-23 hardening): timeline event + cross-session EventType index row +
|
||||
// AgentEmergencyBreak ops event, once per session. The index upsert is load-bearing:
|
||||
// StoreEventsBatchAsync alone leaves the event invisible to every search-by-eventType
|
||||
// surface (found in the 2026-07-22 incident analysis).
|
||||
// surface (found in the 2026-07-22 incident analysis). A session without a writable
|
||||
// row gets the ops event only.
|
||||
// =========================================================================
|
||||
|
||||
private static TableEntity Row(SessionStatus? status = SessionStatus.InProgress, string? deletionState = null)
|
||||
{
|
||||
var row = new TableEntity(TenantId, SessionId);
|
||||
if (status.HasValue) row["Status"] = status.Value.ToString();
|
||||
if (deletionState != null) row["DeletionState"] = deletionState;
|
||||
return row;
|
||||
}
|
||||
|
||||
private static ReportAgentErrorFunction.SessionRowRead Read(TableEntity? row) => new(row, Failed: false);
|
||||
|
||||
private static readonly ReportAgentErrorFunction.SessionRowRead ReadFailed = new(null, Failed: true);
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public readonly Mock<ISessionRepository> SessionRepo = new();
|
||||
public readonly List<OpsEventEntry> OpsEvents = new();
|
||||
public readonly OpsEventService OpsService;
|
||||
private readonly SessionDeletionGuard _deletionGuard = new(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(), NullLogger<SessionDeletionGuard>.Instance);
|
||||
|
||||
public Harness(List<EnrollmentEvent>? existingEvents = null)
|
||||
{
|
||||
@@ -125,9 +142,23 @@ public class ReportAgentErrorFunctionTests
|
||||
OpsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
}
|
||||
|
||||
/// <summary>Default read: an existing, open session row — the historical path.</summary>
|
||||
public Task RunAsync(AgentErrorReport report)
|
||||
=> RunAsync(report, Read(Row()));
|
||||
|
||||
public Task RunAsync(AgentErrorReport report, ReportAgentErrorFunction.SessionRowRead? sessionRead)
|
||||
=> ReportAgentErrorFunction.MaterializeEmergencyBreakArtifactsAsync(
|
||||
report, TenantId, SessionRepo.Object, OpsService, NullLogger<ReportAgentErrorFunction>.Instance);
|
||||
report, TenantId, sessionRead, SessionRepo.Object, _deletionGuard, OpsService,
|
||||
NullLogger<ReportAgentErrorFunction>.Instance);
|
||||
|
||||
public void VerifyNothingWrittenIntoTheSession()
|
||||
{
|
||||
SessionRepo.Verify(r => r.GetSessionEventsAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>()), Times.Never);
|
||||
SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Never);
|
||||
SessionRepo.Verify(r => r.UpsertEventTypeIndexBatchAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<IEnumerable<EnrollmentEvent>>()), Times.Never);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -165,7 +196,7 @@ public class ReportAgentErrorFunctionTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Non_break_error_types_and_missing_session_id_are_ignored()
|
||||
public async Task Non_break_error_types_and_ids_without_a_row_read_are_ignored()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
@@ -173,12 +204,10 @@ public class ReportAgentErrorFunctionTests
|
||||
other.ErrorType = AgentErrorType.ConfigFetchFailed;
|
||||
await h.RunAsync(other);
|
||||
|
||||
var noSession = Report(Now);
|
||||
noSession.SessionId = null!;
|
||||
await h.RunAsync(noSession);
|
||||
// No read = the ids were not GUIDs (ReadSessionRowAsync) — no session can exist to break.
|
||||
await h.RunAsync(Report(Now), sessionRead: null);
|
||||
|
||||
h.SessionRepo.Verify(r => r.GetSessionEventsAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>()), Times.Never);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
Assert.Empty(h.OpsEvents);
|
||||
}
|
||||
|
||||
@@ -192,10 +221,8 @@ public class ReportAgentErrorFunctionTests
|
||||
public async Task Break_on_already_terminal_session_is_info_late_cleanup(SessionStatus status)
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ReturnsAsync(new SessionSummary { SessionId = SessionId, TenantId = TenantId, Status = status });
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
await h.RunAsync(Report(Now), Read(Row(status)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
@@ -216,44 +243,131 @@ public class ReportAgentErrorFunctionTests
|
||||
public async Task Break_on_open_session_stays_warning(SessionStatus status)
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ReturnsAsync(new SessionSummary { SessionId = SessionId, TenantId = TenantId, Status = status });
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
await h.RunAsync(Report(Now), Read(Row(status)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains($"still {status}", ops.Message);
|
||||
Assert.Contains("\"lateCleanup\":false", ops.Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Once);
|
||||
}
|
||||
|
||||
// 2026-09-28 sweep: every "unknown" Warning named a session that never registered. The break is
|
||||
// the only thing the backend ever hears of it — nothing to close, and a timeline event would be
|
||||
// an orphan the maintenance sweep deletes a day later (with a Warning of its own).
|
||||
[Fact]
|
||||
public async Task Break_on_session_without_row_is_info_and_writes_nothing_into_the_session()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(null));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
Assert.Contains("no session row", ops.Message);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"missing\"", ops.Details);
|
||||
Assert.Contains("\"lateCleanup\":false", ops.Details);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Preparing)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Queued)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Running)]
|
||||
[InlineData(AutopilotMonitor.Shared.Models.Deletion.SessionDeletionState.Poisoned)]
|
||||
public async Task Break_on_session_under_cascade_delete_writes_nothing_past_the_lock(string deletionState)
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(Row(SessionStatus.InProgress, deletionState)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Info, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"deleting\"", ops.Details);
|
||||
h.VerifyNothingWrittenIntoTheSession();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Unknown_or_unreadable_session_status_keeps_warning()
|
||||
public async Task Unreadable_row_keeps_the_historical_warning_and_write()
|
||||
{
|
||||
// No row (Harness default: GetSessionAsync → null) — the historical Warning semantics.
|
||||
// Existence unknown — fail safe: the break may be the only signal for an open session.
|
||||
var h = new Harness();
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
await h.RunAsync(Report(Now), ReadFailed);
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"unknown\"", ops.Details);
|
||||
|
||||
// Row read throws — the break must still be recorded, still as Warning.
|
||||
var h2 = new Harness();
|
||||
h2.SessionRepo.Setup(r => r.GetSessionAsync(TenantId, SessionId))
|
||||
.ThrowsAsync(new InvalidOperationException("table down"));
|
||||
await h2.RunAsync(Report(Now));
|
||||
Assert.Equal(OpsEventSeverity.Warning, Assert.Single(h2.OpsEvents).Severity);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClassifyEmergencyBreak_is_terminal_set_only()
|
||||
public async Task Row_without_parseable_status_stays_warning()
|
||||
{
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Succeeded));
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Failed));
|
||||
Assert.Equal((OpsEventSeverity.Info, true), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Incomplete));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.AwaitingUser));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(SessionStatus.Unknown));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false), ReportAgentErrorFunction.ClassifyEmergencyBreak(null));
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now), Read(Row(status: null)));
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal(OpsEventSeverity.Warning, ops.Severity);
|
||||
Assert.Contains("\"sessionStatusAtBreak\":\"unknown\"", ops.Details);
|
||||
}
|
||||
|
||||
// The agent-measured age separates a break just past the cap from a device that slept for
|
||||
// weeks; the backend cannot derive it (no row, or a row registered seconds before the break).
|
||||
[Fact]
|
||||
public async Task Reported_session_age_lands_in_ops_details_and_timeline_data()
|
||||
{
|
||||
var h = new Harness();
|
||||
var report = Report(Now);
|
||||
report.SessionAgeHours = 478.2;
|
||||
|
||||
await h.RunAsync(report);
|
||||
|
||||
Assert.Contains("\"sessionAgeHours\":478.2", Assert.Single(h.OpsEvents).Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => (double)e[0].Data["sessionAgeHours"] == 478.2)), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_without_session_age_leaves_it_null_and_out_of_the_timeline()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
Assert.Contains("\"sessionAgeHours\":null", Assert.Single(h.OpsEvents).Details);
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => !e[0].Data.ContainsKey("sessionAgeHours"))), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClassifyEmergencyBreak_follows_row_state()
|
||||
{
|
||||
static ReportAgentErrorFunction.EmergencyBreakVerdict Classify(TableEntity? row, bool failed = false, bool locked = false)
|
||||
=> ReportAgentErrorFunction.ClassifyEmergencyBreak(row, failed, locked);
|
||||
|
||||
foreach (var terminal in new[] { SessionStatus.Succeeded, SessionStatus.Failed, SessionStatus.Incomplete })
|
||||
{
|
||||
var v = Classify(Row(terminal));
|
||||
Assert.Equal((OpsEventSeverity.Info, true, true, terminal.ToString()), (v.Severity, v.LateCleanup, v.Materialize, v.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
foreach (var open in new[] { SessionStatus.InProgress, SessionStatus.Pending, SessionStatus.AwaitingUser, SessionStatus.Stalled, SessionStatus.Unknown })
|
||||
{
|
||||
var v = Classify(Row(open));
|
||||
Assert.Equal((OpsEventSeverity.Warning, false, true, open.ToString()), (v.Severity, v.LateCleanup, v.Materialize, v.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
var missing = Classify(null);
|
||||
Assert.Equal((OpsEventSeverity.Info, false, false, "missing"), (missing.Severity, missing.LateCleanup, missing.Materialize, missing.SessionStatusAtBreak));
|
||||
|
||||
var deleting = Classify(Row(SessionStatus.Succeeded), locked: true);
|
||||
Assert.Equal((OpsEventSeverity.Info, false, false, "deleting"), (deleting.Severity, deleting.LateCleanup, deleting.Materialize, deleting.SessionStatusAtBreak));
|
||||
|
||||
var unreadable = Classify(null, failed: true);
|
||||
Assert.Equal((OpsEventSeverity.Warning, false, true, "unknown"), (unreadable.Severity, unreadable.LateCleanup, unreadable.Materialize, unreadable.SessionStatusAtBreak));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+156
-80
@@ -1,13 +1,16 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.ApplicationInsights;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
@@ -41,6 +44,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly OpsEventService _opsEventService;
|
||||
private readonly Services.Deletion.ISessionDeletionInventoryReader _sessionRowReader;
|
||||
private readonly SessionDeletionGuard _deletionGuard;
|
||||
private readonly SessionOwnerBindingObserver _ownerBinding;
|
||||
|
||||
public ReportAgentErrorFunction(
|
||||
@@ -58,6 +62,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
ISessionRepository sessionRepo,
|
||||
OpsEventService opsEventService,
|
||||
Services.Deletion.ISessionDeletionInventoryReader sessionRowReader,
|
||||
SessionDeletionGuard deletionGuard,
|
||||
SessionOwnerBindingObserver ownerBinding)
|
||||
{
|
||||
_logger = logger;
|
||||
@@ -74,6 +79,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
_sessionRepo = sessionRepo;
|
||||
_opsEventService = opsEventService;
|
||||
_sessionRowReader = sessionRowReader;
|
||||
_deletionGuard = deletionGuard;
|
||||
_ownerBinding = ownerBinding;
|
||||
}
|
||||
|
||||
@@ -184,16 +190,20 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
["AgentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["Message"] = report.Message ?? string.Empty,
|
||||
["AgentTimestamp"] = report.Timestamp.ToString("O"),
|
||||
["SessionAgeHours"] = report.SessionAgeHours?.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
});
|
||||
|
||||
// One Sessions-row point-read serves the owner binding and the emergency-break verdict.
|
||||
var sessionRead = await ReadSessionRowAsync(tenantId, report.SessionId);
|
||||
|
||||
// SESSION-OWNER-BINDING: the emergency-break path below writes timeline events into
|
||||
// whatever session the report names (one point-read — this function has no guard row
|
||||
// in hand; no stamping from this path). A report naming a session bound to another
|
||||
// device identity keeps its device-scoped diagnostics above and the binary-integrity
|
||||
// check below, but must not write into that session. Still 200: the channel is never
|
||||
// retried, and the agent ignores its status.
|
||||
if (await IsSessionOwnerAllowedAsync(req, tenantId, report.SessionId, validation))
|
||||
await MaterializeEmergencyBreakArtifactsAsync(report, tenantId, _sessionRepo, _opsEventService, _logger);
|
||||
// whatever session the report names (no stamping from this path). A report naming a
|
||||
// session bound to another device identity keeps its device-scoped diagnostics above
|
||||
// and the binary-integrity check below, but must not write into that session. Still
|
||||
// 200: the channel is never retried, and the agent ignores its status.
|
||||
if (IsSessionOwnerAllowed(req, tenantId, report.SessionId, sessionRead, validation))
|
||||
await MaterializeEmergencyBreakArtifactsAsync(
|
||||
report, tenantId, sessionRead, _sessionRepo, _deletionGuard, _opsEventService, _logger);
|
||||
|
||||
var adminConfig = await _adminConfigService.GetConfigurationAsync();
|
||||
await MaterializeIntegrityMismatchAsync(
|
||||
@@ -204,18 +214,42 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// False only when the owner binding refuses the caller for the named session. Ids that are
|
||||
/// not GUIDs have no row to compare against; a failed row read fails open (our defect is
|
||||
/// not evidence of a foreign device).
|
||||
/// Outcome of the one Sessions-row point-read: <see cref="Row"/> is null when the row does
|
||||
/// not exist, <see cref="Failed"/> marks a read that threw (existence unknown).
|
||||
/// </summary>
|
||||
private async Task<bool> IsSessionOwnerAllowedAsync(HttpRequestData req, string tenantId, string? sessionId, SecurityValidationResult validation)
|
||||
internal readonly record struct SessionRowRead(TableEntity? Row, bool Failed);
|
||||
|
||||
/// <summary>
|
||||
/// Null when the ids are not GUIDs: no row can exist for them, so neither the owner binding
|
||||
/// nor the emergency break has anything to act on.
|
||||
/// </summary>
|
||||
private async Task<SessionRowRead?> ReadSessionRowAsync(string tenantId, string? sessionId)
|
||||
{
|
||||
if (!SecurityValidator.IsValidGuid(sessionId) || !SecurityValidator.IsValidGuid(tenantId))
|
||||
return null;
|
||||
try
|
||||
{
|
||||
return new SessionRowRead(await _sessionRowReader.GetSessionRowAsync(tenantId, sessionId!), Failed: false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "ReportAgentError: session row read failed for session {SessionId}", sessionId);
|
||||
return new SessionRowRead(null, Failed: true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// False only when the owner binding refuses the caller for the named session. Ids that are
|
||||
/// not GUIDs have no row to compare against; a failed row read or evaluation fails open (our
|
||||
/// defect is not evidence of a foreign device).
|
||||
/// </summary>
|
||||
private bool IsSessionOwnerAllowed(HttpRequestData req, string tenantId, string? sessionId, SessionRowRead? sessionRead, SecurityValidationResult validation)
|
||||
{
|
||||
if (sessionRead is not { Failed: false } read)
|
||||
return true;
|
||||
try
|
||||
{
|
||||
var row = await _sessionRowReader.GetSessionRowAsync(tenantId, sessionId!);
|
||||
return !_ownerBinding.Observe(req, tenantId, sessionId!, row, validation, "agent/error").Rejected;
|
||||
return !_ownerBinding.Observe(req, tenantId, sessionId!, read.Row, validation, "agent/error").Rejected;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -304,74 +338,65 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
/// Materializes the agent's silent 48h emergency break as (1) a timeline event so it shows
|
||||
/// in the session and the timeout classifier can see it, (2) the cross-session EventType
|
||||
/// index row, and (3) an <c>AgentEmergencyBreak</c> ops event for operator visibility
|
||||
/// (tasks/enrollment-status-reclassification.md). Static seam with explicit dependencies so
|
||||
/// tests can pin the artifact set without booting the Functions HTTP stack.
|
||||
/// Best-effort — a failure here must never turn the always-200 emergency channel into a
|
||||
/// retry loop.
|
||||
/// (tasks/enrollment-status-reclassification.md). (1) and (2) only when the session has a
|
||||
/// writable row (<see cref="ClassifyEmergencyBreak"/>); the ops event always, because the
|
||||
/// feed counts breaks. Static seam with explicit dependencies so tests can pin the artifact
|
||||
/// set without booting the Functions HTTP stack. Best-effort — a failure here must never
|
||||
/// turn the always-200 emergency channel into a retry loop.
|
||||
/// </summary>
|
||||
internal static async Task MaterializeEmergencyBreakArtifactsAsync(
|
||||
AgentErrorReport report,
|
||||
string tenantId,
|
||||
SessionRowRead? sessionRead,
|
||||
ISessionRepository sessionRepo,
|
||||
SessionDeletionGuard deletionGuard,
|
||||
OpsEventService opsEventService,
|
||||
ILogger logger)
|
||||
{
|
||||
if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak
|
||||
|| string.IsNullOrEmpty(report.SessionId))
|
||||
// No read means the ids are not GUIDs — no session can exist to break.
|
||||
if (report.ErrorType != AgentErrorType.SessionAgeEmergencyBreak || sessionRead is not { } read)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000);
|
||||
// Idempotency: the agent's emergency channel can send up to a few reports per session;
|
||||
// only ever materialize one timeline event (and one ops event).
|
||||
var alreadyMaterialized = existing.Any(e =>
|
||||
string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase));
|
||||
if (alreadyMaterialized)
|
||||
var verdict = ClassifyEmergencyBreak(read.Row, read.Failed, IsDeletionLocked(read.Row, deletionGuard));
|
||||
var message = BreakMessage(report);
|
||||
|
||||
if (verdict.Materialize)
|
||||
{
|
||||
return;
|
||||
var existing = await sessionRepo.GetSessionEventsAsync(tenantId, report.SessionId, maxResults: 1000);
|
||||
// Idempotency: the agent's emergency channel can send up to a few reports per
|
||||
// session; only ever materialize one timeline event (and one ops event).
|
||||
var alreadyMaterialized = existing.Any(e =>
|
||||
string.Equals(e.EventType, Constants.EventTypes.AgentEmergencyBreak, StringComparison.OrdinalIgnoreCase));
|
||||
if (alreadyMaterialized)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow);
|
||||
await sessionRepo.StoreEventsBatchAsync(new List<EnrollmentEvent> { evt });
|
||||
|
||||
// StoreEventsBatchAsync writes the Events partition only — the cross-session
|
||||
// EventType index is normally written by EventIngestProcessor, which this
|
||||
// backend-materialized event never passes through. Without the upsert the event
|
||||
// exists on the session timeline but is invisible to every search-by-eventType
|
||||
// surface (portal cross-session search, MCP search_sessions_by_event /
|
||||
// query_raw_events) — found the hard way in the 2026-07-22 incident analysis.
|
||||
await sessionRepo.UpsertEventTypeIndexBatchAsync(
|
||||
tenantId, report.SessionId, new List<EnrollmentEvent> { evt });
|
||||
}
|
||||
|
||||
var evt = BuildAgentEmergencyBreakEvent(report, tenantId, existing, DateTime.UtcNow);
|
||||
await sessionRepo.StoreEventsBatchAsync(new List<EnrollmentEvent> { evt });
|
||||
|
||||
// StoreEventsBatchAsync writes the Events partition only — the cross-session
|
||||
// EventType index is normally written by EventIngestProcessor, which this
|
||||
// backend-materialized event never passes through. Without the upsert the event
|
||||
// exists on the session timeline but is invisible to every search-by-eventType
|
||||
// surface (portal cross-session search, MCP search_sessions_by_event /
|
||||
// query_raw_events) — found the hard way in the 2026-07-22 incident analysis.
|
||||
await sessionRepo.UpsertEventTypeIndexBatchAsync(
|
||||
tenantId, report.SessionId, new List<EnrollmentEvent> { evt });
|
||||
|
||||
// Operator visibility: an emergency break means an agent silently gave up at its
|
||||
// absolute age cap — exactly the "are we losing agents?" signal. Emitted only on
|
||||
// first materialization so repeat reports for the same session cannot flood the
|
||||
// ops feed. OpsEventService never throws.
|
||||
//
|
||||
// Severity follows what the platform already knows about the session (ops audit
|
||||
// 2026-09-03): the break is wall-clock, so a device that was powered off for weeks
|
||||
// fires it on its first boot — long after the sweep reconciled the session. That
|
||||
// break is a late cleanup, not news; only a break on a still-open session is the
|
||||
// decision itself (classifier rule 3). The row read is best-effort: unknown status
|
||||
// keeps the historical Warning.
|
||||
SessionStatus? statusAtBreak = null;
|
||||
try
|
||||
{
|
||||
statusAtBreak = (await sessionRepo.GetSessionAsync(tenantId, report.SessionId))?.Status;
|
||||
}
|
||||
catch (Exception readEx)
|
||||
{
|
||||
logger.LogDebug(readEx,
|
||||
"ReportAgentError: session status read failed for emergency-break severity on {SessionId}", report.SessionId);
|
||||
}
|
||||
|
||||
var verdict = ClassifyEmergencyBreak(statusAtBreak);
|
||||
// absolute age cap — exactly the "are we losing agents?" signal. For a session with
|
||||
// a row it is emitted only on first materialization, so repeat reports cannot flood
|
||||
// the feed. OpsEventService never throws.
|
||||
await opsEventService.RecordAgentEmergencyBreakAsync(
|
||||
tenantId, report.SessionId, report.AgentVersion, evt.Message,
|
||||
verdict.Severity, statusAtBreak, verdict.LateCleanup);
|
||||
tenantId, report.SessionId, report.AgentVersion, message,
|
||||
verdict.Severity, verdict.SessionStatusAtBreak, verdict.LateCleanup, verdict.Context,
|
||||
report.SessionAgeHours);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -381,17 +406,66 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ops-event verdict for an emergency break, keyed on the session status the platform holds
|
||||
/// when the report arrives. Terminal (Succeeded / Failed / Incomplete) = the sweep already
|
||||
/// decided the enrollment and the agent merely cleaned up late (device powered on after a
|
||||
/// shelf period) — Info, <c>LateCleanup=true</c>. Open or unknown = the break is the first
|
||||
/// and only signal that nothing more will arrive — Warning, the historical semantics.
|
||||
/// The cascade-delete writer-block invariant, checked at its chokepoint on the row already in
|
||||
/// hand (no second read). A missing row needs no tombstone check: nothing is written for it.
|
||||
/// </summary>
|
||||
private static bool IsDeletionLocked(TableEntity? sessionRow, SessionDeletionGuard deletionGuard)
|
||||
{
|
||||
try
|
||||
{
|
||||
deletionGuard.ThrowIfLocked(sessionRow, "V2.ReportAgentError.EmergencyBreak");
|
||||
return false;
|
||||
}
|
||||
catch (SessionDeletionLockedException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>What the ops event says and whether the break is written into the session.</summary>
|
||||
internal readonly record struct EmergencyBreakVerdict(
|
||||
string Severity, bool LateCleanup, bool Materialize, string SessionStatusAtBreak, string Context);
|
||||
|
||||
/// <summary>
|
||||
/// Ops-event verdict for an emergency break, keyed on what the platform holds for the session
|
||||
/// when the report arrives. The break is wall-clock and checked at agent start, so it usually
|
||||
/// arrives long after the fact:
|
||||
/// <list type="bullet">
|
||||
/// <item>Terminal (Succeeded / Failed / Incomplete): the sweep already decided the enrollment
|
||||
/// and the agent merely cleaned up late after a shelf period — Info, <c>LateCleanup=true</c>
|
||||
/// (ops audit 2026-09-03).</item>
|
||||
/// <item>No row: in practice a session that never registered (the agent went dark before
|
||||
/// register-session) — nothing to close and nobody who could open it. Info, and no
|
||||
/// timeline event or index row: the orphan sweep would delete the event a day later with
|
||||
/// a Warning of its own, and the index row would point at a session that does not
|
||||
/// exist.</item>
|
||||
/// <item>Row under cascade delete: nothing may be written past the lock — Info.</item>
|
||||
/// <item>Open or unparseable status, or a failed read: the break is the first and only
|
||||
/// signal that nothing more will arrive (classifier rule 3) — Warning, and a failed read
|
||||
/// keeps the historical write.</item>
|
||||
/// </list>
|
||||
/// Pure so the rule is unit-testable.
|
||||
/// </summary>
|
||||
internal static (string Severity, bool LateCleanup) ClassifyEmergencyBreak(SessionStatus? statusAtBreak)
|
||||
=> statusAtBreak.HasValue && Helpers.DeviceJourneyCalculator.IsTerminal(statusAtBreak.Value)
|
||||
? (OpsEventSeverity.Info, true)
|
||||
: (OpsEventSeverity.Warning, false);
|
||||
internal static EmergencyBreakVerdict ClassifyEmergencyBreak(TableEntity? sessionRow, bool rowReadFailed, bool deletionLocked)
|
||||
{
|
||||
if (rowReadFailed)
|
||||
return new(OpsEventSeverity.Warning, false, true, "unknown", "session row unreadable when the break arrived");
|
||||
if (sessionRow == null)
|
||||
return new(OpsEventSeverity.Info, false, false, "missing", "no session row (never registered, or deleted long ago) — nothing to close");
|
||||
if (deletionLocked)
|
||||
return new(OpsEventSeverity.Info, false, false, "deleting", "session is being deleted — nothing written");
|
||||
|
||||
var status = IngestTelemetryFunction.TryReadSessionStatus(sessionRow);
|
||||
var statusText = status?.ToString() ?? "unknown";
|
||||
return status.HasValue && Helpers.DeviceJourneyCalculator.IsTerminal(status.Value)
|
||||
? new(OpsEventSeverity.Info, true, true, statusText, $"late cleanup — session already {statusText} when the break arrived")
|
||||
: new(OpsEventSeverity.Warning, false, true, statusText, $"session still {statusText} when the break arrived");
|
||||
}
|
||||
|
||||
private static string BreakMessage(AgentErrorReport report)
|
||||
=> string.IsNullOrWhiteSpace(report.Message)
|
||||
? "Agent absolute session-age emergency break fired — agent cleaned up and exited"
|
||||
: report.Message;
|
||||
|
||||
/// <summary>
|
||||
/// Builds the backend-materialized <c>agent_emergency_break</c> timeline event from the agent's
|
||||
@@ -415,6 +489,15 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
? report.Timestamp.ToUniversalTime()
|
||||
: nowUtc;
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
["source"] = "emergency_channel",
|
||||
["agentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["reportedAtUtc"] = report.Timestamp.ToString("o"),
|
||||
};
|
||||
if (report.SessionAgeHours.HasValue)
|
||||
data["sessionAgeHours"] = report.SessionAgeHours.Value;
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
TenantId = tenantId,
|
||||
@@ -425,15 +508,8 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Timestamp = breakAt,
|
||||
Sequence = maxSequence + 1,
|
||||
Message = string.IsNullOrWhiteSpace(report.Message)
|
||||
? "Agent absolute session-age emergency break fired — agent cleaned up and exited"
|
||||
: report.Message,
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
["source"] = "emergency_channel",
|
||||
["agentVersion"] = report.AgentVersion ?? string.Empty,
|
||||
["reportedAtUtc"] = report.Timestamp.ToString("o"),
|
||||
},
|
||||
Message = BreakMessage(report),
|
||||
Data = data,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -954,31 +954,26 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
/// <summary>
|
||||
/// An agent reported its absolute session-age emergency break (48h cap) over the
|
||||
/// emergency channel — it cleaned itself up and exited on a session that never reached
|
||||
/// a terminal state. This is the "are we silently losing agents?" signal; emitted by
|
||||
/// <see cref="Functions.Ingest.ReportAgentErrorFunction"/> once per session (guarded by
|
||||
/// the timeline-event idempotency check). Severity is decided by the caller from the
|
||||
/// session status at break time (<c>ReportAgentErrorFunction.ClassifyEmergencyBreak</c>):
|
||||
/// Warning when the session was still open (the break IS the verdict — Telegram-rule
|
||||
/// worthy), Info when the sweep had already terminalized it and the break is a late
|
||||
/// cleanup after the device was powered on again. The 2026-09-03 audit found 224 of 269
|
||||
/// breaks in three days were such late cleanups (kiosk devices stored for weeks); as
|
||||
/// Warnings they buried the real ones. <paramref name="statusAtBreak"/> lands in the
|
||||
/// details as <c>sessionStatusAtBreak</c> so the feed shows which case it was.
|
||||
/// emergency channel — it cleaned itself up and exited. This is the "are we silently
|
||||
/// losing agents?" signal; emitted by <see cref="Functions.Ingest.ReportAgentErrorFunction"/>
|
||||
/// once per break report (for a session with a row, guarded by the timeline-event
|
||||
/// idempotency check). Severity, <paramref name="sessionStatusAtBreak"/> and
|
||||
/// <paramref name="context"/> come from <c>ReportAgentErrorFunction.ClassifyEmergencyBreak</c>:
|
||||
/// Warning only when the session was still open (the break IS the verdict — Telegram-rule
|
||||
/// worthy) or its row could not be read. The 2026-09-03 audit found 224 of 269 breaks in
|
||||
/// three days were late cleanups on terminal sessions, the 2026-09-28 sweep that 115 of 160
|
||||
/// remaining Warnings named sessions that never registered; as Warnings both buried the real
|
||||
/// ones. <paramref name="sessionAgeHours"/> is the agent-measured age (null for agents that
|
||||
/// predate the field) — it separates a break just past the cap from a device that slept for
|
||||
/// weeks, which the backend cannot tell from its own rows.
|
||||
/// </summary>
|
||||
public Task RecordAgentEmergencyBreakAsync(
|
||||
string tenantId, string sessionId, string? agentVersion, string message,
|
||||
string severity, SessionStatus? statusAtBreak, bool lateCleanup)
|
||||
{
|
||||
var statusText = statusAtBreak?.ToString() ?? "unknown";
|
||||
var context = lateCleanup
|
||||
? $"late cleanup — session already {statusText} when the break arrived"
|
||||
: $"session still {statusText} when the break arrived";
|
||||
return WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
|
||||
string severity, string sessionStatusAtBreak, bool lateCleanup, string context, double? sessionAgeHours)
|
||||
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
|
||||
$"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}, {context}): {message}",
|
||||
tenantId, "System.EmergencyChannel",
|
||||
new { sessionId, agentVersion, sessionStatusAtBreak = statusText, lateCleanup });
|
||||
}
|
||||
new { sessionId, agentVersion, sessionStatusAtBreak, lateCleanup, sessionAgeHours });
|
||||
|
||||
/// <summary>
|
||||
/// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend
|
||||
|
||||
@@ -81,6 +81,15 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public long? SequenceNumber { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Session age in hours the agent measured when its absolute session-age emergency break
|
||||
/// fired (<see cref="AgentErrorType.SessionAgeEmergencyBreak"/> only). The backend cannot
|
||||
/// derive it: a never-registered session has no row, and a registered one may have
|
||||
/// registered seconds before the break (hibernate resume). Null for other error types and
|
||||
/// for agents that predate the field.
|
||||
/// </summary>
|
||||
public double? SessionAgeHours { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Agent version string for diagnosing version-specific issues.
|
||||
/// </summary>
|
||||
|
||||
Reference in New Issue
Block a user