From 3d10011460aa9199a1c3b0afcf5e450bf5c4b9a9 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Fri, 9 Oct 2026 22:55:27 +0200 Subject: [PATCH] Orphan sessions: ingest refuses sessions without a row, the sweep becomes their cascade in its own function (D-339) A session without a Sessions row can never be reached by the cascade, so every row written for it was an orphan by construction. Three layers now keep the platform clean: - Prevent: IngestTelemetry answers a batch for an unknown session with 410 (same status and body as the cascade lock; request-row marker IngestRefusal = session_unknown | cascade_locked). The derived writers - vulnerability correlation, analyze, manual rescan - stop on SessionDeletionGuard.GetLivenessAsync != Live. The agent treats every 410 as terminal: one TelemetryRejectedSessionUnknown report (exempt from the reporter cooldown), then terminate_session with forced self-destruct (origin session_gone, no final drain, no diagnostics upload); spool items of a foreign session id are dropped at startup and never terminate the current session. - Detect: OrphanSessionSweepFunction (every 4h at :45, own lease) diffs the Sessions keys against the EventSessionIndex handles - two range drains instead of ~16.8k Sessions point reads per 2h maintenance run - and re-checks every candidate (handle unchanged, row still absent, tombstone) moments before deleting. The in-maintenance sweep raced the retention cascade on their shared 00:00/12:00 tick and reported false orphans; it is gone. - Clean: every table of the cascade manifest minus the tombstone pair (ratchet in TableLifecycleBucketTests), the inventory counters with the stamp on the handle (claim, then decrement), the handle last with If-Match. A weekly read-only reconciliation of EventTypeIndex against the Sessions keys (Sunday mornings UTC, and on every manual maintenance run) reports residue. MaintenanceCompleted now carries stepsMs. MCP search_sessions_by_event auto-exhausts empty-but-continuable pages. Manifests regenerated for the new ops event types and AdminConfiguration.OrphanDeepCheckLastRunUtc. Co-Authored-By: Claude Fable 5.1 --- .../Transport/EmergencyReporterTests.cs | 83 +++ .../ServerControlPlaneSessionGoneTests.cs | 170 ++++++ .../EnrollmentTerminationHandlerTests.cs | 59 ++ .../BackendTelemetryUploaderTests.cs | 35 ++ .../Transport/FakeBackendTelemetryUploader.cs | 6 + .../Transport/TelemetrySpoolTests.cs | 89 ++- .../TelemetryUploadOrchestratorTests.cs | 120 +++- .../Monitoring/Transport/EmergencyReporter.cs | 21 +- .../Orchestration/EnrollmentOrchestrator.cs | 9 +- .../EnrollmentTerminatedEventArgs.cs | 26 +- .../Telemetry/Events/TelemetryEventEmitter.cs | 2 +- .../Signals/TelemetrySignalEmitter.cs | 2 +- .../Transitions/TelemetryTransitionEmitter.cs | 2 +- .../EnrollmentTerminationHandler.cs | 35 +- .../Telemetry/BackendTelemetryUploader.cs | 10 + .../Telemetry/TelemetryPartitionKey.cs | 14 + .../Transport/Telemetry/TelemetrySpool.cs | 75 ++- .../Telemetry/TelemetryUploadOrchestrator.cs | 82 ++- .../Transport/Telemetry/UploadResult.cs | 23 +- .../Runtime/AgentRuntimeHost.cs | 2 + .../Runtime/ServerControlPlane.cs | 99 +++- .../DerivedWriterLivenessTests.cs | 126 +++++ .../IngestTelemetryAuthPathTests.cs | 22 + .../MaintenanceStepTimerTests.cs | 33 ++ .../OffboardingSessionEnumeratorTests.cs | 10 +- .../OrphanSessionRowFiltersTests.cs | 116 ++++ .../OrphanSessionSweeperTests.cs | 528 ++++++++++++++++++ .../OrphanSweepOpsEventTests.cs | 111 ++++ .../SessionDeletionGuardTests.cs | 55 ++ .../TableLifecycleBucketTests.cs | 26 + .../TableStorage/TableConfigRepository.cs | 2 + .../TableMaintenanceRepository.cs | 26 +- .../Config/PatchAdminConfigurationFunction.cs | 1 + .../Ingest/IngestTelemetryFunction.cs | 37 +- .../Maintenance/OrphanSessionSweepFunction.cs | 42 ++ .../GetVulnerabilityReportFunction.cs | 14 +- .../Middleware/RequestTelemetryMiddleware.cs | 4 + .../AutopilotMonitor.Functions/Program.cs | 4 + .../Security/RequestRowMarkers.cs | 16 + .../Analyze/AnalyzeOnEnrollmentEndHandler.cs | 17 + .../Services/Deletion/SessionDeletionGuard.cs | 40 ++ .../Maintenance/MaintenanceStepTimer.cs | 47 ++ .../Maintenance/OrphanSessionRowFilters.cs | 93 +++ .../Maintenance/OrphanSessionSweeper.cs | 429 ++++++++++++++ .../Maintenance/OrphanSweepLockStore.cs | 21 + .../MaintenanceService.Aggregation.cs | 74 --- .../Services/MaintenanceService.cs | 55 +- .../Services/OpsEventService.cs | 119 +++- .../TableStorageService.Maintenance.cs | 93 --- .../TableStorageService.OrphanSweep.cs | 143 +++++ .../VulnerabilityCorrelateHandler.cs | 16 + .../VulnerabilityCorrelationService.cs | 7 +- .../sessions-by-event-exhaust.test.ts | 56 ++ .../src/generated/wire-types.generated.ts | 2 + .../generated/wire-vocabularies.generated.ts | 4 + .../src/tools/sessions.ts | 10 +- .../DataAccess/IMaintenanceRepository.cs | 72 ++- .../DataAccess/OpsEventTypes.cs | 10 +- .../Models/Config/AdminConfiguration.cs | 7 + .../Models/Diagnostics/AgentErrorReport.cs | 9 + .../admin/components/OpsAlertRulesSection.tsx | 12 +- .../utils/shared-manifests.generated.ts | 10 +- .../utils/shared-manifests.json | 18 +- .../utils/wire-types.generated.ts | 2 + 64 files changed, 3228 insertions(+), 275 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/ServerControlPlaneSessionGoneTests.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryPartitionKey.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/DerivedWriterLivenessTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceStepTimerTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/OrphanSessionRowFiltersTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/OrphanSessionSweeperTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/OrphanSweepOpsEventTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/OrphanSessionSweepFunction.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceStepTimer.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSessionRowFilters.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSessionSweeper.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSweepLockStore.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.OrphanSweep.cs create mode 100644 src/McpServer/autopilot-monitor-mcp/src/__tests__/sessions-by-event-exhaust.test.ts diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs index 690960da..0a3fce24 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/EmergencyReporterTests.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Net.Http; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; using AutopilotMonitor.Shared.Models; using Xunit; @@ -49,6 +50,74 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport private static EmergencyReporter Reporter(FakeApiClient client) => new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null); + private static EmergencyReporter Reporter(FakeApiClient client, VirtualClock clock) + => new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null, clock: clock); + + private static VirtualClock Clock() => new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc)); + + [Fact] + public async Task Session_unknown_report_is_not_suppressed_by_the_cooldown_of_an_earlier_report() + { + // Terminal report, sent at most once per process right before the termination — a + // routine report one minute earlier must not cost it. Every other type keeps the cooldown. + var clock = Clock(); + var client = new FakeApiClient(); + client.Results.Enqueue(true); + client.Results.Enqueue(true); + client.Results.Enqueue(true); + var reporter = Reporter(client, clock); + + await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: 503); + clock.Advance(TimeSpan.FromMinutes(1)); + await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410); + + Assert.Equal(2, client.Calls); + Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, client.Reports[1].ErrorType); + + // Control: a third report of another type inside the window is still held back. + clock.Advance(TimeSpan.FromMinutes(1)); + await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "m"); + Assert.Equal(2, client.Calls); + } + + [Fact] + public async Task Second_session_unknown_report_within_the_window_is_still_suppressed() + { + var clock = Clock(); + var client = new FakeApiClient(); + client.Results.Enqueue(true); + client.Results.Enqueue(true); + var reporter = Reporter(client, clock); + + await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410); + clock.Advance(TimeSpan.FromMinutes(1)); + await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410); + + Assert.Equal(1, client.Calls); + } + + [Fact] + public async Task Session_unknown_report_still_counts_against_the_per_session_maximum() + { + // Five delivered reports (ten minutes apart, distinct keys) exhaust the budget; the + // exemption waives only the cooldown, not the maximum. + var clock = Clock(); + var client = new FakeApiClient(); + for (var i = 0; i < 6; i++) client.Results.Enqueue(true); + var reporter = Reporter(client, clock); + + foreach (var status in new[] { 500, 502, 503, 504, 429 }) + { + await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: status); + clock.Advance(TimeSpan.FromMinutes(10)); + } + Assert.Equal(5, client.Calls); + + await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410); + + Assert.Equal(5, client.Calls); + } + [Fact] public async Task Failed_send_rolls_back_reservation_so_a_later_call_retries() { @@ -142,6 +211,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport Assert.Equal(2, client.Calls); } + [Fact] + public async Task Session_unknown_report_carries_error_type_and_http_410() + { + var client = new FakeApiClient(); + client.Results.Enqueue(true); + var reporter = Reporter(client); + + await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410); + + var report = Assert.Single(client.Reports); + Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType); + Assert.Equal(410, report.HttpStatusCode); + } + [Fact] public async Task Session_age_is_carried_in_the_report_and_absent_by_default() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/ServerControlPlaneSessionGoneTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/ServerControlPlaneSessionGoneTests.cs new file mode 100644 index 00000000..a8b89349 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/ServerControlPlaneSessionGoneTests.cs @@ -0,0 +1,170 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Configuration; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Program +{ + /// + /// Host wiring for TelemetryUploadOrchestrator.SessionGone + /// (HTTP 410 on the current session's telemetry): exactly one + /// emergency report carrying the + /// status, then a synthetic forced terminate_session tagged origin=session_gone + /// — and nothing at all once a shutdown is already in progress. + /// + public sealed class ServerControlPlaneSessionGoneTests + { + private sealed class RecordingEmergencyReporter : EmergencyReporter + { + private readonly List _sequence; + public List<(AgentErrorType ErrorType, string Message, int? HttpStatusCode)> Reports { get; } = + new List<(AgentErrorType, string, int?)>(); + + public RecordingEmergencyReporter(List sequence) + : base(apiClient: null!, sessionId: "S1", tenantId: "T1", agentVersion: "0.0.0", logger: null!) + { + _sequence = sequence; + } + + public override Task TrySendAsync( + AgentErrorType errorType, + string message, + int? httpStatusCode = null, + long? sequenceNumber = null, + int attempts = 1, + TimeSpan? perAttemptTimeout = null, + TimeSpan? retryDelay = null, + double? sessionAgeHours = null, + RegistrationFailureSummary? priorRegistrationFailure = null) + { + Reports.Add((errorType, message, httpStatusCode)); + _sequence.Add("report"); + return Task.CompletedTask; + } + } + + private sealed class RecordingDispatcher : ServerActionDispatcher + { + private readonly List _sequence; + public List Dispatched { get; } = new List(); + + public RecordingDispatcher(AgentConfiguration config, AgentLogger logger, InformationalEventPost post, List sequence) + : base( + configuration: config, + logger: logger, + rotateConfigAsync: () => Task.FromResult(true), + uploadDiagnosticsAsync: _ => Task.FromResult(null!), + onTerminateRequested: _ => Task.CompletedTask, + post: post) + { + _sequence = sequence; + } + + public override Task DispatchAsync(List actions) + { + Dispatched.AddRange(actions); + _sequence.Add("dispatch"); + return Task.CompletedTask; + } + } + + private sealed class Rig : IDisposable + { + public TempDirectory Tmp { get; } = new TempDirectory(); + public AgentLogger Logger { get; } + public List Sequence { get; } = new List(); + public RecordingEmergencyReporter Reporter { get; } + public RecordingDispatcher Dispatcher { get; } + public ManualResetEventSlim Shutdown { get; } = new ManualResetEventSlim(false); + public ManualResetEventSlim ShutdownComplete { get; } = new ManualResetEventSlim(false); + + public Rig() + { + Logger = new AgentLogger(Path.Combine(Tmp.Path, "logs"), AgentLogLevel.Info); + var config = new AgentConfiguration { SessionId = "S1", TenantId = "T1", ApiBaseUrl = "http://localhost" }; + var post = new InformationalEventPost( + new FakeSignalIngressSink(), + new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc))); + Reporter = new RecordingEmergencyReporter(Sequence); + Dispatcher = new RecordingDispatcher(config, Logger, post, Sequence); + } + + public void Handle(string reason = "session gone: http 410") => + global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.HandleSessionGone( + reason, Reporter, Dispatcher, Shutdown, ShutdownComplete, Logger); + + public void Dispose() + { + Shutdown.Dispose(); + ShutdownComplete.Dispose(); + Tmp.Dispose(); + } + } + + [Fact] + public void Reports_once_with_410_then_dispatches_forced_terminate_tagged_session_gone() + { + using var rig = new Rig(); + + rig.Handle(); + + Assert.Equal(new[] { "report", "dispatch" }, rig.Sequence); + + var report = Assert.Single(rig.Reporter.Reports); + Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType); + Assert.Equal(410, report.HttpStatusCode); + Assert.Contains("410", report.Message); + + var action = Assert.Single(rig.Dispatcher.Dispatched); + Assert.Equal(ServerActionTypes.TerminateSession, action.Type); + Assert.Contains("410", action.Reason); + Assert.NotNull(action.Params); + var parameters = action.Params!; + Assert.Equal("true", parameters["forceSelfDestruct"]); + Assert.Equal("0", parameters["gracePeriodSeconds"]); + Assert.Equal(TerminationOrigins.SessionGone, parameters["origin"]); + } + + [Theory] + [InlineData(true, false)] + [InlineData(false, true)] + public void Does_nothing_when_a_shutdown_is_already_in_progress(bool shutdownSet, bool shutdownCompleteSet) + { + using var rig = new Rig(); + if (shutdownSet) rig.Shutdown.Set(); + if (shutdownCompleteSet) rig.ShutdownComplete.Set(); + + rig.Handle(); + + Assert.Empty(rig.Reporter.Reports); + Assert.Empty(rig.Dispatcher.Dispatched); + } + + [Fact] + public void ReadOrigin_returns_the_origin_param_or_null() + { + Assert.Equal(TerminationOrigins.SessionGone, + global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin( + new Dictionary { ["origin"] = TerminationOrigins.SessionGone })); + Assert.Equal(TerminationOrigins.KillSignal, + global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin( + new Dictionary { ["origin"] = TerminationOrigins.KillSignal, ["forceSelfDestruct"] = "true" })); + Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin( + new Dictionary { ["adminOutcome"] = "Failed" })); + Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin( + new Dictionary { ["origin"] = string.Empty })); + Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(null!)); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs index 25b06b5f..aa8b3b86 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs @@ -310,6 +310,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination new EnrollmentTerminatedEventArgs( reason, outcome, stage.ToString(), at ?? EndUtc); + /// + /// The args ServerControlPlane.OnTerminateRequested synthesises for a + /// terminate_session action tagged with . + /// + private static EnrollmentTerminatedEventArgs ServerArgs(string origin) => + new EnrollmentTerminatedEventArgs( + EnrollmentTerminationReason.DecisionTerminalStage, + EnrollmentTerminationOutcome.Failed, + SessionStage.Failed.ToString(), + EndUtc, + details: "Server-requested termination", + origin: origin); + [Fact] public void Handle_writes_final_status_json_in_state_directory() { @@ -458,6 +471,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination Assert.Equal(0, rig.CleanupService.Invocations); } + // ============================================================= origin=session_gone (HTTP 410) + + [Fact] + public void Handle_session_gone_skips_spool_drain_and_diagnostics_but_still_cleans_up() + { + // The backend no longer knows the session: every drain would only burn its budget on + // 410s and a diagnostics package would describe a deleted session. Marker, cleanup + // and the (forced) self-destruct still run. + using var rig = new Rig(); + rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build(); + rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5); + var spoolPolls = 0; + rig.PendingItemCountAccessor = () => { spoolPolls++; return 5; }; // never drains — a poll would burn the budget + var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true); + + rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.SessionGone)); + + Assert.Equal(0, spoolPolls); + Assert.Equal(0, rig.DiagnosticsUploads); + Assert.DoesNotContain(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes); + Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes); + Assert.True(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker"))); + Assert.Equal(1, rig.CleanupService.Invocations); + Assert.Equal(1, rig.ShutdownSignalled); + } + + [Fact] + public void Handle_kill_signal_origin_still_drains_spool_and_uploads_diagnostics() + { + // The administrator kill signal keeps its full sequence — only session_gone skips. + using var rig = new Rig(); + rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build(); + rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5); + var spoolPolls = 0; + rig.PendingItemCountAccessor = () => { spoolPolls++; return 0; }; + var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true); + + rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.KillSignal)); + + Assert.True(spoolPolls > 0); + Assert.Equal(1, rig.DiagnosticsUploads); + Assert.Contains(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes); + Assert.Equal(1, rig.CleanupService.Invocations); + Assert.Equal(1, rig.ShutdownSignalled); + } + [Fact] public void Handle_signals_shutdown_exactly_once_even_on_error() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderTests.cs index 53e3b84c..32cf0386 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderTests.cs @@ -275,9 +275,44 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport Assert.False(result.Success); Assert.False(result.IsTransient); + Assert.False(result.IsSessionGone); Assert.Contains(((int)status).ToString(), result.ErrorReason); } + [Fact] + public async Task Gone_410_maps_to_SessionGone() + { + // 410 = the backend no longer knows this session. Not transient (no retry), not auth, + // not poison, not a size problem — a distinct terminal kind the drain reacts to. + using var rig = new Rig(); + rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "session unknown"); + + var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + + Assert.False(result.Success); + Assert.True(result.IsSessionGone); + Assert.False(result.IsTransient); + Assert.False(result.IsAuthFailure); + Assert.False(result.IsPoison); + Assert.False(result.RequiresSplit); + Assert.Contains("410", result.ErrorReason); + } + + [Fact] + public async Task Gone_410_with_poison_body_is_still_SessionGone_never_poison() + { + // A poison body on 410 must not be honoured: dropping items + emitting + // telemetry_upload_poisoned would only produce the next 410 (emit/reject loop). + using var rig = new Rig(); + rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "{\"poison\":true,\"rejectedRowKeys\":[\"rk1\"],\"reason\":\"x\"}"); + + var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + + Assert.True(result.IsSessionGone); + Assert.False(result.IsPoison); + Assert.Null(result.PoisonRowKeys); + } + [Theory] [InlineData(HttpStatusCode.RequestTimeout)] // 408 [InlineData((HttpStatusCode)429)] // TooManyRequests diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/FakeBackendTelemetryUploader.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/FakeBackendTelemetryUploader.cs index 8d959548..e76601ef 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/FakeBackendTelemetryUploader.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/FakeBackendTelemetryUploader.cs @@ -68,6 +68,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport return this; } + public FakeBackendTelemetryUploader QueueSessionGone(string reason = "session gone: http 410") + { + _script.Enqueue(_ => UploadResult.SessionGone(reason)); + return this; + } + public FakeBackendTelemetryUploader QueueThrow(Exception ex) { _script.Enqueue(_ => throw ex); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetrySpoolTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetrySpoolTests.cs index 66af48bb..761e044d 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetrySpoolTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetrySpoolTests.cs @@ -10,10 +10,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport { public sealed class TelemetrySpoolTests { - private static TelemetryItemDraft EventDraft(string rowKey = "row-1") => + private const string OwnKey = "tenant_session"; + private const string ForeignKey = "tenant_old-session"; + + private static TelemetryItemDraft EventDraft(string rowKey = "row-1") => EventDraft(rowKey, OwnKey); + + private static TelemetryItemDraft EventDraft(string rowKey, string partitionKey) => new TelemetryItemDraft( kind: TelemetryItemKind.Event, - partitionKey: "tenant_session", + partitionKey: partitionKey, rowKey: rowKey, payloadJson: "{\"EventType\":\"agent_started\"}", isSessionScoped: true); @@ -195,6 +200,86 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport Assert.Equal(1, next.TelemetryItemId); } + // ============================================================ Startup partition-key filter + + [Fact] + public void Ctor_drops_leading_pending_items_of_another_session_and_persists_cursor() + { + // Same spool directory, new session id: the previous session's unsent items would only + // draw a 410 that must not end the new session. They are dropped on startup; the own + // item behind them stays pending. + using var tmp = new TempDirectory(); + var s1 = new TelemetrySpool(tmp.Path, Clock()); + s1.Enqueue(EventDraft("old-a", ForeignKey)); + s1.Enqueue(EventDraft("old-b", ForeignKey)); + s1.Enqueue(EventDraft("own", OwnKey)); + + var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey); + + Assert.Equal(1, s2.LastUploadedItemId); + Assert.Equal(2, s2.LastAssignedItemId); + Assert.Equal(1, s2.PendingItemCount); + Assert.Equal("own", Assert.Single(s2.Peek(10)).RowKey); + + // Cursor persisted: a later start without the filter sees the same tail with the same bytes. + var s3 = new TelemetrySpool(tmp.Path, Clock()); + Assert.Equal(1, s3.LastUploadedItemId); + Assert.Equal(1, s3.PendingItemCount); + Assert.Equal(s3.PendingBytes, s2.PendingBytes); + + var next = s2.Enqueue(EventDraft("next", OwnKey)); + Assert.Equal(3, next.TelemetryItemId); + } + + [Fact] + public void Ctor_drops_nothing_when_a_foreign_item_follows_an_own_item() + { + // Impossible by construction (one process = one session); if it happens the order is + // not trustworthy, so the filter keeps everything instead of guessing. + using var tmp = new TempDirectory(); + var s1 = new TelemetrySpool(tmp.Path, Clock()); + s1.Enqueue(EventDraft("old-a", ForeignKey)); + s1.Enqueue(EventDraft("own", OwnKey)); + s1.Enqueue(EventDraft("old-b", ForeignKey)); + + var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey); + + Assert.Equal(-1, s2.LastUploadedItemId); + Assert.Equal(3, s2.PendingItemCount); + Assert.Equal(s1.PendingBytes, s2.PendingBytes); + } + + [Fact] + public void Ctor_keeps_pending_items_that_carry_the_expected_key() + { + // WhiteGlove Part 2 reuses the Part-1 session id: its leftovers carry the expected key. + using var tmp = new TempDirectory(); + var s1 = new TelemetrySpool(tmp.Path, Clock()); + s1.Enqueue(EventDraft("a", OwnKey)); + s1.Enqueue(EventDraft("b", OwnKey)); + s1.Enqueue(EventDraft("c", OwnKey)); + + var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey); + + Assert.Equal(-1, s2.LastUploadedItemId); + Assert.Equal(3, s2.PendingItemCount); + Assert.Equal(s1.PendingBytes, s2.PendingBytes); + } + + [Fact] + public void Ctor_without_expected_key_keeps_foreign_items() + { + using var tmp = new TempDirectory(); + var s1 = new TelemetrySpool(tmp.Path, Clock()); + s1.Enqueue(EventDraft("old-a", ForeignKey)); + s1.Enqueue(EventDraft("old-b", ForeignKey)); + + var s2 = new TelemetrySpool(tmp.Path, Clock()); + + Assert.Equal(-1, s2.LastUploadedItemId); + Assert.Equal(2, s2.PendingItemCount); + } + [Fact] public void Enqueue_with_RequiresImmediateFlush_raises_ImmediateFlushRequested() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetryUploadOrchestratorTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetryUploadOrchestratorTests.cs index 1a61f1b1..2651acc6 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetryUploadOrchestratorTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetryUploadOrchestratorTests.cs @@ -10,10 +10,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport { public sealed class TelemetryUploadOrchestratorTests { - private static TelemetryItemDraft Draft(string rowKey) => + private const string OwnKey = "tenant_session"; + + private static TelemetryItemDraft Draft(string rowKey) => Draft(rowKey, OwnKey); + + private static TelemetryItemDraft Draft(string rowKey, string partitionKey) => new TelemetryItemDraft( kind: TelemetryItemKind.Event, - partitionKey: "tenant_session", + partitionKey: partitionKey, rowKey: rowKey, payloadJson: $"{{\"rk\":\"{rowKey}\"}}", isSessionScoped: true); @@ -157,6 +161,118 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport Assert.Equal(1, blockedCount); } + // ============================================================ HTTP 410 — session gone + + [Fact] + public async Task SessionGone_for_current_session_stops_drain_retains_cursor_and_raises_once() + { + using var tmp = new TempDirectory(); + var spool = new TelemetrySpool(tmp.Path, Clock()); + spool.Enqueue(Draft("a")); + spool.Enqueue(Draft("b")); + + var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone(); + using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey); + var gone = new System.Collections.Generic.List(); + int blockedCount = 0; + PoisonReport? poison = null; + sut.SessionGone += (_, reason) => gone.Add(reason); + sut.UploadBlocked += (_, __) => blockedCount++; + sut.BatchPoisoned += (_, p) => poison = p; + + var first = await sut.DrainAllAsync(); + + Assert.False(first.Success); + Assert.Equal(1, first.FailedBatches); + Assert.Equal(0, first.UploadedItems); + Assert.Equal(1, uploader.CallCount); // no retries — nothing can make a 410 succeed + Assert.Equal(-1, sut.LastUploadedItemId); // retained — items stay on disk for the diag ZIP + Assert.Single(gone); + Assert.Contains("410", gone[0]); + Assert.Equal(0, blockedCount); // not a "blocked" pipeline + Assert.Null(poison); + + // One-shot: the next drain hits the 410 again but does not raise again. + var second = await sut.DrainAllAsync(); + Assert.False(second.Success); + Assert.Equal(2, uploader.CallCount); + Assert.Equal(-1, sut.LastUploadedItemId); + Assert.Single(gone); + Assert.Equal(0, blockedCount); + } + + [Fact] + public async Task SessionGone_for_leading_foreign_items_drops_only_them_and_continues() + { + // Leftovers of a previous session id at the head of the spool (defence in depth — the + // spool filters these at startup). A 410 for them must not end the current session: + // drop exactly the leading foreign run, then carry on with the current session's items. + using var tmp = new TempDirectory(); + var spool = new TelemetrySpool(tmp.Path, Clock()); + spool.Enqueue(Draft("old-a", "tenant_old-session")); + spool.Enqueue(Draft("old-b", "tenant_old-session")); + spool.Enqueue(Draft("new", OwnKey)); + + var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueOk(); + using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey); + int goneCount = 0, blockedCount = 0; + sut.SessionGone += (_, __) => goneCount++; + sut.UploadBlocked += (_, __) => blockedCount++; + + var result = await sut.DrainAllAsync(); + + Assert.True(result.Success); + Assert.Equal(0, result.FailedBatches); + Assert.Equal(1, result.UploadedItems); + Assert.Equal(2, uploader.CallCount); + Assert.Equal(3, uploader.Received[0].Count); // first batch carried all three + Assert.Equal("new", Assert.Single(uploader.Received[1]).RowKey); // retry carried only the own item + Assert.Equal(2, sut.LastUploadedItemId); + Assert.Equal(0, goneCount); + Assert.Equal(0, blockedCount); + } + + [Fact] + public async Task SessionGone_is_not_latched_before_a_handler_is_subscribed() + { + // The host subscribes after EnrollmentOrchestrator.Start; a 410 on the very first drain + // must still reach it on the next drain instead of being swallowed by the one-shot guard. + using var tmp = new TempDirectory(); + var spool = new TelemetrySpool(tmp.Path, Clock()); + spool.Enqueue(Draft("only")); + + var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone(); + using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey); + + await sut.DrainAllAsync(); // nobody listening yet + + int goneCount = 0; + sut.SessionGone += (_, __) => goneCount++; + await sut.DrainAllAsync(); + + Assert.Equal(1, goneCount); + Assert.Equal(-1, sut.LastUploadedItemId); + } + + [Fact] + public async Task SessionGone_without_a_current_key_treats_every_batch_as_own() + { + using var tmp = new TempDirectory(); + var spool = new TelemetrySpool(tmp.Path, Clock()); + spool.Enqueue(Draft("x", "tenant_whatever")); + + var uploader = new FakeBackendTelemetryUploader().QueueSessionGone(); + using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock()); + int goneCount = 0; + sut.SessionGone += (_, __) => goneCount++; + + var result = await sut.DrainAllAsync(); + + Assert.False(result.Success); + Assert.Equal(1, goneCount); + Assert.Equal(-1, sut.LastUploadedItemId); + } + [Fact] public async Task TooLarge_splits_batch_and_uploads_everything() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs index 19d88456..a884aa57 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/EmergencyReporter.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.DecisionCore.Engine; using AutopilotMonitor.Shared.Models; namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport @@ -14,6 +15,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// - Same error key (ErrorType + HTTP status code) is only ever sent once per session /// - At most emergency reports total per session /// - At most one report every minutes regardless of error type + /// (the terminal report is exempt) /// /// A failure in this reporter is silently swallowed — it must never cascade into the main loop. /// @@ -32,6 +34,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport private readonly string _tenantId; private readonly string _agentVersion; private readonly AgentLogger _logger; + private readonly IClock _clock; private readonly HashSet _reportedKeys = new HashSet(); private readonly object _antiFloodLock = new object(); @@ -43,13 +46,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport string sessionId, string tenantId, string agentVersion, - AgentLogger logger) + AgentLogger logger, + IClock clock = null) { _apiClient = apiClient; _sessionId = sessionId; _tenantId = tenantId; _agentVersion = agentVersion; _logger = logger; + _clock = clock ?? SystemClock.Instance; } /// @@ -100,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport // All anti-flood checks are protected by a lock to prevent race conditions when // TrySendAsync is called concurrently (fire-and-forget from multiple threads). var key = $"{errorType}:{httpStatusCode}"; + var now = _clock.UtcNow; int currentReportCount; DateTime? previousLastReportTime; @@ -117,8 +123,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport return; } - if (_lastReportTime != null && - (DateTime.UtcNow - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes) + // The session-unknown report is terminal and sent at most once per process (the + // termination follows immediately): a routine report minutes earlier must not cost + // the last word about the session. Only the cooldown is waived; key dedup and the + // per-session maximum still apply. + var exemptFromCooldown = errorType == AgentErrorType.TelemetryRejectedSessionUnknown; + if (!exemptFromCooldown && _lastReportTime != null && + (now - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes) { _logger?.Debug($"[EmergencyChannel] Suppressed (cooldown active, last report at {_lastReportTime.Value:HH:mm:ss}): {key}"); return; @@ -127,7 +138,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport previousLastReportTime = _lastReportTime; _reportedKeys.Add(key); _reportCount++; - _lastReportTime = DateTime.UtcNow; + _lastReportTime = now; currentReportCount = _reportCount; } @@ -145,7 +156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport SessionAgeHours = sessionAgeHours, PriorRegistrationFailure = priorRegistrationFailure, AgentVersion = _agentVersion, - Timestamp = DateTime.UtcNow + Timestamp = now }; var delivered = false; diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs index c4c6c4b5..89da3464 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs @@ -371,8 +371,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration // 3) Telemetry-Transport. Batch size flows from AgentConfiguration.MaxBatchSize // via Program.cs (P1 fix: previously the remote-config knob was merged but // never applied, so tenants saw drainInterval/MaxBatchSize have no effect). - _spool = new TelemetrySpool(_transportDirectory, _clock, _logger); - _transport = new TelemetryUploadOrchestrator(_spool, _uploader, _clock, batchSize: _uploadBatchSize); + // The partition key names the registered session: the spool drops leftovers of a + // previous session id on startup, and the orchestrator tells a 410 for this session + // (SessionGone → host terminates) from a 410 for such leftovers (dropped). + var partitionKey = TelemetryPartitionKey.ForSession(_tenantId, _sessionId); + _spool = new TelemetrySpool(_transportDirectory, _clock, _logger, expectedPartitionKey: partitionKey); + _transport = new TelemetryUploadOrchestrator( + _spool, _uploader, _clock, batchSize: _uploadBatchSize, currentPartitionKey: partitionKey, logger: _logger); // TRACE-H1 — surface skipped oversized items + blocked-pipeline conditions as timeline // events. Raised after the drain guard releases, so posting back through the ingress diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentTerminatedEventArgs.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentTerminatedEventArgs.cs index f8ae9c2e..9a42139e 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentTerminatedEventArgs.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentTerminatedEventArgs.cs @@ -73,13 +73,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration /// public string? StopReason { get; } + /// + /// The origin param of a server-requested terminate_session action + /// (); null for every termination the agent decided + /// itself. makes the termination handler + /// skip the spool drain and the diagnostics upload — the backend no longer knows the + /// session, so both could only fail. + /// + public string? Origin { get; } + public EnrollmentTerminatedEventArgs( EnrollmentTerminationReason reason, EnrollmentTerminationOutcome outcome, string? stageName, DateTime terminatedAtUtc, string? details = null, - string? stopReason = null) + string? stopReason = null, + string? origin = null) { Reason = reason; Outcome = outcome; @@ -87,6 +97,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration TerminatedAtUtc = terminatedAtUtc; Details = details; StopReason = stopReason; + Origin = origin; } } + + /// + /// Values of the origin param on a synthesised terminate_session + /// ServerAction, carried into . + /// + public static class TerminationOrigins + { + /// Administrator kill signal (DeviceKillSignal on a 2xx telemetry response). + public const string KillSignal = "kill_signal"; + + /// The backend answered HTTP 410 for the current session's telemetry. + public const string SessionGone = "session_gone"; + } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Events/TelemetryEventEmitter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Events/TelemetryEventEmitter.cs index faa08123..83119f4f 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Events/TelemetryEventEmitter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Events/TelemetryEventEmitter.cs @@ -45,7 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Events _sequenceCounter = sequenceCounter ?? throw new ArgumentNullException(nameof(sequenceCounter)); _sessionId = sessionId; _tenantId = tenantId; - _partitionKey = $"{tenantId}_{sessionId}"; + _partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId); _traceEventsEnabled = traceEventsEnabled; } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Signals/TelemetrySignalEmitter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Signals/TelemetrySignalEmitter.cs index b6282a43..c267b4b9 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Signals/TelemetrySignalEmitter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Signals/TelemetrySignalEmitter.cs @@ -42,7 +42,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Signals if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId)); if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId)); _transport = transport ?? throw new ArgumentNullException(nameof(transport)); - _partitionKey = $"{tenantId}_{sessionId}"; + _partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId); } /// diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Transitions/TelemetryTransitionEmitter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Transitions/TelemetryTransitionEmitter.cs index 137b1c48..d215bbf9 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Transitions/TelemetryTransitionEmitter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Telemetry/Transitions/TelemetryTransitionEmitter.cs @@ -43,7 +43,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Transitions if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId)); if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId)); _transport = transport ?? throw new ArgumentNullException(nameof(transport)); - _partitionKey = $"{tenantId}_{sessionId}"; + _partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId); } /// diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs index df5c869f..073631f3 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs @@ -306,7 +306,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination }); DelayLateEventGrace(); - DrainSpool(); + DrainSpool(args); // Option 2 (WG Part 1 graceful-exit hardening, 2026-04-30): write the // clean-exit marker BEFORE _signalShutdown returns control to the main @@ -329,7 +329,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination RunUploadDiagnosticsWithEvents(args); WriteEnrollmentCompleteMarker(args); - RunStandaloneRebootIfRequested(); + RunStandaloneRebootIfRequested(args); RunSelfDestructIfAppropriate(args); // Option 2 (same hardening) — covers the standard Completed / Failed terminal // path too. Cleanup PowerShell does not touch this marker, so writing it before @@ -979,6 +979,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination /// private void RunUploadDiagnosticsWithEvents(EnrollmentTerminatedEventArgs args, string suffixOverride = null) { + if (IsSessionGone(args)) + { + // The backend answered 410 for this session: the package would describe a session + // that no longer exists, and its diagnostics_* events could never be accepted. + _logger.Info("EnrollmentTerminationHandler: diagnostics upload skipped — backend no longer knows this session."); + return; + } + var mode = _configuration.DiagnosticsUploadMode ?? "Off"; if (!_configuration.DiagnosticsUploadEnabled || string.Equals(mode, "Off", StringComparison.OrdinalIgnoreCase)) { @@ -1124,7 +1132,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination /// but enables RebootOnComplete, the agent's final act is shutdown.exe /r /// with the configured delay, giving the user a visible countdown. /// - private void RunStandaloneRebootIfRequested() + private void RunStandaloneRebootIfRequested(EnrollmentTerminatedEventArgs args) { if (_configuration.SelfDestructOnComplete) return; if (!_configuration.RebootOnComplete) return; @@ -1148,7 +1156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination ImmediateUpload = true, }); - DrainSpool(); + DrainSpool(args); try { @@ -1185,7 +1193,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination // just above. The WG-Part-1 and standalone-reboot paths already drain first; this // closes the gap on the self-destruct path. Bounded + best-effort (never blocks // termination on a wedged backend). - DrainSpool(); + DrainSpool(args); var service = _cleanupServiceFactory(); service.ExecuteSelfDestruct(); @@ -1318,8 +1326,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination } } - private void DrainSpool() + /// + /// : the backend no longer knows the session. + /// Every drain would only collect more 410s against a bounded budget, and a diagnostics + /// package would describe a deleted session — both are skipped; marker, cleanup and + /// self-destruct run as usual. + /// + private static bool IsSessionGone(EnrollmentTerminatedEventArgs args) => + string.Equals(args.Origin, TerminationOrigins.SessionGone, StringComparison.Ordinal); + + private void DrainSpool(EnrollmentTerminatedEventArgs args) { + if (IsSessionGone(args)) + { + _logger.Info("EnrollmentTerminationHandler: spool drain skipped — backend no longer knows this session."); + return; + } + // Block briefly so pending events can land before the next destructive step // (shutdown.exe, self-destruct). Two phases share the same bounded budget: // diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs index ecf102d6..ff7b4590 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs @@ -165,6 +165,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry { _logger?.Warning($"BackendTelemetryUploader: ingest TRANSIENT (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}"); } + else if (result.IsSessionGone) + { + _logger?.Error($"BackendTelemetryUploader: ingest rejected — backend no longer knows this session (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}"); + } else { _logger?.Error($"BackendTelemetryUploader: ingest PERMANENT fail (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}"); @@ -208,6 +212,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry case HttpStatusCode.RequestEntityTooLarge: // 413 — the data is fine, only the batch return UploadResult.TooLarge(shortReason); // size is wrong → split & retry (P1). + + case HttpStatusCode.Gone: // 410 — the backend no longer knows this session. + // Decided BEFORE the poison-body check: a poison body on 410 would make the + // drain drop items and emit telemetry_upload_poisoned, which the backend + // rejects with 410 again — an emit/reject loop for a session that is over. + return UploadResult.SessionGone($"session gone: {shortReason}"); } if (statusCode >= 500 && statusCode <= 599) return UploadResult.Transient(shortReason); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryPartitionKey.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryPartitionKey.cs new file mode 100644 index 00000000..7b1b4b2d --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryPartitionKey.cs @@ -0,0 +1,14 @@ +#nullable enable +namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry +{ + /// + /// The one composition of a session's Azure-Table partition key. The three emitters stamp + /// it on every ; the spool and the upload orchestrator compare + /// against it to tell the current session's items from leftovers of a previous session id. + /// A drift between producer and comparer would silently drop or retain the wrong items. + /// + public static class TelemetryPartitionKey + { + public static string ForSession(string tenantId, string sessionId) => $"{tenantId}_{sessionId}"; + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetrySpool.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetrySpool.cs index b7b39924..3dc6fc7a 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetrySpool.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetrySpool.cs @@ -74,7 +74,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry public int ByteLength { get; } } - public TelemetrySpool(string directoryPath, IClock clock, Logging.AgentLogger? logger = null) + /// + /// Partition key of the session this process serves (). + /// When set, pending items of another session id found at the head of the spool are dropped + /// on startup — see . Null disables the filter. + /// + public TelemetrySpool( + string directoryPath, + IClock clock, + Logging.AgentLogger? logger = null, + string? expectedPartitionKey = null) { if (string.IsNullOrEmpty(directoryPath)) { @@ -96,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry // count as "pending" and need to be rehydrated into _pending. _lastUploadedItemId = _cursor.Load(); _lastAssignedItemId = ScanSpoolAndRehydratePending(); + DropLeadingForeignPending(expectedPartitionKey); _peakPendingCount = _pending.Count; } @@ -312,5 +322,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry } return highest; } + + /// + /// Startup filter: drops the LEADING run of pending items whose partition key is not + /// , advancing and persisting the cursor past them. + /// Such items are leftovers of a previous session id in the same spool directory; the + /// backend answers 410 for them, and that 410 must never end the session this process + /// serves. One process serves one session, so a foreign item AFTER an own item cannot + /// occur — if it does, the order is not trustworthy and nothing is dropped. WhiteGlove + /// Part 2 reuses the Part-1 session id, so its leftovers carry the expected key and stay. + /// Called once from the ctor, after rehydration. + /// + private void DropLeadingForeignPending(string? expectedPartitionKey) + { + if (string.IsNullOrEmpty(expectedPartitionKey) || _pending.Count == 0) return; + + var leadingForeign = 0; + long lastForeignId = -1; + var seenOwn = false; + foreach (var entry in _pending) + { + var isForeign = !string.Equals(entry.Item.PartitionKey, expectedPartitionKey, StringComparison.Ordinal); + if (!isForeign) + { + seenOwn = true; + continue; + } + + if (!seenOwn) + { + leadingForeign++; + lastForeignId = entry.Item.TelemetryItemId; + continue; + } + + _logger?.Error( + $"TelemetrySpool: pending itemId={entry.Item.TelemetryItemId} carries a foreign partition key after items of the current session — spool order is inconsistent; keeping all {_pending.Count} pending item(s)."); + return; + } + + if (leadingForeign == 0) return; + + try + { + _cursor.Save(lastForeignId); + } + catch (Exception ex) + { + // Keep the items rather than risk a cursor that disagrees with disk; the drain + // loop drops the same leading run on its first 410 (defence in depth). + _logger?.Warning($"TelemetrySpool: could not persist the cursor past {leadingForeign} foreign pending item(s): {ex.Message}"); + return; + } + + _lastUploadedItemId = lastForeignId; + for (var i = 0; i < leadingForeign; i++) + { + _pendingBytes -= _pending.First!.Value.ByteLength; + _pending.RemoveFirst(); + } + + _logger?.Warning( + $"TelemetrySpool: dropped {leadingForeign} pending item(s) of a previous session id (cursor advanced to {lastForeignId}, {_pending.Count} pending item(s) of the current session kept)."); + } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryUploadOrchestrator.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryUploadOrchestrator.cs index 9875ab0e..2a499dcd 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryUploadOrchestrator.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryUploadOrchestrator.cs @@ -45,12 +45,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry // telemetry_upload_blocked exactly once per process rather than on every 30 s drain. private bool _uploadBlockedReported; + // Partition key of the session this process serves; null = every batch counts as own. + private readonly string? _currentPartitionKey; + private readonly Logging.AgentLogger? _logger; + + // One-shot guard for SessionGone. Latched only when a handler is subscribed: the host + // wires the handler after EnrollmentOrchestrator.Start, and a 410 on the very first + // drain must not be swallowed before anyone listens. + private bool _sessionGoneReported; + + /// + /// of the registered session. A 410 on a + /// batch that starts with this key ends the session (); a 410 on + /// a batch that starts with any other key only drops those leftover items. + /// public TelemetryUploadOrchestrator( ITelemetrySpool spool, IBackendTelemetryUploader uploader, IClock clock, int batchSize = 100, - IReadOnlyList? retryBackoffs = null) + IReadOnlyList? retryBackoffs = null, + string? currentPartitionKey = null, + Logging.AgentLogger? logger = null) { _spool = spool ?? throw new ArgumentNullException(nameof(spool)); _uploader = uploader ?? throw new ArgumentNullException(nameof(uploader)); @@ -61,6 +77,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry _effectiveBatchSize = batchSize; _retryBackoffs = retryBackoffs ?? DefaultRetryBackoffs; + _currentPartitionKey = currentPartitionKey; + _logger = logger; } /// @@ -97,6 +115,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry /// public event EventHandler? UploadBlocked; + /// + /// Raised once per process (after the drain guard releases) when the backend answered + /// HTTP 410 for a batch of the CURRENT session: the session is over server-side (never + /// registered, deleted, or being deleted) and no retry can succeed. The batch is retained + /// (cursor stays) and the drain stops; the host reports once over the emergency channel + /// and terminates the agent with self-destruct. Carries the backend error reason. + /// + public event EventHandler? SessionGone; + /// /// true when the last successful upload returned /// and the paused-until window has not expired. Drain is a no-op in this state. @@ -124,6 +151,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry List? pendingSignals = null; List? pendingPoison = null; string? pendingBlockedReason = null; + string? pendingSessionGoneReason = null; DrainResult result; try @@ -247,6 +275,35 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry new PoisonReport(1, oversizeId, "oversize: " + outcome.ErrorReason, PoisonKind.Oversize)); break; } + else if (outcome.IsSessionGone) + { + // HTTP 410 — the backend no longer knows the session these items belong to. + var foreignRun = CountLeadingForeignItems(batch); + if (foreignRun > 0) + { + // Leftovers of a previous session id (the spool drops these at startup; + // this is defence in depth). Drop only that leading run and carry on: a + // legitimate new session must never be ended because of old items. + var throughId = batch[foreignRun - 1].TelemetryItemId; + _spool.MarkUploaded(throughId); + _logger?.Warning( + $"TelemetryUploadOrchestrator: dropped {foreignRun} item(s) of a previous session id the backend rejected with 410 (cursor advanced to {throughId})."); + continue; + } + + // The current session is over server-side. RETAIN (cursor stays, the items + // remain on disk for the diagnostics ZIP), stop this drain, and surface it + // once — not as telemetry_upload_blocked, which would only queue another + // item behind a batch that can never be accepted. + failedBatches++; + lastError = outcome.ErrorReason; + if (!_sessionGoneReported && SessionGone != null) + { + _sessionGoneReported = true; + pendingSessionGoneReason = outcome.ErrorReason; + } + break; + } else { // P1 — RETAIN. Transient (retry next drain), auth-permanent (uploader already @@ -301,9 +358,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry catch { /* handler must not abort the drain result */ } } + if (pendingSessionGoneReason != null) + { + try { SessionGone?.Invoke(this, pendingSessionGoneReason); } + catch { /* handler must not abort the drain result */ } + } + return result; } + /// + /// Number of items at the head of whose partition key is not the + /// current session's. 0 when no current key is configured (every batch counts as own). + /// + private int CountLeadingForeignItems(IReadOnlyList batch) + { + if (_currentPartitionKey == null) return 0; + + var count = 0; + while (count < batch.Count + && !string.Equals(batch[count].PartitionKey, _currentPartitionKey, StringComparison.Ordinal)) + { + count++; + } + return count; + } + private static bool CarriesSignal(UploadResult outcome) => outcome.DeviceBlocked || outcome.DeviceKillSignal diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/UploadResult.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/UploadResult.cs index ae4f14c8..9f138f89 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/UploadResult.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/UploadResult.cs @@ -30,7 +30,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry bool isAuthFailure = false, bool requiresSplit = false, bool isPoison = false, - IReadOnlyList? poisonRowKeys = null) + IReadOnlyList? poisonRowKeys = null, + bool isSessionGone = false) { Success = success; ErrorReason = errorReason; @@ -44,6 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry RequiresSplit = requiresSplit; IsPoison = isPoison; PoisonRowKeys = poisonRowKeys; + IsSessionGone = isSessionGone; } public bool Success { get; } @@ -81,6 +83,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry /// The RowKeys the backend named as poison (item-level). Non-empty when . public IReadOnlyList? PoisonRowKeys { get; } + /// + /// HTTP 410: the backend no longer knows the session the batch belongs to (never + /// registered, deleted, or being deleted). Not transient, not auth, not poison — retrying + /// can never succeed. The orchestrator stops the drain (cursor stays) and raises + /// when the batch belongs to the + /// current session, or drops the batch's leading items when they are leftovers of a + /// previous session id. + /// + public bool IsSessionGone { get; } + /// /// Non-terminal quarantine signalled by the backend. The agent should stop draining /// uploads until (if set) elapses — the session stays alive so @@ -165,5 +177,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false, deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null, isPoison: true, poisonRowKeys: rejectedRowKeys ?? throw new ArgumentNullException(nameof(rejectedRowKeys))); + + /// + /// HTTP 410 — the backend no longer knows the session. Flagged ; + /// never retried, never split, never treated as poison. + /// + public static UploadResult SessionGone(string reason) => + new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false, + deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null, + isSessionGone: true); } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs index 472a3e9f..0466e1bc 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs @@ -439,6 +439,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime lifecyclePost, () => terminationHandler, agentConfig, + auth.EmergencyReporter, + shutdown, shutdownComplete, logger); diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/ServerControlPlane.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/ServerControlPlane.cs index 885e4197..ccc04e81 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/ServerControlPlane.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/ServerControlPlane.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.Net; using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Configuration; using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Orchestration; using AutopilotMonitor.Agent.V2.Core.Termination; using AutopilotMonitor.Shared; @@ -103,6 +105,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime /// dispatcher because these are real actions, not admin-button echoes. /// DeviceBlocked — non-terminal upload pause. Transport already reacted in /// TelemetryUploadOrchestrator.ApplyControlSignals; here we only log. + /// SessionGone (, HTTP 410 + /// on the current session's telemetry) — one emergency report, then the kill-signal + /// termination path tagged origin=session_gone. See . /// /// /// Shutdown race guard: if the agent has already finished its own termination @@ -117,9 +122,14 @@ namespace AutopilotMonitor.Agent.V2.Runtime InformationalEventPost post, Func terminationHandlerAccessor, AgentConfiguration agentConfig, + EmergencyReporter emergencyReporter, + ManualResetEventSlim shutdown, ManualResetEventSlim shutdownComplete, AgentLogger logger) { + orchestrator.Transport.SessionGone += (sender, reason) => + HandleSessionGone(reason, emergencyReporter, dispatcher, shutdown, shutdownComplete, logger); + orchestrator.Transport.ServerResponseReceived += (sender, upload) => { if (shutdownComplete.IsSet) @@ -140,7 +150,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime { { "forceSelfDestruct", "true" }, { "gracePeriodSeconds", "0" }, - { "origin", "kill_signal" }, + { "origin", TerminationOrigins.KillSignal }, }, }; try { dispatcher.DispatchAsync(new List { killAction }).GetAwaiter().GetResult(); } @@ -181,6 +191,76 @@ namespace AutopilotMonitor.Agent.V2.Runtime }; } + // Bounded blocking send for the one session-gone report: the terminate that follows ends + // the process and the cleanup script deletes the agent, so a fire-and-forget send would + // be abandoned mid-flight. Retries stay inside one anti-flood reservation (ONE report). + private const int SessionGoneReportAttempts = 2; + private static readonly TimeSpan SessionGoneReportTimeout = TimeSpan.FromSeconds(15); + private static readonly TimeSpan SessionGoneReportRetryDelay = TimeSpan.FromSeconds(3); + private static readonly TimeSpan SessionGoneReportBudget = TimeSpan.FromSeconds(40); + + /// + /// The backend answered HTTP 410 for the current session's telemetry: the session is over + /// server-side. (a) Send ONE + /// report over the emergency channel — the telemetry channel is exactly what was refused, + /// and the backend accepts this report for unknown sessions. (b) Terminate through the same + /// synthesised terminate_session path as the administrator kill signal (forced + /// self-destruct, no grace), tagged so the + /// termination handler skips the spool drain and the diagnostics upload. A shutdown that is + /// already in progress owns the outcome — then nothing happens here. + /// + internal static void HandleSessionGone( + string reason, + EmergencyReporter emergencyReporter, + ServerActionDispatcher dispatcher, + ManualResetEventSlim shutdown, + ManualResetEventSlim shutdownComplete, + AgentLogger logger) + { + if (shutdownComplete.IsSet || shutdown.IsSet) + { + logger.Debug("ServerControlPlane: session gone (HTTP 410) while shutdown is already in progress — ignoring."); + return; + } + + logger.Error($"Backend no longer knows this session (HTTP 410: {reason}) — reporting once over the emergency channel, then terminating with self-destruct."); + + if (emergencyReporter != null) + { + try + { + emergencyReporter + .TrySendAsync( + AgentErrorType.TelemetryRejectedSessionUnknown, + $"Telemetry upload rejected with HTTP 410 — backend no longer knows this session ({reason}). Agent terminating with self-destruct.", + httpStatusCode: (int)HttpStatusCode.Gone, + attempts: SessionGoneReportAttempts, + perAttemptTimeout: SessionGoneReportTimeout, + retryDelay: SessionGoneReportRetryDelay) + .Wait(SessionGoneReportBudget); + } + catch (Exception ex) + { + logger.Debug($"Session-gone report send failed (best-effort): {ex.Message}"); + } + } + + var terminateAction = new ServerAction + { + Type = ServerActionTypes.TerminateSession, + Reason = "Backend no longer knows this session (HTTP 410)", + QueuedAt = DateTime.UtcNow, + Params = new Dictionary + { + { "forceSelfDestruct", "true" }, + { "gracePeriodSeconds", "0" }, + { "origin", TerminationOrigins.SessionGone }, + }, + }; + try { dispatcher.DispatchAsync(new List { terminateAction }).GetAwaiter().GetResult(); } + catch (Exception ex) { logger.Error("ServerActionDispatcher.DispatchAsync (session gone) threw during server-response wiring.", ex); } + } + /// /// Soft admin-override handling. Portal Mark-Succeeded / Mark-Failed is an echo of an /// admin button click, not a ServerAction: it bypasses the dispatcher (no @@ -281,6 +361,17 @@ namespace AutopilotMonitor.Agent.V2.Runtime : EnrollmentTerminationOutcome.Failed; } + /// + /// The origin param of a synthesised terminate_session action + /// (), or null when absent — it travels into + /// . + /// + internal static string ReadOrigin(IReadOnlyDictionary parameters) + { + if (parameters == null) return null; + return parameters.TryGetValue("origin", out var origin) && !string.IsNullOrEmpty(origin) ? origin : null; + } + // ============================================================ Private terminate-callback /// @@ -326,8 +417,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime // before, masquerading every admin override as an error in SummaryDialog + // firing spurious diagnostics uploads). var mappedOutcome = MapAdminOutcome(action?.Params); + var origin = ReadOrigin(action?.Params); - logger.Warning($"ServerAction terminate_session received (ruleId={action?.RuleId}, reason={action?.Reason}, outcome={mappedOutcome}) — invoking termination handler."); + logger.Warning($"ServerAction terminate_session received (ruleId={action?.RuleId}, reason={action?.Reason}, outcome={mappedOutcome}, origin={origin ?? "none"}) — invoking termination handler."); // Synthesise a Terminated event as if the kernel fired it. terminationHandler.Handle( sender: null, @@ -336,7 +428,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime outcome: mappedOutcome, stageName: orchestrator.CurrentState?.Stage.ToString(), terminatedAtUtc: DateTime.UtcNow, - details: $"Server-requested termination: ruleId={action?.RuleId}, reason={action?.Reason}")); + details: $"Server-requested termination: ruleId={action?.RuleId}, reason={action?.Reason}", + origin: origin)); // Plan §6.2 synchronous-shutdown — block the ingest dispatcher thread until the // main-thread cleanup (orchestrator.Stop, client disposal) has fully run. This diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/DerivedWriterLivenessTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/DerivedWriterLivenessTests.cs new file mode 100644 index 00000000..1360ef53 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/DerivedWriterLivenessTests.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Analyze; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Functions.Services.Vulnerability; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Deletion; +using Azure.Data.Tables; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// The two queue handlers derive rows for a session (rule results, reports, inventory counters, +/// CveIndex). For a session the backend does not know, or one a cascade is tearing down, they +/// must stop before the first read of session data — nothing derived may ever outlive the +/// session's cascade. Both handlers are driven up to that gate with every later dependency +/// unavailable, so a regression that moves the gate shows up as a null dereference or an +/// unexpected mock call. +/// +public class DerivedWriterLivenessTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + + private static SessionDeletionGuard GuardFor(TableEntity? row) + { + var reader = new Mock(); + reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny>(), It.IsAny())) + .ReturnsAsync(row); + return new SessionDeletionGuard(reader.Object, NullLogger.Instance); + } + + private static Mock AdminConfigWithCorrelationEnabled() + { + var adminConfig = new Mock( + Mock.Of(), NullLogger.Instance, + new MemoryCache(new MemoryCacheOptions())); + adminConfig.Setup(a => a.GetConfigurationAsync()) + .ReturnsAsync(new AdminConfiguration { VulnerabilityCorrelationEnabled = true }); + return adminConfig; + } + + public static IEnumerable NotLiveRows() + { + yield return new object?[] { null }; + yield return new object?[] { new TableEntity(TenantId, SessionId) { ["DeletionState"] = SessionDeletionState.Running } }; + } + + [Theory] + [MemberData(nameof(NotLiveRows))] + public async Task VulnerabilityCorrelate_skips_a_missing_or_locked_session_before_loading_inventory(TableEntity? row) + { + var inventoryLoader = new Mock(MockBehavior.Strict); + var vulnRepo = new Mock(MockBehavior.Strict); + var handler = new VulnerabilityCorrelateHandler( + AdminConfigWithCorrelationEnabled().Object, + vulnerabilityCorrelation: null!, + vulnRepo.Object, + Mock.Of(), + inventoryLoader.Object, + Mock.Of(), + signalRNotification: null!, + GuardFor(row), + NullLogger.Instance); + + await handler.HandleAsync(new VulnerabilityCorrelateEnvelope + { + TenantId = TenantId, SessionId = SessionId, Reason = VulnerabilityCorrelateHandler.ReasonShutdownInventory, EnqueuedAt = DateTime.UtcNow, + }); + + inventoryLoader.VerifyNoOtherCalls(); + vulnRepo.VerifyNoOtherCalls(); + } + + [Theory] + [MemberData(nameof(NotLiveRows))] + public async Task Analyze_skips_a_missing_or_locked_session_before_the_rule_engine_runs(TableEntity? row) + { + var ruleRepo = new Mock(MockBehavior.Strict); + var metricsRepo = new Mock(MockBehavior.Strict); + var handler = new AnalyzeOnEnrollmentEndHandler( + ruleService: null!, + ruleRepo.Object, + Mock.Of(), + metricsRepo.Object, + signalRNotification: null!, + configService: null!, + channelDispatcher: null!, + GuardFor(row), + NullLogger.Instance); + + await handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope + { + TenantId = TenantId, SessionId = SessionId, Reason = AnalyzeOnEnrollmentEndHandler.ReasonEnrollmentComplete, EnqueuedAt = DateTime.UtcNow, + }); + + ruleRepo.VerifyNoOtherCalls(); + metricsRepo.VerifyNoOtherCalls(); + } + + [Fact] + public async Task A_guard_read_error_propagates_so_the_queue_retries_instead_of_skipping() + { + var reader = new Mock(); + reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny>(), It.IsAny())) + .ThrowsAsync(new Azure.RequestFailedException(503, "busy")); + var handler = new AnalyzeOnEnrollmentEndHandler( + ruleService: null!, Mock.Of(), Mock.Of(), Mock.Of(), + signalRNotification: null!, configService: null!, channelDispatcher: null!, + new SessionDeletionGuard(reader.Object, NullLogger.Instance), + NullLogger.Instance); + + await Assert.ThrowsAsync(() => handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope + { + TenantId = TenantId, SessionId = SessionId, Reason = AnalyzeOnEnrollmentEndHandler.ReasonEnrollmentComplete, EnqueuedAt = DateTime.UtcNow, + })); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IngestTelemetryAuthPathTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IngestTelemetryAuthPathTests.cs index 8984714b..defc0848 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/IngestTelemetryAuthPathTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/IngestTelemetryAuthPathTests.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Azure.Data.Tables; using AutopilotMonitor.Functions.Functions.Ingest; using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Security; using AutopilotMonitor.Functions.Services.Deletion; using AutopilotMonitor.Shared.Models; using AutopilotMonitor.Shared.Models.Deletion; @@ -155,6 +156,27 @@ public class IngestTelemetryAuthPathTests Assert.Equal(SessionStatus.Succeeded, SessionRowProjections.TryReadStatus(returned)); } + // ============================================================ Session unknown → 410 + + [Fact] + public async Task SessionUnknown_guard_returns_null_that_Run_maps_to_410_session_unknown() + { + // Run: a null guard row (no Sessions row, no active tombstone) is refused with 410 via + // WriteSessionUnknownAsync and stamped IngestRefusal=session_unknown; the cascade lock keeps + // its own value so the two 410 reasons stay apart on the request row. + var guard = NewGuard(out var reader); + reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync((TableEntity?)null); + reader.Setup(r => r.GetActiveSessionTombstoneAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync((TableEntity?)null); + + var row = await guard.EnsureWritableAndGetRowAsync(TenantId, SessionId, callerContext: "V2.IngestTelemetry"); + + Assert.Null(row); + Assert.NotEqual(RequestRowMarkers.IngestRefusal.SessionUnknown, RequestRowMarkers.IngestRefusal.CascadeLocked); + Assert.Equal("IngestRefusal", RequestRowMarkers.IngestRefusalKey); + } + // ============================================================ TryReadStatus (unique helper) [Theory] diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceStepTimerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceStepTimerTests.cs new file mode 100644 index 00000000..7c24b8f2 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceStepTimerTests.cs @@ -0,0 +1,33 @@ +using System; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Maintenance; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +public class MaintenanceStepTimerTests +{ + [Fact] + public async Task Records_every_step_in_order_and_returns_the_body_result() + { + var timer = new MaintenanceStepTimer(); + + await timer.RunAsync("first", () => Task.CompletedTask); + var value = await timer.RunAsync("second", () => Task.FromResult(42)); + + Assert.Equal(42, value); + Assert.Equal(new[] { "first", "second" }, timer.Steps.Keys); + Assert.All(timer.Steps.Values, ms => Assert.True(ms >= 0)); + } + + [Fact] + public async Task A_throwing_step_is_still_recorded_and_the_exception_propagates() + { + var timer = new MaintenanceStepTimer(); + + await Assert.ThrowsAsync( + () => timer.RunAsync("boom", () => throw new InvalidOperationException("x"))); + + Assert.True(timer.Steps.ContainsKey("boom")); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs index 6beccb16..ed790239 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs @@ -121,13 +121,17 @@ public class OffboardingSessionEnumeratorTests public Task> GetSessionsLeanAsync(string tenantId) => throw new NotImplementedException(); public Task> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException(); public Task> GetAllTenantIdsAsync() => throw new NotImplementedException(); - public Task DeleteSessionEventsAsync(string tenantId, string sessionId) => throw new NotImplementedException(); public Task DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => throw new NotImplementedException(); public Task BackfillSessionIndexAsync() => throw new NotImplementedException(); public Task CleanupGhostSessionIndexEntriesAsync() => throw new NotImplementedException(); public Task IsSessionIndexEmptyAsync() => throw new NotImplementedException(); - public Task> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => throw new NotImplementedException(); - public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => throw new NotImplementedException(); + public Task> GetSessionKeysAsync(CancellationToken ct = default) => throw new NotImplementedException(); + public Task> GetEventSessionIndexHandlesAsync(CancellationToken ct = default) => throw new NotImplementedException(); + public Task GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default) => throw new NotImplementedException(); + public Task StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) => throw new NotImplementedException(); + public Task DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) => throw new NotImplementedException(); + public Task DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default) => throw new NotImplementedException(); + public Task> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default) => throw new NotImplementedException(); public Task> GetAllSessionTombstoneKeysAsync() => throw new NotImplementedException(); } } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSessionRowFiltersTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSessionRowFiltersTests.cs new file mode 100644 index 00000000..03a5d1ed --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSessionRowFiltersTests.cs @@ -0,0 +1,116 @@ +using System; +using System.Linq; +using AutopilotMonitor.Functions.Services.Maintenance; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Pins the per-table filter of the orphan-session sweep to the writer's key shape. A filter +/// that names the wrong side (tenant where the session belongs, or vice versa) would either +/// miss the session's rows or touch another session's — both silently. +/// +public class OrphanSessionRowFiltersTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + + [Fact] + public void Every_filter_table_has_a_filter_naming_both_ids() + { + foreach (var table in OrphanSessionRowFilters.Tables) + { + var filter = OrphanSessionRowFilters.For(table, TenantId, SessionId); + Assert.Contains(TenantId, filter); + Assert.Contains(SessionId, filter); + } + } + + [Theory] + [InlineData(Constants.TableNames.Events)] + [InlineData(Constants.TableNames.RuleResults)] + [InlineData(Constants.TableNames.Signals)] + [InlineData(Constants.TableNames.DecisionTransitions)] + [InlineData(Constants.TableNames.VulnerabilityReports)] + public void Session_partition_tables_filter_on_the_composite_partition_key(string table) + { + Assert.Equal($"PartitionKey eq '{TenantId}_{SessionId}'", OrphanSessionRowFilters.For(table, TenantId, SessionId)); + } + + [Theory] + [InlineData(Constants.TableNames.DeviceSnapshot)] + [InlineData(Constants.TableNames.SessionTimeBreakdowns)] + public void One_row_per_session_tables_filter_on_tenant_partition_and_session_row_key(string table) + { + Assert.Equal($"PartitionKey eq '{TenantId}' and RowKey eq '{SessionId}'", OrphanSessionRowFilters.For(table, TenantId, SessionId)); + } + + [Fact] + public void AppInstallSummaries_filter_on_the_SessionId_column() + { + Assert.Equal($"PartitionKey eq '{TenantId}' and SessionId eq '{SessionId}'", + OrphanSessionRowFilters.For(Constants.TableNames.AppInstallSummaries, TenantId, SessionId)); + } + + [Fact] + public void EventTypeIndex_filter_scans_the_tenant_range_and_matches_the_SessionId_column() + { + // RowKey is {invertedTicks}_{sessionId}; without a Sessions row the ticks are not reconstructible. + Assert.Equal($"PartitionKey ge '{TenantId}_' and PartitionKey lt '{TenantId}_~' and SessionId eq '{SessionId}'", + OrphanSessionRowFilters.For(Constants.TableNames.EventTypeIndex, TenantId, SessionId)); + } + + [Fact] + public void CveIndex_filter_scans_the_tenant_range_and_matches_the_session_row_key() + { + Assert.Equal($"PartitionKey ge '{TenantId}_' and PartitionKey lt '{TenantId}_~' and RowKey eq '{SessionId}'", + OrphanSessionRowFilters.For(Constants.TableNames.CveIndex, TenantId, SessionId)); + } + + [Fact] + public void SessionAnnotations_filter_names_every_lane() + { + var filter = OrphanSessionRowFilters.For(Constants.TableNames.SessionAnnotations, TenantId, SessionId); + Assert.StartsWith($"PartitionKey eq '{TenantId}' and (", filter); + foreach (var lane in AnnotationLanes.All) + Assert.Contains($"RowKey eq '{SessionId}_{lane}'", filter); + Assert.Equal(AnnotationLanes.All.Length - 1, filter.Split(" or ").Length - 1); + } + + [Fact] + public void SessionTenantLookup_filter_pins_the_row_to_this_tenant() + { + Assert.Equal($"PartitionKey eq '{SessionId}' and RowKey eq 'tenant' and TenantId eq '{TenantId}'", + OrphanSessionRowFilters.For(Constants.TableNames.SessionTenantLookup, TenantId, SessionId)); + } + + [Fact] + public void Unknown_table_throws_instead_of_guessing() + { + Assert.Throws(() => OrphanSessionRowFilters.For(Constants.TableNames.Sessions, TenantId, SessionId)); + } + + [Theory] + [InlineData("not-a-guid", SessionId)] + [InlineData(TenantId, "'; drop")] + public void Non_guid_ids_throw(string tenantId, string sessionId) + { + Assert.ThrowsAny(() => OrphanSessionRowFilters.For(Constants.TableNames.Events, tenantId, sessionId)); + } + + [Fact] + public void Swept_tables_are_the_filter_tables_plus_inventory_pair_plus_handle() + { + var expected = OrphanSessionRowFilters.Tables + .Concat(new[] + { + Constants.TableNames.SessionInventoryContributions, + Constants.TableNames.SoftwareInventory, + Constants.TableNames.EventSessionIndex, + }) + .ToHashSet(StringComparer.Ordinal); + Assert.Equal(expected, OrphanSessionSweeper.SweptTables.ToHashSet(StringComparer.Ordinal)); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSessionSweeperTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSessionSweeperTests.cs new file mode 100644 index 00000000..513165dd --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSessionSweeperTests.cs @@ -0,0 +1,528 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Backup; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Functions.Services.Maintenance; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Deletion; +using Azure.Data.Tables; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Behaviour of the orphan-session sweep: which handles become candidates, the re-checks right +/// before the deletes, the inventory claim-then-decrement order, the handle as the last delete, +/// the cap/budget, and the weekly read-only reconciliation. Storage is mocked at the repository +/// seams; the ordered action log is the evidence for every ordering invariant. +/// +public class OrphanSessionSweeperTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + private const string SessionB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + private const string SessionC = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; + + // A Wednesday: the weekly reconciliation is not due unless forced or overdue. + private static readonly DateTime Now = new DateTime(2026, 10, 14, 10, 0, 0, DateTimeKind.Utc); + + private sealed class Harness + { + public readonly Mock Maint = new(); + public readonly Mock Vuln = new(); + public readonly Mock Reader = new(); + public readonly Mock AdminConfig; + public readonly Mock LockStore; + public readonly List Ops = new(); + public readonly List Log = new(); + + public readonly HashSet<(string TenantId, string SessionId)> SessionKeys = new(); + public readonly List Handles = new(); + public readonly Dictionary LiveHandles = new(StringComparer.Ordinal); + public readonly HashSet RowsPresent = new(StringComparer.Ordinal); + public readonly HashSet Tombstoned = new(StringComparer.Ordinal); + public readonly Dictionary> Inventory = new(StringComparer.Ordinal); + public readonly HashSet<(string TenantId, string SessionId)> IndexKeys = new(); + public readonly Dictionary RowsPerTable = new(StringComparer.Ordinal); + public readonly AdminConfiguration Config = new AdminConfiguration(); + + public DateTime Clock = Now; + public TimeSpan Budget = TimeSpan.FromMinutes(5); + public bool StampFails; + public bool HandleDeleteFails; + public bool DecrementFails; + public bool LeaseHeld; + public string? FailingTable; + + public Harness() + { + Maint.Setup(m => m.GetSessionKeysAsync(It.IsAny())) + .Callback(() => Log.Add("sessions")) + .ReturnsAsync(() => new HashSet<(string, string)>(SessionKeys)); + Maint.Setup(m => m.GetEventSessionIndexHandlesAsync(It.IsAny())) + .Callback(() => Log.Add("handles")) + .ReturnsAsync(() => Handles.ToList()); + Maint.Setup(m => m.GetEventSessionIndexHandleAsync(TenantId, It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, string s, CancellationToken _) => + LiveHandles.TryGetValue(s, out var live) ? live : Handles.FirstOrDefault(h => h.SessionId == s)); + Maint.Setup(m => m.StampHandleInventoryDecrementedAsync(TenantId, It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, s, _, _) => Log.Add($"stamp:{s}")) + .ReturnsAsync(() => StampFails ? null : "etag-stamped"); + Maint.Setup(m => m.DeleteOrphanSessionRowsAsync(It.IsAny(), TenantId, It.IsAny(), It.IsAny())) + .Returns((string table, string _, string s, CancellationToken _) => + { + Log.Add($"delete:{table}:{s}"); + if (string.Equals(table, FailingTable, StringComparison.Ordinal)) + throw new InvalidOperationException("storage said no"); + return Task.FromResult(RowsPerTable.TryGetValue(table, out var n) ? n : 1); + }); + Maint.Setup(m => m.DeleteEventSessionIndexHandleAsync(TenantId, It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, s, _, _) => Log.Add($"handle:{s}")) + .ReturnsAsync(() => !HandleDeleteFails); + Maint.Setup(m => m.GetEventTypeIndexSessionKeysAsync(It.IsAny())) + .ReturnsAsync(() => new HashSet<(string, string)>(IndexKeys)); + + Vuln.Setup(v => v.GetSessionInventoryContributionsAsync(TenantId, It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, string s, CancellationToken _) => + Inventory.TryGetValue(s, out var keys) ? (IReadOnlyList?)keys : null); + Vuln.Setup(v => v.DecrementSoftwareInventoryEntryAsync(TenantId, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((string _, string vendor, string name, string version, CancellationToken _) => + { + Log.Add($"decrement:{name}"); + if (DecrementFails) throw new InvalidOperationException("counter busy"); + return Task.CompletedTask; + }); + Vuln.Setup(v => v.DeleteSessionInventoryContributionsAsync(TenantId, It.IsAny(), It.IsAny())) + .Callback((_, s, _) => Log.Add($"sideRow:{s}")) + .Returns(Task.CompletedTask); + + Reader.Setup(r => r.GetSessionRowAsync(TenantId, It.IsAny(), It.IsAny>(), It.IsAny())) + .ReturnsAsync((string _, string s, IEnumerable _, CancellationToken _) => + RowsPresent.Contains(s) ? new TableEntity(TenantId, s) : null); + Reader.Setup(r => r.GetActiveSessionTombstoneAsync(TenantId, It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, string s, CancellationToken _) => + Tombstoned.Contains(s) ? new TableEntity(TenantId, s) : null); + + AdminConfig = new Mock( + Mock.Of(), NullLogger.Instance, + new MemoryCache(new MemoryCacheOptions())); + AdminConfig.Setup(a => a.GetConfigurationAsync()).ReturnsAsync(() => Config); + AdminConfig.Setup(a => a.UpdateAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns((Func mutate, string _, string? _) => + { + mutate(Config); + return Task.FromResult(new AdminConfigurationUpdateResult { Before = Config, After = Config }); + }); + + LockStore = new Mock(new SessionDeletionMaintenanceFunctionTests.MaintenanceBlobStub()); + LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny(), It.IsAny())) + .Returns(() => LeaseHeld + ? throw new LeaseHeldException("held", new InvalidOperationException("lease")) + : Task.FromResult(new Mock().Object)); + } + + public OrphanSessionHandle AddHandle(string sessionId, TimeSpan ingestAge, TimeSpan? writtenAge = null, string etag = "etag-1", bool decremented = false) + { + var handle = new OrphanSessionHandle + { + TenantId = TenantId, + SessionId = sessionId, + LastIngestAt = Now - ingestAge, + WrittenAt = Now - (writtenAge ?? TimeSpan.FromDays(1)), + ETag = etag, + InventoryDecremented = decremented, + }; + Handles.Add(handle); + return handle; + } + + public OrphanSessionSweeper Build() + { + var opsRepo = new Mock(); + opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny())) + .Callback(e => { lock (Ops) Ops.Add(e); }) + .Returns(Task.CompletedTask); + var ops = new OpsEventService(opsRepo.Object, NullLogger.Instance, + TestNotifications.InertOpsAlertDispatch(AdminConfig.Object)); + return new OrphanSessionSweeper(Maint.Object, Vuln.Object, Reader.Object, AdminConfig.Object, LockStore.Object, ops, + NullLogger.Instance, () => Clock, Budget); + } + + public Task RunCoreAsync(bool forceDeepCheck = false) => Build().RunCoreAsync(forceDeepCheck, CancellationToken.None); + } + + private static DeletionDecrementKey Key(string name) => new DeletionDecrementKey { Vendor = "v", Name = name, Version = "1" }; + + // ============================================================ candidates + + [Fact] + public async Task Candidates_are_handles_without_a_session_key_outside_both_grace_windows() + { + var h = new Harness(); + h.AddHandle(SessionA, TimeSpan.FromDays(3)); // orphan, old + h.AddHandle(SessionB, TimeSpan.FromDays(3)); // has a row + h.SessionKeys.Add((TenantId, SessionB)); + h.AddHandle(SessionC, TimeSpan.FromHours(1)); // ingest grace + h.AddHandle("dddddddd-dddd-4ddd-8ddd-dddddddddddd", TimeSpan.FromDays(3), writtenAge: TimeSpan.FromMinutes(30)); // restore grace + + var result = await h.RunCoreAsync(); + + Assert.Equal(1, result.Candidates); + Assert.Equal(SessionA, Assert.Single(result.Cleaned).SessionId); + Assert.Equal(4, result.HandlesScanned); + Assert.Equal(1, result.SessionKeys); + } + + [Fact] + public async Task Sessions_are_drained_before_handles() + { + var h = new Harness(); + await h.RunCoreAsync(); + + Assert.Equal(new[] { "sessions", "handles" }, h.Log.Take(2)); + } + + [Fact] + public async Task Legacy_handles_are_flagged_and_come_first() + { + var h = new Harness(); + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.AddHandle(SessionB, Now - OrphanSessionSweeper.LegacyHandleLastIngestAt); + + var result = await h.RunCoreAsync(); + + Assert.Equal(new[] { SessionB, SessionA }, result.Cleaned.Select(c => c.SessionId)); + Assert.True(result.Cleaned[0].Legacy); + Assert.False(result.Cleaned[1].Legacy); + } + + // ============================================================ re-checks + + [Fact] + public async Task Handle_gone_at_recheck_is_skipped_silently() + { + var h = new Harness(); + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.LiveHandles[SessionA] = null; + + var result = await h.RunCoreAsync(); + + Assert.Equal(1, result.SkippedHandleGone); + Assert.Empty(result.Cleaned); + Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal)); + } + + [Fact] + public async Task Handle_changed_since_the_scan_is_skipped() + { + var h = new Harness(); + var scanned = h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.LiveHandles[SessionA] = new OrphanSessionHandle + { + TenantId = TenantId, SessionId = SessionA, LastIngestAt = scanned.LastIngestAt, WrittenAt = scanned.WrittenAt, ETag = "etag-later", + }; + + var result = await h.RunCoreAsync(); + + Assert.Equal(1, result.SkippedHandleChanged); + Assert.Empty(result.Cleaned); + Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal)); + } + + [Fact] + public async Task Row_present_at_recheck_skips_everything() + { + var h = new Harness(); + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.RowsPresent.Add(SessionA); + + var result = await h.RunCoreAsync(); + + Assert.Equal(1, result.SkippedRowReturned); + Assert.Empty(result.Cleaned); + Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal) || l.StartsWith("stamp:", StringComparison.Ordinal)); + } + + [Fact] + public async Task Tombstone_counts_as_afterCascade_and_the_session_is_still_cleaned() + { + var h = new Harness(); + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.Tombstoned.Add(SessionA); + + var result = await h.RunCoreAsync(); + + Assert.Equal(1, result.AfterCascade); + Assert.True(Assert.Single(result.Cleaned).AfterCascade); + } + + // ============================================================ inventory + ordering + + [Fact] + public async Task Inventory_is_claimed_then_decremented_then_the_side_row_goes_before_the_tables() + { + var h = new Harness(); + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.Inventory[SessionA] = new List { Key("app-1"), Key("app-2") }; + + var result = await h.RunCoreAsync(); + + var stamp = h.Log.IndexOf($"stamp:{SessionA}"); + var firstDecrement = h.Log.IndexOf("decrement:app-1"); + var sideRow = h.Log.IndexOf($"sideRow:{SessionA}"); + var firstTable = h.Log.FindIndex(l => l.StartsWith("delete:", StringComparison.Ordinal)); + var handle = h.Log.IndexOf($"handle:{SessionA}"); + Assert.True(0 <= stamp && stamp < firstDecrement && firstDecrement < sideRow && sideRow < firstTable && firstTable < handle, + string.Join(" | ", h.Log)); + + var cleaned = Assert.Single(result.Cleaned); + Assert.Equal(2, cleaned.InventoryKeysDecremented); + Assert.Equal(2, result.TotalsByTable[Constants.TableNames.SoftwareInventory]); + Assert.Equal(1, result.TotalsByTable[Constants.TableNames.SessionInventoryContributions]); + } + + [Fact] + public async Task A_rerun_with_the_stamp_set_skips_the_decrements_but_still_removes_the_side_row() + { + var h = new Harness(); + h.AddHandle(SessionA, TimeSpan.FromDays(3), decremented: true); + h.Inventory[SessionA] = new List { Key("app-1") }; + + var result = await h.RunCoreAsync(); + + Assert.DoesNotContain(h.Log, l => l.StartsWith("stamp:", StringComparison.Ordinal) || l.StartsWith("decrement:", StringComparison.Ordinal)); + Assert.Contains($"sideRow:{SessionA}", h.Log); + Assert.Single(result.Cleaned); + Assert.Equal(0, result.TotalsByTable[Constants.TableNames.SoftwareInventory]); + } + + [Fact] + public async Task A_failed_stamp_aborts_the_candidate_before_any_decrement_or_delete() + { + var h = new Harness { StampFails = true }; + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.Inventory[SessionA] = new List { Key("app-1") }; + + var result = await h.RunCoreAsync(); + + Assert.Equal(1, result.SkippedHandleChanged); + Assert.Empty(result.Cleaned); + Assert.DoesNotContain(h.Log, l => l.StartsWith("decrement:", StringComparison.Ordinal) || l.StartsWith("delete:", StringComparison.Ordinal)); + } + + [Fact] + public async Task A_failed_decrement_is_counted_and_the_session_is_still_cleaned() + { + var h = new Harness { DecrementFails = true }; + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.Inventory[SessionA] = new List { Key("app-1"), Key("app-2") }; + + var result = await h.RunCoreAsync(); + + Assert.Equal(2, result.InventoryDecrementsFailed); + Assert.Single(result.Cleaned); + Assert.Contains($"handle:{SessionA}", h.Log); + } + + // ============================================================ the handle last + + [Fact] + public async Task Every_filter_table_is_cleared_and_the_handle_is_deleted_last_with_the_stamped_etag() + { + var h = new Harness(); + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.Inventory[SessionA] = new List { Key("app-1") }; + + var result = await h.RunCoreAsync(); + + foreach (var table in OrphanSessionRowFilters.Tables) + Assert.Contains($"delete:{table}:{SessionA}", h.Log); + Assert.Equal($"handle:{SessionA}", h.Log.Last()); + h.Maint.Verify(m => m.DeleteEventSessionIndexHandleAsync(TenantId, SessionA, "etag-stamped", It.IsAny()), Times.Once); + Assert.Equal(1, result.TotalsByTable[Constants.TableNames.EventSessionIndex]); + Assert.Equal(OrphanSessionRowFilters.Tables.Length, Assert.Single(result.Cleaned).TotalRows); + } + + [Fact] + public async Task A_handle_rewritten_before_the_final_delete_is_kept_and_reported() + { + var h = new Harness { HandleDeleteFails = true }; + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + + var result = await h.RunCoreAsync(); + + var cleaned = Assert.Single(result.Cleaned); + Assert.True(cleaned.HandleKept); + Assert.Equal(0, result.TotalsByTable[Constants.TableNames.EventSessionIndex]); + } + + [Fact] + public async Task A_failing_table_leaves_the_handle_and_the_run_continues_with_the_next_candidate() + { + var h = new Harness { FailingTable = Constants.TableNames.Signals }; + h.AddHandle(SessionA, TimeSpan.FromDays(4)); + h.AddHandle(SessionB, TimeSpan.FromDays(3)); + // Only session A's Signals delete fails: the harness throws for the table regardless of + // session, so flip the failure off once A has been attempted. + h.Maint.Setup(m => m.DeleteOrphanSessionRowsAsync(Constants.TableNames.Signals, TenantId, SessionB, It.IsAny())) + .ReturnsAsync(1); + + var result = await h.RunCoreAsync(); + + Assert.Equal(1, result.FailedSessions); + Assert.DoesNotContain($"handle:{SessionA}", h.Log); + Assert.Contains($"handle:{SessionB}", h.Log); + Assert.Equal(SessionB, Assert.Single(result.Cleaned).SessionId); + } + + // ============================================================ cap + budget + + [Fact] + public async Task The_cap_leaves_the_rest_for_the_next_run() + { + var h = new Harness(); + for (var i = 0; i < OrphanSessionSweeper.MaxCandidatesPerRun + 3; i++) + h.AddHandle(Guid.NewGuid().ToString(), TimeSpan.FromDays(2) + TimeSpan.FromMinutes(i)); + + var result = await h.RunCoreAsync(); + + Assert.Equal(OrphanSessionSweeper.MaxCandidatesPerRun, result.Cleaned.Count); + Assert.Equal(3, result.Remaining); + Assert.False(result.BudgetExhausted); + } + + [Fact] + public async Task An_exhausted_budget_stops_before_the_next_candidate() + { + var h = new Harness { Budget = TimeSpan.Zero }; + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + h.AddHandle(SessionB, TimeSpan.FromDays(3)); + + var result = await h.RunCoreAsync(); + + Assert.True(result.BudgetExhausted); + Assert.Empty(result.Cleaned); + Assert.Equal(2, result.Remaining); + } + + // ============================================================ weekly reconciliation + + public static IEnumerable DeepCheckCases() + { + var sunday0845 = new DateTime(2026, 10, 18, 8, 45, 0, DateTimeKind.Utc); + var sunday0445 = new DateTime(2026, 10, 18, 4, 45, 0, DateTimeKind.Utc); + var tuesday = new DateTime(2026, 10, 20, 12, 45, 0, DateTimeKind.Utc); + yield return new object?[] { null, sunday0845, true }; // never ran: first Sunday window + yield return new object?[] { null, tuesday, false }; // never ran: wait for Sunday + yield return new object?[] { sunday0845.AddDays(-7), sunday0845, true }; // weekly + yield return new object?[] { sunday0845.AddDays(-7), sunday0445, false }; // Sunday, but before 08:00 + yield return new object?[] { sunday0845.AddDays(-3), sunday0845, false }; // ran this week already + yield return new object?[] { tuesday.AddDays(-9), tuesday, true }; // missed Sunday: overdue + } + + [Theory] + [MemberData(nameof(DeepCheckCases))] + public void Deep_check_is_due_on_Sunday_mornings_or_when_overdue(DateTime? lastRun, DateTime now, bool due) + { + Assert.Equal(due, OrphanSessionSweeper.IsDeepCheckDue(lastRun, now)); + } + + [Fact] + public async Task Reconcile_reports_only_sessions_without_a_row_and_without_a_handle_and_stamps_the_marker() + { + var h = new Harness(); + const string noHandleNoRow = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee"; + const string noHandleButRow = "ffffffff-ffff-4fff-8fff-ffffffffffff"; + h.AddHandle(SessionA, TimeSpan.FromDays(3)); // in the sweep's own pipeline + h.SessionKeys.Add((TenantId, SessionB)); // live + h.RowsPresent.Add(noHandleButRow); // registered after the drain + h.IndexKeys.UnionWith(new[] { (TenantId, SessionA), (TenantId, SessionB), (TenantId, noHandleNoRow), (TenantId, noHandleButRow) }); + + var result = await h.RunCoreAsync(forceDeepCheck: true); + + var reconcile = Assert.IsType(result.Reconcile); + Assert.Equal(4, reconcile.IndexSessions); + Assert.Equal(1, reconcile.ResidueSessions); + Assert.Equal((TenantId, noHandleNoRow), Assert.Single(reconcile.Sample)); + var evt = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted); + Assert.Equal(OpsEventSeverity.Warning, evt.Severity); + Assert.Equal(Now.ToString("o"), h.Config.OrphanDeepCheckLastRunUtc); + } + + [Fact] + public async Task Reconcile_without_residue_is_Info_and_is_not_due_again_mid_week() + { + var h = new Harness(); + h.SessionKeys.Add((TenantId, SessionB)); + h.IndexKeys.Add((TenantId, SessionB)); + + var first = await h.RunCoreAsync(forceDeepCheck: true); + var evt = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted); + Assert.Equal(OpsEventSeverity.Info, evt.Severity); + Assert.Equal(0, first.Reconcile!.ResidueSessions); + + var second = await h.RunCoreAsync(); + Assert.Null(second.Reconcile); + } + + // ============================================================ RunAsync: lease + ops events + + [Fact] + public async Task A_timer_run_writes_the_heartbeat_and_EventsCleaned_only_when_something_was_cleaned() + { + var quiet = new Harness(); + Assert.True(await quiet.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None)); + Assert.Single(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanSweepCompleted && e.Severity == OpsEventSeverity.Info); + Assert.DoesNotContain(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanEventsCleaned); + Assert.DoesNotContain(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted); + + var busy = new Harness(); + busy.AddHandle(SessionA, TimeSpan.FromDays(3)); + Assert.True(await busy.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None)); + var cleaned = Assert.Single(busy.Ops, e => e.EventType == OpsEventTypes.OrphanEventsCleaned); + Assert.Equal(OpsEventSeverity.Warning, cleaned.Severity); + Assert.Contains(SessionA, cleaned.Details ?? string.Empty); + } + + [Fact] + public async Task A_manual_run_always_includes_the_reconciliation() + { + var h = new Harness(); + Assert.True(await h.Build().RunAsync("alice@contoso.com", CancellationToken.None)); + Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted); + } + + [Fact] + public async Task A_held_lease_records_SkippedLocked_and_touches_nothing() + { + var h = new Harness { LeaseHeld = true }; + h.AddHandle(SessionA, TimeSpan.FromDays(3)); + + Assert.False(await h.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None)); + + Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepSkippedLocked); + Assert.Empty(h.Log); + } + + [Fact] + public async Task A_failing_scan_records_OrphanSweepFailed() + { + var h = new Harness(); + h.Maint.Setup(m => m.GetSessionKeysAsync(It.IsAny())).ThrowsAsync(new InvalidOperationException("storage down")); + + Assert.True(await h.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None)); + + var failed = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepFailed); + Assert.Equal(OpsEventSeverity.Error, failed.Severity); + Assert.DoesNotContain(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepCompleted); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSweepOpsEventTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSweepOpsEventTests.cs new file mode 100644 index 00000000..6783f942 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/OrphanSweepOpsEventTests.cs @@ -0,0 +1,111 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Maintenance; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Shape of the ops events the orphan sweep and the maintenance run write: the dashboards and the +/// health pass read these details by name. +/// +public class OrphanSweepOpsEventTests +{ + private static (OpsEventService Service, List Saved) Rig() + { + var saved = new List(); + var opsRepo = new Mock(); + opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny())) + .Callback(e => { lock (saved) saved.Add(e); }) + .Returns(Task.CompletedTask); + var adminConfig = new Mock( + Mock.Of(), NullLogger.Instance, + new MemoryCache(new MemoryCacheOptions())) { CallBase = false }; + var alertDispatch = TestNotifications.InertOpsAlertDispatch(adminConfig.Object); + return (new OpsEventService(opsRepo.Object, NullLogger.Instance, alertDispatch), saved); + } + + [Fact] + public async Task MaintenanceCompleted_carries_the_step_timings() + { + var (service, saved) = Rig(); + var steps = new Dictionary { ["stalledSessions"] = 1200, ["aggregateMetrics"] = 90000 }; + + await service.RecordMaintenanceCompletedAsync(123456, "Timer", stepsMs: steps); + + var evt = Assert.Single(saved); + Assert.Equal(OpsEventTypes.MaintenanceCompleted, evt.EventType); + Assert.Contains("\"stepsMs\"", evt.Details); + Assert.Contains("\"aggregateMetrics\":90000", evt.Details); + } + + [Fact] + public async Task OrphanEventsCleaned_is_a_Warning_with_per_table_totals_and_the_legacy_count() + { + var (service, saved) = Rig(); + var result = new OrphanSweepRunResult { Remaining = 7 }; + var legacy = new OrphanSweepCleanedSession { TenantId = "t", SessionId = "s-legacy", Legacy = true, InventoryKeysDecremented = 3, HadInventorySideRow = true }; + legacy.RowsByTable[Constants.TableNames.Events] = 18; + legacy.RowsByTable[Constants.TableNames.EventTypeIndex] = 13; + result.Add(legacy); + var fresh = new OrphanSweepCleanedSession { TenantId = "t", SessionId = "s-fresh" }; + fresh.RowsByTable[Constants.TableNames.Events] = 1; + result.Add(fresh); + + await service.RecordOrphanEventsCleanedAsync(result); + + var evt = Assert.Single(saved); + Assert.Equal(OpsEventTypes.OrphanEventsCleaned, evt.EventType); + Assert.Equal(OpsEventSeverity.Warning, evt.Severity); + Assert.Contains("\"orphanSessions\":2", evt.Details); + Assert.Contains("\"totalEventsDeleted\":19", evt.Details); + Assert.Contains("\"legacySessions\":1", evt.Details); + Assert.Contains("\"remaining\":7", evt.Details); + Assert.Contains($"\"{Constants.TableNames.SoftwareInventory}\":3", evt.Details); + Assert.Contains($"\"{Constants.TableNames.EventSessionIndex}\":2", evt.Details); + Assert.Contains("\"legacy\":true", evt.Details); + } + + [Fact] + public async Task OrphanSweepCompleted_is_the_Info_heartbeat_with_the_counters() + { + var (service, saved) = Rig(); + var result = new OrphanSweepRunResult + { + HandlesScanned = 16784, SessionKeys = 16700, Candidates = 3, Remaining = 1, SkippedHandleGone = 2, DurationMs = 4200, + Reconcile = new OrphanReconcileResult { IndexSessions = 16000, ResidueSessions = 0, DurationMs = 180000 }, + }; + + await service.RecordOrphanSweepCompletedAsync(result, "Timer"); + + var evt = Assert.Single(saved); + Assert.Equal(OpsEventTypes.OrphanSweepCompleted, evt.EventType); + Assert.Equal(OpsEventSeverity.Info, evt.Severity); + Assert.Contains("\"handlesScanned\":16784", evt.Details); + Assert.Contains("\"skippedHandleGone\":2", evt.Details); + Assert.Contains("\"residueSessions\":0", evt.Details); + } + + [Fact] + public async Task OrphanReconcileCompleted_flips_to_Warning_with_a_sample_when_residue_exists() + { + var (service, saved) = Rig(); + var clean = new OrphanReconcileResult { IndexSessions = 10, ResidueSessions = 0 }; + var dirty = new OrphanReconcileResult { IndexSessions = 10, ResidueSessions = 2, Sample = { ("t1", "s1"), ("t1", "s2") } }; + + await service.RecordOrphanReconcileCompletedAsync(clean); + await service.RecordOrphanReconcileCompletedAsync(dirty); + + Assert.Equal(OpsEventSeverity.Info, saved[0].Severity); + Assert.Equal(OpsEventSeverity.Warning, saved[1].Severity); + Assert.Contains("\"sessionId\":\"s2\"", saved[1].Details); + Assert.Contains("\"sampleTruncated\":false", saved[1].Details); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionGuardTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionGuardTests.cs index c9628e6d..3676e604 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionGuardTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionGuardTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Azure.Data.Tables; @@ -133,6 +134,60 @@ public class SessionDeletionGuardTests Assert.Equal(SessionId, ex.SessionId); } + // ============================================================ GetLivenessAsync (derived writers) ==== + + [Fact] + public async Task GetLivenessAsync_reports_Missing_when_the_row_does_not_exist() + { + // Unlike the ingest contract, a derived writer (queue handler, rescan) stops here: nothing + // may be derived for a session the backend does not know. + var guard = NewGuard(out var reader); + reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny>(), It.IsAny())) + .ReturnsAsync((TableEntity?)null); + + Assert.Equal(SessionLiveness.Missing, await guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityCorrelate")); + reader.Verify(r => r.GetActiveSessionTombstoneAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [Theory] + [InlineData(SessionDeletionState.Preparing)] + [InlineData(SessionDeletionState.Queued)] + [InlineData(SessionDeletionState.Running)] + [InlineData(SessionDeletionState.Poisoned)] + public async Task GetLivenessAsync_reports_Locked_for_every_lock_state(string state) + { + var guard = NewGuard(out var reader); + reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny>(), It.IsAny())) + .ReturnsAsync(new TableEntity(TenantId, SessionId) { ["DeletionState"] = state }); + + Assert.Equal(SessionLiveness.Locked, await guard.GetLivenessAsync(TenantId, SessionId, "AnalyzeOnEnrollmentEnd")); + } + + [Theory] + [InlineData(null)] + [InlineData("None")] + public async Task GetLivenessAsync_reports_Live_for_an_unlocked_row(string? state) + { + var guard = NewGuard(out var reader); + var row = new TableEntity(TenantId, SessionId); + if (state != null) row["DeletionState"] = state; + reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny>(), It.IsAny())) + .ReturnsAsync(row); + + Assert.Equal(SessionLiveness.Live, await guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityRescan")); + } + + [Fact] + public async Task GetLivenessAsync_propagates_a_read_error() + { + // A storage failure must surface to the queue worker (retry/poison), never read as "missing". + var guard = NewGuard(out var reader); + reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny>(), It.IsAny())) + .ThrowsAsync(new Azure.RequestFailedException(503, "busy")); + + await Assert.ThrowsAsync(() => guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityCorrelate")); + } + // ============================================================ Codex F3: tombstone-marker disambiguation ==== [Fact] diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TableLifecycleBucketTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TableLifecycleBucketTests.cs index 666d7426..d41d6655 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/TableLifecycleBucketTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TableLifecycleBucketTests.cs @@ -81,6 +81,32 @@ public class TableLifecycleBucketTests Constants.TableNames.DelegationInvitations, }; + /// + /// The orphan-session sweep is the cascade for sessions without a Sessions row, so it must + /// clear exactly the tables the cascade manifest clears — minus the tombstone pair, which does + /// not exist for such a session. A new manifest step without a sweep entry (or the reverse) + /// fails here until someone decides the table's orphan lifecycle. + /// + [Fact] + public async Task OrphanSweep_ClearsEveryPerSessionCascadeTable_ExceptTheTombstonePair() + { + var cascade = await PerSessionCascadeTablesAsync(); + cascade.Remove(Constants.TableNames.Sessions); + cascade.Remove(Constants.TableNames.SessionsIndex); + + var swept = AutopilotMonitor.Functions.Services.Maintenance.OrphanSessionSweeper.SweptTables.ToHashSet(StringComparer.Ordinal); + + var missingFromSweep = cascade.Except(swept, StringComparer.Ordinal).OrderBy(t => t, StringComparer.Ordinal).ToList(); + var sweptButNotCascaded = swept.Except(cascade, StringComparer.Ordinal).OrderBy(t => t, StringComparer.Ordinal).ToList(); + + Assert.True(missingFromSweep.Count == 0, + "Per-session cascade tables the orphan sweep does not clear — add them to OrphanSessionRowFilters.Tables " + + "(with the writer's key shape) or handle them in OrphanSessionSweeper:\n - " + string.Join("\n - ", missingFromSweep)); + Assert.True(sweptButNotCascaded.Count == 0, + "Orphan sweep clears tables the cascade manifest does not know — add a manifest step first:\n - " + + string.Join("\n - ", sweptButNotCascaded)); + } + [Fact] public void TableNamesConstants_SetEquals_TableNamesAll() { diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs index 1e4bfa7f..9e8373a3 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs @@ -1019,6 +1019,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage { "MsrcLastSyncUtc", config.MsrcLastSyncUtc }, { "NvdCacheLastRefreshUtc", config.NvdCacheLastRefreshUtc }, { "EpssLastSyncUtc", config.EpssLastSyncUtc }, + { "OrphanDeepCheckLastRunUtc", config.OrphanDeepCheckLastRunUtc }, { "MaxDiagnosticsDownloadSizeMB", config.MaxDiagnosticsDownloadSizeMB }, { "DiagnosticsDownloadTimeoutSeconds", config.DiagnosticsDownloadTimeoutSeconds }, { "OpsEventRetentionDays", config.OpsEventRetentionDays }, @@ -1094,6 +1095,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage MsrcLastSyncUtc = entity.GetString("MsrcLastSyncUtc"), NvdCacheLastRefreshUtc = entity.GetString("NvdCacheLastRefreshUtc"), EpssLastSyncUtc = entity.GetString("EpssLastSyncUtc"), + OrphanDeepCheckLastRunUtc = entity.GetString("OrphanDeepCheckLastRunUtc"), MaxDiagnosticsDownloadSizeMB = entity.GetInt32("MaxDiagnosticsDownloadSizeMB") ?? 500, DiagnosticsDownloadTimeoutSeconds = entity.GetInt32("DiagnosticsDownloadTimeoutSeconds") ?? 120, OpsEventRetentionDays = entity.GetInt32("OpsEventRetentionDays") ?? 90, diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs index 1e8e6c4e..2fdcab64 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs @@ -87,9 +87,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task> GetAllTenantIdsAsync() => _storage.GetAllTenantIdsAsync(); - public Task DeleteSessionEventsAsync(string tenantId, string sessionId) - => _storage.DeleteSessionEventsAsync(tenantId, sessionId); - public Task DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => _storage.DeleteSessionRuleResultsAsync(tenantId, sessionId); @@ -102,11 +99,26 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task IsSessionIndexEmptyAsync() => _storage.IsSessionIndexEmptyAsync(); - public Task> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) - => _storage.GetOrphanedEventSessionsAsync(gracePeriod); + public Task> GetSessionKeysAsync(CancellationToken ct = default) + => _storage.GetSessionKeysAsync(ct); - public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) - => _storage.DeleteEventSessionIndexEntryAsync(tenantId, sessionId); + public Task> GetEventSessionIndexHandlesAsync(CancellationToken ct = default) + => _storage.GetEventSessionIndexHandlesAsync(ct); + + public Task GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default) + => _storage.GetEventSessionIndexHandleAsync(tenantId, sessionId, ct); + + public Task StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) + => _storage.StampHandleInventoryDecrementedAsync(tenantId, sessionId, etag, ct); + + public Task DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) + => _storage.DeleteEventSessionIndexHandleAsync(tenantId, sessionId, etag, ct); + + public Task DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default) + => _storage.DeleteOrphanSessionRowsAsync(table, tenantId, sessionId, ct); + + public Task> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default) + => _storage.GetEventTypeIndexSessionKeysAsync(ct); public Task> GetAllSessionTombstoneKeysAsync() => _storage.GetAllSessionTombstoneKeysAsync(); diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Config/PatchAdminConfigurationFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Config/PatchAdminConfigurationFunction.cs index d5a42c1c..0e806e22 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Config/PatchAdminConfigurationFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Config/PatchAdminConfigurationFunction.cs @@ -55,6 +55,7 @@ namespace AutopilotMonitor.Functions.Functions.Config "LatestAgentV2Version", "LatestAgentV2Sha256", "LatestAgentV2ExeSha256", "LatestBootstrapV2ScriptVersion", "PlanTierDefinitionsJson", "VulnerabilityDataLastSyncUtc", "MsrcLastSyncUtc", "NvdCacheLastRefreshUtc", "EpssLastSyncUtc", + "OrphanDeepCheckLastRunUtc", }; /// diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs index 4a46a4ef..ee7bb02b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs @@ -228,12 +228,14 @@ namespace AutopilotMonitor.Functions.Functions.Ingest return AsOutput(await WriteDeviceBlockedAsync(req, sessionVerdict)); } - // Cascade-delete guard: refuse the batch with 410 Gone when a V2 cascade owns the - // Sessions row (states Preparing/Queued/Running/Poisoned). Without this check, the - // hot-path writers below (StoreEventsBatchAsync, signal/transition StoreBatchAsync, - // UpdateSessionImeAgentVersionAsync upsert, …) would land rows past the lock and - // leave orphan data the manifest cannot describe. One read per batch; absent - // Sessions row → silent pass (caller handles session-not-found in its own write). + // Session guard: refuse the batch with 410 Gone when a V2 cascade owns the Sessions + // row (states Preparing/Queued/Running/Poisoned) OR when there is no Sessions row at + // all. Both mean "this session is over server-side": the hot-path writers below + // (StoreEventsBatchAsync, signal/transition StoreBatchAsync, the index upserts, …) + // would otherwise land rows no cascade can ever reach, because a cascade needs the + // Sessions row. Registration always precedes the first upload (and re-creates an + // absent row), so a live session never sees this. One read per batch; the row's + // IngestRefusal marker tells the two 410 reasons apart on the request row. Azure.Data.Tables.TableEntity? guardSessionRow; try { @@ -241,12 +243,22 @@ namespace AutopilotMonitor.Functions.Functions.Ingest } catch (SessionDeletionLockedException locked) { + RequestRowMarkers.Stamp(req, RequestRowMarkers.IngestRefusalKey, RequestRowMarkers.IngestRefusal.CascadeLocked); _logger.LogInformation( "IngestTelemetry: refused batch — cascade in flight tenant={Tenant} session={Session} state={State} manifestId={ManifestId}", bodyTenantId, sessionId, locked.CurrentState, locked.ManifestId); return AsOutput(await WriteSessionLockedAsync(req, locked)); } + if (guardSessionRow == null) + { + RequestRowMarkers.Stamp(req, RequestRowMarkers.IngestRefusalKey, RequestRowMarkers.IngestRefusal.SessionUnknown); + _logger.LogInformation( + "IngestTelemetry: refused batch — session unknown tenant={Tenant} session={Session}", + bodyTenantId, sessionId); + return AsOutput(await WriteSessionUnknownAsync(req)); + } + // The header serial is caller-declared; the Sessions row carries the serial the // session was registered under. When they differ, the ROW's serial is held against // the block list too — a mid-session header change must not dodge a block placed @@ -415,6 +427,19 @@ namespace AutopilotMonitor.Functions.Functions.Ingest ProcessedAt = DateTime.UtcNow, }); + /// + /// 410 for a session without a Sessions row: same status and body shape as the cascade lock, + /// because the agent's reaction is the same (stop, clean up). Never a poison body — the + /// agent would drop the items and re-emit a marker event into the same dead session. + /// + private static Task WriteSessionUnknownAsync(HttpRequestData req) + => req.JsonAsync(HttpStatusCode.Gone, new IngestEventsResponse + { + Success = false, + Message = "Session is unknown to the backend (never registered, or already deleted); further telemetry will be rejected.", + ProcessedAt = DateTime.UtcNow, + }); + /// /// 422 + poison body: the backend end of the agent's item-level poison protocol /// (BackendTelemetryUploader.TryReadPoisonSignalAsync). Logged as a Warning with diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/OrphanSessionSweepFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/OrphanSessionSweepFunction.cs new file mode 100644 index 00000000..81c7a617 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/OrphanSessionSweepFunction.cs @@ -0,0 +1,42 @@ +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Maintenance; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Maintenance +{ + /// + /// Timer for the orphan-session sweep (): rows of sessions + /// that have no Sessions row — which no cascade can ever reach — are found through their + /// EventSessionIndex handle and removed from every session table. Its own timer, lease and + /// ops events: the sweep needs a slower cadence than the 2h Maintenance run, must never + /// race the retention cascade's 00:00/12:00 tick, and its cost (two key drains instead of one + /// Sessions point-read per handle) should stay visible on its own heartbeat. + /// + public sealed class OrphanSessionSweepFunction + { + // Minute 45 of every fourth hour: no other timer fires at :45 (Maintenance :00 of even + // hours, SessionSweep :30, WhatsNew :20, backup watchdog :00/:30, retention 00:00/12:00). + // const per repo convention (see SessionSweepFunction.cs). + private const string Cron = "0 45 */4 * * *"; + + private readonly OrphanSessionSweeper _sweeper; + private readonly ILogger _logger; + + public OrphanSessionSweepFunction(OrphanSessionSweeper sweeper, ILogger logger) + { + _sweeper = sweeper; + _logger = logger; + } + + [Function("OrphanSessionSweep")] + public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken) + { + _logger.LogInformation("OrphanSessionSweep timer trigger fired"); + // Lease, run and every ops event (Completed / Failed / SkippedLocked) live in the sweeper, + // shared with the manual maintenance run. + await _sweeper.RunAsync("Timer", cancellationToken); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Vulnerability/GetVulnerabilityReportFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Vulnerability/GetVulnerabilityReportFunction.cs index 28633e5c..e59165c4 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Vulnerability/GetVulnerabilityReportFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Vulnerability/GetVulnerabilityReportFunction.cs @@ -24,19 +24,22 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability private readonly ISessionRepository _sessionRepo; private readonly VulnerabilityCorrelationService _correlationService; private readonly IVulnerabilityInventoryLoader _inventoryLoader; + private readonly Services.Deletion.SessionDeletionGuard _deletionGuard; public GetVulnerabilityReportFunction( ILogger logger, IVulnerabilityRepository vulnRepo, ISessionRepository sessionRepo, VulnerabilityCorrelationService correlationService, - IVulnerabilityInventoryLoader inventoryLoader) + IVulnerabilityInventoryLoader inventoryLoader, + Services.Deletion.SessionDeletionGuard deletionGuard) { _logger = logger; _vulnRepo = vulnRepo; _sessionRepo = sessionRepo; _correlationService = correlationService; _inventoryLoader = inventoryLoader; + _deletionGuard = deletionGuard; } [Function("GetVulnerabilityReport")] @@ -65,6 +68,15 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability if (rescan) { + // A re-scan derives rows (report, CveIndex, inventory counters): only for a live + // session, like the queue handler — never for one the backend does not know or a + // cascade is tearing down. + var liveness = await _deletionGuard.GetLivenessAsync(effectiveTenantId, sessionId, "VulnerabilityRescan"); + if (liveness == Services.Deletion.SessionLiveness.Missing) + return await req.NotFoundAsync("Session not found."); + if (liveness == Services.Deletion.SessionLiveness.Locked) + return await req.ConflictAsync("Session is being deleted; a re-scan is not possible."); + try { _logger.LogInformation("Vulnerability re-scan requested for session {SessionId} by {User} (query: {Query})", diff --git a/src/Backend/AutopilotMonitor.Functions/Middleware/RequestTelemetryMiddleware.cs b/src/Backend/AutopilotMonitor.Functions/Middleware/RequestTelemetryMiddleware.cs index 85ea1b84..67f5452b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Middleware/RequestTelemetryMiddleware.cs +++ b/src/Backend/AutopilotMonitor.Functions/Middleware/RequestTelemetryMiddleware.cs @@ -198,6 +198,10 @@ public class RequestTelemetryMiddleware : IFunctionsWorkerMiddleware if (context.Items.TryGetValue(RequestRowMarkers.DeviceValidationKey, out var deviceValidation) && deviceValidation is string deviceValidationOutcome) requestTelemetry.Properties[RequestRowMarkers.DeviceValidationKey] = deviceValidationOutcome; + // INGEST-REFUSAL — set by IngestTelemetry on its 410 paths (cascade lock vs unknown session). + if (context.Items.TryGetValue(RequestRowMarkers.IngestRefusalKey, out var ingestRefusal) && ingestRefusal is string ingestRefusalReason) + requestTelemetry.Properties[RequestRowMarkers.IngestRefusalKey] = ingestRefusalReason; + // CERT-DEVICE-BINDING — the Intune device lookup behind the certificate (admitting or // observing), set by SecurityValidator whenever a lookup result exists for the request. foreach (var key in RequestRowMarkers.CertDeviceBindingKeys) diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index 6a57123f..216812ad 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -294,6 +294,10 @@ builder.Services.AddHttpClient() // BackgroundService worker that runs a queued manual run. builder.Services.AddSingleton(); builder.Services.AddSingleton(); +// Orphan-session sweep: own 4h timer (OrphanSessionSweepFunction), own lease, also run by the +// manual maintenance trigger. The cascade for sessions that have no Sessions row. +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton< AutopilotMonitor.Functions.Services.Maintenance.IMaintenanceTriggerProducer, AutopilotMonitor.Functions.Services.Maintenance.AzureQueueMaintenanceTriggerProducer>(); diff --git a/src/Backend/AutopilotMonitor.Functions/Security/RequestRowMarkers.cs b/src/Backend/AutopilotMonitor.Functions/Security/RequestRowMarkers.cs index edcf729d..d0547e31 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/RequestRowMarkers.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/RequestRowMarkers.cs @@ -41,6 +41,22 @@ namespace AutopilotMonitor.Functions.Security public static string DeviceValidationValue(ValidatorType validatedBy) => validatedBy == ValidatorType.Unknown ? DeviceValidation.None : validatedBy.ToString(); + /// + /// Why a telemetry batch was refused with 410: the session is locked by a cascade delete, + /// or the backend has no Sessions row for it. Both share the status so the agent treats + /// them alike; the row keeps them apart. + /// + public const string IngestRefusalKey = "IngestRefusal"; + + /// Closed set of values. + public static class IngestRefusal + { + /// The Sessions row is in a cascade lock state (or freshly tombstoned). + public const string CascadeLocked = "cascade_locked"; + /// No Sessions row and no tombstone: never registered, or deleted longer ago than the marker lives. + public const string SessionUnknown = "session_unknown"; + } + /// /// Outcome of the Intune device lookup behind the client certificate /// ( name). Absent when no lookup result exists for diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AnalyzeOnEnrollmentEndHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AnalyzeOnEnrollmentEndHandler.cs index d5c93a12..97fae50c 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AnalyzeOnEnrollmentEndHandler.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AnalyzeOnEnrollmentEndHandler.cs @@ -50,6 +50,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze private readonly SignalRNotificationService _signalRNotification; private readonly TenantConfigurationService _configService; private readonly Notifications.NotificationChannelDispatcher _channelDispatcher; + private readonly Deletion.SessionDeletionGuard _deletionGuard; private readonly ILogger _logger; public const string ReasonEnrollmentComplete = "enrollment_complete"; @@ -66,6 +67,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze SignalRNotificationService signalRNotification, TenantConfigurationService configService, Notifications.NotificationChannelDispatcher channelDispatcher, + Deletion.SessionDeletionGuard deletionGuard, ILogger logger) { _ruleService = ruleService; @@ -75,6 +77,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze _signalRNotification = signalRNotification; _configService = configService; _channelDispatcher = channelDispatcher; + _deletionGuard = deletionGuard; _logger = logger; } @@ -94,6 +97,20 @@ namespace AutopilotMonitor.Functions.Services.Analyze } var sessionPrefix = $"[Session: {envelope.SessionId.Substring(0, Math.Min(8, envelope.SessionId.Length))}]"; + + // Rule results, stats and channel alerts only for a live session: a session the backend + // does not know, or one a cascade is tearing down, gets nothing derived (no cascade could + // reach the rows afterwards, and an alert would name a session that does not exist). + var liveness = await _deletionGuard.GetLivenessAsync( + envelope.TenantId, envelope.SessionId, "AnalyzeOnEnrollmentEnd", cancellationToken).ConfigureAwait(false); + if (liveness != Deletion.SessionLiveness.Live) + { + _logger.LogWarning( + "{Prefix} Analyze skipped — session {Liveness} (reason={Reason})", + sessionPrefix, liveness, envelope.Reason); + return; + } + var isVulnerabilityRerun = string.Equals( envelope.Reason, ReasonVulnerabilityCorrelated, StringComparison.OrdinalIgnoreCase); var isWhitegloveSealed = string.Equals( diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionGuard.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionGuard.cs index c8491dbf..97ddb58e 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionGuard.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionGuard.cs @@ -8,6 +8,17 @@ using Microsoft.Extensions.Logging; namespace AutopilotMonitor.Functions.Services.Deletion { + /// Outcome of . + public enum SessionLiveness + { + /// Sessions row present and not in a cascade lock state. + Live, + /// No Sessions row: never registered, or deleted. + Missing, + /// Sessions row present but owned by a cascade (Preparing / Queued / Running / Poisoned). + Locked, + } + /// /// Single chokepoint for the cascade-delete writer-block invariant (Plan §1 P7 / §5 PR3). /// Two APIs split for hot-path I/O cost: @@ -88,6 +99,35 @@ namespace AutopilotMonitor.Functions.Services.Deletion /// Columns the guard reads off the row itself; a projected read adds them to the caller's list. public static readonly string[] LockColumns = { "DeletionState", "PendingDeletionManifestId" }; + /// + /// For writers that DERIVE rows for a session (queue handlers, the manual rescan). Unlike + /// the registration/ingest contract, a missing Sessions row is a stop here, not a pass: + /// a derived row for a session the backend does not know can never be reached by a + /// cascade and would only feed the orphan sweep. Read errors propagate (queue retry). + /// + public async Task GetLivenessAsync(string tenantId, string sessionId, string callerContext, CancellationToken cancellationToken = default) + { + var sessionRow = await _reader.GetSessionRowAsync(tenantId, sessionId, LockColumns, cancellationToken); + if (sessionRow == null) + { + _logger.LogWarning( + "SessionDeletionGuard: no Sessions row — derived write skipped: tenant={TenantId} session={SessionId} caller={Caller}", + tenantId, sessionId, callerContext); + return SessionLiveness.Missing; + } + + var state = sessionRow.GetString("DeletionState"); + if (SessionDeletionState.IsLocked(state)) + { + _logger.LogInformation( + "SessionDeletionGuard: session locked — derived write skipped: tenant={TenantId} session={SessionId} state={State} caller={Caller}", + tenantId, sessionId, state, callerContext); + return SessionLiveness.Locked; + } + + return SessionLiveness.Live; + } + /// /// with the /// row projected to plus — the agent hot diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceStepTimer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceStepTimer.cs new file mode 100644 index 00000000..14b2d5fc --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/MaintenanceStepTimer.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; + +namespace AutopilotMonitor.Functions.Services.Maintenance +{ + /// + /// Per-step wall-clock of one maintenance run, reported as stepsMs on the + /// MaintenanceCompleted ops event. Worker-side timing logs never reach App Insights, + /// and the run's composition is what decides which steps leave the 2h timer next. + /// A step that throws is still recorded (its time counts), then the exception propagates. + /// + public sealed class MaintenanceStepTimer + { + private readonly Dictionary _steps = new Dictionary(StringComparer.Ordinal); + + /// Durations in declaration order (ms, rounded). + public IReadOnlyDictionary Steps => _steps; + + public async Task RunAsync(string step, Func body) + { + var sw = Stopwatch.StartNew(); + try + { + await body().ConfigureAwait(false); + } + finally + { + _steps[step] = sw.ElapsedMilliseconds; + } + } + + public async Task RunAsync(string step, Func> body) + { + var sw = Stopwatch.StartNew(); + try + { + return await body().ConfigureAwait(false); + } + finally + { + _steps[step] = sw.ElapsedMilliseconds; + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSessionRowFilters.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSessionRowFilters.cs new file mode 100644 index 00000000..69207fc0 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSessionRowFilters.cs @@ -0,0 +1,93 @@ +using System; +using System.Linq; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Functions.Services.Maintenance +{ + /// + /// The OData filter that selects every row of ONE session in each table the orphan sweep + /// clears — the key shape of the table's writer, not of the cascade manifest (a session + /// without a Sessions row has no manifest). Pure, so the shapes are unit-tested against the + /// writers once instead of trusted per call. + /// + public static class OrphanSessionRowFilters + { + /// + /// Tables cleared through a filter, in the order the sweep runs them. The handle + /// (EventSessionIndex) and the inventory pair (SessionInventoryContributions + the + /// SoftwareInventory decrements) are handled by the sweeper itself and are not listed here. + /// + public static readonly string[] Tables = + { + Constants.TableNames.Events, + Constants.TableNames.RuleResults, + Constants.TableNames.AppInstallSummaries, + Constants.TableNames.VulnerabilityReports, + Constants.TableNames.DeviceSnapshot, + Constants.TableNames.Signals, + Constants.TableNames.DecisionTransitions, + Constants.TableNames.EventTypeIndex, + Constants.TableNames.CveIndex, + Constants.TableNames.SessionTimeBreakdowns, + Constants.TableNames.SessionAnnotations, + Constants.TableNames.SessionTenantLookup, + }; + + /// + /// Filter for ; throws for a table that has no declared shape so a + /// new session table can never be swept with a guessed filter. + /// + public static string For(string table, string tenantId, string sessionId) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); + var t = ODataSanitizer.EscapeValue(tenantId); + var s = ODataSanitizer.EscapeValue(sessionId); + + switch (table) + { + // Partition per session: {tenantId}_{sessionId}. + case Constants.TableNames.Events: + case Constants.TableNames.RuleResults: + case Constants.TableNames.Signals: + case Constants.TableNames.DecisionTransitions: + case Constants.TableNames.VulnerabilityReports: + return $"PartitionKey eq '{t}_{s}'"; + + // One row per session under the tenant partition. + case Constants.TableNames.DeviceSnapshot: + case Constants.TableNames.SessionTimeBreakdowns: + return $"PartitionKey eq '{t}' and RowKey eq '{s}'"; + + // Tenant partition, session in a column. + case Constants.TableNames.AppInstallSummaries: + return $"PartitionKey eq '{t}' and SessionId eq '{s}'"; + + // PK {tenantId}_{eventType}, RK {invertedTicks}_{sessionId}: the RowKey of a session + // without a Sessions row is not reconstructible (one row per batch), so the session + // is found by its column across the tenant's partitions. Tenant ids are GUIDs, so the + // '_'..'_~' range cannot reach another tenant. + case Constants.TableNames.EventTypeIndex: + return $"PartitionKey ge '{t}_' and PartitionKey lt '{t}_~' and SessionId eq '{s}'"; + + // PK {tenantId}_{cveId}, RK sessionId. + case Constants.TableNames.CveIndex: + return $"PartitionKey ge '{t}_' and PartitionKey lt '{t}_~' and RowKey eq '{s}'"; + + // PK tenantId, RK {sessionId}_{lane}; the lanes are a closed set. + case Constants.TableNames.SessionAnnotations: + var lanes = string.Join(" or ", AnnotationLanes.All.Select(lane => $"RowKey eq '{s}_{lane}'")); + return $"PartitionKey eq '{t}' and ({lanes})"; + + // PK sessionId, RK 'tenant'; the TenantId column pins the row to this tenant's session. + case Constants.TableNames.SessionTenantLookup: + return $"PartitionKey eq '{s}' and RowKey eq 'tenant' and TenantId eq '{t}'"; + + default: + throw new ArgumentException($"No orphan-sweep filter is declared for table '{table}'.", nameof(table)); + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSessionSweeper.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSessionSweeper.cs new file mode 100644 index 00000000..dc5c9370 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSessionSweeper.cs @@ -0,0 +1,429 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Backup; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using Azure.Storage.Blobs.Specialized; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Maintenance +{ + /// + /// The cascade for sessions that have no Sessions row. A cascade needs the row (CAS lock, + /// manifest, tombstone); rows a writer leaves behind for a session the backend does not know + /// are therefore reachable only through the discovery handle every events write creates + /// (EventSessionIndex, PK tenantId / RK sessionId). The sweep diffs those handles against the + /// Sessions keys, re-checks every candidate moments before touching it, clears every session + /// table the cascade manifest knows (ratchet in TableLifecycleBucketTests), and deletes + /// the handle LAST — a failure before that point leaves the session discoverable for the next + /// run; every delete is idempotent. + /// + /// Once a week (Sunday morning UTC, or on any manual maintenance run) it additionally + /// reconciles EventTypeIndex against the Sessions keys, read-only: the proof that nothing + /// without a handle is left behind. + /// + /// + public sealed class OrphanSessionSweeper + { + /// A handle younger than this may belong to a session whose registration is still in flight. + public static readonly TimeSpan IngestGrace = TimeSpan.FromHours(24); + + /// + /// A restore re-inserts the handle (old LastIngestAt, fresh write time) and the Sessions row + /// LAST; a handle written this recently is left alone until the restore has either finished + /// or failed. + /// + public static readonly TimeSpan HandleWriteGrace = TimeSpan.FromHours(2); + + public const int MaxCandidatesPerRun = 50; + public static readonly TimeSpan RunBudget = TimeSpan.FromMinutes(5); + + /// Sunday at or after this hour (UTC) is the weekly reconciliation window — the quiet part of the week. + public const int DeepCheckSundayHourUtc = 8; + public static readonly TimeSpan DeepCheckInterval = TimeSpan.FromDays(6); + /// A missed Sunday (deploy, outage) does not postpone the reconciliation past this age. + public static readonly TimeSpan DeepCheckOverdue = TimeSpan.FromDays(8); + public const int DeepCheckSampleSize = 20; + + /// LastIngestAt of a handle written by hand for a legacy orphan: no ingest ever happened at that time. + public static readonly DateTime LegacyHandleLastIngestAt = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + public const string Actor = "System.OrphanSweep"; + public const string TimerTrigger = "Timer"; + + /// + /// Every table the sweep clears: the filter tables, the inventory pair (side row + the + /// SoftwareInventory decrements) and the handle. Pinned against the cascade manifest. + /// + public static readonly IReadOnlyList SweptTables = OrphanSessionRowFilters.Tables + .Concat(new[] + { + Constants.TableNames.SessionInventoryContributions, + Constants.TableNames.SoftwareInventory, + Constants.TableNames.EventSessionIndex, + }) + .ToArray(); + + private readonly IMaintenanceRepository _maintenanceRepo; + private readonly IVulnerabilityRepository _vulnRepo; + private readonly ISessionDeletionInventoryReader _reader; + private readonly AdminConfigurationService _adminConfig; + private readonly OrphanSweepLockStore _lockStore; + private readonly OpsEventService _opsEvents; + private readonly ILogger _logger; + private readonly Func _clock; + private readonly TimeSpan _runBudget; + + public OrphanSessionSweeper( + IMaintenanceRepository maintenanceRepo, + IVulnerabilityRepository vulnRepo, + ISessionDeletionInventoryReader reader, + AdminConfigurationService adminConfig, + OrphanSweepLockStore lockStore, + OpsEventService opsEvents, + ILogger logger) + : this(maintenanceRepo, vulnRepo, reader, adminConfig, lockStore, opsEvents, logger, () => DateTime.UtcNow, RunBudget) + { + } + + /// Test seam: injectable clock and run budget. + internal OrphanSessionSweeper( + IMaintenanceRepository maintenanceRepo, + IVulnerabilityRepository vulnRepo, + ISessionDeletionInventoryReader reader, + AdminConfigurationService adminConfig, + OrphanSweepLockStore lockStore, + OpsEventService opsEvents, + ILogger logger, + Func clock, + TimeSpan runBudget) + { + _maintenanceRepo = maintenanceRepo; + _vulnRepo = vulnRepo; + _reader = reader; + _adminConfig = adminConfig; + _lockStore = lockStore; + _opsEvents = opsEvents; + _logger = logger; + _clock = clock; + _runBudget = runBudget; + } + + /// + /// One run under the sweep lease: timer and manual maintenance enter here. Returns false + /// when another run held the lease. A manual run always includes the reconciliation. + /// Every outcome is an ops event; nothing propagates to the caller. + /// + public async Task RunAsync(string triggeredBy, CancellationToken cancellationToken) + { + BlobLeaseClient lease; + try + { + lease = await _lockStore.AcquireLeaseAsync(ct: cancellationToken).ConfigureAwait(false); + } + catch (LeaseHeldException) + { + _logger.LogWarning("OrphanSweep: lease held by another run — skipping (triggeredBy={TriggeredBy})", triggeredBy); + await _opsEvents.RecordOrphanSweepSkippedLockedAsync(triggeredBy).ConfigureAwait(false); + return false; + } + + using var runCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var leaseHolder = new MaintenanceLeaseHolder(lease, runCts, _logger); + try + { + OrphanSweepRunResult result; + try + { + result = await RunCoreAsync(forceDeepCheck: triggeredBy != TimerTrigger, runCts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "OrphanSweep: run failed (triggeredBy={TriggeredBy})", triggeredBy); + await _opsEvents.RecordOrphanSweepFailedAsync(ex.Message, triggeredBy).ConfigureAwait(false); + return true; + } + + await _opsEvents.RecordOrphanSweepCompletedAsync(result, triggeredBy).ConfigureAwait(false); + if (result.Cleaned.Count > 0) + await _opsEvents.RecordOrphanEventsCleanedAsync(result).ConfigureAwait(false); + return true; + } + finally + { + if (leaseHolder.RenewalFailureReason != null) + _logger.LogWarning("OrphanSweep: the lease could not be renewed during the run ({Reason})", leaseHolder.RenewalFailureReason); + await leaseHolder.DisposeAsync().ConfigureAwait(false); + } + } + + /// The run without lease and ops events (unit-testable). + internal async Task RunCoreAsync(bool forceDeepCheck, CancellationToken ct) + { + var sw = Stopwatch.StartNew(); + var now = _clock(); + var result = new OrphanSweepRunResult(); + + // Sessions first: a cascade that finishes between the two drains removes the handle + // (step 6) before the row (step 18), so this order cannot produce a candidate the + // cascade is still working on. + var sessionKeys = await _maintenanceRepo.GetSessionKeysAsync(ct).ConfigureAwait(false); + var handles = await _maintenanceRepo.GetEventSessionIndexHandlesAsync(ct).ConfigureAwait(false); + result.SessionKeys = sessionKeys.Count; + result.HandlesScanned = handles.Count; + + var candidates = handles + .Where(h => !sessionKeys.Contains((h.TenantId, h.SessionId))) + .Where(h => h.LastIngestAt <= now - IngestGrace && h.WrittenAt <= now - HandleWriteGrace) + .OrderBy(h => h.LastIngestAt) + .ToList(); + result.Candidates = candidates.Count; + + var processed = 0; + foreach (var candidate in candidates) + { + if (processed >= MaxCandidatesPerRun) break; + if (sw.Elapsed >= _runBudget) + { + result.BudgetExhausted = true; + break; + } + ct.ThrowIfCancellationRequested(); + processed++; + try + { + await CleanCandidateAsync(candidate, result, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + result.FailedSessions++; + _logger.LogError(ex, + "OrphanSweep: cleaning tenant={TenantId} session={SessionId} failed — the handle stays for the next run", + candidate.TenantId, candidate.SessionId); + } + } + result.Remaining = candidates.Count - processed; + + if (forceDeepCheck || IsDeepCheckDue(await ReadDeepCheckLastRunAsync().ConfigureAwait(false), now)) + result.Reconcile = await ReconcileAsync(sessionKeys, handles, now, ct).ConfigureAwait(false); + + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + + private async Task CleanCandidateAsync(OrphanSessionHandle candidate, OrphanSweepRunResult result, CancellationToken ct) + { + var tenantId = candidate.TenantId; + var sessionId = candidate.SessionId; + + // Re-checks moments before the deletes — the scan above is minutes old. + var handle = await _maintenanceRepo.GetEventSessionIndexHandleAsync(tenantId, sessionId, ct).ConfigureAwait(false); + if (handle == null) + { + result.SkippedHandleGone++; // a cascade got there first + return; + } + if (!string.Equals(handle.ETag, candidate.ETag, StringComparison.Ordinal)) + { + result.SkippedHandleChanged++; // a later write; its own grace starts over + return; + } + var row = await _reader.GetSessionRowAsync(tenantId, sessionId, SessionDeletionGuard.LockColumns, ct).ConfigureAwait(false); + if (row != null) + { + result.SkippedRowReturned++; // registered or restored meanwhile + return; + } + var afterCascade = await _reader.GetActiveSessionTombstoneAsync(tenantId, sessionId, ct).ConfigureAwait(false) != null; + if (afterCascade) result.AfterCascade++; + + var cleaned = new OrphanSweepCleanedSession + { + TenantId = tenantId, + SessionId = sessionId, + Legacy = candidate.LastIngestAt <= LegacyHandleLastIngestAt, + AfterCascade = afterCascade, + }; + var etag = handle.ETag; + + // Inventory first, with the handle as the progress record. The stamp is the claim: a + // crash after it loses at most this session's decrements (the counts stay high, as they + // did before the sweep existed), and a rerun can never decrement twice. + if (!handle.InventoryDecremented) + { + var stamped = await _maintenanceRepo.StampHandleInventoryDecrementedAsync(tenantId, sessionId, etag, ct).ConfigureAwait(false); + if (stamped == null) + { + result.SkippedHandleChanged++; + return; + } + etag = stamped; + + var keys = await _vulnRepo.GetSessionInventoryContributionsAsync(tenantId, sessionId, ct).ConfigureAwait(false); + if (keys != null) + { + cleaned.HadInventorySideRow = true; + foreach (var key in keys) + { + try + { + await _vulnRepo.DecrementSoftwareInventoryEntryAsync(tenantId, key.Vendor, key.Name, key.Version, ct).ConfigureAwait(false); + cleaned.InventoryKeysDecremented++; + } + catch (Exception ex) + { + result.InventoryDecrementsFailed++; + _logger.LogWarning(ex, + "OrphanSweep: inventory decrement failed tenant={TenantId} session={SessionId} key={Vendor}/{Name}/{Version}", + tenantId, sessionId, key.Vendor, key.Name, key.Version); + } + } + } + } + await _vulnRepo.DeleteSessionInventoryContributionsAsync(tenantId, sessionId, ct).ConfigureAwait(false); + + foreach (var table in OrphanSessionRowFilters.Tables) + { + var rows = await _maintenanceRepo.DeleteOrphanSessionRowsAsync(table, tenantId, sessionId, ct).ConfigureAwait(false); + cleaned.RowsByTable[table] = rows; + } + + // The handle last, and only while unchanged: a batch that re-wrote it meanwhile has + // rows the next run must still find. + var handleDeleted = await _maintenanceRepo.DeleteEventSessionIndexHandleAsync(tenantId, sessionId, etag, ct).ConfigureAwait(false); + cleaned.HandleKept = !handleDeleted; + + result.Add(cleaned); + _logger.LogInformation( + "OrphanSweep: cleaned tenant={TenantId} session={SessionId} rows={Rows} inventoryKeys={Keys} legacy={Legacy} afterCascade={AfterCascade} handleKept={HandleKept}", + tenantId, sessionId, cleaned.TotalRows, cleaned.InventoryKeysDecremented, cleaned.Legacy, cleaned.AfterCascade, cleaned.HandleKept); + } + + /// + /// Sunday at/after 08:00 UTC once the last run is at least six days old; a run older than + /// eight days is due at the next tick regardless. Never run before: the first Sunday. + /// + internal static bool IsDeepCheckDue(DateTime? lastRunUtc, DateTime nowUtc) + { + var inSundayWindow = nowUtc.DayOfWeek == DayOfWeek.Sunday && nowUtc.Hour >= DeepCheckSundayHourUtc; + if (lastRunUtc == null) return inSundayWindow; + var age = nowUtc - lastRunUtc.Value; + if (age >= DeepCheckOverdue) return true; + return inSundayWindow && age >= DeepCheckInterval; + } + + private async Task ReadDeepCheckLastRunAsync() + { + var config = await _adminConfig.GetConfigurationAsync().ConfigureAwait(false); + return DateTime.TryParse(config?.OrphanDeepCheckLastRunUtc, CultureInfo.InvariantCulture, + DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed + : (DateTime?)null; + } + + /// + /// Read-only: every (tenant, session) in EventTypeIndex that has neither a Sessions row nor + /// a handle (handles are the sweep's own pipeline) is residue — confirmed against the live + /// Sessions row, because the drains are minutes old. Residue is reported, never cleaned + /// here: the operator gives it a handle, and the normal path with all its checks does the rest. + /// + private async Task ReconcileAsync( + HashSet<(string TenantId, string SessionId)> sessionKeys, + IReadOnlyList handles, + DateTime now, + CancellationToken ct) + { + var sw = Stopwatch.StartNew(); + var indexKeys = await _maintenanceRepo.GetEventTypeIndexSessionKeysAsync(ct).ConfigureAwait(false); + var handleKeys = new HashSet<(string TenantId, string SessionId)>(handles.Select(h => (h.TenantId, h.SessionId))); + + var residue = new List<(string TenantId, string SessionId)>(); + foreach (var key in indexKeys) + { + if (sessionKeys.Contains(key) || handleKeys.Contains(key)) continue; + ct.ThrowIfCancellationRequested(); + var row = await _reader.GetSessionRowAsync(key.TenantId, key.SessionId, SessionDeletionGuard.LockColumns, ct).ConfigureAwait(false); + if (row != null) continue; + residue.Add(key); + } + + var result = new OrphanReconcileResult + { + IndexSessions = indexKeys.Count, + ResidueSessions = residue.Count, + Sample = residue.Take(DeepCheckSampleSize).ToList(), + DurationMs = sw.ElapsedMilliseconds, + }; + await _opsEvents.RecordOrphanReconcileCompletedAsync(result).ConfigureAwait(false); + + var stamp = now.ToString("o", CultureInfo.InvariantCulture); + await _adminConfig.UpdateAsync(c => { c.OrphanDeepCheckLastRunUtc = stamp; return null; }, Actor, "orphan-sweep").ConfigureAwait(false); + return result; + } + } + + /// One session the sweep cleaned in a run. + public sealed class OrphanSweepCleanedSession + { + public string TenantId { get; set; } = string.Empty; + public string SessionId { get; set; } = string.Empty; + /// The handle was written by hand for a legacy orphan (LastIngestAt 2000-01-01). + public bool Legacy { get; set; } + /// A tombstone marker existed: a cascade ran, rows were written after it. + public bool AfterCascade { get; set; } + /// The handle changed between the deletes and the final delete; the next run sees the session again. + public bool HandleKept { get; set; } + public bool HadInventorySideRow { get; set; } + public int InventoryKeysDecremented { get; set; } + public Dictionary RowsByTable { get; } = new Dictionary(StringComparer.Ordinal); + public int TotalRows => RowsByTable.Values.Sum(); + } + + /// Outcome of the weekly reconciliation. + public sealed class OrphanReconcileResult + { + public int IndexSessions { get; set; } + public int ResidueSessions { get; set; } + public List<(string TenantId, string SessionId)> Sample { get; set; } = new List<(string TenantId, string SessionId)>(); + public long DurationMs { get; set; } + } + + /// Counters of one sweep run; the Completed ops event carries them verbatim. + public sealed class OrphanSweepRunResult + { + public int HandlesScanned { get; set; } + public int SessionKeys { get; set; } + public int Candidates { get; set; } + public List Cleaned { get; } = new List(); + public int SkippedRowReturned { get; set; } + public int SkippedHandleGone { get; set; } + public int SkippedHandleChanged { get; set; } + public int AfterCascade { get; set; } + public int Remaining { get; set; } + public int FailedSessions { get; set; } + public int InventoryDecrementsFailed { get; set; } + public bool BudgetExhausted { get; set; } + public long DurationMs { get; set; } + public Dictionary TotalsByTable { get; } = new Dictionary(StringComparer.Ordinal); + public OrphanReconcileResult? Reconcile { get; set; } + + public void Add(OrphanSweepCleanedSession cleaned) + { + Cleaned.Add(cleaned); + foreach (var kv in cleaned.RowsByTable) + TotalsByTable[kv.Key] = TotalsByTable.TryGetValue(kv.Key, out var n) ? n + kv.Value : kv.Value; + Bump(Constants.TableNames.SessionInventoryContributions, cleaned.HadInventorySideRow ? 1 : 0); + Bump(Constants.TableNames.SoftwareInventory, cleaned.InventoryKeysDecremented); + Bump(Constants.TableNames.EventSessionIndex, cleaned.HandleKept ? 0 : 1); + } + + private void Bump(string table, int by) + => TotalsByTable[table] = TotalsByTable.TryGetValue(table, out var n) ? n + by : by; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSweepLockStore.cs b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSweepLockStore.cs new file mode 100644 index 00000000..636202f8 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Maintenance/OrphanSweepLockStore.cs @@ -0,0 +1,21 @@ +using AutopilotMonitor.Functions.Services.Locking; +using AutopilotMonitor.Shared; + +namespace AutopilotMonitor.Functions.Services.Maintenance +{ + /// + /// Blob-lease lock for the orphan-session sweep: the 4h timer, the manual maintenance run and + /// a second host instance enter through it. Its own sentinel — the sweep has no reason to + /// exclude the platform maintenance run or the retention cascade, and they none to exclude it. + /// + public class OrphanSweepLockStore : BlobLeaseLockStore + { + /// Sentinel blob path. _lock/… does not parse as a manifest blob name, so the manifest-TTL sweep skips it. + public const string LockBlobName = "_lock/orphan-session-sweep.lock"; + + public OrphanSweepLockStore(BlobStorageService blobs) + : base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "orphan session sweep") + { + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs index 91b290a9..40b3166e 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.Aggregation.cs @@ -1217,80 +1217,6 @@ namespace AutopilotMonitor.Functions.Services _logger.LogWarning(ex, "Failed to cleanup old verdict-calibration aggregates"); } } - /// - /// Detects and cleans up orphaned events — events stored in the Events table - /// whose session no longer exists in the Sessions table. - /// Uses the EventSessionIndex side-table for efficient detection (no full Events scan). - /// Grace period: 24 hours to protect against register+ingest race conditions. - /// - private async Task CleanupOrphanedEventsAsync() - { - try - { - _logger.LogInformation("Starting orphaned events cleanup..."); - var sw = Stopwatch.StartNew(); - - var orphans = await _maintenanceRepo.GetOrphanedEventSessionsAsync(TimeSpan.FromHours(24)); - - if (orphans.Count == 0) - { - _logger.LogInformation("No orphaned events found"); - return; - } - - _logger.LogWarning("Found {Count} orphaned event sessions, cleaning up...", orphans.Count); - - int totalEventsDeleted = 0; - int sessionsCleanedUp = 0; - // Per-orphan breakdown for the OpsEvent so the dashboard shows which tenant/session - // was cleaned (the worker LogInformation below does not reach App Insights). - var cleanedOrphans = new List(); - - foreach (var orphan in orphans) - { - try - { - // Ordering invariant: the index entry is removed ONLY after the event - // delete completed cleanly (DeleteSessionEventsAsync throws on failure). - // Orphan detection scans only the index — deleting it first would make - // surviving event rows permanently undiscoverable. - var deletedEvents = await _maintenanceRepo.DeleteSessionEventsAsync(orphan.TenantId, orphan.SessionId); - await _maintenanceRepo.DeleteEventSessionIndexEntryAsync(orphan.TenantId, orphan.SessionId); - - totalEventsDeleted += deletedEvents; - sessionsCleanedUp++; - // Report the actual rows deleted (may differ from the index's recorded count). - cleanedOrphans.Add(new OrphanedEventSession - { - TenantId = orphan.TenantId, - SessionId = orphan.SessionId, - LastIngestAt = orphan.LastIngestAt, - EventCount = deletedEvents - }); - - _logger.LogInformation( - "Cleaned orphan: TenantId={TenantId}, SessionId={SessionId}, Events={Events}", - orphan.TenantId, orphan.SessionId, deletedEvents); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to cleanup orphan: TenantId={TenantId}, SessionId={SessionId}", - orphan.TenantId, orphan.SessionId); - } - } - - sw.Stop(); - _logger.LogInformation( - "Orphaned events cleanup completed in {Ms}ms: {Sessions} sessions, {Events} events deleted", - sw.ElapsedMilliseconds, sessionsCleanedUp, totalEventsDeleted); - - await _opsEventService.RecordOrphanEventsCleanedAsync(sessionsCleanedUp, totalEventsDeleted, cleanedOrphans); - } - catch (Exception ex) - { - _logger.LogError(ex, "Orphaned events cleanup failed"); - } - } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 0ed07846..5aa2cec5 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Functions.Services.Monitoring; using AutopilotMonitor.Shared.DataAccess; @@ -56,6 +57,7 @@ namespace AutopilotMonitor.Functions.Services private readonly PreviewWhitelistService _previewWhitelistService; private readonly IStorageInitializer _storageInitializer; private readonly Maintenance.MaintenanceRunGate _runGate; + private readonly Maintenance.OrphanSessionSweeper _orphanSweeper; private readonly ILogger _logger; private const string PlatformStatsAliasFileName = "platform-stats.json"; @@ -93,6 +95,7 @@ namespace AutopilotMonitor.Functions.Services PreviewWhitelistService previewWhitelistService, IStorageInitializer storageInitializer, Maintenance.MaintenanceRunGate runGate, + Maintenance.OrphanSessionSweeper orphanSweeper, ILogger logger) { _maintenanceRepo = maintenanceRepo; @@ -124,6 +127,7 @@ namespace AutopilotMonitor.Functions.Services _previewWhitelistService = previewWhitelistService; _storageInitializer = storageInitializer; _runGate = runGate; + _orphanSweeper = orphanSweeper; _logger = logger; } @@ -150,53 +154,56 @@ namespace AutopilotMonitor.Functions.Services _logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}"); var maintenanceStart = Stopwatch.StartNew(); + // Every step is timed into the Completed event (stepsMs): the run's composition is + // what decides which steps leave this timer next, and worker logs never show it. + var steps = new Maintenance.MaintenanceStepTimer(); try { // Startup only point-reads the table schema sentinel (see // TableStorageService.InitializeTablesAsync); this daily full pass is what // recreates a table that was deleted out-of-band. Cheap (idempotent creates). - await EnsureAllTablesAsync(); - await MarkStalledSessionsAsTimedOutAsync(); - await DetectExcessiveEventSessionsAsync(); - await AggregateMetricsWithCatchUpAsync(); + await steps.RunAsync("ensureTables", async () => { await EnsureAllTablesAsync(); }); + await steps.RunAsync("stalledSessions", async () => { await MarkStalledSessionsAsTimedOutAsync(); }); + await steps.RunAsync("excessiveEvents", async () => { await DetectExcessiveEventSessionsAsync(); }); + await steps.RunAsync("aggregateMetrics", async () => { await AggregateMetricsWithCatchUpAsync(); }); // F3 PR6: rule-frequency regression radar, right after the rule-stats // aggregation so the window rows are fresh. Anchored on YESTERDAY — whole // days only, a partial today would understate the window rate. Fail-soft. - await RunRuleRegressionRadarAsync(DateTime.UtcNow.Date.AddDays(-1)); + await steps.RunAsync("ruleRegressionRadar", async () => { await RunRuleRegressionRadarAsync(DateTime.UtcNow.Date.AddDays(-1)); }); // App-version duration regression radar: same episode/tracker pattern over the // install summaries (trailing 35d horizon loaded internally). Fail-soft. - await RunAppVersionRegressionRadarAsync(); + await steps.RunAsync("appVersionRadar", async () => { await RunAppVersionRegressionRadarAsync(); }); // ONE projected cross-tenant window scan feeds every rolling sweep below (the // StartedAt-only filter is a full-table drain in Table Storage — four of them per // tick was four times the same read). Each sweep slices its own window. - var sweepWindow = await LoadSweepWindowSessionsAsync(); + var sweepWindow = await steps.RunAsync>("sweepWindow", () => LoadSweepWindowSessionsAsync()); // F1 PR2: rolling 30d breakdown backfill + daily attribution aggregates. Owns // its own window (NOT the snapshot-gated catch-up above) so late-terminating // sessions still reach their StartedAt-date's aggregate. Fail-soft internally. - await SweepTimeAttributionAsync(sweepWindow); + await steps.RunAsync("timeAttribution", async () => { await SweepTimeAttributionAsync(sweepWindow); }); // F2 PR4: device-history chain heal (incl. deleted-session ref cleanup) + daily // FTR aggregates over the same rolling window. Fail-soft internally. - await SweepDeviceJourneysAsync(sweepWindow); + await steps.RunAsync("deviceJourneys", async () => { await SweepDeviceJourneysAsync(sweepWindow); }); // Verdict calibration: per-verdict-path daily buckets over the same rolling // window, AFTER the device-journey sweep so the re-enrollment proxy reads // freshly merged chains. Fail-soft internally. - await SweepVerdictCalibrationAsync(sweepWindow); + await steps.RunAsync("verdictCalibration", async () => { await SweepVerdictCalibrationAsync(sweepWindow); }); // Verdict-calibration drift radar over the rows the sweep just refreshed; anchored // on yesterday like the rule radar (whole days only). Fail-soft. - await RunVerdictCalibrationRadarAsync(DateTime.UtcNow.Date.AddDays(-1), sweepWindow); + await steps.RunAsync("verdictCalibrationRadar", async () => { await RunVerdictCalibrationRadarAsync(DateTime.UtcNow.Date.AddDays(-1), sweepWindow); }); // Plan §5 PR6 / §16 R14: session retention fanout extracted out of the 2h timer // into the dedicated 12h SessionDeletionMaintenanceFunction so cascade-lifecycle // work has independent cadence + kill-switch + OpsEvent watchdogs. The non-session - // tail of the old CleanupOldDataAsync (UserUsageLog + RuleStats) stays here. - await CleanupOldUsageDataAsync(); - await CleanupOldDistressReportsAsync(); - await CleanupOldOpsEventsAsync(); - await CleanupUnboundedTablesAsync(); - await CleanupOrphanedEventsAsync(); - await CheckAgentBlobStorageAsync(); - await CheckEmbeddedCertExpiryAsync(); - await CheckPoisonQueueBacklogAsync(); - await RecomputePlatformStatsAsync(); + // tail of the old CleanupOldDataAsync (UserUsageLog + RuleStats) stays here. The + // orphan-session sweep left this timer too (OrphanSessionSweepFunction, every 4h). + await steps.RunAsync("usageData", async () => { await CleanupOldUsageDataAsync(); }); + await steps.RunAsync("distressReports", async () => { await CleanupOldDistressReportsAsync(); }); + await steps.RunAsync("opsEvents", async () => { await CleanupOldOpsEventsAsync(); }); + await steps.RunAsync("unboundedTables", async () => { await CleanupUnboundedTablesAsync(); }); + await steps.RunAsync("agentBlobStorage", async () => { await CheckAgentBlobStorageAsync(); }); + await steps.RunAsync("embeddedCertExpiry", async () => { await CheckEmbeddedCertExpiryAsync(); }); + await steps.RunAsync("poisonQueues", async () => { await CheckPoisonQueueBacklogAsync(); }); + await steps.RunAsync("platformStats", async () => { await RecomputePlatformStatsAsync(); }); // Backfill and repair tasks run only via manual trigger (RunManualAsync) // to keep the timer-triggered path lightweight. See RunManualAsync for: @@ -205,7 +212,7 @@ namespace AutopilotMonitor.Functions.Services maintenanceStart.Stop(); _logger.LogInformation($"Daily maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms"); - await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer"); + await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer", stepsMs: steps.Steps); // Soft watchdog: the run completed, but if it is climbing toward the host's 60min // functionTimeout (e.g. a large first-time retention backlog deleting row-by-row), @@ -316,6 +323,10 @@ namespace AutopilotMonitor.Functions.Services await CleanupUnboundedTablesAsync(); result.DataCleanupExecuted = true; + // The orphan-session sweep has its own timer and lease; a manual run includes it + // (with the reconciliation) so an operator can drive it from the same button. + await _orphanSweeper.RunAsync(triggeredBy, CancellationToken.None); + // --- Backfill & repair tasks (manual-only, not in timer path) --- // Safety net: backfill any sessions missing from SessionsIndex diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 6583de04..99e58684 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -84,15 +84,17 @@ namespace AutopilotMonitor.Functions.Services /// is the run report of a manual run: the trigger answers 202 /// before the run starts, so this event is where the operator reads what the run did. /// - public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null) + public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null, + IReadOnlyDictionary? stepsMs = null) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info, $"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})", null, triggeredBy, report == null - ? (object)new { durationMs } + ? (object)new { durationMs, stepsMs } : new { durationMs, + stepsMs, aggregatedDate = report.AggregatedDate, stalledSessionsChecked = report.StalledSessionsChecked, dataCleanupExecuted = report.DataCleanupExecuted, @@ -138,36 +140,113 @@ namespace AutopilotMonitor.Functions.Services $"Cleaned up {deletedCount} ops events older than {retentionDays} days", null, "System.Maintenance", new { deletedCount, retentionDays }); + // ── Orphan session sweep (OrphanSessionSweeper) ───────────────────────── + /// - /// Records the orphan-cleanup result. carries the - /// per-session breakdown (which tenant + session lost lingering events and how many), - /// so the ops dashboard shows *what* was cleaned, not just a count. The list is capped - /// to keep the OpsEvents Table row under the per-property (32 KB) / entity (~1 MB) limits; - /// the full count always survives in orphanSessions and a detailsTruncated - /// flag flags any clipping. Note: an "orphan" is a session row that no longer exists while - /// its events lingered past the 24h grace — the tenant itself usually still exists. + /// Heartbeat of every sweep run: scan sizes, candidates, what was cleaned, skipped and + /// left for the next run, the per-table totals and — when it ran — the reconciliation. /// - public Task RecordOrphanEventsCleanedAsync(int orphanSessions, int totalEventsDeleted, - IReadOnlyList? cleanedOrphans = null) + public Task RecordOrphanSweepCompletedAsync(Maintenance.OrphanSweepRunResult result, string triggeredBy) + => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepCompleted, OpsEventSeverity.Info, + $"Orphan session sweep completed in {result.DurationMs}ms — handles={result.HandlesScanned} sessions={result.SessionKeys} " + + $"candidates={result.Candidates} cleaned={result.Cleaned.Count} remaining={result.Remaining} failed={result.FailedSessions}", + null, Maintenance.OrphanSessionSweeper.Actor, + new + { + triggeredBy, + durationMs = result.DurationMs, + handlesScanned = result.HandlesScanned, + sessionKeys = result.SessionKeys, + candidates = result.Candidates, + cleaned = result.Cleaned.Count, + remaining = result.Remaining, + failedSessions = result.FailedSessions, + skippedRowReturned = result.SkippedRowReturned, + skippedHandleGone = result.SkippedHandleGone, + skippedHandleChanged = result.SkippedHandleChanged, + afterCascade = result.AfterCascade, + inventoryDecrementsFailed = result.InventoryDecrementsFailed, + budgetExhausted = result.BudgetExhausted, + totalsByTable = result.TotalsByTable, + reconcile = result.Reconcile == null ? null : new + { + indexSessions = result.Reconcile.IndexSessions, + residueSessions = result.Reconcile.ResidueSessions, + durationMs = result.Reconcile.DurationMs, + }, + }); + + public Task RecordOrphanSweepFailedAsync(string error, string triggeredBy) + => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepFailed, OpsEventSeverity.Error, + $"Orphan session sweep failed: {error}", + null, Maintenance.OrphanSessionSweeper.Actor, new { error, triggeredBy }); + + public Task RecordOrphanSweepSkippedLockedAsync(string triggeredBy) + => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepSkippedLocked, OpsEventSeverity.Warning, + $"Orphan session sweep skipped — another run holds the lease (triggered by {triggeredBy})", + null, Maintenance.OrphanSessionSweeper.Actor, new { reason = "lease held by another run", triggeredBy }); + + /// + /// Weekly read-only reconciliation of EventTypeIndex against the Sessions keys. Info at + /// zero residue; Warning with a capped sample otherwise — residue means rows exist for a + /// session that has neither a Sessions row nor a handle, which the sweep cannot reach on its own. + /// + public Task RecordOrphanReconcileCompletedAsync(Maintenance.OrphanReconcileResult result) + => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanReconcileCompleted, + result.ResidueSessions == 0 ? OpsEventSeverity.Info : OpsEventSeverity.Warning, + result.ResidueSessions == 0 + ? $"Orphan reconciliation: no residue across {result.IndexSessions} indexed sessions ({result.DurationMs}ms)" + : $"Orphan reconciliation: {result.ResidueSessions} indexed sessions without a Sessions row or a handle ({result.DurationMs}ms)", + null, Maintenance.OrphanSessionSweeper.Actor, + new + { + indexSessions = result.IndexSessions, + residueSessions = result.ResidueSessions, + sample = result.Sample.Select(k => new { tenantId = k.TenantId, sessionId = k.SessionId }).ToList(), + sampleTruncated = result.ResidueSessions > result.Sample.Count, + durationMs = result.DurationMs, + }); + + /// + /// Rows of sessions without a Sessions row were removed (Warning: after the ingest refusal + /// every real hit is a writer outside the registration path or a restore residue; legacy + /// handles are expected while the backlog drains). totalEventsDeleted keeps its old + /// meaning (Events rows) next to the per-table totals. The per-session list is capped to + /// keep the row inside the Table Storage property limits; the counts are never clipped. + /// + public Task RecordOrphanEventsCleanedAsync(Maintenance.OrphanSweepRunResult result) { const int maxDetailRows = 50; - var orphanList = cleanedOrphans ?? Array.Empty(); - var detail = orphanList - .OrderByDescending(o => o.EventCount) + var detail = result.Cleaned + .OrderByDescending(c => c.TotalRows) .Take(maxDetailRows) - .Select(o => new { tenantId = o.TenantId, sessionId = o.SessionId, eventCount = o.EventCount }) + .Select(c => new + { + tenantId = c.TenantId, + sessionId = c.SessionId, + rows = c.TotalRows, + inventoryKeys = c.InventoryKeysDecremented, + legacy = c.Legacy, + afterCascade = c.AfterCascade, + handleKept = c.HandleKept, + }) .ToList(); + result.TotalsByTable.TryGetValue(Shared.Constants.TableNames.Events, out var eventsDeleted); + var legacy = result.Cleaned.Count(c => c.Legacy); return WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanEventsCleaned, OpsEventSeverity.Warning, - $"Cleaned {totalEventsDeleted} orphaned events across {orphanSessions} sessions", - null, "System.Maintenance", + $"Cleaned {result.TotalsByTable.Values.Sum()} rows of {result.Cleaned.Count} orphan sessions ({legacy} legacy handles)", + null, Maintenance.OrphanSessionSweeper.Actor, new { - orphanSessions, - totalEventsDeleted, + orphanSessions = result.Cleaned.Count, + totalEventsDeleted = eventsDeleted, + legacySessions = legacy, + remaining = result.Remaining, + totalsByTable = result.TotalsByTable, orphans = detail, - detailsTruncated = orphanList.Count > maxDetailRows + detailsTruncated = result.Cleaned.Count > maxDetailRows, }); } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs index b4f3d5e3..f78c31c3 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs @@ -935,23 +935,6 @@ namespace AutopilotMonitor.Functions.Services // ===== DELETION HELPERS ===== - /// - /// Deletes all events for a session from storage - /// - public async Task DeleteSessionEventsAsync(string tenantId, string sessionId) - { - SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); - SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); - - var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Events); - var partitionKey = $"{tenantId}_{sessionId}"; - var filter = $"PartitionKey eq '{partitionKey}'"; - var deleted = await DeleteByFilterInBatchesAsync(tableClient, filter, $"events for session {sessionId}"); - if (deleted > 0) - _logger.LogInformation($"Deleted {deleted} events for session {sessionId}"); - return deleted; - } - /// /// Deletes all rule results for a session /// @@ -1188,81 +1171,5 @@ namespace AutopilotMonitor.Functions.Services return false; } } - - // ===== ORPHAN EVENT DETECTION ===== - - /// - /// Scans EventSessionIndex, checks each entry against the Sessions table, - /// and returns entries where no session exists and LastIngestAt is older than the grace period. - /// - public async Task> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) - { - var orphans = new List(); - var cutoff = DateTime.UtcNow - gracePeriod; - - try - { - var indexClient = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex); - var sessionsClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); - - await foreach (var entity in indexClient.QueryAsync()) - { - var tenantId = entity.PartitionKey; - var sessionId = entity.RowKey; - var lastIngestAt = entity.GetDateTimeOffset("LastIngestAt")?.UtcDateTime ?? DateTime.MinValue; - var eventCount = entity.GetInt32("EventCount") ?? 0; - - // Grace period: skip recent entries (race condition protection) - if (lastIngestAt > cutoff) - continue; - - // Check if session exists - try - { - var session = await sessionsClient.GetEntityIfExistsAsync(tenantId, sessionId, select: new[] { "PartitionKey" }); - if (!session.HasValue) - { - orphans.Add(new OrphanedEventSession - { - TenantId = tenantId, - SessionId = sessionId, - LastIngestAt = lastIngestAt, - EventCount = eventCount - }); - } - } - catch (RequestFailedException ex) - { - // GetEntityIfExistsAsync does NOT throw on 404 (handled via HasValue above), - // so this only fires on transient errors (429/500/503). Treating those as - // "orphan" would delete events of live sessions — skip the entry instead. - _logger.LogWarning(ex, "Transient error checking session existence for {TenantId}/{SessionId} (status {Status}); skipping orphan classification this run", tenantId, sessionId, ex.Status); - } - } - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to scan EventSessionIndex for orphans"); - } - - return orphans; - } - - public async Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) - { - try - { - var indexClient = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex); - await indexClient.DeleteEntityAsync(tenantId, sessionId); - } - catch (RequestFailedException ex) when (ex.Status == 404) - { - // Already deleted, ignore - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to delete EventSessionIndex entry for {TenantId}/{SessionId}", tenantId, sessionId); - } - } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.OrphanSweep.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.OrphanSweep.cs new file mode 100644 index 00000000..a1b13eab --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.OrphanSweep.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using Azure; +using Azure.Data.Tables; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Storage side of the orphan-session sweep (): + /// the two key drains the diff runs on, the EventSessionIndex handle as the sweep's progress + /// row, and the per-table delete by the writer's key shape. + /// + public partial class TableStorageService + { + private const string HandleInventoryDecrementedColumn = "InventoryDecremented"; + + // Timestamp must be named explicitly in a projection; ETag always travels with the row. + private static readonly string[] HandleColumns = + { "PartitionKey", "RowKey", "Timestamp", "LastIngestAt", HandleInventoryDecrementedColumn }; + + public async Task> GetSessionKeysAsync(CancellationToken ct = default) + { + var client = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); + var keys = new HashSet<(string TenantId, string SessionId)>(); + await foreach (var row in client.QueryAsync( + maxPerPage: 1000, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct)) + { + keys.Add((row.PartitionKey, row.RowKey)); + } + return keys; + } + + public async Task> GetEventSessionIndexHandlesAsync(CancellationToken ct = default) + { + var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex); + var handles = new List(); + await foreach (var row in client.QueryAsync(maxPerPage: 1000, select: HandleColumns, cancellationToken: ct)) + { + handles.Add(ToHandle(row)); + } + return handles; + } + + public async Task GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); + var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex); + var row = await client.GetEntityIfExistsAsync(tenantId, sessionId, select: HandleColumns, cancellationToken: ct); + return row.HasValue ? ToHandle(row.Value!) : null; + } + + public async Task StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); + var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex); + var stamp = new TableEntity(tenantId, sessionId) { [HandleInventoryDecrementedColumn] = true }; + try + { + var response = await client.UpdateEntityAsync(stamp, new ETag(etag), TableUpdateMode.Merge, ct); + return response.Headers.ETag?.ToString(); + } + catch (RequestFailedException ex) when (ex.Status == 412 || ex.Status == 404) + { + return null; + } + } + + public async Task DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); + var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex); + try + { + await client.DeleteEntityAsync(tenantId, sessionId, new ETag(etag), ct); + return true; + } + catch (RequestFailedException ex) when (ex.Status == 412 || ex.Status == 404) + { + return false; + } + } + + public async Task DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default) + { + var filter = Maintenance.OrphanSessionRowFilters.For(table, tenantId, sessionId); + var client = _tableServiceClient.GetTableClient(table); + var keys = new List<(string Pk, string Rk)>(); + await foreach (var row in client.QueryAsync( + filter: filter, maxPerPage: 1000, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct)) + { + keys.Add((row.PartitionKey, row.RowKey)); + } + if (keys.Count == 0) return 0; + + // Groups by partition and falls back to per-row deletes on a 404 inside a batch, so a + // rerun after a partial failure is idempotent; any other failure propagates. + await DeleteByExactKeysInBatchesAsync(table, keys, ct); + return keys.Count; + } + + public async Task> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default) + { + var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventTypeIndex); + var keys = new HashSet<(string TenantId, string SessionId)>(); + await foreach (var row in client.QueryAsync( + maxPerPage: 1000, select: new[] { "PartitionKey", "SessionId", "TenantId" }, cancellationToken: ct)) + { + var sessionId = row.GetString("SessionId"); + if (string.IsNullOrEmpty(sessionId)) continue; // the readers skip such rows as well + + var tenantId = row.GetString("TenantId"); + if (string.IsNullOrEmpty(tenantId)) + { + // Rows older than the TenantId column: PartitionKey is {tenantId}_{eventType} and a + // tenant id (a GUID) never contains an underscore. + var cut = row.PartitionKey.IndexOf('_'); + if (cut <= 0) continue; + tenantId = row.PartitionKey.Substring(0, cut); + } + keys.Add((tenantId!, sessionId!)); + } + return keys; + } + + private static OrphanSessionHandle ToHandle(TableEntity row) => new OrphanSessionHandle + { + TenantId = row.PartitionKey, + SessionId = row.RowKey, + LastIngestAt = row.GetDateTimeOffset("LastIngestAt")?.UtcDateTime ?? DateTime.MinValue, + WrittenAt = row.Timestamp?.UtcDateTime ?? DateTime.MinValue, + ETag = row.ETag.ToString(), + InventoryDecremented = row.GetBoolean(HandleInventoryDecrementedColumn) ?? false, + }; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelateHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelateHandler.cs index b747dd24..40be5c66 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelateHandler.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelateHandler.cs @@ -57,6 +57,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability private readonly IVulnerabilityInventoryLoader _inventoryLoader; private readonly IAnalyzeOnEnrollmentEndProducer _analyzeProducer; private readonly SignalRNotificationService _signalRNotification; + private readonly Deletion.SessionDeletionGuard _deletionGuard; private readonly ILogger _logger; public VulnerabilityCorrelateHandler( @@ -67,6 +68,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability IVulnerabilityInventoryLoader inventoryLoader, IAnalyzeOnEnrollmentEndProducer analyzeProducer, SignalRNotificationService signalRNotification, + Deletion.SessionDeletionGuard deletionGuard, ILogger logger) { _adminConfigService = adminConfigService; @@ -76,6 +78,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability _inventoryLoader = inventoryLoader; _analyzeProducer = analyzeProducer; _signalRNotification = signalRNotification; + _deletionGuard = deletionGuard; _logger = logger; } @@ -107,6 +110,19 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability return; } + // Derived rows only for a live session: a session the backend does not know, or one a + // cascade is tearing down, must not gain an inventory side-row, CveIndex rows or a + // report — no cascade could reach them afterwards. + var liveness = await _deletionGuard.GetLivenessAsync( + envelope.TenantId, envelope.SessionId, "VulnerabilityCorrelate", cancellationToken).ConfigureAwait(false); + if (liveness != Deletion.SessionLiveness.Live) + { + _logger.LogWarning( + "{Prefix} VulnerabilityCorrelate skipped — session {Liveness} (reason={Reason})", + sessionPrefix, liveness, envelope.Reason); + return; + } + // Filter inventory by the envelope's WhiteGlovePart so a Part-2 correlation // does not pick up Part-1 inventory items (which would mis-tag findings as // user_enrollment) and a delayed Part-1 retry does not later overwrite a diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs b/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs index fc5195ca..83ea4754 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs @@ -271,9 +271,10 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability // Persist software inventory via the contributions side-row (Plan §17). Awaited inside // the invocation: a detached task can be cut off by a scale-in after the side-row has // already promised the key set, leaving the counters short with no trace. The method - // never throws, so the report — the primary product — is stored regardless. PR3 wires - // SessionDeletionGuard above this layer (at VulnerabilityCorrelateHandler) so a session - // locked by a cascade never reaches this code path. + // never throws, so the report — the primary product — is stored regardless. Both callers + // (VulnerabilityCorrelateHandler, GetVulnerabilityReportFunction's rescan) check + // SessionDeletionGuard.GetLivenessAsync first, so a session the backend does not know + // or a cascade is tearing down never reaches this write. await ApplyInventoryContributionsAsync(tenantId, sessionId, inventoryItems, cpeMappingsDict); // Summary log for production monitoring diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/sessions-by-event-exhaust.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/sessions-by-event-exhaust.test.ts new file mode 100644 index 00000000..df926a66 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/sessions-by-event-exhaust.test.ts @@ -0,0 +1,56 @@ +/** + * search_sessions_by_event reads the EventTypeIndex, and the backend drops index rows whose + * session no longer exists (deleted, or not yet removed by the orphan sweep). A page can therefore + * be empty and still carry a nextLink, so the tool must go through the same auto-exhausting + * scan as search_sessions instead of a single fetch. The scan seam is the observable here. + */ +import { describe, it, expect, beforeEach, vi } from 'vitest'; + +const { scanMock } = vi.hoisted(() => ({ scanMock: vi.fn() })); + +vi.mock('../client.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, scanWithTimeoutFallback: scanMock }; +}); + +import { registerSessionTools } from '../tools/sessions.js'; + +type Handler = (args: Record) => Promise<{ content: Array<{ text: string }>; isError?: boolean }>; + +function searchSessionsByEvent(): Handler { + const handlers: Record = {}; + const fake = { registerTool: (name: string, _def: unknown, handler: Handler) => { handlers[name] = handler; } }; + registerSessionTools(fake as never, true); + return handlers.search_sessions_by_event; +} + +const PAGE = { count: 1, sessions: [{ sessionId: 's1', tenantId: 't1' }], scannedPages: 3 }; + +describe('search_sessions_by_event — auto-exhausts empty-but-continuable pages', () => { + beforeEach(() => { + scanMock.mockReset(); + scanMock.mockResolvedValue(PAGE); + }); + + it('reaches the backend through the scanning fetch, not a single apiFetch', async () => { + const result = await searchSessionsByEvent()({ eventType: 'app_install_failed' }); + + expect(result.isError).toBeFalsy(); + expect(scanMock).toHaveBeenCalledTimes(1); + // Global vs tenant route follows the request's routing context, not the registration flag. + const [path, basePath] = scanMock.mock.calls[0] as [string, string]; + expect(basePath).toMatch(/^\/api\/(global\/)?search\/sessions-by-event$/); + expect(path.startsWith(`${basePath}?`)).toBe(true); + expect(path).toContain('eventType=app_install_failed'); + expect(result.content[0].text).toContain('"scannedPages":3'); + }); + + it('keeps the nextLink cursor when continuing', async () => { + const next = '/api/search/sessions-by-event?eventType=app_install_failed&pageSize=50&continuation=c1'; + await searchSessionsByEvent()({ eventType: 'app_install_failed', tenantId: 't1', continuation: next }); + + const [path] = scanMock.mock.calls[0] as [string, string]; + expect(path).toContain('continuation=c1'); + expect(path).toContain('eventType=app_install_failed'); + }); +}); diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index d924de06..b4dc7fde 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -200,6 +200,8 @@ export interface AdminConfiguration { nvdCacheLastRefreshUtc: string; /** Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss. */ epssLastSyncUtc: string; + /** Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal. */ + orphanDeepCheckLastRunUtc: string; } /** The immutable Entra identity behind a cross-tenant-role UPN: the HOME tenant the UPN was granted for and, once known, the user's object id. A role row (GlobalAdmins / DelegatedAdmins / TenantGroupAssignments) confers nothing unless the caller's validated JWT matches this binding. */ diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts index 2a0a70d2..25a8fd49 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts @@ -70,6 +70,10 @@ export const OPS_EVENT_TYPES = [ "SessionSweepFailed", "OpsEventCleanup", "OrphanEventsCleaned", + "OrphanSweepCompleted", + "OrphanSweepFailed", + "OrphanSweepSkippedLocked", + "OrphanReconcileCompleted", "SessionDeletionMaintenanceStarted", "SessionDeletionMaintenanceBudgetExceeded", "SessionDeletionMaintenanceSkippedLocked", diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts index 49c886e0..7b667061 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts @@ -236,9 +236,11 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated: const { eventType, tenantId: rawTenantId, continuation } = args; const pageSize = pageSizeForCall(args.pageSize, continuation, DEFAULT_FIRST_PAGE_SIZE); const tenantId = enforceDelegatedTenantForPage(rawTenantId, continuation); - // eventType is the sole filter and is applied server-side (EventTypeIndex OData), - // so an empty page never carries a nextLink — no auto-exhaust needed. An empty result - // for an uncatalogued type says so, so a typo does not read as "no such sessions". + // eventType is applied server-side (EventTypeIndex OData), but the backend drops index + // rows whose session no longer exists (deleted, or not yet removed by the orphan sweep), + // so a page can be empty and still carry a nextLink — auto-exhaust forward like + // search_sessions. An empty result for an uncatalogued type says so, so a typo does not + // read as "no such sessions". const basePath = pickGlobalOrTenantPath('/api/global/search/sessions-by-event', '/api/search/sessions-by-event', tenantId); const path = followNextLink( basePath, @@ -246,7 +248,7 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated: continuation, { pageSize }, ); - const data = await apiFetch(path); + const data = await scanWithTimeoutFallback(path, basePath, effectivePageSize(pageSize, continuation)); return toolResultText(withEventTypeNote(data, eventType, !continuation), MAX_RESULT_SIZE_CHARS.indexSessions); } catch (error: unknown) { return toolError('search_sessions_by_event', args, error); diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs index 89b552a7..8948e701 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs @@ -130,7 +130,6 @@ namespace AutopilotMonitor.Shared.DataAccess Task> GetAllTenantIdsAsync(); // --- Cleanup --- - Task DeleteSessionEventsAsync(string tenantId, string sessionId); Task DeleteSessionRuleResultsAsync(string tenantId, string sessionId); // --- Index Maintenance --- @@ -138,13 +137,51 @@ namespace AutopilotMonitor.Shared.DataAccess Task CleanupGhostSessionIndexEntriesAsync(); Task IsSessionIndexEmptyAsync(); - // --- Orphan Event Detection --- + // --- Orphan session sweep (rows of sessions that have no Sessions row) --- + /// - /// Returns EventSessionIndex entries whose session no longer exists in the Sessions table - /// and whose last ingest is older than the grace period. + /// Every (tenantId, sessionId) key of the Sessions table, projected to the keys (one range + /// request per 1000 rows). The sweep diffs the EventSessionIndex handles against this set + /// instead of point-reading one Sessions row per handle. /// - Task> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod); - Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId); + Task> GetSessionKeysAsync(System.Threading.CancellationToken ct = default); + + /// + /// Every EventSessionIndex handle with the columns the sweep decides on: last ingest, the + /// row's own write time, ETag and the inventory progress stamp. + /// + Task> GetEventSessionIndexHandlesAsync(System.Threading.CancellationToken ct = default); + + /// Point-reads one handle; null when it no longer exists (a cascade took it). + Task GetEventSessionIndexHandleAsync(string tenantId, string sessionId, System.Threading.CancellationToken ct = default); + + /// + /// Marks the handle's software-inventory decrements as applied (ETag-conditional merge). + /// Returns the handle's new ETag, or null when the row changed or vanished since + /// was read. + /// + Task StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, System.Threading.CancellationToken ct = default); + + /// + /// Deletes the handle only while it still carries : false when a + /// later write changed it (412) or it is already gone (404). + /// + Task DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, System.Threading.CancellationToken ct = default); + + /// + /// Deletes every row of that belongs to the session, by that table's + /// own key shape. Fail-loud on query or delete errors; rows already missing count as done. + /// Returns the number of rows found. + /// + Task DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, System.Threading.CancellationToken ct = default); + + /// + /// Distinct (tenantId, sessionId) pairs present in EventTypeIndex — the input of the weekly + /// reconciliation (a full projected drain, roughly 65 rows per session). The tenant comes + /// from the TenantId column, falling back to the PartitionKey prefix for rows written + /// before that column existed. + /// + Task> GetEventTypeIndexSessionKeysAsync(System.Threading.CancellationToken ct = default); // --- Session Tombstones (F2 device-journey sweep input) --- /// @@ -167,12 +204,31 @@ namespace AutopilotMonitor.Shared.DataAccess IAsyncEnumerable EnumerateSessionsForOffboardingAsync(string tenantId, System.Threading.CancellationToken ct = default); } - public class OrphanedEventSession + /// + /// One EventSessionIndex row as the orphan sweep sees it — the discovery handle that every + /// events write creates (PK tenantId, RK sessionId). + /// + public sealed class OrphanSessionHandle { public string TenantId { get; set; } = string.Empty; public string SessionId { get; set; } = string.Empty; + + /// Server time of the last ingest that touched the handle. public DateTime LastIngestAt { get; set; } - public int EventCount { get; set; } + + /// + /// System write time of the row. A restore re-inserts the handle with its old + /// LastIngestAt but a fresh write time, so the sweep waits on this one too. + /// + public DateTime WrittenAt { get; set; } + + public string ETag { get; set; } = string.Empty; + + /// + /// True once the sweep applied the session's software-inventory decrements. Progress + /// marker across runs: a rerun after a crash skips the decrements and only removes the rows. + /// + public bool InventoryDecremented { get; set; } } /// diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs index f472fa20..75b62a1a 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs @@ -38,6 +38,14 @@ namespace AutopilotMonitor.Shared.DataAccess public const string SessionSweepFailed = "SessionSweepFailed"; public const string OpsEventCleanup = "OpsEventCleanup"; public const string OrphanEventsCleaned = "OrphanEventsCleaned"; + /// Every orphan-session sweep run (Info heartbeat): scanned handles, candidates, cleaned, remaining, per-table totals. + public const string OrphanSweepCompleted = "OrphanSweepCompleted"; + /// The sweep run itself failed before or while enumerating (Error); per-candidate failures stay inside Completed. + public const string OrphanSweepFailed = "OrphanSweepFailed"; + /// Another sweep run (timer, manual maintenance or a second host instance) held the sweep lease (Warning). + public const string OrphanSweepSkippedLocked = "OrphanSweepSkippedLocked"; + /// Weekly read-only reconciliation of EventTypeIndex against the Sessions keys: Info at zero residue, Warning with a sample otherwise. + public const string OrphanReconcileCompleted = "OrphanReconcileCompleted"; public const string SessionDeletionMaintenanceStarted = "SessionDeletionMaintenanceStarted"; public const string SessionDeletionMaintenanceBudgetExceeded = "SessionDeletionMaintenanceBudgetExceeded"; public const string SessionDeletionMaintenanceSkippedLocked = "SessionDeletionMaintenanceSkippedLocked"; @@ -184,7 +192,7 @@ namespace AutopilotMonitor.Shared.DataAccess public static readonly IReadOnlyList All = new[] { ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles, - MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed, + MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, OrphanSweepCompleted, OrphanSweepFailed, OrphanSweepSkippedLocked, OrphanReconcileCompleted, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed, DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen, OffboardingFeedbackReceived, FeedbackReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled, SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected, diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs b/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs index 6d59860e..b4879c68 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs @@ -640,6 +640,13 @@ namespace AutopilotMonitor.Shared.Models /// public string EpssLastSyncUtc { get; set; } = default!; + /// + /// Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only + /// EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, + /// never patched from the portal. + /// + public string OrphanDeepCheckLastRunUtc { get; set; } = default!; + /// /// Returns a shallow copy with all secret-bearing string fields replaced by /// (empty values are left empty). Used when diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/AgentErrorReport.cs b/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/AgentErrorReport.cs index 9ae13ac2..5f2472ab 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/AgentErrorReport.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/AgentErrorReport.cs @@ -38,6 +38,15 @@ namespace AutopilotMonitor.Shared.Models /// out the silence grace. See tasks/enrollment-status-reclassification.md. /// SessionAgeEmergencyBreak = 4, + + /// + /// POST /api/agent/telemetry answered 410 Gone for the agent's own session: the backend + /// no longer knows it (never registered, deleted, or being deleted). The agent stops + /// uploading and terminates with self-destruct; this single report is the last thing the + /// device says about the session. Sent over the emergency channel because the telemetry + /// channel is exactly what was refused. + /// + TelemetryRejectedSessionUnknown = 5, } /// diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index 83f52c4f..a78e6fc3 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -88,8 +88,16 @@ const OPS_EVENT_TYPES: Record = { // PR2: GA restored a single row from a backup. Warning severity so operators // can wire a Telegram rule and see the audit in near-real-time. "BackupRowRestored", - // Orphan cleanup sweep (Warning): events whose parent session no longer exists were - // deleted during maintenance aggregation. Backend helper RecordOrphanEventsCleanedAsync. + // Orphan-session sweep (OrphanSessionSweepFunction, every 4h at :45) — dual-register per + // memory feedback_ops_event_types_dual_register. Backend helpers RecordOrphanSweep{Completed, + // Failed,SkippedLocked}Async, RecordOrphanReconcileCompletedAsync, RecordOrphanEventsCleanedAsync. + // Completed is the 4h heartbeat (Info); EventsCleaned (Warning) fires only when rows of a + // session without a Sessions row were removed; ReconcileCompleted is the weekly read-only + // EventTypeIndex check (Info at zero residue, Warning otherwise). + "OrphanSweepCompleted", + "OrphanSweepFailed", + "OrphanSweepSkippedLocked", + "OrphanReconcileCompleted", "OrphanEventsCleaned", ], // SLA breach evaluation (SlaBreachEvaluationService). Dual-register per memory diff --git a/src/Web/autopilot-monitor-web/utils/shared-manifests.generated.ts b/src/Web/autopilot-monitor-web/utils/shared-manifests.generated.ts index 812b4032..d8a5ba5f 100644 --- a/src/Web/autopilot-monitor-web/utils/shared-manifests.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/shared-manifests.generated.ts @@ -61,7 +61,8 @@ export const SHARED_MANIFEST = { "vulnerabilityDataLastSyncUtc", "msrcLastSyncUtc", "nvdCacheLastRefreshUtc", - "epssLastSyncUtc" + "epssLastSyncUtc", + "orphanDeepCheckLastRunUtc" ], "defaults": { "partitionKey": "GlobalConfig", @@ -118,7 +119,8 @@ export const SHARED_MANIFEST = { "vulnerabilityDataLastSyncUtc": null, "msrcLastSyncUtc": null, "nvdCacheLastRefreshUtc": null, - "epssLastSyncUtc": null + "epssLastSyncUtc": null, + "orphanDeepCheckLastRunUtc": null } }, "tenantConfiguration": { @@ -604,6 +606,10 @@ export const SHARED_MANIFEST = { "SessionSweepFailed", "OpsEventCleanup", "OrphanEventsCleaned", + "OrphanSweepCompleted", + "OrphanSweepFailed", + "OrphanSweepSkippedLocked", + "OrphanReconcileCompleted", "SessionDeletionMaintenanceStarted", "SessionDeletionMaintenanceBudgetExceeded", "SessionDeletionMaintenanceSkippedLocked", diff --git a/src/Web/autopilot-monitor-web/utils/shared-manifests.json b/src/Web/autopilot-monitor-web/utils/shared-manifests.json index 9857818a..470bc319 100644 --- a/src/Web/autopilot-monitor-web/utils/shared-manifests.json +++ b/src/Web/autopilot-monitor-web/utils/shared-manifests.json @@ -57,7 +57,8 @@ "vulnerabilityDataLastSyncUtc", "msrcLastSyncUtc", "nvdCacheLastRefreshUtc", - "epssLastSyncUtc" + "epssLastSyncUtc", + "orphanDeepCheckLastRunUtc" ], "defaults": { "partitionKey": "GlobalConfig", @@ -114,7 +115,8 @@ "vulnerabilityDataLastSyncUtc": null, "msrcLastSyncUtc": null, "nvdCacheLastRefreshUtc": null, - "epssLastSyncUtc": null + "epssLastSyncUtc": null, + "orphanDeepCheckLastRunUtc": null } }, "tenantConfiguration": { @@ -600,6 +602,10 @@ "SessionSweepFailed", "OpsEventCleanup", "OrphanEventsCleaned", + "OrphanSweepCompleted", + "OrphanSweepFailed", + "OrphanSweepSkippedLocked", + "OrphanReconcileCompleted", "SessionDeletionMaintenanceStarted", "SessionDeletionMaintenanceBudgetExceeded", "SessionDeletionMaintenanceSkippedLocked", @@ -1750,6 +1756,14 @@ "name": "string" }, "doc": "Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss." + }, + { + "name": "orphanDeepCheckLastRunUtc", + "type": { + "kind": "primitive", + "name": "string" + }, + "doc": "Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal." } ] }, diff --git a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts index 553fd5f6..563dd1a3 100644 --- a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts @@ -200,6 +200,8 @@ export interface AdminConfiguration { nvdCacheLastRefreshUtc: string; /** Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss. */ epssLastSyncUtc: string; + /** Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal. */ + orphanDeepCheckLastRunUtc: string; } /** The immutable Entra identity behind a cross-tenant-role UPN: the HOME tenant the UPN was granted for and, once known, the user's object id. A role row (GlobalAdmins / DelegatedAdmins / TenantGroupAssignments) confers nothing unless the caller's validated JWT matches this binding. */