Orphan sessions: ingest refuses sessions without a row, the sweep becomes their cascade in its own function (D-339)

A session without a Sessions row can never be reached by the cascade, so every row written for it was an orphan by construction. Three layers now keep the platform clean:

- Prevent: IngestTelemetry answers a batch for an unknown session with 410 (same status and body as the cascade lock; request-row marker IngestRefusal = session_unknown | cascade_locked). The derived writers - vulnerability correlation, analyze, manual rescan - stop on SessionDeletionGuard.GetLivenessAsync != Live. The agent treats every 410 as terminal: one TelemetryRejectedSessionUnknown report (exempt from the reporter cooldown), then terminate_session with forced self-destruct (origin session_gone, no final drain, no diagnostics upload); spool items of a foreign session id are dropped at startup and never terminate the current session.
- Detect: OrphanSessionSweepFunction (every 4h at :45, own lease) diffs the Sessions keys against the EventSessionIndex handles - two range drains instead of ~16.8k Sessions point reads per 2h maintenance run - and re-checks every candidate (handle unchanged, row still absent, tombstone) moments before deleting. The in-maintenance sweep raced the retention cascade on their shared 00:00/12:00 tick and reported false orphans; it is gone.
- Clean: every table of the cascade manifest minus the tombstone pair (ratchet in TableLifecycleBucketTests), the inventory counters with the stamp on the handle (claim, then decrement), the handle last with If-Match. A weekly read-only reconciliation of EventTypeIndex against the Sessions keys (Sunday mornings UTC, and on every manual maintenance run) reports residue.

MaintenanceCompleted now carries stepsMs. MCP search_sessions_by_event auto-exhausts empty-but-continuable pages. Manifests regenerated for the new ops event types and AdminConfiguration.OrphanDeepCheckLastRunUtc.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-10-09 22:55:27 +02:00
parent edad842a0b
commit 3d10011460
64 changed files with 3228 additions and 275 deletions
@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Shared.Models;
using Xunit;
@@ -49,6 +50,74 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
private static EmergencyReporter Reporter(FakeApiClient client)
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null);
private static EmergencyReporter Reporter(FakeApiClient client, VirtualClock clock)
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null, clock: clock);
private static VirtualClock Clock() => new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc));
[Fact]
public async Task Session_unknown_report_is_not_suppressed_by_the_cooldown_of_an_earlier_report()
{
// Terminal report, sent at most once per process right before the termination — a
// routine report one minute earlier must not cost it. Every other type keeps the cooldown.
var clock = Clock();
var client = new FakeApiClient();
client.Results.Enqueue(true);
client.Results.Enqueue(true);
client.Results.Enqueue(true);
var reporter = Reporter(client, clock);
await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: 503);
clock.Advance(TimeSpan.FromMinutes(1));
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
Assert.Equal(2, client.Calls);
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, client.Reports[1].ErrorType);
// Control: a third report of another type inside the window is still held back.
clock.Advance(TimeSpan.FromMinutes(1));
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
Assert.Equal(2, client.Calls);
}
[Fact]
public async Task Second_session_unknown_report_within_the_window_is_still_suppressed()
{
var clock = Clock();
var client = new FakeApiClient();
client.Results.Enqueue(true);
client.Results.Enqueue(true);
var reporter = Reporter(client, clock);
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
clock.Advance(TimeSpan.FromMinutes(1));
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
Assert.Equal(1, client.Calls);
}
[Fact]
public async Task Session_unknown_report_still_counts_against_the_per_session_maximum()
{
// Five delivered reports (ten minutes apart, distinct keys) exhaust the budget; the
// exemption waives only the cooldown, not the maximum.
var clock = Clock();
var client = new FakeApiClient();
for (var i = 0; i < 6; i++) client.Results.Enqueue(true);
var reporter = Reporter(client, clock);
foreach (var status in new[] { 500, 502, 503, 504, 429 })
{
await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: status);
clock.Advance(TimeSpan.FromMinutes(10));
}
Assert.Equal(5, client.Calls);
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
Assert.Equal(5, client.Calls);
}
[Fact]
public async Task Failed_send_rolls_back_reservation_so_a_later_call_retries()
{
@@ -142,6 +211,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
Assert.Equal(2, client.Calls);
}
[Fact]
public async Task Session_unknown_report_carries_error_type_and_http_410()
{
var client = new FakeApiClient();
client.Results.Enqueue(true);
var reporter = Reporter(client);
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
var report = Assert.Single(client.Reports);
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType);
Assert.Equal(410, report.HttpStatusCode);
}
[Fact]
public async Task Session_age_is_carried_in_the_report_and_absent_by_default()
{
@@ -0,0 +1,170 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Agent.V2.Core.Configuration;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
using AutopilotMonitor.Shared.Models;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
{
/// <summary>
/// Host wiring for <c>TelemetryUploadOrchestrator.SessionGone</c>
/// (HTTP 410 on the current session's telemetry): exactly one
/// <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/> emergency report carrying the
/// status, then a synthetic forced <c>terminate_session</c> tagged <c>origin=session_gone</c>
/// — and nothing at all once a shutdown is already in progress.
/// </summary>
public sealed class ServerControlPlaneSessionGoneTests
{
private sealed class RecordingEmergencyReporter : EmergencyReporter
{
private readonly List<string> _sequence;
public List<(AgentErrorType ErrorType, string Message, int? HttpStatusCode)> Reports { get; } =
new List<(AgentErrorType, string, int?)>();
public RecordingEmergencyReporter(List<string> sequence)
: base(apiClient: null!, sessionId: "S1", tenantId: "T1", agentVersion: "0.0.0", logger: null!)
{
_sequence = sequence;
}
public override Task TrySendAsync(
AgentErrorType errorType,
string message,
int? httpStatusCode = null,
long? sequenceNumber = null,
int attempts = 1,
TimeSpan? perAttemptTimeout = null,
TimeSpan? retryDelay = null,
double? sessionAgeHours = null,
RegistrationFailureSummary? priorRegistrationFailure = null)
{
Reports.Add((errorType, message, httpStatusCode));
_sequence.Add("report");
return Task.CompletedTask;
}
}
private sealed class RecordingDispatcher : ServerActionDispatcher
{
private readonly List<string> _sequence;
public List<ServerAction> Dispatched { get; } = new List<ServerAction>();
public RecordingDispatcher(AgentConfiguration config, AgentLogger logger, InformationalEventPost post, List<string> sequence)
: base(
configuration: config,
logger: logger,
rotateConfigAsync: () => Task.FromResult(true),
uploadDiagnosticsAsync: _ => Task.FromResult<DiagnosticsUploadResult>(null!),
onTerminateRequested: _ => Task.CompletedTask,
post: post)
{
_sequence = sequence;
}
public override Task DispatchAsync(List<ServerAction> actions)
{
Dispatched.AddRange(actions);
_sequence.Add("dispatch");
return Task.CompletedTask;
}
}
private sealed class Rig : IDisposable
{
public TempDirectory Tmp { get; } = new TempDirectory();
public AgentLogger Logger { get; }
public List<string> Sequence { get; } = new List<string>();
public RecordingEmergencyReporter Reporter { get; }
public RecordingDispatcher Dispatcher { get; }
public ManualResetEventSlim Shutdown { get; } = new ManualResetEventSlim(false);
public ManualResetEventSlim ShutdownComplete { get; } = new ManualResetEventSlim(false);
public Rig()
{
Logger = new AgentLogger(Path.Combine(Tmp.Path, "logs"), AgentLogLevel.Info);
var config = new AgentConfiguration { SessionId = "S1", TenantId = "T1", ApiBaseUrl = "http://localhost" };
var post = new InformationalEventPost(
new FakeSignalIngressSink(),
new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc)));
Reporter = new RecordingEmergencyReporter(Sequence);
Dispatcher = new RecordingDispatcher(config, Logger, post, Sequence);
}
public void Handle(string reason = "session gone: http 410") =>
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.HandleSessionGone(
reason, Reporter, Dispatcher, Shutdown, ShutdownComplete, Logger);
public void Dispose()
{
Shutdown.Dispose();
ShutdownComplete.Dispose();
Tmp.Dispose();
}
}
[Fact]
public void Reports_once_with_410_then_dispatches_forced_terminate_tagged_session_gone()
{
using var rig = new Rig();
rig.Handle();
Assert.Equal(new[] { "report", "dispatch" }, rig.Sequence);
var report = Assert.Single(rig.Reporter.Reports);
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType);
Assert.Equal(410, report.HttpStatusCode);
Assert.Contains("410", report.Message);
var action = Assert.Single(rig.Dispatcher.Dispatched);
Assert.Equal(ServerActionTypes.TerminateSession, action.Type);
Assert.Contains("410", action.Reason);
Assert.NotNull(action.Params);
var parameters = action.Params!;
Assert.Equal("true", parameters["forceSelfDestruct"]);
Assert.Equal("0", parameters["gracePeriodSeconds"]);
Assert.Equal(TerminationOrigins.SessionGone, parameters["origin"]);
}
[Theory]
[InlineData(true, false)]
[InlineData(false, true)]
public void Does_nothing_when_a_shutdown_is_already_in_progress(bool shutdownSet, bool shutdownCompleteSet)
{
using var rig = new Rig();
if (shutdownSet) rig.Shutdown.Set();
if (shutdownCompleteSet) rig.ShutdownComplete.Set();
rig.Handle();
Assert.Empty(rig.Reporter.Reports);
Assert.Empty(rig.Dispatcher.Dispatched);
}
[Fact]
public void ReadOrigin_returns_the_origin_param_or_null()
{
Assert.Equal(TerminationOrigins.SessionGone,
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
new Dictionary<string, string> { ["origin"] = TerminationOrigins.SessionGone }));
Assert.Equal(TerminationOrigins.KillSignal,
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
new Dictionary<string, string> { ["origin"] = TerminationOrigins.KillSignal, ["forceSelfDestruct"] = "true" }));
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
new Dictionary<string, string> { ["adminOutcome"] = "Failed" }));
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
new Dictionary<string, string> { ["origin"] = string.Empty }));
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(null!));
}
}
}
@@ -310,6 +310,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
new EnrollmentTerminatedEventArgs(
reason, outcome, stage.ToString(), at ?? EndUtc);
/// <summary>
/// The args <c>ServerControlPlane.OnTerminateRequested</c> synthesises for a
/// <c>terminate_session</c> action tagged with <paramref name="origin"/>.
/// </summary>
private static EnrollmentTerminatedEventArgs ServerArgs(string origin) =>
new EnrollmentTerminatedEventArgs(
EnrollmentTerminationReason.DecisionTerminalStage,
EnrollmentTerminationOutcome.Failed,
SessionStage.Failed.ToString(),
EndUtc,
details: "Server-requested termination",
origin: origin);
[Fact]
public void Handle_writes_final_status_json_in_state_directory()
{
@@ -458,6 +471,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
Assert.Equal(0, rig.CleanupService.Invocations);
}
// ============================================================= origin=session_gone (HTTP 410)
[Fact]
public void Handle_session_gone_skips_spool_drain_and_diagnostics_but_still_cleans_up()
{
// The backend no longer knows the session: every drain would only burn its budget on
// 410s and a diagnostics package would describe a deleted session. Marker, cleanup
// and the (forced) self-destruct still run.
using var rig = new Rig();
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build();
rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5);
var spoolPolls = 0;
rig.PendingItemCountAccessor = () => { spoolPolls++; return 5; }; // never drains — a poll would burn the budget
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true);
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.SessionGone));
Assert.Equal(0, spoolPolls);
Assert.Equal(0, rig.DiagnosticsUploads);
Assert.DoesNotContain(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes);
Assert.True(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
Assert.Equal(1, rig.CleanupService.Invocations);
Assert.Equal(1, rig.ShutdownSignalled);
}
[Fact]
public void Handle_kill_signal_origin_still_drains_spool_and_uploads_diagnostics()
{
// The administrator kill signal keeps its full sequence — only session_gone skips.
using var rig = new Rig();
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build();
rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5);
var spoolPolls = 0;
rig.PendingItemCountAccessor = () => { spoolPolls++; return 0; };
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true);
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.KillSignal));
Assert.True(spoolPolls > 0);
Assert.Equal(1, rig.DiagnosticsUploads);
Assert.Contains(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
Assert.Equal(1, rig.CleanupService.Invocations);
Assert.Equal(1, rig.ShutdownSignalled);
}
[Fact]
public void Handle_signals_shutdown_exactly_once_even_on_error()
{
@@ -275,9 +275,44 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
Assert.False(result.Success);
Assert.False(result.IsTransient);
Assert.False(result.IsSessionGone);
Assert.Contains(((int)status).ToString(), result.ErrorReason);
}
[Fact]
public async Task Gone_410_maps_to_SessionGone()
{
// 410 = the backend no longer knows this session. Not transient (no retry), not auth,
// not poison, not a size problem — a distinct terminal kind the drain reacts to.
using var rig = new Rig();
rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "session unknown");
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
Assert.False(result.Success);
Assert.True(result.IsSessionGone);
Assert.False(result.IsTransient);
Assert.False(result.IsAuthFailure);
Assert.False(result.IsPoison);
Assert.False(result.RequiresSplit);
Assert.Contains("410", result.ErrorReason);
}
[Fact]
public async Task Gone_410_with_poison_body_is_still_SessionGone_never_poison()
{
// A poison body on 410 must not be honoured: dropping items + emitting
// telemetry_upload_poisoned would only produce the next 410 (emit/reject loop).
using var rig = new Rig();
rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "{\"poison\":true,\"rejectedRowKeys\":[\"rk1\"],\"reason\":\"x\"}");
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
Assert.True(result.IsSessionGone);
Assert.False(result.IsPoison);
Assert.Null(result.PoisonRowKeys);
}
[Theory]
[InlineData(HttpStatusCode.RequestTimeout)] // 408
[InlineData((HttpStatusCode)429)] // TooManyRequests
@@ -68,6 +68,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
return this;
}
public FakeBackendTelemetryUploader QueueSessionGone(string reason = "session gone: http 410")
{
_script.Enqueue(_ => UploadResult.SessionGone(reason));
return this;
}
public FakeBackendTelemetryUploader QueueThrow(Exception ex)
{
_script.Enqueue(_ => throw ex);
@@ -10,10 +10,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
{
public sealed class TelemetrySpoolTests
{
private static TelemetryItemDraft EventDraft(string rowKey = "row-1") =>
private const string OwnKey = "tenant_session";
private const string ForeignKey = "tenant_old-session";
private static TelemetryItemDraft EventDraft(string rowKey = "row-1") => EventDraft(rowKey, OwnKey);
private static TelemetryItemDraft EventDraft(string rowKey, string partitionKey) =>
new TelemetryItemDraft(
kind: TelemetryItemKind.Event,
partitionKey: "tenant_session",
partitionKey: partitionKey,
rowKey: rowKey,
payloadJson: "{\"EventType\":\"agent_started\"}",
isSessionScoped: true);
@@ -195,6 +200,86 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
Assert.Equal(1, next.TelemetryItemId);
}
// ============================================================ Startup partition-key filter
[Fact]
public void Ctor_drops_leading_pending_items_of_another_session_and_persists_cursor()
{
// Same spool directory, new session id: the previous session's unsent items would only
// draw a 410 that must not end the new session. They are dropped on startup; the own
// item behind them stays pending.
using var tmp = new TempDirectory();
var s1 = new TelemetrySpool(tmp.Path, Clock());
s1.Enqueue(EventDraft("old-a", ForeignKey));
s1.Enqueue(EventDraft("old-b", ForeignKey));
s1.Enqueue(EventDraft("own", OwnKey));
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
Assert.Equal(1, s2.LastUploadedItemId);
Assert.Equal(2, s2.LastAssignedItemId);
Assert.Equal(1, s2.PendingItemCount);
Assert.Equal("own", Assert.Single(s2.Peek(10)).RowKey);
// Cursor persisted: a later start without the filter sees the same tail with the same bytes.
var s3 = new TelemetrySpool(tmp.Path, Clock());
Assert.Equal(1, s3.LastUploadedItemId);
Assert.Equal(1, s3.PendingItemCount);
Assert.Equal(s3.PendingBytes, s2.PendingBytes);
var next = s2.Enqueue(EventDraft("next", OwnKey));
Assert.Equal(3, next.TelemetryItemId);
}
[Fact]
public void Ctor_drops_nothing_when_a_foreign_item_follows_an_own_item()
{
// Impossible by construction (one process = one session); if it happens the order is
// not trustworthy, so the filter keeps everything instead of guessing.
using var tmp = new TempDirectory();
var s1 = new TelemetrySpool(tmp.Path, Clock());
s1.Enqueue(EventDraft("old-a", ForeignKey));
s1.Enqueue(EventDraft("own", OwnKey));
s1.Enqueue(EventDraft("old-b", ForeignKey));
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
Assert.Equal(-1, s2.LastUploadedItemId);
Assert.Equal(3, s2.PendingItemCount);
Assert.Equal(s1.PendingBytes, s2.PendingBytes);
}
[Fact]
public void Ctor_keeps_pending_items_that_carry_the_expected_key()
{
// WhiteGlove Part 2 reuses the Part-1 session id: its leftovers carry the expected key.
using var tmp = new TempDirectory();
var s1 = new TelemetrySpool(tmp.Path, Clock());
s1.Enqueue(EventDraft("a", OwnKey));
s1.Enqueue(EventDraft("b", OwnKey));
s1.Enqueue(EventDraft("c", OwnKey));
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
Assert.Equal(-1, s2.LastUploadedItemId);
Assert.Equal(3, s2.PendingItemCount);
Assert.Equal(s1.PendingBytes, s2.PendingBytes);
}
[Fact]
public void Ctor_without_expected_key_keeps_foreign_items()
{
using var tmp = new TempDirectory();
var s1 = new TelemetrySpool(tmp.Path, Clock());
s1.Enqueue(EventDraft("old-a", ForeignKey));
s1.Enqueue(EventDraft("old-b", ForeignKey));
var s2 = new TelemetrySpool(tmp.Path, Clock());
Assert.Equal(-1, s2.LastUploadedItemId);
Assert.Equal(2, s2.PendingItemCount);
}
[Fact]
public void Enqueue_with_RequiresImmediateFlush_raises_ImmediateFlushRequested()
{
@@ -10,10 +10,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
{
public sealed class TelemetryUploadOrchestratorTests
{
private static TelemetryItemDraft Draft(string rowKey) =>
private const string OwnKey = "tenant_session";
private static TelemetryItemDraft Draft(string rowKey) => Draft(rowKey, OwnKey);
private static TelemetryItemDraft Draft(string rowKey, string partitionKey) =>
new TelemetryItemDraft(
kind: TelemetryItemKind.Event,
partitionKey: "tenant_session",
partitionKey: partitionKey,
rowKey: rowKey,
payloadJson: $"{{\"rk\":\"{rowKey}\"}}",
isSessionScoped: true);
@@ -157,6 +161,118 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
Assert.Equal(1, blockedCount);
}
// ============================================================ HTTP 410 — session gone
[Fact]
public async Task SessionGone_for_current_session_stops_drain_retains_cursor_and_raises_once()
{
using var tmp = new TempDirectory();
var spool = new TelemetrySpool(tmp.Path, Clock());
spool.Enqueue(Draft("a"));
spool.Enqueue(Draft("b"));
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone();
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
var gone = new System.Collections.Generic.List<string>();
int blockedCount = 0;
PoisonReport? poison = null;
sut.SessionGone += (_, reason) => gone.Add(reason);
sut.UploadBlocked += (_, __) => blockedCount++;
sut.BatchPoisoned += (_, p) => poison = p;
var first = await sut.DrainAllAsync();
Assert.False(first.Success);
Assert.Equal(1, first.FailedBatches);
Assert.Equal(0, first.UploadedItems);
Assert.Equal(1, uploader.CallCount); // no retries — nothing can make a 410 succeed
Assert.Equal(-1, sut.LastUploadedItemId); // retained — items stay on disk for the diag ZIP
Assert.Single(gone);
Assert.Contains("410", gone[0]);
Assert.Equal(0, blockedCount); // not a "blocked" pipeline
Assert.Null(poison);
// One-shot: the next drain hits the 410 again but does not raise again.
var second = await sut.DrainAllAsync();
Assert.False(second.Success);
Assert.Equal(2, uploader.CallCount);
Assert.Equal(-1, sut.LastUploadedItemId);
Assert.Single(gone);
Assert.Equal(0, blockedCount);
}
[Fact]
public async Task SessionGone_for_leading_foreign_items_drops_only_them_and_continues()
{
// Leftovers of a previous session id at the head of the spool (defence in depth — the
// spool filters these at startup). A 410 for them must not end the current session:
// drop exactly the leading foreign run, then carry on with the current session's items.
using var tmp = new TempDirectory();
var spool = new TelemetrySpool(tmp.Path, Clock());
spool.Enqueue(Draft("old-a", "tenant_old-session"));
spool.Enqueue(Draft("old-b", "tenant_old-session"));
spool.Enqueue(Draft("new", OwnKey));
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueOk();
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
int goneCount = 0, blockedCount = 0;
sut.SessionGone += (_, __) => goneCount++;
sut.UploadBlocked += (_, __) => blockedCount++;
var result = await sut.DrainAllAsync();
Assert.True(result.Success);
Assert.Equal(0, result.FailedBatches);
Assert.Equal(1, result.UploadedItems);
Assert.Equal(2, uploader.CallCount);
Assert.Equal(3, uploader.Received[0].Count); // first batch carried all three
Assert.Equal("new", Assert.Single(uploader.Received[1]).RowKey); // retry carried only the own item
Assert.Equal(2, sut.LastUploadedItemId);
Assert.Equal(0, goneCount);
Assert.Equal(0, blockedCount);
}
[Fact]
public async Task SessionGone_is_not_latched_before_a_handler_is_subscribed()
{
// The host subscribes after EnrollmentOrchestrator.Start; a 410 on the very first drain
// must still reach it on the next drain instead of being swallowed by the one-shot guard.
using var tmp = new TempDirectory();
var spool = new TelemetrySpool(tmp.Path, Clock());
spool.Enqueue(Draft("only"));
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone();
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
await sut.DrainAllAsync(); // nobody listening yet
int goneCount = 0;
sut.SessionGone += (_, __) => goneCount++;
await sut.DrainAllAsync();
Assert.Equal(1, goneCount);
Assert.Equal(-1, sut.LastUploadedItemId);
}
[Fact]
public async Task SessionGone_without_a_current_key_treats_every_batch_as_own()
{
using var tmp = new TempDirectory();
var spool = new TelemetrySpool(tmp.Path, Clock());
spool.Enqueue(Draft("x", "tenant_whatever"));
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone();
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock());
int goneCount = 0;
sut.SessionGone += (_, __) => goneCount++;
var result = await sut.DrainAllAsync();
Assert.False(result.Success);
Assert.Equal(1, goneCount);
Assert.Equal(-1, sut.LastUploadedItemId);
}
[Fact]
public async Task TooLarge_splits_batch_and_uploads_everything()
{
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.DecisionCore.Engine;
using AutopilotMonitor.Shared.Models;
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
@@ -14,6 +15,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
/// - Same error key (ErrorType + HTTP status code) is only ever sent once per session
/// - At most <see cref="MaxReportsPerSession"/> emergency reports total per session
/// - At most one report every <see cref="MinIntervalMinutes"/> minutes regardless of error type
/// (the terminal <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/> report is exempt)
///
/// A failure in this reporter is silently swallowed — it must never cascade into the main loop.
/// </summary>
@@ -32,6 +34,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
private readonly string _tenantId;
private readonly string _agentVersion;
private readonly AgentLogger _logger;
private readonly IClock _clock;
private readonly HashSet<string> _reportedKeys = new HashSet<string>();
private readonly object _antiFloodLock = new object();
@@ -43,13 +46,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
string sessionId,
string tenantId,
string agentVersion,
AgentLogger logger)
AgentLogger logger,
IClock clock = null)
{
_apiClient = apiClient;
_sessionId = sessionId;
_tenantId = tenantId;
_agentVersion = agentVersion;
_logger = logger;
_clock = clock ?? SystemClock.Instance;
}
/// <summary>
@@ -100,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
// All anti-flood checks are protected by a lock to prevent race conditions when
// TrySendAsync is called concurrently (fire-and-forget from multiple threads).
var key = $"{errorType}:{httpStatusCode}";
var now = _clock.UtcNow;
int currentReportCount;
DateTime? previousLastReportTime;
@@ -117,8 +123,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
return;
}
if (_lastReportTime != null &&
(DateTime.UtcNow - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
// The session-unknown report is terminal and sent at most once per process (the
// termination follows immediately): a routine report minutes earlier must not cost
// the last word about the session. Only the cooldown is waived; key dedup and the
// per-session maximum still apply.
var exemptFromCooldown = errorType == AgentErrorType.TelemetryRejectedSessionUnknown;
if (!exemptFromCooldown && _lastReportTime != null &&
(now - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
{
_logger?.Debug($"[EmergencyChannel] Suppressed (cooldown active, last report at {_lastReportTime.Value:HH:mm:ss}): {key}");
return;
@@ -127,7 +138,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
previousLastReportTime = _lastReportTime;
_reportedKeys.Add(key);
_reportCount++;
_lastReportTime = DateTime.UtcNow;
_lastReportTime = now;
currentReportCount = _reportCount;
}
@@ -145,7 +156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
SessionAgeHours = sessionAgeHours,
PriorRegistrationFailure = priorRegistrationFailure,
AgentVersion = _agentVersion,
Timestamp = DateTime.UtcNow
Timestamp = now
};
var delivered = false;
@@ -371,8 +371,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
// 3) Telemetry-Transport. Batch size flows from AgentConfiguration.MaxBatchSize
// via Program.cs (P1 fix: previously the remote-config knob was merged but
// never applied, so tenants saw drainInterval/MaxBatchSize have no effect).
_spool = new TelemetrySpool(_transportDirectory, _clock, _logger);
_transport = new TelemetryUploadOrchestrator(_spool, _uploader, _clock, batchSize: _uploadBatchSize);
// The partition key names the registered session: the spool drops leftovers of a
// previous session id on startup, and the orchestrator tells a 410 for this session
// (SessionGone → host terminates) from a 410 for such leftovers (dropped).
var partitionKey = TelemetryPartitionKey.ForSession(_tenantId, _sessionId);
_spool = new TelemetrySpool(_transportDirectory, _clock, _logger, expectedPartitionKey: partitionKey);
_transport = new TelemetryUploadOrchestrator(
_spool, _uploader, _clock, batchSize: _uploadBatchSize, currentPartitionKey: partitionKey, logger: _logger);
// TRACE-H1 — surface skipped oversized items + blocked-pipeline conditions as timeline
// events. Raised after the drain guard releases, so posting back through the ingress
@@ -73,13 +73,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
/// </summary>
public string? StopReason { get; }
/// <summary>
/// The <c>origin</c> param of a server-requested <c>terminate_session</c> action
/// (<see cref="TerminationOrigins"/>); null for every termination the agent decided
/// itself. <see cref="TerminationOrigins.SessionGone"/> makes the termination handler
/// skip the spool drain and the diagnostics upload — the backend no longer knows the
/// session, so both could only fail.
/// </summary>
public string? Origin { get; }
public EnrollmentTerminatedEventArgs(
EnrollmentTerminationReason reason,
EnrollmentTerminationOutcome outcome,
string? stageName,
DateTime terminatedAtUtc,
string? details = null,
string? stopReason = null)
string? stopReason = null,
string? origin = null)
{
Reason = reason;
Outcome = outcome;
@@ -87,6 +97,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
TerminatedAtUtc = terminatedAtUtc;
Details = details;
StopReason = stopReason;
Origin = origin;
}
}
/// <summary>
/// Values of the <c>origin</c> param on a synthesised <c>terminate_session</c>
/// <c>ServerAction</c>, carried into <see cref="EnrollmentTerminatedEventArgs.Origin"/>.
/// </summary>
public static class TerminationOrigins
{
/// <summary>Administrator kill signal (<c>DeviceKillSignal</c> on a 2xx telemetry response).</summary>
public const string KillSignal = "kill_signal";
/// <summary>The backend answered HTTP 410 for the current session's telemetry.</summary>
public const string SessionGone = "session_gone";
}
}
@@ -45,7 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Events
_sequenceCounter = sequenceCounter ?? throw new ArgumentNullException(nameof(sequenceCounter));
_sessionId = sessionId;
_tenantId = tenantId;
_partitionKey = $"{tenantId}_{sessionId}";
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
_traceEventsEnabled = traceEventsEnabled;
}
@@ -42,7 +42,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Signals
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId));
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId));
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
_partitionKey = $"{tenantId}_{sessionId}";
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
}
/// <summary>
@@ -43,7 +43,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Transitions
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId));
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId));
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
_partitionKey = $"{tenantId}_{sessionId}";
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
}
/// <summary>
@@ -306,7 +306,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
});
DelayLateEventGrace();
DrainSpool();
DrainSpool(args);
// Option 2 (WG Part 1 graceful-exit hardening, 2026-04-30): write the
// clean-exit marker BEFORE _signalShutdown returns control to the main
@@ -329,7 +329,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
RunUploadDiagnosticsWithEvents(args);
WriteEnrollmentCompleteMarker(args);
RunStandaloneRebootIfRequested();
RunStandaloneRebootIfRequested(args);
RunSelfDestructIfAppropriate(args);
// Option 2 (same hardening) — covers the standard Completed / Failed terminal
// path too. Cleanup PowerShell does not touch this marker, so writing it before
@@ -979,6 +979,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
/// </param>
private void RunUploadDiagnosticsWithEvents(EnrollmentTerminatedEventArgs args, string suffixOverride = null)
{
if (IsSessionGone(args))
{
// The backend answered 410 for this session: the package would describe a session
// that no longer exists, and its diagnostics_* events could never be accepted.
_logger.Info("EnrollmentTerminationHandler: diagnostics upload skipped — backend no longer knows this session.");
return;
}
var mode = _configuration.DiagnosticsUploadMode ?? "Off";
if (!_configuration.DiagnosticsUploadEnabled || string.Equals(mode, "Off", StringComparison.OrdinalIgnoreCase))
{
@@ -1124,7 +1132,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
/// but enables <c>RebootOnComplete</c>, the agent's final act is <c>shutdown.exe /r</c>
/// with the configured delay, giving the user a visible countdown.
/// </summary>
private void RunStandaloneRebootIfRequested()
private void RunStandaloneRebootIfRequested(EnrollmentTerminatedEventArgs args)
{
if (_configuration.SelfDestructOnComplete) return;
if (!_configuration.RebootOnComplete) return;
@@ -1148,7 +1156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
ImmediateUpload = true,
});
DrainSpool();
DrainSpool(args);
try
{
@@ -1185,7 +1193,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
// just above. The WG-Part-1 and standalone-reboot paths already drain first; this
// closes the gap on the self-destruct path. Bounded + best-effort (never blocks
// termination on a wedged backend).
DrainSpool();
DrainSpool(args);
var service = _cleanupServiceFactory();
service.ExecuteSelfDestruct();
@@ -1318,8 +1326,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
}
}
private void DrainSpool()
/// <summary>
/// <see cref="TerminationOrigins.SessionGone"/>: the backend no longer knows the session.
/// Every drain would only collect more 410s against a bounded budget, and a diagnostics
/// package would describe a deleted session — both are skipped; marker, cleanup and
/// self-destruct run as usual.
/// </summary>
private static bool IsSessionGone(EnrollmentTerminatedEventArgs args) =>
string.Equals(args.Origin, TerminationOrigins.SessionGone, StringComparison.Ordinal);
private void DrainSpool(EnrollmentTerminatedEventArgs args)
{
if (IsSessionGone(args))
{
_logger.Info("EnrollmentTerminationHandler: spool drain skipped — backend no longer knows this session.");
return;
}
// Block briefly so pending events can land before the next destructive step
// (shutdown.exe, self-destruct). Two phases share the same bounded budget:
//
@@ -165,6 +165,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
{
_logger?.Warning($"BackendTelemetryUploader: ingest TRANSIENT (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
}
else if (result.IsSessionGone)
{
_logger?.Error($"BackendTelemetryUploader: ingest rejected — backend no longer knows this session (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
}
else
{
_logger?.Error($"BackendTelemetryUploader: ingest PERMANENT fail (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
@@ -208,6 +212,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
case HttpStatusCode.RequestEntityTooLarge: // 413 — the data is fine, only the batch
return UploadResult.TooLarge(shortReason); // size is wrong → split & retry (P1).
case HttpStatusCode.Gone: // 410 — the backend no longer knows this session.
// Decided BEFORE the poison-body check: a poison body on 410 would make the
// drain drop items and emit telemetry_upload_poisoned, which the backend
// rejects with 410 again — an emit/reject loop for a session that is over.
return UploadResult.SessionGone($"session gone: {shortReason}");
}
if (statusCode >= 500 && statusCode <= 599) return UploadResult.Transient(shortReason);
@@ -0,0 +1,14 @@
#nullable enable
namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
{
/// <summary>
/// The one composition of a session's Azure-Table partition key. The three emitters stamp
/// it on every <see cref="TelemetryItem"/>; the spool and the upload orchestrator compare
/// against it to tell the current session's items from leftovers of a previous session id.
/// A drift between producer and comparer would silently drop or retain the wrong items.
/// </summary>
public static class TelemetryPartitionKey
{
public static string ForSession(string tenantId, string sessionId) => $"{tenantId}_{sessionId}";
}
}
@@ -74,7 +74,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
public int ByteLength { get; }
}
public TelemetrySpool(string directoryPath, IClock clock, Logging.AgentLogger? logger = null)
/// <param name="expectedPartitionKey">
/// Partition key of the session this process serves (<see cref="TelemetryPartitionKey.ForSession"/>).
/// When set, pending items of another session id found at the head of the spool are dropped
/// on startup — see <see cref="DropLeadingForeignPending"/>. Null disables the filter.
/// </param>
public TelemetrySpool(
string directoryPath,
IClock clock,
Logging.AgentLogger? logger = null,
string? expectedPartitionKey = null)
{
if (string.IsNullOrEmpty(directoryPath))
{
@@ -96,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
// count as "pending" and need to be rehydrated into _pending.
_lastUploadedItemId = _cursor.Load();
_lastAssignedItemId = ScanSpoolAndRehydratePending();
DropLeadingForeignPending(expectedPartitionKey);
_peakPendingCount = _pending.Count;
}
@@ -312,5 +322,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
}
return highest;
}
/// <summary>
/// Startup filter: drops the LEADING run of pending items whose partition key is not
/// <paramref name="expectedPartitionKey"/>, advancing and persisting the cursor past them.
/// Such items are leftovers of a previous session id in the same spool directory; the
/// backend answers 410 for them, and that 410 must never end the session this process
/// serves. One process serves one session, so a foreign item AFTER an own item cannot
/// occur — if it does, the order is not trustworthy and nothing is dropped. WhiteGlove
/// Part 2 reuses the Part-1 session id, so its leftovers carry the expected key and stay.
/// Called once from the ctor, after rehydration.
/// </summary>
private void DropLeadingForeignPending(string? expectedPartitionKey)
{
if (string.IsNullOrEmpty(expectedPartitionKey) || _pending.Count == 0) return;
var leadingForeign = 0;
long lastForeignId = -1;
var seenOwn = false;
foreach (var entry in _pending)
{
var isForeign = !string.Equals(entry.Item.PartitionKey, expectedPartitionKey, StringComparison.Ordinal);
if (!isForeign)
{
seenOwn = true;
continue;
}
if (!seenOwn)
{
leadingForeign++;
lastForeignId = entry.Item.TelemetryItemId;
continue;
}
_logger?.Error(
$"TelemetrySpool: pending itemId={entry.Item.TelemetryItemId} carries a foreign partition key after items of the current session — spool order is inconsistent; keeping all {_pending.Count} pending item(s).");
return;
}
if (leadingForeign == 0) return;
try
{
_cursor.Save(lastForeignId);
}
catch (Exception ex)
{
// Keep the items rather than risk a cursor that disagrees with disk; the drain
// loop drops the same leading run on its first 410 (defence in depth).
_logger?.Warning($"TelemetrySpool: could not persist the cursor past {leadingForeign} foreign pending item(s): {ex.Message}");
return;
}
_lastUploadedItemId = lastForeignId;
for (var i = 0; i < leadingForeign; i++)
{
_pendingBytes -= _pending.First!.Value.ByteLength;
_pending.RemoveFirst();
}
_logger?.Warning(
$"TelemetrySpool: dropped {leadingForeign} pending item(s) of a previous session id (cursor advanced to {lastForeignId}, {_pending.Count} pending item(s) of the current session kept).");
}
}
}
@@ -45,12 +45,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
// telemetry_upload_blocked exactly once per process rather than on every 30 s drain.
private bool _uploadBlockedReported;
// Partition key of the session this process serves; null = every batch counts as own.
private readonly string? _currentPartitionKey;
private readonly Logging.AgentLogger? _logger;
// One-shot guard for SessionGone. Latched only when a handler is subscribed: the host
// wires the handler after EnrollmentOrchestrator.Start, and a 410 on the very first
// drain must not be swallowed before anyone listens.
private bool _sessionGoneReported;
/// <param name="currentPartitionKey">
/// <see cref="TelemetryPartitionKey.ForSession"/> of the registered session. A 410 on a
/// batch that starts with this key ends the session (<see cref="SessionGone"/>); a 410 on
/// a batch that starts with any other key only drops those leftover items.
/// </param>
public TelemetryUploadOrchestrator(
ITelemetrySpool spool,
IBackendTelemetryUploader uploader,
IClock clock,
int batchSize = 100,
IReadOnlyList<TimeSpan>? retryBackoffs = null)
IReadOnlyList<TimeSpan>? retryBackoffs = null,
string? currentPartitionKey = null,
Logging.AgentLogger? logger = null)
{
_spool = spool ?? throw new ArgumentNullException(nameof(spool));
_uploader = uploader ?? throw new ArgumentNullException(nameof(uploader));
@@ -61,6 +77,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
_effectiveBatchSize = batchSize;
_retryBackoffs = retryBackoffs ?? DefaultRetryBackoffs;
_currentPartitionKey = currentPartitionKey;
_logger = logger;
}
/// <summary>
@@ -97,6 +115,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
/// </summary>
public event EventHandler<string>? UploadBlocked;
/// <summary>
/// Raised once per process (after the drain guard releases) when the backend answered
/// HTTP 410 for a batch of the CURRENT session: the session is over server-side (never
/// registered, deleted, or being deleted) and no retry can succeed. The batch is retained
/// (cursor stays) and the drain stops; the host reports once over the emergency channel
/// and terminates the agent with self-destruct. Carries the backend error reason.
/// </summary>
public event EventHandler<string>? SessionGone;
/// <summary>
/// <c>true</c> when the last successful upload returned <see cref="UploadResult.DeviceBlocked"/>
/// and the paused-until window has not expired. Drain is a no-op in this state.
@@ -124,6 +151,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
List<UploadResult>? pendingSignals = null;
List<PoisonReport>? pendingPoison = null;
string? pendingBlockedReason = null;
string? pendingSessionGoneReason = null;
DrainResult result;
try
@@ -247,6 +275,35 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
new PoisonReport(1, oversizeId, "oversize: " + outcome.ErrorReason, PoisonKind.Oversize));
break;
}
else if (outcome.IsSessionGone)
{
// HTTP 410 — the backend no longer knows the session these items belong to.
var foreignRun = CountLeadingForeignItems(batch);
if (foreignRun > 0)
{
// Leftovers of a previous session id (the spool drops these at startup;
// this is defence in depth). Drop only that leading run and carry on: a
// legitimate new session must never be ended because of old items.
var throughId = batch[foreignRun - 1].TelemetryItemId;
_spool.MarkUploaded(throughId);
_logger?.Warning(
$"TelemetryUploadOrchestrator: dropped {foreignRun} item(s) of a previous session id the backend rejected with 410 (cursor advanced to {throughId}).");
continue;
}
// The current session is over server-side. RETAIN (cursor stays, the items
// remain on disk for the diagnostics ZIP), stop this drain, and surface it
// once — not as telemetry_upload_blocked, which would only queue another
// item behind a batch that can never be accepted.
failedBatches++;
lastError = outcome.ErrorReason;
if (!_sessionGoneReported && SessionGone != null)
{
_sessionGoneReported = true;
pendingSessionGoneReason = outcome.ErrorReason;
}
break;
}
else
{
// P1 — RETAIN. Transient (retry next drain), auth-permanent (uploader already
@@ -301,9 +358,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
catch { /* handler must not abort the drain result */ }
}
if (pendingSessionGoneReason != null)
{
try { SessionGone?.Invoke(this, pendingSessionGoneReason); }
catch { /* handler must not abort the drain result */ }
}
return result;
}
/// <summary>
/// Number of items at the head of <paramref name="batch"/> whose partition key is not the
/// current session's. 0 when no current key is configured (every batch counts as own).
/// </summary>
private int CountLeadingForeignItems(IReadOnlyList<TelemetryItem> batch)
{
if (_currentPartitionKey == null) return 0;
var count = 0;
while (count < batch.Count
&& !string.Equals(batch[count].PartitionKey, _currentPartitionKey, StringComparison.Ordinal))
{
count++;
}
return count;
}
private static bool CarriesSignal(UploadResult outcome) =>
outcome.DeviceBlocked
|| outcome.DeviceKillSignal
@@ -30,7 +30,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
bool isAuthFailure = false,
bool requiresSplit = false,
bool isPoison = false,
IReadOnlyList<string>? poisonRowKeys = null)
IReadOnlyList<string>? poisonRowKeys = null,
bool isSessionGone = false)
{
Success = success;
ErrorReason = errorReason;
@@ -44,6 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
RequiresSplit = requiresSplit;
IsPoison = isPoison;
PoisonRowKeys = poisonRowKeys;
IsSessionGone = isSessionGone;
}
public bool Success { get; }
@@ -81,6 +83,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
/// <summary>The RowKeys the backend named as poison (item-level). Non-empty when <see cref="IsPoison"/>.</summary>
public IReadOnlyList<string>? PoisonRowKeys { get; }
/// <summary>
/// HTTP 410: the backend no longer knows the session the batch belongs to (never
/// registered, deleted, or being deleted). Not transient, not auth, not poison — retrying
/// can never succeed. The orchestrator stops the drain (cursor stays) and raises
/// <see cref="TelemetryUploadOrchestrator.SessionGone"/> when the batch belongs to the
/// current session, or drops the batch's leading items when they are leftovers of a
/// previous session id.
/// </summary>
public bool IsSessionGone { get; }
/// <summary>
/// Non-terminal quarantine signalled by the backend. The agent should stop draining
/// uploads until <see cref="UnblockAt"/> (if set) elapses — the session stays alive so
@@ -165,5 +177,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
isPoison: true, poisonRowKeys: rejectedRowKeys ?? throw new ArgumentNullException(nameof(rejectedRowKeys)));
/// <summary>
/// HTTP 410 — the backend no longer knows the session. Flagged <see cref="IsSessionGone"/>;
/// never retried, never split, never treated as poison.
/// </summary>
public static UploadResult SessionGone(string reason) =>
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
isSessionGone: true);
}
}
@@ -439,6 +439,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
lifecyclePost,
() => terminationHandler,
agentConfig,
auth.EmergencyReporter,
shutdown,
shutdownComplete,
logger);
@@ -1,10 +1,12 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Agent.V2.Core.Configuration;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Agent.V2.Core.Termination;
using AutopilotMonitor.Shared;
@@ -103,6 +105,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
/// dispatcher because these are real actions, not admin-button echoes.</item>
/// <item><b>DeviceBlocked</b> — non-terminal upload pause. Transport already reacted in
/// <c>TelemetryUploadOrchestrator.ApplyControlSignals</c>; here we only log.</item>
/// <item><b>SessionGone</b> (<see cref="TelemetryUploadOrchestrator.SessionGone"/>, HTTP 410
/// on the current session's telemetry) — one emergency report, then the kill-signal
/// termination path tagged <c>origin=session_gone</c>. See <see cref="HandleSessionGone"/>.</item>
/// </list>
/// <para>
/// <b>Shutdown race guard</b>: if the agent has already finished its own termination
@@ -117,9 +122,14 @@ namespace AutopilotMonitor.Agent.V2.Runtime
InformationalEventPost post,
Func<EnrollmentTerminationHandler> terminationHandlerAccessor,
AgentConfiguration agentConfig,
EmergencyReporter emergencyReporter,
ManualResetEventSlim shutdown,
ManualResetEventSlim shutdownComplete,
AgentLogger logger)
{
orchestrator.Transport.SessionGone += (sender, reason) =>
HandleSessionGone(reason, emergencyReporter, dispatcher, shutdown, shutdownComplete, logger);
orchestrator.Transport.ServerResponseReceived += (sender, upload) =>
{
if (shutdownComplete.IsSet)
@@ -140,7 +150,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
{
{ "forceSelfDestruct", "true" },
{ "gracePeriodSeconds", "0" },
{ "origin", "kill_signal" },
{ "origin", TerminationOrigins.KillSignal },
},
};
try { dispatcher.DispatchAsync(new List<ServerAction> { killAction }).GetAwaiter().GetResult(); }
@@ -181,6 +191,76 @@ namespace AutopilotMonitor.Agent.V2.Runtime
};
}
// Bounded blocking send for the one session-gone report: the terminate that follows ends
// the process and the cleanup script deletes the agent, so a fire-and-forget send would
// be abandoned mid-flight. Retries stay inside one anti-flood reservation (ONE report).
private const int SessionGoneReportAttempts = 2;
private static readonly TimeSpan SessionGoneReportTimeout = TimeSpan.FromSeconds(15);
private static readonly TimeSpan SessionGoneReportRetryDelay = TimeSpan.FromSeconds(3);
private static readonly TimeSpan SessionGoneReportBudget = TimeSpan.FromSeconds(40);
/// <summary>
/// The backend answered HTTP 410 for the current session's telemetry: the session is over
/// server-side. (a) Send ONE <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/>
/// report over the emergency channel — the telemetry channel is exactly what was refused,
/// and the backend accepts this report for unknown sessions. (b) Terminate through the same
/// synthesised <c>terminate_session</c> path as the administrator kill signal (forced
/// self-destruct, no grace), tagged <see cref="TerminationOrigins.SessionGone"/> so the
/// termination handler skips the spool drain and the diagnostics upload. A shutdown that is
/// already in progress owns the outcome — then nothing happens here.
/// </summary>
internal static void HandleSessionGone(
string reason,
EmergencyReporter emergencyReporter,
ServerActionDispatcher dispatcher,
ManualResetEventSlim shutdown,
ManualResetEventSlim shutdownComplete,
AgentLogger logger)
{
if (shutdownComplete.IsSet || shutdown.IsSet)
{
logger.Debug("ServerControlPlane: session gone (HTTP 410) while shutdown is already in progress — ignoring.");
return;
}
logger.Error($"Backend no longer knows this session (HTTP 410: {reason}) — reporting once over the emergency channel, then terminating with self-destruct.");
if (emergencyReporter != null)
{
try
{
emergencyReporter
.TrySendAsync(
AgentErrorType.TelemetryRejectedSessionUnknown,
$"Telemetry upload rejected with HTTP 410 — backend no longer knows this session ({reason}). Agent terminating with self-destruct.",
httpStatusCode: (int)HttpStatusCode.Gone,
attempts: SessionGoneReportAttempts,
perAttemptTimeout: SessionGoneReportTimeout,
retryDelay: SessionGoneReportRetryDelay)
.Wait(SessionGoneReportBudget);
}
catch (Exception ex)
{
logger.Debug($"Session-gone report send failed (best-effort): {ex.Message}");
}
}
var terminateAction = new ServerAction
{
Type = ServerActionTypes.TerminateSession,
Reason = "Backend no longer knows this session (HTTP 410)",
QueuedAt = DateTime.UtcNow,
Params = new Dictionary<string, string>
{
{ "forceSelfDestruct", "true" },
{ "gracePeriodSeconds", "0" },
{ "origin", TerminationOrigins.SessionGone },
},
};
try { dispatcher.DispatchAsync(new List<ServerAction> { terminateAction }).GetAwaiter().GetResult(); }
catch (Exception ex) { logger.Error("ServerActionDispatcher.DispatchAsync (session gone) threw during server-response wiring.", ex); }
}
/// <summary>
/// Soft admin-override handling. Portal Mark-Succeeded / Mark-Failed is an echo of an
/// admin button click, not a ServerAction: it bypasses the dispatcher (no
@@ -281,6 +361,17 @@ namespace AutopilotMonitor.Agent.V2.Runtime
: EnrollmentTerminationOutcome.Failed;
}
/// <summary>
/// The <c>origin</c> param of a synthesised <c>terminate_session</c> action
/// (<see cref="TerminationOrigins"/>), or null when absent — it travels into
/// <see cref="EnrollmentTerminatedEventArgs.Origin"/>.
/// </summary>
internal static string ReadOrigin(IReadOnlyDictionary<string, string> parameters)
{
if (parameters == null) return null;
return parameters.TryGetValue("origin", out var origin) && !string.IsNullOrEmpty(origin) ? origin : null;
}
// ============================================================ Private terminate-callback
/// <summary>
@@ -326,8 +417,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
// before, masquerading every admin override as an error in SummaryDialog +
// firing spurious diagnostics uploads).
var mappedOutcome = MapAdminOutcome(action?.Params);
var origin = ReadOrigin(action?.Params);
logger.Warning($"ServerAction terminate_session received (ruleId={action?.RuleId}, reason={action?.Reason}, outcome={mappedOutcome}) — invoking termination handler.");
logger.Warning($"ServerAction terminate_session received (ruleId={action?.RuleId}, reason={action?.Reason}, outcome={mappedOutcome}, origin={origin ?? "none"}) — invoking termination handler.");
// Synthesise a Terminated event as if the kernel fired it.
terminationHandler.Handle(
sender: null,
@@ -336,7 +428,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
outcome: mappedOutcome,
stageName: orchestrator.CurrentState?.Stage.ToString(),
terminatedAtUtc: DateTime.UtcNow,
details: $"Server-requested termination: ruleId={action?.RuleId}, reason={action?.Reason}"));
details: $"Server-requested termination: ruleId={action?.RuleId}, reason={action?.Reason}",
origin: origin));
// Plan §6.2 synchronous-shutdown — block the ingest dispatcher thread until the
// main-thread cleanup (orchestrator.Stop, client disposal) has fully run. This
@@ -0,0 +1,126 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Analyze;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Deletion;
using Azure.Data.Tables;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The two queue handlers derive rows for a session (rule results, reports, inventory counters,
/// CveIndex). For a session the backend does not know, or one a cascade is tearing down, they
/// must stop before the first read of session data — nothing derived may ever outlive the
/// session's cascade. Both handlers are driven up to that gate with every later dependency
/// unavailable, so a regression that moves the gate shows up as a null dereference or an
/// unexpected mock call.
/// </summary>
public class DerivedWriterLivenessTests
{
private const string TenantId = "11111111-1111-1111-1111-111111111111";
private const string SessionId = "22222222-2222-2222-2222-222222222222";
private static SessionDeletionGuard GuardFor(TableEntity? row)
{
var reader = new Mock<ISessionDeletionInventoryReader>();
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(row);
return new SessionDeletionGuard(reader.Object, NullLogger<SessionDeletionGuard>.Instance);
}
private static Mock<AdminConfigurationService> AdminConfigWithCorrelationEnabled()
{
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
new MemoryCache(new MemoryCacheOptions()));
adminConfig.Setup(a => a.GetConfigurationAsync())
.ReturnsAsync(new AdminConfiguration { VulnerabilityCorrelationEnabled = true });
return adminConfig;
}
public static IEnumerable<object?[]> NotLiveRows()
{
yield return new object?[] { null };
yield return new object?[] { new TableEntity(TenantId, SessionId) { ["DeletionState"] = SessionDeletionState.Running } };
}
[Theory]
[MemberData(nameof(NotLiveRows))]
public async Task VulnerabilityCorrelate_skips_a_missing_or_locked_session_before_loading_inventory(TableEntity? row)
{
var inventoryLoader = new Mock<IVulnerabilityInventoryLoader>(MockBehavior.Strict);
var vulnRepo = new Mock<IVulnerabilityRepository>(MockBehavior.Strict);
var handler = new VulnerabilityCorrelateHandler(
AdminConfigWithCorrelationEnabled().Object,
vulnerabilityCorrelation: null!,
vulnRepo.Object,
Mock.Of<ISessionRepository>(),
inventoryLoader.Object,
Mock.Of<IAnalyzeOnEnrollmentEndProducer>(),
signalRNotification: null!,
GuardFor(row),
NullLogger<VulnerabilityCorrelateHandler>.Instance);
await handler.HandleAsync(new VulnerabilityCorrelateEnvelope
{
TenantId = TenantId, SessionId = SessionId, Reason = VulnerabilityCorrelateHandler.ReasonShutdownInventory, EnqueuedAt = DateTime.UtcNow,
});
inventoryLoader.VerifyNoOtherCalls();
vulnRepo.VerifyNoOtherCalls();
}
[Theory]
[MemberData(nameof(NotLiveRows))]
public async Task Analyze_skips_a_missing_or_locked_session_before_the_rule_engine_runs(TableEntity? row)
{
var ruleRepo = new Mock<IRuleRepository>(MockBehavior.Strict);
var metricsRepo = new Mock<IMetricsRepository>(MockBehavior.Strict);
var handler = new AnalyzeOnEnrollmentEndHandler(
ruleService: null!,
ruleRepo.Object,
Mock.Of<ISessionRepository>(),
metricsRepo.Object,
signalRNotification: null!,
configService: null!,
channelDispatcher: null!,
GuardFor(row),
NullLogger<AnalyzeOnEnrollmentEndHandler>.Instance);
await handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope
{
TenantId = TenantId, SessionId = SessionId, Reason = AnalyzeOnEnrollmentEndHandler.ReasonEnrollmentComplete, EnqueuedAt = DateTime.UtcNow,
});
ruleRepo.VerifyNoOtherCalls();
metricsRepo.VerifyNoOtherCalls();
}
[Fact]
public async Task A_guard_read_error_propagates_so_the_queue_retries_instead_of_skipping()
{
var reader = new Mock<ISessionDeletionInventoryReader>();
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
var handler = new AnalyzeOnEnrollmentEndHandler(
ruleService: null!, Mock.Of<IRuleRepository>(), Mock.Of<ISessionRepository>(), Mock.Of<IMetricsRepository>(),
signalRNotification: null!, configService: null!, channelDispatcher: null!,
new SessionDeletionGuard(reader.Object, NullLogger<SessionDeletionGuard>.Instance),
NullLogger<AnalyzeOnEnrollmentEndHandler>.Instance);
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope
{
TenantId = TenantId, SessionId = SessionId, Reason = AnalyzeOnEnrollmentEndHandler.ReasonEnrollmentComplete, EnqueuedAt = DateTime.UtcNow,
}));
}
}
@@ -3,6 +3,7 @@ using System.Threading.Tasks;
using Azure.Data.Tables;
using AutopilotMonitor.Functions.Functions.Ingest;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Deletion;
@@ -155,6 +156,27 @@ public class IngestTelemetryAuthPathTests
Assert.Equal(SessionStatus.Succeeded, SessionRowProjections.TryReadStatus(returned));
}
// ============================================================ Session unknown → 410
[Fact]
public async Task SessionUnknown_guard_returns_null_that_Run_maps_to_410_session_unknown()
{
// Run: a null guard row (no Sessions row, no active tombstone) is refused with 410 via
// WriteSessionUnknownAsync and stamped IngestRefusal=session_unknown; the cascade lock keeps
// its own value so the two 410 reasons stay apart on the request row.
var guard = NewGuard(out var reader);
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
.ReturnsAsync((TableEntity?)null);
reader.Setup(r => r.GetActiveSessionTombstoneAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
.ReturnsAsync((TableEntity?)null);
var row = await guard.EnsureWritableAndGetRowAsync(TenantId, SessionId, callerContext: "V2.IngestTelemetry");
Assert.Null(row);
Assert.NotEqual(RequestRowMarkers.IngestRefusal.SessionUnknown, RequestRowMarkers.IngestRefusal.CascadeLocked);
Assert.Equal("IngestRefusal", RequestRowMarkers.IngestRefusalKey);
}
// ============================================================ TryReadStatus (unique helper)
[Theory]
@@ -0,0 +1,33 @@
using System;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Maintenance;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
public class MaintenanceStepTimerTests
{
[Fact]
public async Task Records_every_step_in_order_and_returns_the_body_result()
{
var timer = new MaintenanceStepTimer();
await timer.RunAsync("first", () => Task.CompletedTask);
var value = await timer.RunAsync<int>("second", () => Task.FromResult(42));
Assert.Equal(42, value);
Assert.Equal(new[] { "first", "second" }, timer.Steps.Keys);
Assert.All(timer.Steps.Values, ms => Assert.True(ms >= 0));
}
[Fact]
public async Task A_throwing_step_is_still_recorded_and_the_exception_propagates()
{
var timer = new MaintenanceStepTimer();
await Assert.ThrowsAsync<InvalidOperationException>(
() => timer.RunAsync("boom", () => throw new InvalidOperationException("x")));
Assert.True(timer.Steps.ContainsKey("boom"));
}
}
@@ -121,13 +121,17 @@ public class OffboardingSessionEnumeratorTests
public Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException();
public Task<List<string>> GetAllTenantIdsAsync() => throw new NotImplementedException();
public Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
public Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
public Task<int> BackfillSessionIndexAsync() => throw new NotImplementedException();
public Task<int> CleanupGhostSessionIndexEntriesAsync() => throw new NotImplementedException();
public Task<bool> IsSessionIndexEmptyAsync() => throw new NotImplementedException();
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => throw new NotImplementedException();
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => throw new NotImplementedException();
public Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(CancellationToken ct = default) => throw new NotImplementedException();
public Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(CancellationToken ct = default) => throw new NotImplementedException();
public Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default) => throw new NotImplementedException();
public Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) => throw new NotImplementedException();
public Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) => throw new NotImplementedException();
public Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default) => throw new NotImplementedException();
public Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default) => throw new NotImplementedException();
public Task<List<(string TenantId, string SessionId)>> GetAllSessionTombstoneKeysAsync() => throw new NotImplementedException();
}
}
@@ -0,0 +1,116 @@
using System;
using System.Linq;
using AutopilotMonitor.Functions.Services.Maintenance;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Pins the per-table filter of the orphan-session sweep to the writer's key shape. A filter
/// that names the wrong side (tenant where the session belongs, or vice versa) would either
/// miss the session's rows or touch another session's — both silently.
/// </summary>
public class OrphanSessionRowFiltersTests
{
private const string TenantId = "11111111-1111-1111-1111-111111111111";
private const string SessionId = "22222222-2222-2222-2222-222222222222";
[Fact]
public void Every_filter_table_has_a_filter_naming_both_ids()
{
foreach (var table in OrphanSessionRowFilters.Tables)
{
var filter = OrphanSessionRowFilters.For(table, TenantId, SessionId);
Assert.Contains(TenantId, filter);
Assert.Contains(SessionId, filter);
}
}
[Theory]
[InlineData(Constants.TableNames.Events)]
[InlineData(Constants.TableNames.RuleResults)]
[InlineData(Constants.TableNames.Signals)]
[InlineData(Constants.TableNames.DecisionTransitions)]
[InlineData(Constants.TableNames.VulnerabilityReports)]
public void Session_partition_tables_filter_on_the_composite_partition_key(string table)
{
Assert.Equal($"PartitionKey eq '{TenantId}_{SessionId}'", OrphanSessionRowFilters.For(table, TenantId, SessionId));
}
[Theory]
[InlineData(Constants.TableNames.DeviceSnapshot)]
[InlineData(Constants.TableNames.SessionTimeBreakdowns)]
public void One_row_per_session_tables_filter_on_tenant_partition_and_session_row_key(string table)
{
Assert.Equal($"PartitionKey eq '{TenantId}' and RowKey eq '{SessionId}'", OrphanSessionRowFilters.For(table, TenantId, SessionId));
}
[Fact]
public void AppInstallSummaries_filter_on_the_SessionId_column()
{
Assert.Equal($"PartitionKey eq '{TenantId}' and SessionId eq '{SessionId}'",
OrphanSessionRowFilters.For(Constants.TableNames.AppInstallSummaries, TenantId, SessionId));
}
[Fact]
public void EventTypeIndex_filter_scans_the_tenant_range_and_matches_the_SessionId_column()
{
// RowKey is {invertedTicks}_{sessionId}; without a Sessions row the ticks are not reconstructible.
Assert.Equal($"PartitionKey ge '{TenantId}_' and PartitionKey lt '{TenantId}_~' and SessionId eq '{SessionId}'",
OrphanSessionRowFilters.For(Constants.TableNames.EventTypeIndex, TenantId, SessionId));
}
[Fact]
public void CveIndex_filter_scans_the_tenant_range_and_matches_the_session_row_key()
{
Assert.Equal($"PartitionKey ge '{TenantId}_' and PartitionKey lt '{TenantId}_~' and RowKey eq '{SessionId}'",
OrphanSessionRowFilters.For(Constants.TableNames.CveIndex, TenantId, SessionId));
}
[Fact]
public void SessionAnnotations_filter_names_every_lane()
{
var filter = OrphanSessionRowFilters.For(Constants.TableNames.SessionAnnotations, TenantId, SessionId);
Assert.StartsWith($"PartitionKey eq '{TenantId}' and (", filter);
foreach (var lane in AnnotationLanes.All)
Assert.Contains($"RowKey eq '{SessionId}_{lane}'", filter);
Assert.Equal(AnnotationLanes.All.Length - 1, filter.Split(" or ").Length - 1);
}
[Fact]
public void SessionTenantLookup_filter_pins_the_row_to_this_tenant()
{
Assert.Equal($"PartitionKey eq '{SessionId}' and RowKey eq 'tenant' and TenantId eq '{TenantId}'",
OrphanSessionRowFilters.For(Constants.TableNames.SessionTenantLookup, TenantId, SessionId));
}
[Fact]
public void Unknown_table_throws_instead_of_guessing()
{
Assert.Throws<ArgumentException>(() => OrphanSessionRowFilters.For(Constants.TableNames.Sessions, TenantId, SessionId));
}
[Theory]
[InlineData("not-a-guid", SessionId)]
[InlineData(TenantId, "'; drop")]
public void Non_guid_ids_throw(string tenantId, string sessionId)
{
Assert.ThrowsAny<ArgumentException>(() => OrphanSessionRowFilters.For(Constants.TableNames.Events, tenantId, sessionId));
}
[Fact]
public void Swept_tables_are_the_filter_tables_plus_inventory_pair_plus_handle()
{
var expected = OrphanSessionRowFilters.Tables
.Concat(new[]
{
Constants.TableNames.SessionInventoryContributions,
Constants.TableNames.SoftwareInventory,
Constants.TableNames.EventSessionIndex,
})
.ToHashSet(StringComparer.Ordinal);
Assert.Equal(expected, OrphanSessionSweeper.SweptTables.ToHashSet(StringComparer.Ordinal));
}
}
@@ -0,0 +1,528 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Functions.Services.Maintenance;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Deletion;
using Azure.Data.Tables;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Behaviour of the orphan-session sweep: which handles become candidates, the re-checks right
/// before the deletes, the inventory claim-then-decrement order, the handle as the last delete,
/// the cap/budget, and the weekly read-only reconciliation. Storage is mocked at the repository
/// seams; the ordered action log is the evidence for every ordering invariant.
/// </summary>
public class OrphanSessionSweeperTests
{
private const string TenantId = "11111111-1111-1111-1111-111111111111";
private const string SessionA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
private const string SessionB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
private const string SessionC = "cccccccc-cccc-4ccc-8ccc-cccccccccccc";
// A Wednesday: the weekly reconciliation is not due unless forced or overdue.
private static readonly DateTime Now = new DateTime(2026, 10, 14, 10, 0, 0, DateTimeKind.Utc);
private sealed class Harness
{
public readonly Mock<IMaintenanceRepository> Maint = new();
public readonly Mock<IVulnerabilityRepository> Vuln = new();
public readonly Mock<ISessionDeletionInventoryReader> Reader = new();
public readonly Mock<AdminConfigurationService> AdminConfig;
public readonly Mock<OrphanSweepLockStore> LockStore;
public readonly List<OpsEventEntry> Ops = new();
public readonly List<string> Log = new();
public readonly HashSet<(string TenantId, string SessionId)> SessionKeys = new();
public readonly List<OrphanSessionHandle> Handles = new();
public readonly Dictionary<string, OrphanSessionHandle?> LiveHandles = new(StringComparer.Ordinal);
public readonly HashSet<string> RowsPresent = new(StringComparer.Ordinal);
public readonly HashSet<string> Tombstoned = new(StringComparer.Ordinal);
public readonly Dictionary<string, List<DeletionDecrementKey>> Inventory = new(StringComparer.Ordinal);
public readonly HashSet<(string TenantId, string SessionId)> IndexKeys = new();
public readonly Dictionary<string, int> RowsPerTable = new(StringComparer.Ordinal);
public readonly AdminConfiguration Config = new AdminConfiguration();
public DateTime Clock = Now;
public TimeSpan Budget = TimeSpan.FromMinutes(5);
public bool StampFails;
public bool HandleDeleteFails;
public bool DecrementFails;
public bool LeaseHeld;
public string? FailingTable;
public Harness()
{
Maint.Setup(m => m.GetSessionKeysAsync(It.IsAny<CancellationToken>()))
.Callback(() => Log.Add("sessions"))
.ReturnsAsync(() => new HashSet<(string, string)>(SessionKeys));
Maint.Setup(m => m.GetEventSessionIndexHandlesAsync(It.IsAny<CancellationToken>()))
.Callback(() => Log.Add("handles"))
.ReturnsAsync(() => Handles.ToList());
Maint.Setup(m => m.GetEventSessionIndexHandleAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((string _, string s, CancellationToken _) =>
LiveHandles.TryGetValue(s, out var live) ? live : Handles.FirstOrDefault(h => h.SessionId == s));
Maint.Setup(m => m.StampHandleInventoryDecrementedAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Callback<string, string, string, CancellationToken>((_, s, _, _) => Log.Add($"stamp:{s}"))
.ReturnsAsync(() => StampFails ? null : "etag-stamped");
Maint.Setup(m => m.DeleteOrphanSessionRowsAsync(It.IsAny<string>(), TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns((string table, string _, string s, CancellationToken _) =>
{
Log.Add($"delete:{table}:{s}");
if (string.Equals(table, FailingTable, StringComparison.Ordinal))
throw new InvalidOperationException("storage said no");
return Task.FromResult(RowsPerTable.TryGetValue(table, out var n) ? n : 1);
});
Maint.Setup(m => m.DeleteEventSessionIndexHandleAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Callback<string, string, string, CancellationToken>((_, s, _, _) => Log.Add($"handle:{s}"))
.ReturnsAsync(() => !HandleDeleteFails);
Maint.Setup(m => m.GetEventTypeIndexSessionKeysAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync(() => new HashSet<(string, string)>(IndexKeys));
Vuln.Setup(v => v.GetSessionInventoryContributionsAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((string _, string s, CancellationToken _) =>
Inventory.TryGetValue(s, out var keys) ? (IReadOnlyList<DeletionDecrementKey>?)keys : null);
Vuln.Setup(v => v.DecrementSoftwareInventoryEntryAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns((string _, string vendor, string name, string version, CancellationToken _) =>
{
Log.Add($"decrement:{name}");
if (DecrementFails) throw new InvalidOperationException("counter busy");
return Task.CompletedTask;
});
Vuln.Setup(v => v.DeleteSessionInventoryContributionsAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Callback<string, string, CancellationToken>((_, s, _) => Log.Add($"sideRow:{s}"))
.Returns(Task.CompletedTask);
Reader.Setup(r => r.GetSessionRowAsync(TenantId, It.IsAny<string>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((string _, string s, IEnumerable<string> _, CancellationToken _) =>
RowsPresent.Contains(s) ? new TableEntity(TenantId, s) : null);
Reader.Setup(r => r.GetActiveSessionTombstoneAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((string _, string s, CancellationToken _) =>
Tombstoned.Contains(s) ? new TableEntity(TenantId, s) : null);
AdminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
new MemoryCache(new MemoryCacheOptions()));
AdminConfig.Setup(a => a.GetConfigurationAsync()).ReturnsAsync(() => Config);
AdminConfig.Setup(a => a.UpdateAsync(It.IsAny<Func<AdminConfiguration, string?>>(), It.IsAny<string>(), It.IsAny<string?>()))
.Returns((Func<AdminConfiguration, string?> mutate, string _, string? _) =>
{
mutate(Config);
return Task.FromResult(new AdminConfigurationUpdateResult { Before = Config, After = Config });
});
LockStore = new Mock<OrphanSweepLockStore>(new SessionDeletionMaintenanceFunctionTests.MaintenanceBlobStub());
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
.Returns(() => LeaseHeld
? throw new LeaseHeldException("held", new InvalidOperationException("lease"))
: Task.FromResult(new Mock<Azure.Storage.Blobs.Specialized.BlobLeaseClient>().Object));
}
public OrphanSessionHandle AddHandle(string sessionId, TimeSpan ingestAge, TimeSpan? writtenAge = null, string etag = "etag-1", bool decremented = false)
{
var handle = new OrphanSessionHandle
{
TenantId = TenantId,
SessionId = sessionId,
LastIngestAt = Now - ingestAge,
WrittenAt = Now - (writtenAge ?? TimeSpan.FromDays(1)),
ETag = etag,
InventoryDecremented = decremented,
};
Handles.Add(handle);
return handle;
}
public OrphanSessionSweeper Build()
{
var opsRepo = new Mock<IOpsEventRepository>();
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
.Callback<OpsEventEntry>(e => { lock (Ops) Ops.Add(e); })
.Returns(Task.CompletedTask);
var ops = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance,
TestNotifications.InertOpsAlertDispatch(AdminConfig.Object));
return new OrphanSessionSweeper(Maint.Object, Vuln.Object, Reader.Object, AdminConfig.Object, LockStore.Object, ops,
NullLogger<OrphanSessionSweeper>.Instance, () => Clock, Budget);
}
public Task<OrphanSweepRunResult> RunCoreAsync(bool forceDeepCheck = false) => Build().RunCoreAsync(forceDeepCheck, CancellationToken.None);
}
private static DeletionDecrementKey Key(string name) => new DeletionDecrementKey { Vendor = "v", Name = name, Version = "1" };
// ============================================================ candidates
[Fact]
public async Task Candidates_are_handles_without_a_session_key_outside_both_grace_windows()
{
var h = new Harness();
h.AddHandle(SessionA, TimeSpan.FromDays(3)); // orphan, old
h.AddHandle(SessionB, TimeSpan.FromDays(3)); // has a row
h.SessionKeys.Add((TenantId, SessionB));
h.AddHandle(SessionC, TimeSpan.FromHours(1)); // ingest grace
h.AddHandle("dddddddd-dddd-4ddd-8ddd-dddddddddddd", TimeSpan.FromDays(3), writtenAge: TimeSpan.FromMinutes(30)); // restore grace
var result = await h.RunCoreAsync();
Assert.Equal(1, result.Candidates);
Assert.Equal(SessionA, Assert.Single(result.Cleaned).SessionId);
Assert.Equal(4, result.HandlesScanned);
Assert.Equal(1, result.SessionKeys);
}
[Fact]
public async Task Sessions_are_drained_before_handles()
{
var h = new Harness();
await h.RunCoreAsync();
Assert.Equal(new[] { "sessions", "handles" }, h.Log.Take(2));
}
[Fact]
public async Task Legacy_handles_are_flagged_and_come_first()
{
var h = new Harness();
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.AddHandle(SessionB, Now - OrphanSessionSweeper.LegacyHandleLastIngestAt);
var result = await h.RunCoreAsync();
Assert.Equal(new[] { SessionB, SessionA }, result.Cleaned.Select(c => c.SessionId));
Assert.True(result.Cleaned[0].Legacy);
Assert.False(result.Cleaned[1].Legacy);
}
// ============================================================ re-checks
[Fact]
public async Task Handle_gone_at_recheck_is_skipped_silently()
{
var h = new Harness();
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.LiveHandles[SessionA] = null;
var result = await h.RunCoreAsync();
Assert.Equal(1, result.SkippedHandleGone);
Assert.Empty(result.Cleaned);
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal));
}
[Fact]
public async Task Handle_changed_since_the_scan_is_skipped()
{
var h = new Harness();
var scanned = h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.LiveHandles[SessionA] = new OrphanSessionHandle
{
TenantId = TenantId, SessionId = SessionA, LastIngestAt = scanned.LastIngestAt, WrittenAt = scanned.WrittenAt, ETag = "etag-later",
};
var result = await h.RunCoreAsync();
Assert.Equal(1, result.SkippedHandleChanged);
Assert.Empty(result.Cleaned);
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal));
}
[Fact]
public async Task Row_present_at_recheck_skips_everything()
{
var h = new Harness();
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.RowsPresent.Add(SessionA);
var result = await h.RunCoreAsync();
Assert.Equal(1, result.SkippedRowReturned);
Assert.Empty(result.Cleaned);
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal) || l.StartsWith("stamp:", StringComparison.Ordinal));
}
[Fact]
public async Task Tombstone_counts_as_afterCascade_and_the_session_is_still_cleaned()
{
var h = new Harness();
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.Tombstoned.Add(SessionA);
var result = await h.RunCoreAsync();
Assert.Equal(1, result.AfterCascade);
Assert.True(Assert.Single(result.Cleaned).AfterCascade);
}
// ============================================================ inventory + ordering
[Fact]
public async Task Inventory_is_claimed_then_decremented_then_the_side_row_goes_before_the_tables()
{
var h = new Harness();
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1"), Key("app-2") };
var result = await h.RunCoreAsync();
var stamp = h.Log.IndexOf($"stamp:{SessionA}");
var firstDecrement = h.Log.IndexOf("decrement:app-1");
var sideRow = h.Log.IndexOf($"sideRow:{SessionA}");
var firstTable = h.Log.FindIndex(l => l.StartsWith("delete:", StringComparison.Ordinal));
var handle = h.Log.IndexOf($"handle:{SessionA}");
Assert.True(0 <= stamp && stamp < firstDecrement && firstDecrement < sideRow && sideRow < firstTable && firstTable < handle,
string.Join(" | ", h.Log));
var cleaned = Assert.Single(result.Cleaned);
Assert.Equal(2, cleaned.InventoryKeysDecremented);
Assert.Equal(2, result.TotalsByTable[Constants.TableNames.SoftwareInventory]);
Assert.Equal(1, result.TotalsByTable[Constants.TableNames.SessionInventoryContributions]);
}
[Fact]
public async Task A_rerun_with_the_stamp_set_skips_the_decrements_but_still_removes_the_side_row()
{
var h = new Harness();
h.AddHandle(SessionA, TimeSpan.FromDays(3), decremented: true);
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
var result = await h.RunCoreAsync();
Assert.DoesNotContain(h.Log, l => l.StartsWith("stamp:", StringComparison.Ordinal) || l.StartsWith("decrement:", StringComparison.Ordinal));
Assert.Contains($"sideRow:{SessionA}", h.Log);
Assert.Single(result.Cleaned);
Assert.Equal(0, result.TotalsByTable[Constants.TableNames.SoftwareInventory]);
}
[Fact]
public async Task A_failed_stamp_aborts_the_candidate_before_any_decrement_or_delete()
{
var h = new Harness { StampFails = true };
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
var result = await h.RunCoreAsync();
Assert.Equal(1, result.SkippedHandleChanged);
Assert.Empty(result.Cleaned);
Assert.DoesNotContain(h.Log, l => l.StartsWith("decrement:", StringComparison.Ordinal) || l.StartsWith("delete:", StringComparison.Ordinal));
}
[Fact]
public async Task A_failed_decrement_is_counted_and_the_session_is_still_cleaned()
{
var h = new Harness { DecrementFails = true };
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1"), Key("app-2") };
var result = await h.RunCoreAsync();
Assert.Equal(2, result.InventoryDecrementsFailed);
Assert.Single(result.Cleaned);
Assert.Contains($"handle:{SessionA}", h.Log);
}
// ============================================================ the handle last
[Fact]
public async Task Every_filter_table_is_cleared_and_the_handle_is_deleted_last_with_the_stamped_etag()
{
var h = new Harness();
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
var result = await h.RunCoreAsync();
foreach (var table in OrphanSessionRowFilters.Tables)
Assert.Contains($"delete:{table}:{SessionA}", h.Log);
Assert.Equal($"handle:{SessionA}", h.Log.Last());
h.Maint.Verify(m => m.DeleteEventSessionIndexHandleAsync(TenantId, SessionA, "etag-stamped", It.IsAny<CancellationToken>()), Times.Once);
Assert.Equal(1, result.TotalsByTable[Constants.TableNames.EventSessionIndex]);
Assert.Equal(OrphanSessionRowFilters.Tables.Length, Assert.Single(result.Cleaned).TotalRows);
}
[Fact]
public async Task A_handle_rewritten_before_the_final_delete_is_kept_and_reported()
{
var h = new Harness { HandleDeleteFails = true };
h.AddHandle(SessionA, TimeSpan.FromDays(3));
var result = await h.RunCoreAsync();
var cleaned = Assert.Single(result.Cleaned);
Assert.True(cleaned.HandleKept);
Assert.Equal(0, result.TotalsByTable[Constants.TableNames.EventSessionIndex]);
}
[Fact]
public async Task A_failing_table_leaves_the_handle_and_the_run_continues_with_the_next_candidate()
{
var h = new Harness { FailingTable = Constants.TableNames.Signals };
h.AddHandle(SessionA, TimeSpan.FromDays(4));
h.AddHandle(SessionB, TimeSpan.FromDays(3));
// Only session A's Signals delete fails: the harness throws for the table regardless of
// session, so flip the failure off once A has been attempted.
h.Maint.Setup(m => m.DeleteOrphanSessionRowsAsync(Constants.TableNames.Signals, TenantId, SessionB, It.IsAny<CancellationToken>()))
.ReturnsAsync(1);
var result = await h.RunCoreAsync();
Assert.Equal(1, result.FailedSessions);
Assert.DoesNotContain($"handle:{SessionA}", h.Log);
Assert.Contains($"handle:{SessionB}", h.Log);
Assert.Equal(SessionB, Assert.Single(result.Cleaned).SessionId);
}
// ============================================================ cap + budget
[Fact]
public async Task The_cap_leaves_the_rest_for_the_next_run()
{
var h = new Harness();
for (var i = 0; i < OrphanSessionSweeper.MaxCandidatesPerRun + 3; i++)
h.AddHandle(Guid.NewGuid().ToString(), TimeSpan.FromDays(2) + TimeSpan.FromMinutes(i));
var result = await h.RunCoreAsync();
Assert.Equal(OrphanSessionSweeper.MaxCandidatesPerRun, result.Cleaned.Count);
Assert.Equal(3, result.Remaining);
Assert.False(result.BudgetExhausted);
}
[Fact]
public async Task An_exhausted_budget_stops_before_the_next_candidate()
{
var h = new Harness { Budget = TimeSpan.Zero };
h.AddHandle(SessionA, TimeSpan.FromDays(3));
h.AddHandle(SessionB, TimeSpan.FromDays(3));
var result = await h.RunCoreAsync();
Assert.True(result.BudgetExhausted);
Assert.Empty(result.Cleaned);
Assert.Equal(2, result.Remaining);
}
// ============================================================ weekly reconciliation
public static IEnumerable<object?[]> DeepCheckCases()
{
var sunday0845 = new DateTime(2026, 10, 18, 8, 45, 0, DateTimeKind.Utc);
var sunday0445 = new DateTime(2026, 10, 18, 4, 45, 0, DateTimeKind.Utc);
var tuesday = new DateTime(2026, 10, 20, 12, 45, 0, DateTimeKind.Utc);
yield return new object?[] { null, sunday0845, true }; // never ran: first Sunday window
yield return new object?[] { null, tuesday, false }; // never ran: wait for Sunday
yield return new object?[] { sunday0845.AddDays(-7), sunday0845, true }; // weekly
yield return new object?[] { sunday0845.AddDays(-7), sunday0445, false }; // Sunday, but before 08:00
yield return new object?[] { sunday0845.AddDays(-3), sunday0845, false }; // ran this week already
yield return new object?[] { tuesday.AddDays(-9), tuesday, true }; // missed Sunday: overdue
}
[Theory]
[MemberData(nameof(DeepCheckCases))]
public void Deep_check_is_due_on_Sunday_mornings_or_when_overdue(DateTime? lastRun, DateTime now, bool due)
{
Assert.Equal(due, OrphanSessionSweeper.IsDeepCheckDue(lastRun, now));
}
[Fact]
public async Task Reconcile_reports_only_sessions_without_a_row_and_without_a_handle_and_stamps_the_marker()
{
var h = new Harness();
const string noHandleNoRow = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee";
const string noHandleButRow = "ffffffff-ffff-4fff-8fff-ffffffffffff";
h.AddHandle(SessionA, TimeSpan.FromDays(3)); // in the sweep's own pipeline
h.SessionKeys.Add((TenantId, SessionB)); // live
h.RowsPresent.Add(noHandleButRow); // registered after the drain
h.IndexKeys.UnionWith(new[] { (TenantId, SessionA), (TenantId, SessionB), (TenantId, noHandleNoRow), (TenantId, noHandleButRow) });
var result = await h.RunCoreAsync(forceDeepCheck: true);
var reconcile = Assert.IsType<OrphanReconcileResult>(result.Reconcile);
Assert.Equal(4, reconcile.IndexSessions);
Assert.Equal(1, reconcile.ResidueSessions);
Assert.Equal((TenantId, noHandleNoRow), Assert.Single(reconcile.Sample));
var evt = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
Assert.Equal(OpsEventSeverity.Warning, evt.Severity);
Assert.Equal(Now.ToString("o"), h.Config.OrphanDeepCheckLastRunUtc);
}
[Fact]
public async Task Reconcile_without_residue_is_Info_and_is_not_due_again_mid_week()
{
var h = new Harness();
h.SessionKeys.Add((TenantId, SessionB));
h.IndexKeys.Add((TenantId, SessionB));
var first = await h.RunCoreAsync(forceDeepCheck: true);
var evt = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
Assert.Equal(OpsEventSeverity.Info, evt.Severity);
Assert.Equal(0, first.Reconcile!.ResidueSessions);
var second = await h.RunCoreAsync();
Assert.Null(second.Reconcile);
}
// ============================================================ RunAsync: lease + ops events
[Fact]
public async Task A_timer_run_writes_the_heartbeat_and_EventsCleaned_only_when_something_was_cleaned()
{
var quiet = new Harness();
Assert.True(await quiet.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
Assert.Single(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanSweepCompleted && e.Severity == OpsEventSeverity.Info);
Assert.DoesNotContain(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanEventsCleaned);
Assert.DoesNotContain(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
var busy = new Harness();
busy.AddHandle(SessionA, TimeSpan.FromDays(3));
Assert.True(await busy.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
var cleaned = Assert.Single(busy.Ops, e => e.EventType == OpsEventTypes.OrphanEventsCleaned);
Assert.Equal(OpsEventSeverity.Warning, cleaned.Severity);
Assert.Contains(SessionA, cleaned.Details ?? string.Empty);
}
[Fact]
public async Task A_manual_run_always_includes_the_reconciliation()
{
var h = new Harness();
Assert.True(await h.Build().RunAsync("alice@contoso.com", CancellationToken.None));
Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
}
[Fact]
public async Task A_held_lease_records_SkippedLocked_and_touches_nothing()
{
var h = new Harness { LeaseHeld = true };
h.AddHandle(SessionA, TimeSpan.FromDays(3));
Assert.False(await h.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepSkippedLocked);
Assert.Empty(h.Log);
}
[Fact]
public async Task A_failing_scan_records_OrphanSweepFailed()
{
var h = new Harness();
h.Maint.Setup(m => m.GetSessionKeysAsync(It.IsAny<CancellationToken>())).ThrowsAsync(new InvalidOperationException("storage down"));
Assert.True(await h.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
var failed = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepFailed);
Assert.Equal(OpsEventSeverity.Error, failed.Severity);
Assert.DoesNotContain(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepCompleted);
}
}
@@ -0,0 +1,111 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Maintenance;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Shape of the ops events the orphan sweep and the maintenance run write: the dashboards and the
/// health pass read these details by name.
/// </summary>
public class OrphanSweepOpsEventTests
{
private static (OpsEventService Service, List<OpsEventEntry> Saved) Rig()
{
var saved = new List<OpsEventEntry>();
var opsRepo = new Mock<IOpsEventRepository>();
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
.Callback<OpsEventEntry>(e => { lock (saved) saved.Add(e); })
.Returns(Task.CompletedTask);
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
new MemoryCache(new MemoryCacheOptions())) { CallBase = false };
var alertDispatch = TestNotifications.InertOpsAlertDispatch(adminConfig.Object);
return (new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch), saved);
}
[Fact]
public async Task MaintenanceCompleted_carries_the_step_timings()
{
var (service, saved) = Rig();
var steps = new Dictionary<string, long> { ["stalledSessions"] = 1200, ["aggregateMetrics"] = 90000 };
await service.RecordMaintenanceCompletedAsync(123456, "Timer", stepsMs: steps);
var evt = Assert.Single(saved);
Assert.Equal(OpsEventTypes.MaintenanceCompleted, evt.EventType);
Assert.Contains("\"stepsMs\"", evt.Details);
Assert.Contains("\"aggregateMetrics\":90000", evt.Details);
}
[Fact]
public async Task OrphanEventsCleaned_is_a_Warning_with_per_table_totals_and_the_legacy_count()
{
var (service, saved) = Rig();
var result = new OrphanSweepRunResult { Remaining = 7 };
var legacy = new OrphanSweepCleanedSession { TenantId = "t", SessionId = "s-legacy", Legacy = true, InventoryKeysDecremented = 3, HadInventorySideRow = true };
legacy.RowsByTable[Constants.TableNames.Events] = 18;
legacy.RowsByTable[Constants.TableNames.EventTypeIndex] = 13;
result.Add(legacy);
var fresh = new OrphanSweepCleanedSession { TenantId = "t", SessionId = "s-fresh" };
fresh.RowsByTable[Constants.TableNames.Events] = 1;
result.Add(fresh);
await service.RecordOrphanEventsCleanedAsync(result);
var evt = Assert.Single(saved);
Assert.Equal(OpsEventTypes.OrphanEventsCleaned, evt.EventType);
Assert.Equal(OpsEventSeverity.Warning, evt.Severity);
Assert.Contains("\"orphanSessions\":2", evt.Details);
Assert.Contains("\"totalEventsDeleted\":19", evt.Details);
Assert.Contains("\"legacySessions\":1", evt.Details);
Assert.Contains("\"remaining\":7", evt.Details);
Assert.Contains($"\"{Constants.TableNames.SoftwareInventory}\":3", evt.Details);
Assert.Contains($"\"{Constants.TableNames.EventSessionIndex}\":2", evt.Details);
Assert.Contains("\"legacy\":true", evt.Details);
}
[Fact]
public async Task OrphanSweepCompleted_is_the_Info_heartbeat_with_the_counters()
{
var (service, saved) = Rig();
var result = new OrphanSweepRunResult
{
HandlesScanned = 16784, SessionKeys = 16700, Candidates = 3, Remaining = 1, SkippedHandleGone = 2, DurationMs = 4200,
Reconcile = new OrphanReconcileResult { IndexSessions = 16000, ResidueSessions = 0, DurationMs = 180000 },
};
await service.RecordOrphanSweepCompletedAsync(result, "Timer");
var evt = Assert.Single(saved);
Assert.Equal(OpsEventTypes.OrphanSweepCompleted, evt.EventType);
Assert.Equal(OpsEventSeverity.Info, evt.Severity);
Assert.Contains("\"handlesScanned\":16784", evt.Details);
Assert.Contains("\"skippedHandleGone\":2", evt.Details);
Assert.Contains("\"residueSessions\":0", evt.Details);
}
[Fact]
public async Task OrphanReconcileCompleted_flips_to_Warning_with_a_sample_when_residue_exists()
{
var (service, saved) = Rig();
var clean = new OrphanReconcileResult { IndexSessions = 10, ResidueSessions = 0 };
var dirty = new OrphanReconcileResult { IndexSessions = 10, ResidueSessions = 2, Sample = { ("t1", "s1"), ("t1", "s2") } };
await service.RecordOrphanReconcileCompletedAsync(clean);
await service.RecordOrphanReconcileCompletedAsync(dirty);
Assert.Equal(OpsEventSeverity.Info, saved[0].Severity);
Assert.Equal(OpsEventSeverity.Warning, saved[1].Severity);
Assert.Contains("\"sessionId\":\"s2\"", saved[1].Details);
Assert.Contains("\"sampleTruncated\":false", saved[1].Details);
}
}
@@ -1,3 +1,4 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Azure.Data.Tables;
@@ -133,6 +134,60 @@ public class SessionDeletionGuardTests
Assert.Equal(SessionId, ex.SessionId);
}
// ============================================================ GetLivenessAsync (derived writers) ====
[Fact]
public async Task GetLivenessAsync_reports_Missing_when_the_row_does_not_exist()
{
// Unlike the ingest contract, a derived writer (queue handler, rescan) stops here: nothing
// may be derived for a session the backend does not know.
var guard = NewGuard(out var reader);
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((TableEntity?)null);
Assert.Equal(SessionLiveness.Missing, await guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityCorrelate"));
reader.Verify(r => r.GetActiveSessionTombstoneAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Never);
}
[Theory]
[InlineData(SessionDeletionState.Preparing)]
[InlineData(SessionDeletionState.Queued)]
[InlineData(SessionDeletionState.Running)]
[InlineData(SessionDeletionState.Poisoned)]
public async Task GetLivenessAsync_reports_Locked_for_every_lock_state(string state)
{
var guard = NewGuard(out var reader);
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new TableEntity(TenantId, SessionId) { ["DeletionState"] = state });
Assert.Equal(SessionLiveness.Locked, await guard.GetLivenessAsync(TenantId, SessionId, "AnalyzeOnEnrollmentEnd"));
}
[Theory]
[InlineData(null)]
[InlineData("None")]
public async Task GetLivenessAsync_reports_Live_for_an_unlocked_row(string? state)
{
var guard = NewGuard(out var reader);
var row = new TableEntity(TenantId, SessionId);
if (state != null) row["DeletionState"] = state;
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(row);
Assert.Equal(SessionLiveness.Live, await guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityRescan"));
}
[Fact]
public async Task GetLivenessAsync_propagates_a_read_error()
{
// A storage failure must surface to the queue worker (retry/poison), never read as "missing".
var guard = NewGuard(out var reader);
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityCorrelate"));
}
// ============================================================ Codex F3: tombstone-marker disambiguation ====
[Fact]
@@ -81,6 +81,32 @@ public class TableLifecycleBucketTests
Constants.TableNames.DelegationInvitations,
};
/// <summary>
/// The orphan-session sweep is the cascade for sessions without a Sessions row, so it must
/// clear exactly the tables the cascade manifest clears — minus the tombstone pair, which does
/// not exist for such a session. A new manifest step without a sweep entry (or the reverse)
/// fails here until someone decides the table's orphan lifecycle.
/// </summary>
[Fact]
public async Task OrphanSweep_ClearsEveryPerSessionCascadeTable_ExceptTheTombstonePair()
{
var cascade = await PerSessionCascadeTablesAsync();
cascade.Remove(Constants.TableNames.Sessions);
cascade.Remove(Constants.TableNames.SessionsIndex);
var swept = AutopilotMonitor.Functions.Services.Maintenance.OrphanSessionSweeper.SweptTables.ToHashSet(StringComparer.Ordinal);
var missingFromSweep = cascade.Except(swept, StringComparer.Ordinal).OrderBy(t => t, StringComparer.Ordinal).ToList();
var sweptButNotCascaded = swept.Except(cascade, StringComparer.Ordinal).OrderBy(t => t, StringComparer.Ordinal).ToList();
Assert.True(missingFromSweep.Count == 0,
"Per-session cascade tables the orphan sweep does not clear — add them to OrphanSessionRowFilters.Tables "
+ "(with the writer's key shape) or handle them in OrphanSessionSweeper:\n - " + string.Join("\n - ", missingFromSweep));
Assert.True(sweptButNotCascaded.Count == 0,
"Orphan sweep clears tables the cascade manifest does not know — add a manifest step first:\n - "
+ string.Join("\n - ", sweptButNotCascaded));
}
[Fact]
public void TableNamesConstants_SetEquals_TableNamesAll()
{
@@ -1019,6 +1019,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
{ "MsrcLastSyncUtc", config.MsrcLastSyncUtc },
{ "NvdCacheLastRefreshUtc", config.NvdCacheLastRefreshUtc },
{ "EpssLastSyncUtc", config.EpssLastSyncUtc },
{ "OrphanDeepCheckLastRunUtc", config.OrphanDeepCheckLastRunUtc },
{ "MaxDiagnosticsDownloadSizeMB", config.MaxDiagnosticsDownloadSizeMB },
{ "DiagnosticsDownloadTimeoutSeconds", config.DiagnosticsDownloadTimeoutSeconds },
{ "OpsEventRetentionDays", config.OpsEventRetentionDays },
@@ -1094,6 +1095,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
MsrcLastSyncUtc = entity.GetString("MsrcLastSyncUtc"),
NvdCacheLastRefreshUtc = entity.GetString("NvdCacheLastRefreshUtc"),
EpssLastSyncUtc = entity.GetString("EpssLastSyncUtc"),
OrphanDeepCheckLastRunUtc = entity.GetString("OrphanDeepCheckLastRunUtc"),
MaxDiagnosticsDownloadSizeMB = entity.GetInt32("MaxDiagnosticsDownloadSizeMB") ?? 500,
DiagnosticsDownloadTimeoutSeconds = entity.GetInt32("DiagnosticsDownloadTimeoutSeconds") ?? 120,
OpsEventRetentionDays = entity.GetInt32("OpsEventRetentionDays") ?? 90,
@@ -87,9 +87,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task<List<string>> GetAllTenantIdsAsync()
=> _storage.GetAllTenantIdsAsync();
public Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId)
=> _storage.DeleteSessionEventsAsync(tenantId, sessionId);
public Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId)
=> _storage.DeleteSessionRuleResultsAsync(tenantId, sessionId);
@@ -102,11 +99,26 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task<bool> IsSessionIndexEmptyAsync()
=> _storage.IsSessionIndexEmptyAsync();
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod)
=> _storage.GetOrphanedEventSessionsAsync(gracePeriod);
public Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(CancellationToken ct = default)
=> _storage.GetSessionKeysAsync(ct);
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId)
=> _storage.DeleteEventSessionIndexEntryAsync(tenantId, sessionId);
public Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(CancellationToken ct = default)
=> _storage.GetEventSessionIndexHandlesAsync(ct);
public Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default)
=> _storage.GetEventSessionIndexHandleAsync(tenantId, sessionId, ct);
public Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default)
=> _storage.StampHandleInventoryDecrementedAsync(tenantId, sessionId, etag, ct);
public Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default)
=> _storage.DeleteEventSessionIndexHandleAsync(tenantId, sessionId, etag, ct);
public Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default)
=> _storage.DeleteOrphanSessionRowsAsync(table, tenantId, sessionId, ct);
public Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default)
=> _storage.GetEventTypeIndexSessionKeysAsync(ct);
public Task<List<(string TenantId, string SessionId)>> GetAllSessionTombstoneKeysAsync()
=> _storage.GetAllSessionTombstoneKeysAsync();
@@ -55,6 +55,7 @@ namespace AutopilotMonitor.Functions.Functions.Config
"LatestAgentV2Version", "LatestAgentV2Sha256", "LatestAgentV2ExeSha256", "LatestBootstrapV2ScriptVersion",
"PlanTierDefinitionsJson",
"VulnerabilityDataLastSyncUtc", "MsrcLastSyncUtc", "NvdCacheLastRefreshUtc", "EpssLastSyncUtc",
"OrphanDeepCheckLastRunUtc",
};
/// <summary>
@@ -228,12 +228,14 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
return AsOutput(await WriteDeviceBlockedAsync(req, sessionVerdict));
}
// Cascade-delete guard: refuse the batch with 410 Gone when a V2 cascade owns the
// Sessions row (states Preparing/Queued/Running/Poisoned). Without this check, the
// hot-path writers below (StoreEventsBatchAsync, signal/transition StoreBatchAsync,
// UpdateSessionImeAgentVersionAsync upsert, …) would land rows past the lock and
// leave orphan data the manifest cannot describe. One read per batch; absent
// Sessions row → silent pass (caller handles session-not-found in its own write).
// Session guard: refuse the batch with 410 Gone when a V2 cascade owns the Sessions
// row (states Preparing/Queued/Running/Poisoned) OR when there is no Sessions row at
// all. Both mean "this session is over server-side": the hot-path writers below
// (StoreEventsBatchAsync, signal/transition StoreBatchAsync, the index upserts, …)
// would otherwise land rows no cascade can ever reach, because a cascade needs the
// Sessions row. Registration always precedes the first upload (and re-creates an
// absent row), so a live session never sees this. One read per batch; the row's
// IngestRefusal marker tells the two 410 reasons apart on the request row.
Azure.Data.Tables.TableEntity? guardSessionRow;
try
{
@@ -241,12 +243,22 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
}
catch (SessionDeletionLockedException locked)
{
RequestRowMarkers.Stamp(req, RequestRowMarkers.IngestRefusalKey, RequestRowMarkers.IngestRefusal.CascadeLocked);
_logger.LogInformation(
"IngestTelemetry: refused batch — cascade in flight tenant={Tenant} session={Session} state={State} manifestId={ManifestId}",
bodyTenantId, sessionId, locked.CurrentState, locked.ManifestId);
return AsOutput(await WriteSessionLockedAsync(req, locked));
}
if (guardSessionRow == null)
{
RequestRowMarkers.Stamp(req, RequestRowMarkers.IngestRefusalKey, RequestRowMarkers.IngestRefusal.SessionUnknown);
_logger.LogInformation(
"IngestTelemetry: refused batch — session unknown tenant={Tenant} session={Session}",
bodyTenantId, sessionId);
return AsOutput(await WriteSessionUnknownAsync(req));
}
// The header serial is caller-declared; the Sessions row carries the serial the
// session was registered under. When they differ, the ROW's serial is held against
// the block list too — a mid-session header change must not dodge a block placed
@@ -415,6 +427,19 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
ProcessedAt = DateTime.UtcNow,
});
/// <summary>
/// 410 for a session without a Sessions row: same status and body shape as the cascade lock,
/// because the agent's reaction is the same (stop, clean up). Never a poison body — the
/// agent would drop the items and re-emit a marker event into the same dead session.
/// </summary>
private static Task<HttpResponseData> WriteSessionUnknownAsync(HttpRequestData req)
=> req.JsonAsync(HttpStatusCode.Gone, new IngestEventsResponse
{
Success = false,
Message = "Session is unknown to the backend (never registered, or already deleted); further telemetry will be rejected.",
ProcessedAt = DateTime.UtcNow,
});
/// <summary>
/// 422 + poison body: the backend end of the agent's item-level poison protocol
/// (<c>BackendTelemetryUploader.TryReadPoisonSignalAsync</c>). Logged as a Warning with
@@ -0,0 +1,42 @@
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Maintenance;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Maintenance
{
/// <summary>
/// Timer for the orphan-session sweep (<see cref="OrphanSessionSweeper"/>): rows of sessions
/// that have no Sessions row — which no cascade can ever reach — are found through their
/// EventSessionIndex handle and removed from every session table. Its own timer, lease and
/// ops events: the sweep needs a slower cadence than the 2h <c>Maintenance</c> run, must never
/// race the retention cascade's 00:00/12:00 tick, and its cost (two key drains instead of one
/// Sessions point-read per handle) should stay visible on its own heartbeat.
/// </summary>
public sealed class OrphanSessionSweepFunction
{
// Minute 45 of every fourth hour: no other timer fires at :45 (Maintenance :00 of even
// hours, SessionSweep :30, WhatsNew :20, backup watchdog :00/:30, retention 00:00/12:00).
// const per repo convention (see SessionSweepFunction.cs).
private const string Cron = "0 45 */4 * * *";
private readonly OrphanSessionSweeper _sweeper;
private readonly ILogger<OrphanSessionSweepFunction> _logger;
public OrphanSessionSweepFunction(OrphanSessionSweeper sweeper, ILogger<OrphanSessionSweepFunction> logger)
{
_sweeper = sweeper;
_logger = logger;
}
[Function("OrphanSessionSweep")]
public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken)
{
_logger.LogInformation("OrphanSessionSweep timer trigger fired");
// Lease, run and every ops event (Completed / Failed / SkippedLocked) live in the sweeper,
// shared with the manual maintenance run.
await _sweeper.RunAsync("Timer", cancellationToken);
}
}
}
@@ -24,19 +24,22 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
private readonly ISessionRepository _sessionRepo;
private readonly VulnerabilityCorrelationService _correlationService;
private readonly IVulnerabilityInventoryLoader _inventoryLoader;
private readonly Services.Deletion.SessionDeletionGuard _deletionGuard;
public GetVulnerabilityReportFunction(
ILogger<GetVulnerabilityReportFunction> logger,
IVulnerabilityRepository vulnRepo,
ISessionRepository sessionRepo,
VulnerabilityCorrelationService correlationService,
IVulnerabilityInventoryLoader inventoryLoader)
IVulnerabilityInventoryLoader inventoryLoader,
Services.Deletion.SessionDeletionGuard deletionGuard)
{
_logger = logger;
_vulnRepo = vulnRepo;
_sessionRepo = sessionRepo;
_correlationService = correlationService;
_inventoryLoader = inventoryLoader;
_deletionGuard = deletionGuard;
}
[Function("GetVulnerabilityReport")]
@@ -65,6 +68,15 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
if (rescan)
{
// A re-scan derives rows (report, CveIndex, inventory counters): only for a live
// session, like the queue handler — never for one the backend does not know or a
// cascade is tearing down.
var liveness = await _deletionGuard.GetLivenessAsync(effectiveTenantId, sessionId, "VulnerabilityRescan");
if (liveness == Services.Deletion.SessionLiveness.Missing)
return await req.NotFoundAsync("Session not found.");
if (liveness == Services.Deletion.SessionLiveness.Locked)
return await req.ConflictAsync("Session is being deleted; a re-scan is not possible.");
try
{
_logger.LogInformation("Vulnerability re-scan requested for session {SessionId} by {User} (query: {Query})",
@@ -198,6 +198,10 @@ public class RequestTelemetryMiddleware : IFunctionsWorkerMiddleware
if (context.Items.TryGetValue(RequestRowMarkers.DeviceValidationKey, out var deviceValidation) && deviceValidation is string deviceValidationOutcome)
requestTelemetry.Properties[RequestRowMarkers.DeviceValidationKey] = deviceValidationOutcome;
// INGEST-REFUSAL — set by IngestTelemetry on its 410 paths (cascade lock vs unknown session).
if (context.Items.TryGetValue(RequestRowMarkers.IngestRefusalKey, out var ingestRefusal) && ingestRefusal is string ingestRefusalReason)
requestTelemetry.Properties[RequestRowMarkers.IngestRefusalKey] = ingestRefusalReason;
// CERT-DEVICE-BINDING — the Intune device lookup behind the certificate (admitting or
// observing), set by SecurityValidator whenever a lookup result exists for the request.
foreach (var key in RequestRowMarkers.CertDeviceBindingKeys)
@@ -294,6 +294,10 @@ builder.Services.AddHttpClient<GitHubRuleRepository>()
// BackgroundService worker that runs a queued manual run.
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunLockStore>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunGate>();
// Orphan-session sweep: own 4h timer (OrphanSessionSweepFunction), own lease, also run by the
// manual maintenance trigger. The cascade for sessions that have no Sessions row.
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.OrphanSweepLockStore>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.OrphanSessionSweeper>();
builder.Services.AddSingleton<
AutopilotMonitor.Functions.Services.Maintenance.IMaintenanceTriggerProducer,
AutopilotMonitor.Functions.Services.Maintenance.AzureQueueMaintenanceTriggerProducer>();
@@ -41,6 +41,22 @@ namespace AutopilotMonitor.Functions.Security
public static string DeviceValidationValue(ValidatorType validatedBy)
=> validatedBy == ValidatorType.Unknown ? DeviceValidation.None : validatedBy.ToString();
/// <summary>
/// Why a telemetry batch was refused with 410: the session is locked by a cascade delete,
/// or the backend has no Sessions row for it. Both share the status so the agent treats
/// them alike; the row keeps them apart.
/// </summary>
public const string IngestRefusalKey = "IngestRefusal";
/// <summary>Closed set of <see cref="IngestRefusalKey"/> values.</summary>
public static class IngestRefusal
{
/// <summary>The Sessions row is in a cascade lock state (or freshly tombstoned).</summary>
public const string CascadeLocked = "cascade_locked";
/// <summary>No Sessions row and no tombstone: never registered, or deleted longer ago than the marker lives.</summary>
public const string SessionUnknown = "session_unknown";
}
/// <summary>
/// Outcome of the Intune device lookup behind the client certificate
/// (<see cref="IntuneDeviceBindingOutcome"/> name). Absent when no lookup result exists for
@@ -50,6 +50,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
private readonly SignalRNotificationService _signalRNotification;
private readonly TenantConfigurationService _configService;
private readonly Notifications.NotificationChannelDispatcher _channelDispatcher;
private readonly Deletion.SessionDeletionGuard _deletionGuard;
private readonly ILogger<AnalyzeOnEnrollmentEndHandler> _logger;
public const string ReasonEnrollmentComplete = "enrollment_complete";
@@ -66,6 +67,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
SignalRNotificationService signalRNotification,
TenantConfigurationService configService,
Notifications.NotificationChannelDispatcher channelDispatcher,
Deletion.SessionDeletionGuard deletionGuard,
ILogger<AnalyzeOnEnrollmentEndHandler> logger)
{
_ruleService = ruleService;
@@ -75,6 +77,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
_signalRNotification = signalRNotification;
_configService = configService;
_channelDispatcher = channelDispatcher;
_deletionGuard = deletionGuard;
_logger = logger;
}
@@ -94,6 +97,20 @@ namespace AutopilotMonitor.Functions.Services.Analyze
}
var sessionPrefix = $"[Session: {envelope.SessionId.Substring(0, Math.Min(8, envelope.SessionId.Length))}]";
// Rule results, stats and channel alerts only for a live session: a session the backend
// does not know, or one a cascade is tearing down, gets nothing derived (no cascade could
// reach the rows afterwards, and an alert would name a session that does not exist).
var liveness = await _deletionGuard.GetLivenessAsync(
envelope.TenantId, envelope.SessionId, "AnalyzeOnEnrollmentEnd", cancellationToken).ConfigureAwait(false);
if (liveness != Deletion.SessionLiveness.Live)
{
_logger.LogWarning(
"{Prefix} Analyze skipped — session {Liveness} (reason={Reason})",
sessionPrefix, liveness, envelope.Reason);
return;
}
var isVulnerabilityRerun = string.Equals(
envelope.Reason, ReasonVulnerabilityCorrelated, StringComparison.OrdinalIgnoreCase);
var isWhitegloveSealed = string.Equals(
@@ -8,6 +8,17 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Deletion
{
/// <summary>Outcome of <see cref="SessionDeletionGuard.GetLivenessAsync"/>.</summary>
public enum SessionLiveness
{
/// <summary>Sessions row present and not in a cascade lock state.</summary>
Live,
/// <summary>No Sessions row: never registered, or deleted.</summary>
Missing,
/// <summary>Sessions row present but owned by a cascade (Preparing / Queued / Running / Poisoned).</summary>
Locked,
}
/// <summary>
/// Single chokepoint for the cascade-delete writer-block invariant (Plan §1 P7 / §5 PR3).
/// Two APIs split for hot-path I/O cost:
@@ -88,6 +99,35 @@ namespace AutopilotMonitor.Functions.Services.Deletion
/// <summary>Columns the guard reads off the row itself; a projected read adds them to the caller's list.</summary>
public static readonly string[] LockColumns = { "DeletionState", "PendingDeletionManifestId" };
/// <summary>
/// For writers that DERIVE rows for a session (queue handlers, the manual rescan). Unlike
/// the registration/ingest contract, a missing Sessions row is a stop here, not a pass:
/// a derived row for a session the backend does not know can never be reached by a
/// cascade and would only feed the orphan sweep. Read errors propagate (queue retry).
/// </summary>
public async Task<SessionLiveness> GetLivenessAsync(string tenantId, string sessionId, string callerContext, CancellationToken cancellationToken = default)
{
var sessionRow = await _reader.GetSessionRowAsync(tenantId, sessionId, LockColumns, cancellationToken);
if (sessionRow == null)
{
_logger.LogWarning(
"SessionDeletionGuard: no Sessions row — derived write skipped: tenant={TenantId} session={SessionId} caller={Caller}",
tenantId, sessionId, callerContext);
return SessionLiveness.Missing;
}
var state = sessionRow.GetString("DeletionState");
if (SessionDeletionState.IsLocked(state))
{
_logger.LogInformation(
"SessionDeletionGuard: session locked — derived write skipped: tenant={TenantId} session={SessionId} state={State} caller={Caller}",
tenantId, sessionId, state, callerContext);
return SessionLiveness.Locked;
}
return SessionLiveness.Live;
}
/// <summary>
/// <see cref="EnsureWritableAndGetRowAsync(string, string, string, CancellationToken)"/> with the
/// row projected to <paramref name="select"/> plus <see cref="LockColumns"/> — the agent hot
@@ -0,0 +1,47 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// Per-step wall-clock of one maintenance run, reported as <c>stepsMs</c> on the
/// <c>MaintenanceCompleted</c> ops event. Worker-side timing logs never reach App Insights,
/// and the run's composition is what decides which steps leave the 2h timer next.
/// A step that throws is still recorded (its time counts), then the exception propagates.
/// </summary>
public sealed class MaintenanceStepTimer
{
private readonly Dictionary<string, long> _steps = new Dictionary<string, long>(StringComparer.Ordinal);
/// <summary>Durations in declaration order (ms, rounded).</summary>
public IReadOnlyDictionary<string, long> Steps => _steps;
public async Task RunAsync(string step, Func<Task> body)
{
var sw = Stopwatch.StartNew();
try
{
await body().ConfigureAwait(false);
}
finally
{
_steps[step] = sw.ElapsedMilliseconds;
}
}
public async Task<T> RunAsync<T>(string step, Func<Task<T>> body)
{
var sw = Stopwatch.StartNew();
try
{
return await body().ConfigureAwait(false);
}
finally
{
_steps[step] = sw.ElapsedMilliseconds;
}
}
}
}
@@ -0,0 +1,93 @@
using System;
using System.Linq;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// The OData filter that selects every row of ONE session in each table the orphan sweep
/// clears — the key shape of the table's writer, not of the cascade manifest (a session
/// without a Sessions row has no manifest). Pure, so the shapes are unit-tested against the
/// writers once instead of trusted per call.
/// </summary>
public static class OrphanSessionRowFilters
{
/// <summary>
/// Tables cleared through a filter, in the order the sweep runs them. The handle
/// (EventSessionIndex) and the inventory pair (SessionInventoryContributions + the
/// SoftwareInventory decrements) are handled by the sweeper itself and are not listed here.
/// </summary>
public static readonly string[] Tables =
{
Constants.TableNames.Events,
Constants.TableNames.RuleResults,
Constants.TableNames.AppInstallSummaries,
Constants.TableNames.VulnerabilityReports,
Constants.TableNames.DeviceSnapshot,
Constants.TableNames.Signals,
Constants.TableNames.DecisionTransitions,
Constants.TableNames.EventTypeIndex,
Constants.TableNames.CveIndex,
Constants.TableNames.SessionTimeBreakdowns,
Constants.TableNames.SessionAnnotations,
Constants.TableNames.SessionTenantLookup,
};
/// <summary>
/// Filter for <paramref name="table"/>; throws for a table that has no declared shape so a
/// new session table can never be swept with a guessed filter.
/// </summary>
public static string For(string table, string tenantId, string sessionId)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
var t = ODataSanitizer.EscapeValue(tenantId);
var s = ODataSanitizer.EscapeValue(sessionId);
switch (table)
{
// Partition per session: {tenantId}_{sessionId}.
case Constants.TableNames.Events:
case Constants.TableNames.RuleResults:
case Constants.TableNames.Signals:
case Constants.TableNames.DecisionTransitions:
case Constants.TableNames.VulnerabilityReports:
return $"PartitionKey eq '{t}_{s}'";
// One row per session under the tenant partition.
case Constants.TableNames.DeviceSnapshot:
case Constants.TableNames.SessionTimeBreakdowns:
return $"PartitionKey eq '{t}' and RowKey eq '{s}'";
// Tenant partition, session in a column.
case Constants.TableNames.AppInstallSummaries:
return $"PartitionKey eq '{t}' and SessionId eq '{s}'";
// PK {tenantId}_{eventType}, RK {invertedTicks}_{sessionId}: the RowKey of a session
// without a Sessions row is not reconstructible (one row per batch), so the session
// is found by its column across the tenant's partitions. Tenant ids are GUIDs, so the
// '_'..'_~' range cannot reach another tenant.
case Constants.TableNames.EventTypeIndex:
return $"PartitionKey ge '{t}_' and PartitionKey lt '{t}_~' and SessionId eq '{s}'";
// PK {tenantId}_{cveId}, RK sessionId.
case Constants.TableNames.CveIndex:
return $"PartitionKey ge '{t}_' and PartitionKey lt '{t}_~' and RowKey eq '{s}'";
// PK tenantId, RK {sessionId}_{lane}; the lanes are a closed set.
case Constants.TableNames.SessionAnnotations:
var lanes = string.Join(" or ", AnnotationLanes.All.Select(lane => $"RowKey eq '{s}_{lane}'"));
return $"PartitionKey eq '{t}' and ({lanes})";
// PK sessionId, RK 'tenant'; the TenantId column pins the row to this tenant's session.
case Constants.TableNames.SessionTenantLookup:
return $"PartitionKey eq '{s}' and RowKey eq 'tenant' and TenantId eq '{t}'";
default:
throw new ArgumentException($"No orphan-sweep filter is declared for table '{table}'.", nameof(table));
}
}
}
}
@@ -0,0 +1,429 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using Azure.Storage.Blobs.Specialized;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// The cascade for sessions that have no Sessions row. A cascade needs the row (CAS lock,
/// manifest, tombstone); rows a writer leaves behind for a session the backend does not know
/// are therefore reachable only through the discovery handle every events write creates
/// (EventSessionIndex, PK tenantId / RK sessionId). The sweep diffs those handles against the
/// Sessions keys, re-checks every candidate moments before touching it, clears every session
/// table the cascade manifest knows (ratchet in <c>TableLifecycleBucketTests</c>), and deletes
/// the handle LAST — a failure before that point leaves the session discoverable for the next
/// run; every delete is idempotent.
/// <para>
/// Once a week (Sunday morning UTC, or on any manual maintenance run) it additionally
/// reconciles EventTypeIndex against the Sessions keys, read-only: the proof that nothing
/// without a handle is left behind.
/// </para>
/// </summary>
public sealed class OrphanSessionSweeper
{
/// <summary>A handle younger than this may belong to a session whose registration is still in flight.</summary>
public static readonly TimeSpan IngestGrace = TimeSpan.FromHours(24);
/// <summary>
/// A restore re-inserts the handle (old LastIngestAt, fresh write time) and the Sessions row
/// LAST; a handle written this recently is left alone until the restore has either finished
/// or failed.
/// </summary>
public static readonly TimeSpan HandleWriteGrace = TimeSpan.FromHours(2);
public const int MaxCandidatesPerRun = 50;
public static readonly TimeSpan RunBudget = TimeSpan.FromMinutes(5);
/// <summary>Sunday at or after this hour (UTC) is the weekly reconciliation window — the quiet part of the week.</summary>
public const int DeepCheckSundayHourUtc = 8;
public static readonly TimeSpan DeepCheckInterval = TimeSpan.FromDays(6);
/// <summary>A missed Sunday (deploy, outage) does not postpone the reconciliation past this age.</summary>
public static readonly TimeSpan DeepCheckOverdue = TimeSpan.FromDays(8);
public const int DeepCheckSampleSize = 20;
/// <summary>LastIngestAt of a handle written by hand for a legacy orphan: no ingest ever happened at that time.</summary>
public static readonly DateTime LegacyHandleLastIngestAt = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc);
public const string Actor = "System.OrphanSweep";
public const string TimerTrigger = "Timer";
/// <summary>
/// Every table the sweep clears: the filter tables, the inventory pair (side row + the
/// SoftwareInventory decrements) and the handle. Pinned against the cascade manifest.
/// </summary>
public static readonly IReadOnlyList<string> SweptTables = OrphanSessionRowFilters.Tables
.Concat(new[]
{
Constants.TableNames.SessionInventoryContributions,
Constants.TableNames.SoftwareInventory,
Constants.TableNames.EventSessionIndex,
})
.ToArray();
private readonly IMaintenanceRepository _maintenanceRepo;
private readonly IVulnerabilityRepository _vulnRepo;
private readonly ISessionDeletionInventoryReader _reader;
private readonly AdminConfigurationService _adminConfig;
private readonly OrphanSweepLockStore _lockStore;
private readonly OpsEventService _opsEvents;
private readonly ILogger<OrphanSessionSweeper> _logger;
private readonly Func<DateTime> _clock;
private readonly TimeSpan _runBudget;
public OrphanSessionSweeper(
IMaintenanceRepository maintenanceRepo,
IVulnerabilityRepository vulnRepo,
ISessionDeletionInventoryReader reader,
AdminConfigurationService adminConfig,
OrphanSweepLockStore lockStore,
OpsEventService opsEvents,
ILogger<OrphanSessionSweeper> logger)
: this(maintenanceRepo, vulnRepo, reader, adminConfig, lockStore, opsEvents, logger, () => DateTime.UtcNow, RunBudget)
{
}
/// <summary>Test seam: injectable clock and run budget.</summary>
internal OrphanSessionSweeper(
IMaintenanceRepository maintenanceRepo,
IVulnerabilityRepository vulnRepo,
ISessionDeletionInventoryReader reader,
AdminConfigurationService adminConfig,
OrphanSweepLockStore lockStore,
OpsEventService opsEvents,
ILogger<OrphanSessionSweeper> logger,
Func<DateTime> clock,
TimeSpan runBudget)
{
_maintenanceRepo = maintenanceRepo;
_vulnRepo = vulnRepo;
_reader = reader;
_adminConfig = adminConfig;
_lockStore = lockStore;
_opsEvents = opsEvents;
_logger = logger;
_clock = clock;
_runBudget = runBudget;
}
/// <summary>
/// One run under the sweep lease: timer and manual maintenance enter here. Returns false
/// when another run held the lease. A manual run always includes the reconciliation.
/// Every outcome is an ops event; nothing propagates to the caller.
/// </summary>
public async Task<bool> RunAsync(string triggeredBy, CancellationToken cancellationToken)
{
BlobLeaseClient lease;
try
{
lease = await _lockStore.AcquireLeaseAsync(ct: cancellationToken).ConfigureAwait(false);
}
catch (LeaseHeldException)
{
_logger.LogWarning("OrphanSweep: lease held by another run — skipping (triggeredBy={TriggeredBy})", triggeredBy);
await _opsEvents.RecordOrphanSweepSkippedLockedAsync(triggeredBy).ConfigureAwait(false);
return false;
}
using var runCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var leaseHolder = new MaintenanceLeaseHolder(lease, runCts, _logger);
try
{
OrphanSweepRunResult result;
try
{
result = await RunCoreAsync(forceDeepCheck: triggeredBy != TimerTrigger, runCts.Token).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogError(ex, "OrphanSweep: run failed (triggeredBy={TriggeredBy})", triggeredBy);
await _opsEvents.RecordOrphanSweepFailedAsync(ex.Message, triggeredBy).ConfigureAwait(false);
return true;
}
await _opsEvents.RecordOrphanSweepCompletedAsync(result, triggeredBy).ConfigureAwait(false);
if (result.Cleaned.Count > 0)
await _opsEvents.RecordOrphanEventsCleanedAsync(result).ConfigureAwait(false);
return true;
}
finally
{
if (leaseHolder.RenewalFailureReason != null)
_logger.LogWarning("OrphanSweep: the lease could not be renewed during the run ({Reason})", leaseHolder.RenewalFailureReason);
await leaseHolder.DisposeAsync().ConfigureAwait(false);
}
}
/// <summary>The run without lease and ops events (unit-testable).</summary>
internal async Task<OrphanSweepRunResult> RunCoreAsync(bool forceDeepCheck, CancellationToken ct)
{
var sw = Stopwatch.StartNew();
var now = _clock();
var result = new OrphanSweepRunResult();
// Sessions first: a cascade that finishes between the two drains removes the handle
// (step 6) before the row (step 18), so this order cannot produce a candidate the
// cascade is still working on.
var sessionKeys = await _maintenanceRepo.GetSessionKeysAsync(ct).ConfigureAwait(false);
var handles = await _maintenanceRepo.GetEventSessionIndexHandlesAsync(ct).ConfigureAwait(false);
result.SessionKeys = sessionKeys.Count;
result.HandlesScanned = handles.Count;
var candidates = handles
.Where(h => !sessionKeys.Contains((h.TenantId, h.SessionId)))
.Where(h => h.LastIngestAt <= now - IngestGrace && h.WrittenAt <= now - HandleWriteGrace)
.OrderBy(h => h.LastIngestAt)
.ToList();
result.Candidates = candidates.Count;
var processed = 0;
foreach (var candidate in candidates)
{
if (processed >= MaxCandidatesPerRun) break;
if (sw.Elapsed >= _runBudget)
{
result.BudgetExhausted = true;
break;
}
ct.ThrowIfCancellationRequested();
processed++;
try
{
await CleanCandidateAsync(candidate, result, ct).ConfigureAwait(false);
}
catch (Exception ex)
{
result.FailedSessions++;
_logger.LogError(ex,
"OrphanSweep: cleaning tenant={TenantId} session={SessionId} failed — the handle stays for the next run",
candidate.TenantId, candidate.SessionId);
}
}
result.Remaining = candidates.Count - processed;
if (forceDeepCheck || IsDeepCheckDue(await ReadDeepCheckLastRunAsync().ConfigureAwait(false), now))
result.Reconcile = await ReconcileAsync(sessionKeys, handles, now, ct).ConfigureAwait(false);
result.DurationMs = sw.ElapsedMilliseconds;
return result;
}
private async Task CleanCandidateAsync(OrphanSessionHandle candidate, OrphanSweepRunResult result, CancellationToken ct)
{
var tenantId = candidate.TenantId;
var sessionId = candidate.SessionId;
// Re-checks moments before the deletes — the scan above is minutes old.
var handle = await _maintenanceRepo.GetEventSessionIndexHandleAsync(tenantId, sessionId, ct).ConfigureAwait(false);
if (handle == null)
{
result.SkippedHandleGone++; // a cascade got there first
return;
}
if (!string.Equals(handle.ETag, candidate.ETag, StringComparison.Ordinal))
{
result.SkippedHandleChanged++; // a later write; its own grace starts over
return;
}
var row = await _reader.GetSessionRowAsync(tenantId, sessionId, SessionDeletionGuard.LockColumns, ct).ConfigureAwait(false);
if (row != null)
{
result.SkippedRowReturned++; // registered or restored meanwhile
return;
}
var afterCascade = await _reader.GetActiveSessionTombstoneAsync(tenantId, sessionId, ct).ConfigureAwait(false) != null;
if (afterCascade) result.AfterCascade++;
var cleaned = new OrphanSweepCleanedSession
{
TenantId = tenantId,
SessionId = sessionId,
Legacy = candidate.LastIngestAt <= LegacyHandleLastIngestAt,
AfterCascade = afterCascade,
};
var etag = handle.ETag;
// Inventory first, with the handle as the progress record. The stamp is the claim: a
// crash after it loses at most this session's decrements (the counts stay high, as they
// did before the sweep existed), and a rerun can never decrement twice.
if (!handle.InventoryDecremented)
{
var stamped = await _maintenanceRepo.StampHandleInventoryDecrementedAsync(tenantId, sessionId, etag, ct).ConfigureAwait(false);
if (stamped == null)
{
result.SkippedHandleChanged++;
return;
}
etag = stamped;
var keys = await _vulnRepo.GetSessionInventoryContributionsAsync(tenantId, sessionId, ct).ConfigureAwait(false);
if (keys != null)
{
cleaned.HadInventorySideRow = true;
foreach (var key in keys)
{
try
{
await _vulnRepo.DecrementSoftwareInventoryEntryAsync(tenantId, key.Vendor, key.Name, key.Version, ct).ConfigureAwait(false);
cleaned.InventoryKeysDecremented++;
}
catch (Exception ex)
{
result.InventoryDecrementsFailed++;
_logger.LogWarning(ex,
"OrphanSweep: inventory decrement failed tenant={TenantId} session={SessionId} key={Vendor}/{Name}/{Version}",
tenantId, sessionId, key.Vendor, key.Name, key.Version);
}
}
}
}
await _vulnRepo.DeleteSessionInventoryContributionsAsync(tenantId, sessionId, ct).ConfigureAwait(false);
foreach (var table in OrphanSessionRowFilters.Tables)
{
var rows = await _maintenanceRepo.DeleteOrphanSessionRowsAsync(table, tenantId, sessionId, ct).ConfigureAwait(false);
cleaned.RowsByTable[table] = rows;
}
// The handle last, and only while unchanged: a batch that re-wrote it meanwhile has
// rows the next run must still find.
var handleDeleted = await _maintenanceRepo.DeleteEventSessionIndexHandleAsync(tenantId, sessionId, etag, ct).ConfigureAwait(false);
cleaned.HandleKept = !handleDeleted;
result.Add(cleaned);
_logger.LogInformation(
"OrphanSweep: cleaned tenant={TenantId} session={SessionId} rows={Rows} inventoryKeys={Keys} legacy={Legacy} afterCascade={AfterCascade} handleKept={HandleKept}",
tenantId, sessionId, cleaned.TotalRows, cleaned.InventoryKeysDecremented, cleaned.Legacy, cleaned.AfterCascade, cleaned.HandleKept);
}
/// <summary>
/// Sunday at/after 08:00 UTC once the last run is at least six days old; a run older than
/// eight days is due at the next tick regardless. Never run before: the first Sunday.
/// </summary>
internal static bool IsDeepCheckDue(DateTime? lastRunUtc, DateTime nowUtc)
{
var inSundayWindow = nowUtc.DayOfWeek == DayOfWeek.Sunday && nowUtc.Hour >= DeepCheckSundayHourUtc;
if (lastRunUtc == null) return inSundayWindow;
var age = nowUtc - lastRunUtc.Value;
if (age >= DeepCheckOverdue) return true;
return inSundayWindow && age >= DeepCheckInterval;
}
private async Task<DateTime?> ReadDeepCheckLastRunAsync()
{
var config = await _adminConfig.GetConfigurationAsync().ConfigureAwait(false);
return DateTime.TryParse(config?.OrphanDeepCheckLastRunUtc, CultureInfo.InvariantCulture,
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var parsed)
? parsed
: (DateTime?)null;
}
/// <summary>
/// Read-only: every (tenant, session) in EventTypeIndex that has neither a Sessions row nor
/// a handle (handles are the sweep's own pipeline) is residue — confirmed against the live
/// Sessions row, because the drains are minutes old. Residue is reported, never cleaned
/// here: the operator gives it a handle, and the normal path with all its checks does the rest.
/// </summary>
private async Task<OrphanReconcileResult> ReconcileAsync(
HashSet<(string TenantId, string SessionId)> sessionKeys,
IReadOnlyList<OrphanSessionHandle> handles,
DateTime now,
CancellationToken ct)
{
var sw = Stopwatch.StartNew();
var indexKeys = await _maintenanceRepo.GetEventTypeIndexSessionKeysAsync(ct).ConfigureAwait(false);
var handleKeys = new HashSet<(string TenantId, string SessionId)>(handles.Select(h => (h.TenantId, h.SessionId)));
var residue = new List<(string TenantId, string SessionId)>();
foreach (var key in indexKeys)
{
if (sessionKeys.Contains(key) || handleKeys.Contains(key)) continue;
ct.ThrowIfCancellationRequested();
var row = await _reader.GetSessionRowAsync(key.TenantId, key.SessionId, SessionDeletionGuard.LockColumns, ct).ConfigureAwait(false);
if (row != null) continue;
residue.Add(key);
}
var result = new OrphanReconcileResult
{
IndexSessions = indexKeys.Count,
ResidueSessions = residue.Count,
Sample = residue.Take(DeepCheckSampleSize).ToList(),
DurationMs = sw.ElapsedMilliseconds,
};
await _opsEvents.RecordOrphanReconcileCompletedAsync(result).ConfigureAwait(false);
var stamp = now.ToString("o", CultureInfo.InvariantCulture);
await _adminConfig.UpdateAsync(c => { c.OrphanDeepCheckLastRunUtc = stamp; return null; }, Actor, "orphan-sweep").ConfigureAwait(false);
return result;
}
}
/// <summary>One session the sweep cleaned in a run.</summary>
public sealed class OrphanSweepCleanedSession
{
public string TenantId { get; set; } = string.Empty;
public string SessionId { get; set; } = string.Empty;
/// <summary>The handle was written by hand for a legacy orphan (LastIngestAt 2000-01-01).</summary>
public bool Legacy { get; set; }
/// <summary>A tombstone marker existed: a cascade ran, rows were written after it.</summary>
public bool AfterCascade { get; set; }
/// <summary>The handle changed between the deletes and the final delete; the next run sees the session again.</summary>
public bool HandleKept { get; set; }
public bool HadInventorySideRow { get; set; }
public int InventoryKeysDecremented { get; set; }
public Dictionary<string, int> RowsByTable { get; } = new Dictionary<string, int>(StringComparer.Ordinal);
public int TotalRows => RowsByTable.Values.Sum();
}
/// <summary>Outcome of the weekly reconciliation.</summary>
public sealed class OrphanReconcileResult
{
public int IndexSessions { get; set; }
public int ResidueSessions { get; set; }
public List<(string TenantId, string SessionId)> Sample { get; set; } = new List<(string TenantId, string SessionId)>();
public long DurationMs { get; set; }
}
/// <summary>Counters of one sweep run; the Completed ops event carries them verbatim.</summary>
public sealed class OrphanSweepRunResult
{
public int HandlesScanned { get; set; }
public int SessionKeys { get; set; }
public int Candidates { get; set; }
public List<OrphanSweepCleanedSession> Cleaned { get; } = new List<OrphanSweepCleanedSession>();
public int SkippedRowReturned { get; set; }
public int SkippedHandleGone { get; set; }
public int SkippedHandleChanged { get; set; }
public int AfterCascade { get; set; }
public int Remaining { get; set; }
public int FailedSessions { get; set; }
public int InventoryDecrementsFailed { get; set; }
public bool BudgetExhausted { get; set; }
public long DurationMs { get; set; }
public Dictionary<string, int> TotalsByTable { get; } = new Dictionary<string, int>(StringComparer.Ordinal);
public OrphanReconcileResult? Reconcile { get; set; }
public void Add(OrphanSweepCleanedSession cleaned)
{
Cleaned.Add(cleaned);
foreach (var kv in cleaned.RowsByTable)
TotalsByTable[kv.Key] = TotalsByTable.TryGetValue(kv.Key, out var n) ? n + kv.Value : kv.Value;
Bump(Constants.TableNames.SessionInventoryContributions, cleaned.HadInventorySideRow ? 1 : 0);
Bump(Constants.TableNames.SoftwareInventory, cleaned.InventoryKeysDecremented);
Bump(Constants.TableNames.EventSessionIndex, cleaned.HandleKept ? 0 : 1);
}
private void Bump(string table, int by)
=> TotalsByTable[table] = TotalsByTable.TryGetValue(table, out var n) ? n + by : by;
}
}
@@ -0,0 +1,21 @@
using AutopilotMonitor.Functions.Services.Locking;
using AutopilotMonitor.Shared;
namespace AutopilotMonitor.Functions.Services.Maintenance
{
/// <summary>
/// Blob-lease lock for the orphan-session sweep: the 4h timer, the manual maintenance run and
/// a second host instance enter through it. Its own sentinel — the sweep has no reason to
/// exclude the platform maintenance run or the retention cascade, and they none to exclude it.
/// </summary>
public class OrphanSweepLockStore : BlobLeaseLockStore
{
/// <summary>Sentinel blob path. <c>_lock/…</c> does not parse as a manifest blob name, so the manifest-TTL sweep skips it.</summary>
public const string LockBlobName = "_lock/orphan-session-sweep.lock";
public OrphanSweepLockStore(BlobStorageService blobs)
: base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "orphan session sweep")
{
}
}
}
@@ -1217,80 +1217,6 @@ namespace AutopilotMonitor.Functions.Services
_logger.LogWarning(ex, "Failed to cleanup old verdict-calibration aggregates");
}
}
/// <summary>
/// Detects and cleans up orphaned events — events stored in the Events table
/// whose session no longer exists in the Sessions table.
/// Uses the EventSessionIndex side-table for efficient detection (no full Events scan).
/// Grace period: 24 hours to protect against register+ingest race conditions.
/// </summary>
private async Task CleanupOrphanedEventsAsync()
{
try
{
_logger.LogInformation("Starting orphaned events cleanup...");
var sw = Stopwatch.StartNew();
var orphans = await _maintenanceRepo.GetOrphanedEventSessionsAsync(TimeSpan.FromHours(24));
if (orphans.Count == 0)
{
_logger.LogInformation("No orphaned events found");
return;
}
_logger.LogWarning("Found {Count} orphaned event sessions, cleaning up...", orphans.Count);
int totalEventsDeleted = 0;
int sessionsCleanedUp = 0;
// Per-orphan breakdown for the OpsEvent so the dashboard shows which tenant/session
// was cleaned (the worker LogInformation below does not reach App Insights).
var cleanedOrphans = new List<OrphanedEventSession>();
foreach (var orphan in orphans)
{
try
{
// Ordering invariant: the index entry is removed ONLY after the event
// delete completed cleanly (DeleteSessionEventsAsync throws on failure).
// Orphan detection scans only the index — deleting it first would make
// surviving event rows permanently undiscoverable.
var deletedEvents = await _maintenanceRepo.DeleteSessionEventsAsync(orphan.TenantId, orphan.SessionId);
await _maintenanceRepo.DeleteEventSessionIndexEntryAsync(orphan.TenantId, orphan.SessionId);
totalEventsDeleted += deletedEvents;
sessionsCleanedUp++;
// Report the actual rows deleted (may differ from the index's recorded count).
cleanedOrphans.Add(new OrphanedEventSession
{
TenantId = orphan.TenantId,
SessionId = orphan.SessionId,
LastIngestAt = orphan.LastIngestAt,
EventCount = deletedEvents
});
_logger.LogInformation(
"Cleaned orphan: TenantId={TenantId}, SessionId={SessionId}, Events={Events}",
orphan.TenantId, orphan.SessionId, deletedEvents);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to cleanup orphan: TenantId={TenantId}, SessionId={SessionId}",
orphan.TenantId, orphan.SessionId);
}
}
sw.Stop();
_logger.LogInformation(
"Orphaned events cleanup completed in {Ms}ms: {Sessions} sessions, {Events} events deleted",
sw.ElapsedMilliseconds, sessionsCleanedUp, totalEventsDeleted);
await _opsEventService.RecordOrphanEventsCleanedAsync(sessionsCleanedUp, totalEventsDeleted, cleanedOrphans);
}
catch (Exception ex)
{
_logger.LogError(ex, "Orphaned events cleanup failed");
}
}
}
}
@@ -4,6 +4,7 @@ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Monitoring;
using AutopilotMonitor.Shared.DataAccess;
@@ -56,6 +57,7 @@ namespace AutopilotMonitor.Functions.Services
private readonly PreviewWhitelistService _previewWhitelistService;
private readonly IStorageInitializer _storageInitializer;
private readonly Maintenance.MaintenanceRunGate _runGate;
private readonly Maintenance.OrphanSessionSweeper _orphanSweeper;
private readonly ILogger<MaintenanceService> _logger;
private const string PlatformStatsAliasFileName = "platform-stats.json";
@@ -93,6 +95,7 @@ namespace AutopilotMonitor.Functions.Services
PreviewWhitelistService previewWhitelistService,
IStorageInitializer storageInitializer,
Maintenance.MaintenanceRunGate runGate,
Maintenance.OrphanSessionSweeper orphanSweeper,
ILogger<MaintenanceService> logger)
{
_maintenanceRepo = maintenanceRepo;
@@ -124,6 +127,7 @@ namespace AutopilotMonitor.Functions.Services
_previewWhitelistService = previewWhitelistService;
_storageInitializer = storageInitializer;
_runGate = runGate;
_orphanSweeper = orphanSweeper;
_logger = logger;
}
@@ -150,53 +154,56 @@ namespace AutopilotMonitor.Functions.Services
_logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}");
var maintenanceStart = Stopwatch.StartNew();
// Every step is timed into the Completed event (stepsMs): the run's composition is
// what decides which steps leave this timer next, and worker logs never show it.
var steps = new Maintenance.MaintenanceStepTimer();
try
{
// Startup only point-reads the table schema sentinel (see
// TableStorageService.InitializeTablesAsync); this daily full pass is what
// recreates a table that was deleted out-of-band. Cheap (idempotent creates).
await EnsureAllTablesAsync();
await MarkStalledSessionsAsTimedOutAsync();
await DetectExcessiveEventSessionsAsync();
await AggregateMetricsWithCatchUpAsync();
await steps.RunAsync("ensureTables", async () => { await EnsureAllTablesAsync(); });
await steps.RunAsync("stalledSessions", async () => { await MarkStalledSessionsAsTimedOutAsync(); });
await steps.RunAsync("excessiveEvents", async () => { await DetectExcessiveEventSessionsAsync(); });
await steps.RunAsync("aggregateMetrics", async () => { await AggregateMetricsWithCatchUpAsync(); });
// F3 PR6: rule-frequency regression radar, right after the rule-stats
// aggregation so the window rows are fresh. Anchored on YESTERDAY — whole
// days only, a partial today would understate the window rate. Fail-soft.
await RunRuleRegressionRadarAsync(DateTime.UtcNow.Date.AddDays(-1));
await steps.RunAsync("ruleRegressionRadar", async () => { await RunRuleRegressionRadarAsync(DateTime.UtcNow.Date.AddDays(-1)); });
// App-version duration regression radar: same episode/tracker pattern over the
// install summaries (trailing 35d horizon loaded internally). Fail-soft.
await RunAppVersionRegressionRadarAsync();
await steps.RunAsync("appVersionRadar", async () => { await RunAppVersionRegressionRadarAsync(); });
// ONE projected cross-tenant window scan feeds every rolling sweep below (the
// StartedAt-only filter is a full-table drain in Table Storage — four of them per
// tick was four times the same read). Each sweep slices its own window.
var sweepWindow = await LoadSweepWindowSessionsAsync();
var sweepWindow = await steps.RunAsync<List<SessionSummary>>("sweepWindow", () => LoadSweepWindowSessionsAsync());
// F1 PR2: rolling 30d breakdown backfill + daily attribution aggregates. Owns
// its own window (NOT the snapshot-gated catch-up above) so late-terminating
// sessions still reach their StartedAt-date's aggregate. Fail-soft internally.
await SweepTimeAttributionAsync(sweepWindow);
await steps.RunAsync("timeAttribution", async () => { await SweepTimeAttributionAsync(sweepWindow); });
// F2 PR4: device-history chain heal (incl. deleted-session ref cleanup) + daily
// FTR aggregates over the same rolling window. Fail-soft internally.
await SweepDeviceJourneysAsync(sweepWindow);
await steps.RunAsync("deviceJourneys", async () => { await SweepDeviceJourneysAsync(sweepWindow); });
// Verdict calibration: per-verdict-path daily buckets over the same rolling
// window, AFTER the device-journey sweep so the re-enrollment proxy reads
// freshly merged chains. Fail-soft internally.
await SweepVerdictCalibrationAsync(sweepWindow);
await steps.RunAsync("verdictCalibration", async () => { await SweepVerdictCalibrationAsync(sweepWindow); });
// Verdict-calibration drift radar over the rows the sweep just refreshed; anchored
// on yesterday like the rule radar (whole days only). Fail-soft.
await RunVerdictCalibrationRadarAsync(DateTime.UtcNow.Date.AddDays(-1), sweepWindow);
await steps.RunAsync("verdictCalibrationRadar", async () => { await RunVerdictCalibrationRadarAsync(DateTime.UtcNow.Date.AddDays(-1), sweepWindow); });
// Plan §5 PR6 / §16 R14: session retention fanout extracted out of the 2h timer
// into the dedicated 12h SessionDeletionMaintenanceFunction so cascade-lifecycle
// work has independent cadence + kill-switch + OpsEvent watchdogs. The non-session
// tail of the old CleanupOldDataAsync (UserUsageLog + RuleStats) stays here.
await CleanupOldUsageDataAsync();
await CleanupOldDistressReportsAsync();
await CleanupOldOpsEventsAsync();
await CleanupUnboundedTablesAsync();
await CleanupOrphanedEventsAsync();
await CheckAgentBlobStorageAsync();
await CheckEmbeddedCertExpiryAsync();
await CheckPoisonQueueBacklogAsync();
await RecomputePlatformStatsAsync();
// tail of the old CleanupOldDataAsync (UserUsageLog + RuleStats) stays here. The
// orphan-session sweep left this timer too (OrphanSessionSweepFunction, every 4h).
await steps.RunAsync("usageData", async () => { await CleanupOldUsageDataAsync(); });
await steps.RunAsync("distressReports", async () => { await CleanupOldDistressReportsAsync(); });
await steps.RunAsync("opsEvents", async () => { await CleanupOldOpsEventsAsync(); });
await steps.RunAsync("unboundedTables", async () => { await CleanupUnboundedTablesAsync(); });
await steps.RunAsync("agentBlobStorage", async () => { await CheckAgentBlobStorageAsync(); });
await steps.RunAsync("embeddedCertExpiry", async () => { await CheckEmbeddedCertExpiryAsync(); });
await steps.RunAsync("poisonQueues", async () => { await CheckPoisonQueueBacklogAsync(); });
await steps.RunAsync("platformStats", async () => { await RecomputePlatformStatsAsync(); });
// Backfill and repair tasks run only via manual trigger (RunManualAsync)
// to keep the timer-triggered path lightweight. See RunManualAsync for:
@@ -205,7 +212,7 @@ namespace AutopilotMonitor.Functions.Services
maintenanceStart.Stop();
_logger.LogInformation($"Daily maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer");
await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer", stepsMs: steps.Steps);
// Soft watchdog: the run completed, but if it is climbing toward the host's 60min
// functionTimeout (e.g. a large first-time retention backlog deleting row-by-row),
@@ -316,6 +323,10 @@ namespace AutopilotMonitor.Functions.Services
await CleanupUnboundedTablesAsync();
result.DataCleanupExecuted = true;
// The orphan-session sweep has its own timer and lease; a manual run includes it
// (with the reconciliation) so an operator can drive it from the same button.
await _orphanSweeper.RunAsync(triggeredBy, CancellationToken.None);
// --- Backfill & repair tasks (manual-only, not in timer path) ---
// Safety net: backfill any sessions missing from SessionsIndex
@@ -84,15 +84,17 @@ namespace AutopilotMonitor.Functions.Services
/// <paramref name="report"/> is the run report of a manual run: the trigger answers 202
/// before the run starts, so this event is where the operator reads what the run did.
/// </summary>
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null)
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null,
IReadOnlyDictionary<string, long>? stepsMs = null)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info,
$"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})",
null, triggeredBy,
report == null
? (object)new { durationMs }
? (object)new { durationMs, stepsMs }
: new
{
durationMs,
stepsMs,
aggregatedDate = report.AggregatedDate,
stalledSessionsChecked = report.StalledSessionsChecked,
dataCleanupExecuted = report.DataCleanupExecuted,
@@ -138,36 +140,113 @@ namespace AutopilotMonitor.Functions.Services
$"Cleaned up {deletedCount} ops events older than {retentionDays} days",
null, "System.Maintenance", new { deletedCount, retentionDays });
// ── Orphan session sweep (OrphanSessionSweeper) ─────────────────────────
/// <summary>
/// Records the orphan-cleanup result. <paramref name="cleanedOrphans"/> carries the
/// per-session breakdown (which tenant + session lost lingering events and how many),
/// so the ops dashboard shows *what* was cleaned, not just a count. The list is capped
/// to keep the OpsEvents Table row under the per-property (32 KB) / entity (~1 MB) limits;
/// the full count always survives in <c>orphanSessions</c> and a <c>detailsTruncated</c>
/// flag flags any clipping. Note: an "orphan" is a session row that no longer exists while
/// its events lingered past the 24h grace — the tenant itself usually still exists.
/// Heartbeat of every sweep run: scan sizes, candidates, what was cleaned, skipped and
/// left for the next run, the per-table totals and — when it ran — the reconciliation.
/// </summary>
public Task RecordOrphanEventsCleanedAsync(int orphanSessions, int totalEventsDeleted,
IReadOnlyList<OrphanedEventSession>? cleanedOrphans = null)
public Task RecordOrphanSweepCompletedAsync(Maintenance.OrphanSweepRunResult result, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepCompleted, OpsEventSeverity.Info,
$"Orphan session sweep completed in {result.DurationMs}ms — handles={result.HandlesScanned} sessions={result.SessionKeys} "
+ $"candidates={result.Candidates} cleaned={result.Cleaned.Count} remaining={result.Remaining} failed={result.FailedSessions}",
null, Maintenance.OrphanSessionSweeper.Actor,
new
{
triggeredBy,
durationMs = result.DurationMs,
handlesScanned = result.HandlesScanned,
sessionKeys = result.SessionKeys,
candidates = result.Candidates,
cleaned = result.Cleaned.Count,
remaining = result.Remaining,
failedSessions = result.FailedSessions,
skippedRowReturned = result.SkippedRowReturned,
skippedHandleGone = result.SkippedHandleGone,
skippedHandleChanged = result.SkippedHandleChanged,
afterCascade = result.AfterCascade,
inventoryDecrementsFailed = result.InventoryDecrementsFailed,
budgetExhausted = result.BudgetExhausted,
totalsByTable = result.TotalsByTable,
reconcile = result.Reconcile == null ? null : new
{
indexSessions = result.Reconcile.IndexSessions,
residueSessions = result.Reconcile.ResidueSessions,
durationMs = result.Reconcile.DurationMs,
},
});
public Task RecordOrphanSweepFailedAsync(string error, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepFailed, OpsEventSeverity.Error,
$"Orphan session sweep failed: {error}",
null, Maintenance.OrphanSessionSweeper.Actor, new { error, triggeredBy });
public Task RecordOrphanSweepSkippedLockedAsync(string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepSkippedLocked, OpsEventSeverity.Warning,
$"Orphan session sweep skipped — another run holds the lease (triggered by {triggeredBy})",
null, Maintenance.OrphanSessionSweeper.Actor, new { reason = "lease held by another run", triggeredBy });
/// <summary>
/// Weekly read-only reconciliation of EventTypeIndex against the Sessions keys. Info at
/// zero residue; Warning with a capped sample otherwise — residue means rows exist for a
/// session that has neither a Sessions row nor a handle, which the sweep cannot reach on its own.
/// </summary>
public Task RecordOrphanReconcileCompletedAsync(Maintenance.OrphanReconcileResult result)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanReconcileCompleted,
result.ResidueSessions == 0 ? OpsEventSeverity.Info : OpsEventSeverity.Warning,
result.ResidueSessions == 0
? $"Orphan reconciliation: no residue across {result.IndexSessions} indexed sessions ({result.DurationMs}ms)"
: $"Orphan reconciliation: {result.ResidueSessions} indexed sessions without a Sessions row or a handle ({result.DurationMs}ms)",
null, Maintenance.OrphanSessionSweeper.Actor,
new
{
indexSessions = result.IndexSessions,
residueSessions = result.ResidueSessions,
sample = result.Sample.Select(k => new { tenantId = k.TenantId, sessionId = k.SessionId }).ToList(),
sampleTruncated = result.ResidueSessions > result.Sample.Count,
durationMs = result.DurationMs,
});
/// <summary>
/// Rows of sessions without a Sessions row were removed (Warning: after the ingest refusal
/// every real hit is a writer outside the registration path or a restore residue; legacy
/// handles are expected while the backlog drains). <c>totalEventsDeleted</c> keeps its old
/// meaning (Events rows) next to the per-table totals. The per-session list is capped to
/// keep the row inside the Table Storage property limits; the counts are never clipped.
/// </summary>
public Task RecordOrphanEventsCleanedAsync(Maintenance.OrphanSweepRunResult result)
{
const int maxDetailRows = 50;
var orphanList = cleanedOrphans ?? Array.Empty<OrphanedEventSession>();
var detail = orphanList
.OrderByDescending(o => o.EventCount)
var detail = result.Cleaned
.OrderByDescending(c => c.TotalRows)
.Take(maxDetailRows)
.Select(o => new { tenantId = o.TenantId, sessionId = o.SessionId, eventCount = o.EventCount })
.Select(c => new
{
tenantId = c.TenantId,
sessionId = c.SessionId,
rows = c.TotalRows,
inventoryKeys = c.InventoryKeysDecremented,
legacy = c.Legacy,
afterCascade = c.AfterCascade,
handleKept = c.HandleKept,
})
.ToList();
result.TotalsByTable.TryGetValue(Shared.Constants.TableNames.Events, out var eventsDeleted);
var legacy = result.Cleaned.Count(c => c.Legacy);
return WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanEventsCleaned, OpsEventSeverity.Warning,
$"Cleaned {totalEventsDeleted} orphaned events across {orphanSessions} sessions",
null, "System.Maintenance",
$"Cleaned {result.TotalsByTable.Values.Sum()} rows of {result.Cleaned.Count} orphan sessions ({legacy} legacy handles)",
null, Maintenance.OrphanSessionSweeper.Actor,
new
{
orphanSessions,
totalEventsDeleted,
orphanSessions = result.Cleaned.Count,
totalEventsDeleted = eventsDeleted,
legacySessions = legacy,
remaining = result.Remaining,
totalsByTable = result.TotalsByTable,
orphans = detail,
detailsTruncated = orphanList.Count > maxDetailRows
detailsTruncated = result.Cleaned.Count > maxDetailRows,
});
}
@@ -935,23 +935,6 @@ namespace AutopilotMonitor.Functions.Services
// ===== DELETION HELPERS =====
/// <summary>
/// Deletes all events for a session from storage
/// </summary>
public async Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Events);
var partitionKey = $"{tenantId}_{sessionId}";
var filter = $"PartitionKey eq '{partitionKey}'";
var deleted = await DeleteByFilterInBatchesAsync(tableClient, filter, $"events for session {sessionId}");
if (deleted > 0)
_logger.LogInformation($"Deleted {deleted} events for session {sessionId}");
return deleted;
}
/// <summary>
/// Deletes all rule results for a session
/// </summary>
@@ -1188,81 +1171,5 @@ namespace AutopilotMonitor.Functions.Services
return false;
}
}
// ===== ORPHAN EVENT DETECTION =====
/// <summary>
/// Scans EventSessionIndex, checks each entry against the Sessions table,
/// and returns entries where no session exists and LastIngestAt is older than the grace period.
/// </summary>
public async Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod)
{
var orphans = new List<OrphanedEventSession>();
var cutoff = DateTime.UtcNow - gracePeriod;
try
{
var indexClient = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
var sessionsClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
await foreach (var entity in indexClient.QueryAsync<TableEntity>())
{
var tenantId = entity.PartitionKey;
var sessionId = entity.RowKey;
var lastIngestAt = entity.GetDateTimeOffset("LastIngestAt")?.UtcDateTime ?? DateTime.MinValue;
var eventCount = entity.GetInt32("EventCount") ?? 0;
// Grace period: skip recent entries (race condition protection)
if (lastIngestAt > cutoff)
continue;
// Check if session exists
try
{
var session = await sessionsClient.GetEntityIfExistsAsync<TableEntity>(tenantId, sessionId, select: new[] { "PartitionKey" });
if (!session.HasValue)
{
orphans.Add(new OrphanedEventSession
{
TenantId = tenantId,
SessionId = sessionId,
LastIngestAt = lastIngestAt,
EventCount = eventCount
});
}
}
catch (RequestFailedException ex)
{
// GetEntityIfExistsAsync does NOT throw on 404 (handled via HasValue above),
// so this only fires on transient errors (429/500/503). Treating those as
// "orphan" would delete events of live sessions — skip the entry instead.
_logger.LogWarning(ex, "Transient error checking session existence for {TenantId}/{SessionId} (status {Status}); skipping orphan classification this run", tenantId, sessionId, ex.Status);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to scan EventSessionIndex for orphans");
}
return orphans;
}
public async Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId)
{
try
{
var indexClient = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
await indexClient.DeleteEntityAsync(tenantId, sessionId);
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
// Already deleted, ignore
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete EventSessionIndex entry for {TenantId}/{SessionId}", tenantId, sessionId);
}
}
}
}
@@ -0,0 +1,143 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using Azure;
using Azure.Data.Tables;
namespace AutopilotMonitor.Functions.Services
{
/// <summary>
/// Storage side of the orphan-session sweep (<see cref="Maintenance.OrphanSessionSweeper"/>):
/// the two key drains the diff runs on, the EventSessionIndex handle as the sweep's progress
/// row, and the per-table delete by the writer's key shape.
/// </summary>
public partial class TableStorageService
{
private const string HandleInventoryDecrementedColumn = "InventoryDecremented";
// Timestamp must be named explicitly in a projection; ETag always travels with the row.
private static readonly string[] HandleColumns =
{ "PartitionKey", "RowKey", "Timestamp", "LastIngestAt", HandleInventoryDecrementedColumn };
public async Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(CancellationToken ct = default)
{
var client = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
var keys = new HashSet<(string TenantId, string SessionId)>();
await foreach (var row in client.QueryAsync<TableEntity>(
maxPerPage: 1000, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
{
keys.Add((row.PartitionKey, row.RowKey));
}
return keys;
}
public async Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(CancellationToken ct = default)
{
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
var handles = new List<OrphanSessionHandle>();
await foreach (var row in client.QueryAsync<TableEntity>(maxPerPage: 1000, select: HandleColumns, cancellationToken: ct))
{
handles.Add(ToHandle(row));
}
return handles;
}
public async Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
var row = await client.GetEntityIfExistsAsync<TableEntity>(tenantId, sessionId, select: HandleColumns, cancellationToken: ct);
return row.HasValue ? ToHandle(row.Value!) : null;
}
public async Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
var stamp = new TableEntity(tenantId, sessionId) { [HandleInventoryDecrementedColumn] = true };
try
{
var response = await client.UpdateEntityAsync(stamp, new ETag(etag), TableUpdateMode.Merge, ct);
return response.Headers.ETag?.ToString();
}
catch (RequestFailedException ex) when (ex.Status == 412 || ex.Status == 404)
{
return null;
}
}
public async Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
try
{
await client.DeleteEntityAsync(tenantId, sessionId, new ETag(etag), ct);
return true;
}
catch (RequestFailedException ex) when (ex.Status == 412 || ex.Status == 404)
{
return false;
}
}
public async Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default)
{
var filter = Maintenance.OrphanSessionRowFilters.For(table, tenantId, sessionId);
var client = _tableServiceClient.GetTableClient(table);
var keys = new List<(string Pk, string Rk)>();
await foreach (var row in client.QueryAsync<TableEntity>(
filter: filter, maxPerPage: 1000, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
{
keys.Add((row.PartitionKey, row.RowKey));
}
if (keys.Count == 0) return 0;
// Groups by partition and falls back to per-row deletes on a 404 inside a batch, so a
// rerun after a partial failure is idempotent; any other failure propagates.
await DeleteByExactKeysInBatchesAsync(table, keys, ct);
return keys.Count;
}
public async Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default)
{
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventTypeIndex);
var keys = new HashSet<(string TenantId, string SessionId)>();
await foreach (var row in client.QueryAsync<TableEntity>(
maxPerPage: 1000, select: new[] { "PartitionKey", "SessionId", "TenantId" }, cancellationToken: ct))
{
var sessionId = row.GetString("SessionId");
if (string.IsNullOrEmpty(sessionId)) continue; // the readers skip such rows as well
var tenantId = row.GetString("TenantId");
if (string.IsNullOrEmpty(tenantId))
{
// Rows older than the TenantId column: PartitionKey is {tenantId}_{eventType} and a
// tenant id (a GUID) never contains an underscore.
var cut = row.PartitionKey.IndexOf('_');
if (cut <= 0) continue;
tenantId = row.PartitionKey.Substring(0, cut);
}
keys.Add((tenantId!, sessionId!));
}
return keys;
}
private static OrphanSessionHandle ToHandle(TableEntity row) => new OrphanSessionHandle
{
TenantId = row.PartitionKey,
SessionId = row.RowKey,
LastIngestAt = row.GetDateTimeOffset("LastIngestAt")?.UtcDateTime ?? DateTime.MinValue,
WrittenAt = row.Timestamp?.UtcDateTime ?? DateTime.MinValue,
ETag = row.ETag.ToString(),
InventoryDecremented = row.GetBoolean(HandleInventoryDecrementedColumn) ?? false,
};
}
}
@@ -57,6 +57,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
private readonly IVulnerabilityInventoryLoader _inventoryLoader;
private readonly IAnalyzeOnEnrollmentEndProducer _analyzeProducer;
private readonly SignalRNotificationService _signalRNotification;
private readonly Deletion.SessionDeletionGuard _deletionGuard;
private readonly ILogger<VulnerabilityCorrelateHandler> _logger;
public VulnerabilityCorrelateHandler(
@@ -67,6 +68,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
IVulnerabilityInventoryLoader inventoryLoader,
IAnalyzeOnEnrollmentEndProducer analyzeProducer,
SignalRNotificationService signalRNotification,
Deletion.SessionDeletionGuard deletionGuard,
ILogger<VulnerabilityCorrelateHandler> logger)
{
_adminConfigService = adminConfigService;
@@ -76,6 +78,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
_inventoryLoader = inventoryLoader;
_analyzeProducer = analyzeProducer;
_signalRNotification = signalRNotification;
_deletionGuard = deletionGuard;
_logger = logger;
}
@@ -107,6 +110,19 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
return;
}
// Derived rows only for a live session: a session the backend does not know, or one a
// cascade is tearing down, must not gain an inventory side-row, CveIndex rows or a
// report — no cascade could reach them afterwards.
var liveness = await _deletionGuard.GetLivenessAsync(
envelope.TenantId, envelope.SessionId, "VulnerabilityCorrelate", cancellationToken).ConfigureAwait(false);
if (liveness != Deletion.SessionLiveness.Live)
{
_logger.LogWarning(
"{Prefix} VulnerabilityCorrelate skipped — session {Liveness} (reason={Reason})",
sessionPrefix, liveness, envelope.Reason);
return;
}
// Filter inventory by the envelope's WhiteGlovePart so a Part-2 correlation
// does not pick up Part-1 inventory items (which would mis-tag findings as
// user_enrollment) and a delayed Part-1 retry does not later overwrite a
@@ -271,9 +271,10 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
// Persist software inventory via the contributions side-row (Plan §17). Awaited inside
// the invocation: a detached task can be cut off by a scale-in after the side-row has
// already promised the key set, leaving the counters short with no trace. The method
// never throws, so the report — the primary product — is stored regardless. PR3 wires
// SessionDeletionGuard above this layer (at VulnerabilityCorrelateHandler) so a session
// locked by a cascade never reaches this code path.
// never throws, so the report — the primary product — is stored regardless. Both callers
// (VulnerabilityCorrelateHandler, GetVulnerabilityReportFunction's rescan) check
// SessionDeletionGuard.GetLivenessAsync first, so a session the backend does not know
// or a cascade is tearing down never reaches this write.
await ApplyInventoryContributionsAsync(tenantId, sessionId, inventoryItems, cpeMappingsDict);
// Summary log for production monitoring
@@ -0,0 +1,56 @@
/**
* search_sessions_by_event reads the EventTypeIndex, and the backend drops index rows whose
* session no longer exists (deleted, or not yet removed by the orphan sweep). A page can therefore
* be empty and still carry a nextLink, so the tool must go through the same auto-exhausting
* scan as search_sessions instead of a single fetch. The scan seam is the observable here.
*/
import { describe, it, expect, beforeEach, vi } from 'vitest';
const { scanMock } = vi.hoisted(() => ({ scanMock: vi.fn() }));
vi.mock('../client.js', async (importOriginal) => {
const actual = await importOriginal<typeof import('../client.js')>();
return { ...actual, scanWithTimeoutFallback: scanMock };
});
import { registerSessionTools } from '../tools/sessions.js';
type Handler = (args: Record<string, unknown>) => Promise<{ content: Array<{ text: string }>; isError?: boolean }>;
function searchSessionsByEvent(): Handler {
const handlers: Record<string, Handler> = {};
const fake = { registerTool: (name: string, _def: unknown, handler: Handler) => { handlers[name] = handler; } };
registerSessionTools(fake as never, true);
return handlers.search_sessions_by_event;
}
const PAGE = { count: 1, sessions: [{ sessionId: 's1', tenantId: 't1' }], scannedPages: 3 };
describe('search_sessions_by_event — auto-exhausts empty-but-continuable pages', () => {
beforeEach(() => {
scanMock.mockReset();
scanMock.mockResolvedValue(PAGE);
});
it('reaches the backend through the scanning fetch, not a single apiFetch', async () => {
const result = await searchSessionsByEvent()({ eventType: 'app_install_failed' });
expect(result.isError).toBeFalsy();
expect(scanMock).toHaveBeenCalledTimes(1);
// Global vs tenant route follows the request's routing context, not the registration flag.
const [path, basePath] = scanMock.mock.calls[0] as [string, string];
expect(basePath).toMatch(/^\/api\/(global\/)?search\/sessions-by-event$/);
expect(path.startsWith(`${basePath}?`)).toBe(true);
expect(path).toContain('eventType=app_install_failed');
expect(result.content[0].text).toContain('"scannedPages":3');
});
it('keeps the nextLink cursor when continuing', async () => {
const next = '/api/search/sessions-by-event?eventType=app_install_failed&pageSize=50&continuation=c1';
await searchSessionsByEvent()({ eventType: 'app_install_failed', tenantId: 't1', continuation: next });
const [path] = scanMock.mock.calls[0] as [string, string];
expect(path).toContain('continuation=c1');
expect(path).toContain('eventType=app_install_failed');
});
});
@@ -200,6 +200,8 @@ export interface AdminConfiguration {
nvdCacheLastRefreshUtc: string;
/** Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss. */
epssLastSyncUtc: string;
/** Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal. */
orphanDeepCheckLastRunUtc: string;
}
/** The immutable Entra identity behind a cross-tenant-role UPN: the HOME tenant the UPN was granted for and, once known, the user's object id. A role row (GlobalAdmins / DelegatedAdmins / TenantGroupAssignments) confers nothing unless the caller's validated JWT matches this binding. */
@@ -70,6 +70,10 @@ export const OPS_EVENT_TYPES = [
"SessionSweepFailed",
"OpsEventCleanup",
"OrphanEventsCleaned",
"OrphanSweepCompleted",
"OrphanSweepFailed",
"OrphanSweepSkippedLocked",
"OrphanReconcileCompleted",
"SessionDeletionMaintenanceStarted",
"SessionDeletionMaintenanceBudgetExceeded",
"SessionDeletionMaintenanceSkippedLocked",
@@ -236,9 +236,11 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated:
const { eventType, tenantId: rawTenantId, continuation } = args;
const pageSize = pageSizeForCall(args.pageSize, continuation, DEFAULT_FIRST_PAGE_SIZE);
const tenantId = enforceDelegatedTenantForPage(rawTenantId, continuation);
// eventType is the sole filter and is applied server-side (EventTypeIndex OData),
// so an empty page never carries a nextLink — no auto-exhaust needed. An empty result
// for an uncatalogued type says so, so a typo does not read as "no such sessions".
// eventType is applied server-side (EventTypeIndex OData), but the backend drops index
// rows whose session no longer exists (deleted, or not yet removed by the orphan sweep),
// so a page can be empty and still carry a nextLink — auto-exhaust forward like
// search_sessions. An empty result for an uncatalogued type says so, so a typo does not
// read as "no such sessions".
const basePath = pickGlobalOrTenantPath('/api/global/search/sessions-by-event', '/api/search/sessions-by-event', tenantId);
const path = followNextLink(
basePath,
@@ -246,7 +248,7 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated:
continuation,
{ pageSize },
);
const data = await apiFetch<SessionListResponse>(path);
const data = await scanWithTimeoutFallback(path, basePath, effectivePageSize(pageSize, continuation));
return toolResultText(withEventTypeNote(data, eventType, !continuation), MAX_RESULT_SIZE_CHARS.indexSessions);
} catch (error: unknown) {
return toolError('search_sessions_by_event', args, error);
@@ -130,7 +130,6 @@ namespace AutopilotMonitor.Shared.DataAccess
Task<List<string>> GetAllTenantIdsAsync();
// --- Cleanup ---
Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId);
Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId);
// --- Index Maintenance ---
@@ -138,13 +137,51 @@ namespace AutopilotMonitor.Shared.DataAccess
Task<int> CleanupGhostSessionIndexEntriesAsync();
Task<bool> IsSessionIndexEmptyAsync();
// --- Orphan Event Detection ---
// --- Orphan session sweep (rows of sessions that have no Sessions row) ---
/// <summary>
/// Returns EventSessionIndex entries whose session no longer exists in the Sessions table
/// and whose last ingest is older than the grace period.
/// Every (tenantId, sessionId) key of the Sessions table, projected to the keys (one range
/// request per 1000 rows). The sweep diffs the EventSessionIndex handles against this set
/// instead of point-reading one Sessions row per handle.
/// </summary>
Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod);
Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId);
Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(System.Threading.CancellationToken ct = default);
/// <summary>
/// Every EventSessionIndex handle with the columns the sweep decides on: last ingest, the
/// row's own write time, ETag and the inventory progress stamp.
/// </summary>
Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(System.Threading.CancellationToken ct = default);
/// <summary>Point-reads one handle; null when it no longer exists (a cascade took it).</summary>
Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, System.Threading.CancellationToken ct = default);
/// <summary>
/// Marks the handle's software-inventory decrements as applied (ETag-conditional merge).
/// Returns the handle's new ETag, or null when the row changed or vanished since
/// <paramref name="etag"/> was read.
/// </summary>
Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, System.Threading.CancellationToken ct = default);
/// <summary>
/// Deletes the handle only while it still carries <paramref name="etag"/>: false when a
/// later write changed it (412) or it is already gone (404).
/// </summary>
Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, System.Threading.CancellationToken ct = default);
/// <summary>
/// Deletes every row of <paramref name="table"/> that belongs to the session, by that table's
/// own key shape. Fail-loud on query or delete errors; rows already missing count as done.
/// Returns the number of rows found.
/// </summary>
Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, System.Threading.CancellationToken ct = default);
/// <summary>
/// Distinct (tenantId, sessionId) pairs present in EventTypeIndex — the input of the weekly
/// reconciliation (a full projected drain, roughly 65 rows per session). The tenant comes
/// from the TenantId column, falling back to the PartitionKey prefix for rows written
/// before that column existed.
/// </summary>
Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(System.Threading.CancellationToken ct = default);
// --- Session Tombstones (F2 device-journey sweep input) ---
/// <summary>
@@ -167,12 +204,31 @@ namespace AutopilotMonitor.Shared.DataAccess
IAsyncEnumerable<string> EnumerateSessionsForOffboardingAsync(string tenantId, System.Threading.CancellationToken ct = default);
}
public class OrphanedEventSession
/// <summary>
/// One EventSessionIndex row as the orphan sweep sees it — the discovery handle that every
/// events write creates (PK tenantId, RK sessionId).
/// </summary>
public sealed class OrphanSessionHandle
{
public string TenantId { get; set; } = string.Empty;
public string SessionId { get; set; } = string.Empty;
/// <summary>Server time of the last ingest that touched the handle.</summary>
public DateTime LastIngestAt { get; set; }
public int EventCount { get; set; }
/// <summary>
/// System write time of the row. A restore re-inserts the handle with its old
/// LastIngestAt but a fresh write time, so the sweep waits on this one too.
/// </summary>
public DateTime WrittenAt { get; set; }
public string ETag { get; set; } = string.Empty;
/// <summary>
/// True once the sweep applied the session's software-inventory decrements. Progress
/// marker across runs: a rerun after a crash skips the decrements and only removes the rows.
/// </summary>
public bool InventoryDecremented { get; set; }
}
/// <summary>
@@ -38,6 +38,14 @@ namespace AutopilotMonitor.Shared.DataAccess
public const string SessionSweepFailed = "SessionSweepFailed";
public const string OpsEventCleanup = "OpsEventCleanup";
public const string OrphanEventsCleaned = "OrphanEventsCleaned";
/// <summary>Every orphan-session sweep run (Info heartbeat): scanned handles, candidates, cleaned, remaining, per-table totals.</summary>
public const string OrphanSweepCompleted = "OrphanSweepCompleted";
/// <summary>The sweep run itself failed before or while enumerating (Error); per-candidate failures stay inside Completed.</summary>
public const string OrphanSweepFailed = "OrphanSweepFailed";
/// <summary>Another sweep run (timer, manual maintenance or a second host instance) held the sweep lease (Warning).</summary>
public const string OrphanSweepSkippedLocked = "OrphanSweepSkippedLocked";
/// <summary>Weekly read-only reconciliation of EventTypeIndex against the Sessions keys: Info at zero residue, Warning with a sample otherwise.</summary>
public const string OrphanReconcileCompleted = "OrphanReconcileCompleted";
public const string SessionDeletionMaintenanceStarted = "SessionDeletionMaintenanceStarted";
public const string SessionDeletionMaintenanceBudgetExceeded = "SessionDeletionMaintenanceBudgetExceeded";
public const string SessionDeletionMaintenanceSkippedLocked = "SessionDeletionMaintenanceSkippedLocked";
@@ -184,7 +192,7 @@ namespace AutopilotMonitor.Shared.DataAccess
public static readonly IReadOnlyList<string> All = new[]
{
ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles,
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed,
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, OrphanSweepCompleted, OrphanSweepFailed, OrphanSweepSkippedLocked, OrphanReconcileCompleted, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed,
DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen,
OffboardingFeedbackReceived, FeedbackReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled,
SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected,
@@ -640,6 +640,13 @@ namespace AutopilotMonitor.Shared.Models
/// </summary>
public string EpssLastSyncUtc { get; set; } = default!;
/// <summary>
/// Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only
/// EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself,
/// never patched from the portal.
/// </summary>
public string OrphanDeepCheckLastRunUtc { get; set; } = default!;
/// <summary>
/// Returns a shallow copy with all secret-bearing string fields replaced by
/// <see cref="Constants.RedactedSecretPlaceholder"/> (empty values are left empty). Used when
@@ -38,6 +38,15 @@ namespace AutopilotMonitor.Shared.Models
/// out the silence grace. See tasks/enrollment-status-reclassification.md.
/// </summary>
SessionAgeEmergencyBreak = 4,
/// <summary>
/// POST /api/agent/telemetry answered 410 Gone for the agent's own session: the backend
/// no longer knows it (never registered, deleted, or being deleted). The agent stops
/// uploading and terminates with self-destruct; this single report is the last thing the
/// device says about the session. Sent over the emergency channel because the telemetry
/// channel is exactly what was refused.
/// </summary>
TelemetryRejectedSessionUnknown = 5,
}
/// <summary>
@@ -88,8 +88,16 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
// PR2: GA restored a single row from a backup. Warning severity so operators
// can wire a Telegram rule and see the audit in near-real-time.
"BackupRowRestored",
// Orphan cleanup sweep (Warning): events whose parent session no longer exists were
// deleted during maintenance aggregation. Backend helper RecordOrphanEventsCleanedAsync.
// Orphan-session sweep (OrphanSessionSweepFunction, every 4h at :45) — dual-register per
// memory feedback_ops_event_types_dual_register. Backend helpers RecordOrphanSweep{Completed,
// Failed,SkippedLocked}Async, RecordOrphanReconcileCompletedAsync, RecordOrphanEventsCleanedAsync.
// Completed is the 4h heartbeat (Info); EventsCleaned (Warning) fires only when rows of a
// session without a Sessions row were removed; ReconcileCompleted is the weekly read-only
// EventTypeIndex check (Info at zero residue, Warning otherwise).
"OrphanSweepCompleted",
"OrphanSweepFailed",
"OrphanSweepSkippedLocked",
"OrphanReconcileCompleted",
"OrphanEventsCleaned",
],
// SLA breach evaluation (SlaBreachEvaluationService). Dual-register per memory
@@ -61,7 +61,8 @@ export const SHARED_MANIFEST = {
"vulnerabilityDataLastSyncUtc",
"msrcLastSyncUtc",
"nvdCacheLastRefreshUtc",
"epssLastSyncUtc"
"epssLastSyncUtc",
"orphanDeepCheckLastRunUtc"
],
"defaults": {
"partitionKey": "GlobalConfig",
@@ -118,7 +119,8 @@ export const SHARED_MANIFEST = {
"vulnerabilityDataLastSyncUtc": null,
"msrcLastSyncUtc": null,
"nvdCacheLastRefreshUtc": null,
"epssLastSyncUtc": null
"epssLastSyncUtc": null,
"orphanDeepCheckLastRunUtc": null
}
},
"tenantConfiguration": {
@@ -604,6 +606,10 @@ export const SHARED_MANIFEST = {
"SessionSweepFailed",
"OpsEventCleanup",
"OrphanEventsCleaned",
"OrphanSweepCompleted",
"OrphanSweepFailed",
"OrphanSweepSkippedLocked",
"OrphanReconcileCompleted",
"SessionDeletionMaintenanceStarted",
"SessionDeletionMaintenanceBudgetExceeded",
"SessionDeletionMaintenanceSkippedLocked",
@@ -57,7 +57,8 @@
"vulnerabilityDataLastSyncUtc",
"msrcLastSyncUtc",
"nvdCacheLastRefreshUtc",
"epssLastSyncUtc"
"epssLastSyncUtc",
"orphanDeepCheckLastRunUtc"
],
"defaults": {
"partitionKey": "GlobalConfig",
@@ -114,7 +115,8 @@
"vulnerabilityDataLastSyncUtc": null,
"msrcLastSyncUtc": null,
"nvdCacheLastRefreshUtc": null,
"epssLastSyncUtc": null
"epssLastSyncUtc": null,
"orphanDeepCheckLastRunUtc": null
}
},
"tenantConfiguration": {
@@ -600,6 +602,10 @@
"SessionSweepFailed",
"OpsEventCleanup",
"OrphanEventsCleaned",
"OrphanSweepCompleted",
"OrphanSweepFailed",
"OrphanSweepSkippedLocked",
"OrphanReconcileCompleted",
"SessionDeletionMaintenanceStarted",
"SessionDeletionMaintenanceBudgetExceeded",
"SessionDeletionMaintenanceSkippedLocked",
@@ -1750,6 +1756,14 @@
"name": "string"
},
"doc": "Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss."
},
{
"name": "orphanDeepCheckLastRunUtc",
"type": {
"kind": "primitive",
"name": "string"
},
"doc": "Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal."
}
]
},
@@ -200,6 +200,8 @@ export interface AdminConfiguration {
nvdCacheLastRefreshUtc: string;
/** Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss. */
epssLastSyncUtc: string;
/** Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal. */
orphanDeepCheckLastRunUtc: string;
}
/** The immutable Entra identity behind a cross-tenant-role UPN: the HOME tenant the UPN was granted for and, once known, the user's object id. A role row (GlobalAdmins / DelegatedAdmins / TenantGroupAssignments) confers nothing unless the caller's validated JWT matches this binding. */