mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-23 20:26:10 +00:00
Orphan sessions: ingest refuses sessions without a row, the sweep becomes their cascade in its own function (D-339)
A session without a Sessions row can never be reached by the cascade, so every row written for it was an orphan by construction. Three layers now keep the platform clean: - Prevent: IngestTelemetry answers a batch for an unknown session with 410 (same status and body as the cascade lock; request-row marker IngestRefusal = session_unknown | cascade_locked). The derived writers - vulnerability correlation, analyze, manual rescan - stop on SessionDeletionGuard.GetLivenessAsync != Live. The agent treats every 410 as terminal: one TelemetryRejectedSessionUnknown report (exempt from the reporter cooldown), then terminate_session with forced self-destruct (origin session_gone, no final drain, no diagnostics upload); spool items of a foreign session id are dropped at startup and never terminate the current session. - Detect: OrphanSessionSweepFunction (every 4h at :45, own lease) diffs the Sessions keys against the EventSessionIndex handles - two range drains instead of ~16.8k Sessions point reads per 2h maintenance run - and re-checks every candidate (handle unchanged, row still absent, tombstone) moments before deleting. The in-maintenance sweep raced the retention cascade on their shared 00:00/12:00 tick and reported false orphans; it is gone. - Clean: every table of the cascade manifest minus the tombstone pair (ratchet in TableLifecycleBucketTests), the inventory counters with the stamp on the handle (claim, then decrement), the handle last with If-Match. A weekly read-only reconciliation of EventTypeIndex against the Sessions keys (Sunday mornings UTC, and on every manual maintenance run) reports residue. MaintenanceCompleted now carries stepsMs. MCP search_sessions_by_event auto-exhausts empty-but-continuable pages. Manifests regenerated for the new ops event types and AdminConfiguration.OrphanDeepCheckLastRunUtc. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
+83
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
@@ -49,6 +50,74 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
private static EmergencyReporter Reporter(FakeApiClient client)
|
||||
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null);
|
||||
|
||||
private static EmergencyReporter Reporter(FakeApiClient client, VirtualClock clock)
|
||||
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null, clock: clock);
|
||||
|
||||
private static VirtualClock Clock() => new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc));
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_is_not_suppressed_by_the_cooldown_of_an_earlier_report()
|
||||
{
|
||||
// Terminal report, sent at most once per process right before the termination — a
|
||||
// routine report one minute earlier must not cost it. Every other type keeps the cooldown.
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: 503);
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(2, client.Calls);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, client.Reports[1].ErrorType);
|
||||
|
||||
// Control: a third report of another type inside the window is still held back.
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Second_session_unknown_report_within_the_window_is_still_suppressed()
|
||||
{
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(1, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_still_counts_against_the_per_session_maximum()
|
||||
{
|
||||
// Five delivered reports (ten minutes apart, distinct keys) exhaust the budget; the
|
||||
// exemption waives only the cooldown, not the maximum.
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
for (var i = 0; i < 6; i++) client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
foreach (var status in new[] { 500, 502, 503, 504, 429 })
|
||||
{
|
||||
await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: status);
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
}
|
||||
Assert.Equal(5, client.Calls);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(5, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failed_send_rolls_back_reservation_so_a_later_call_retries()
|
||||
{
|
||||
@@ -142,6 +211,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_carries_error_type_and_http_410()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
var report = Assert.Single(client.Reports);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType);
|
||||
Assert.Equal(410, report.HttpStatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_age_is_carried_in_the_report_and_absent_by_default()
|
||||
{
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
{
|
||||
/// <summary>
|
||||
/// Host wiring for <c>TelemetryUploadOrchestrator.SessionGone</c>
|
||||
/// (HTTP 410 on the current session's telemetry): exactly one
|
||||
/// <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/> emergency report carrying the
|
||||
/// status, then a synthetic forced <c>terminate_session</c> tagged <c>origin=session_gone</c>
|
||||
/// — and nothing at all once a shutdown is already in progress.
|
||||
/// </summary>
|
||||
public sealed class ServerControlPlaneSessionGoneTests
|
||||
{
|
||||
private sealed class RecordingEmergencyReporter : EmergencyReporter
|
||||
{
|
||||
private readonly List<string> _sequence;
|
||||
public List<(AgentErrorType ErrorType, string Message, int? HttpStatusCode)> Reports { get; } =
|
||||
new List<(AgentErrorType, string, int?)>();
|
||||
|
||||
public RecordingEmergencyReporter(List<string> sequence)
|
||||
: base(apiClient: null!, sessionId: "S1", tenantId: "T1", agentVersion: "0.0.0", logger: null!)
|
||||
{
|
||||
_sequence = sequence;
|
||||
}
|
||||
|
||||
public override Task TrySendAsync(
|
||||
AgentErrorType errorType,
|
||||
string message,
|
||||
int? httpStatusCode = null,
|
||||
long? sequenceNumber = null,
|
||||
int attempts = 1,
|
||||
TimeSpan? perAttemptTimeout = null,
|
||||
TimeSpan? retryDelay = null,
|
||||
double? sessionAgeHours = null,
|
||||
RegistrationFailureSummary? priorRegistrationFailure = null)
|
||||
{
|
||||
Reports.Add((errorType, message, httpStatusCode));
|
||||
_sequence.Add("report");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class RecordingDispatcher : ServerActionDispatcher
|
||||
{
|
||||
private readonly List<string> _sequence;
|
||||
public List<ServerAction> Dispatched { get; } = new List<ServerAction>();
|
||||
|
||||
public RecordingDispatcher(AgentConfiguration config, AgentLogger logger, InformationalEventPost post, List<string> sequence)
|
||||
: base(
|
||||
configuration: config,
|
||||
logger: logger,
|
||||
rotateConfigAsync: () => Task.FromResult(true),
|
||||
uploadDiagnosticsAsync: _ => Task.FromResult<DiagnosticsUploadResult>(null!),
|
||||
onTerminateRequested: _ => Task.CompletedTask,
|
||||
post: post)
|
||||
{
|
||||
_sequence = sequence;
|
||||
}
|
||||
|
||||
public override Task DispatchAsync(List<ServerAction> actions)
|
||||
{
|
||||
Dispatched.AddRange(actions);
|
||||
_sequence.Add("dispatch");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Rig : IDisposable
|
||||
{
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public AgentLogger Logger { get; }
|
||||
public List<string> Sequence { get; } = new List<string>();
|
||||
public RecordingEmergencyReporter Reporter { get; }
|
||||
public RecordingDispatcher Dispatcher { get; }
|
||||
public ManualResetEventSlim Shutdown { get; } = new ManualResetEventSlim(false);
|
||||
public ManualResetEventSlim ShutdownComplete { get; } = new ManualResetEventSlim(false);
|
||||
|
||||
public Rig()
|
||||
{
|
||||
Logger = new AgentLogger(Path.Combine(Tmp.Path, "logs"), AgentLogLevel.Info);
|
||||
var config = new AgentConfiguration { SessionId = "S1", TenantId = "T1", ApiBaseUrl = "http://localhost" };
|
||||
var post = new InformationalEventPost(
|
||||
new FakeSignalIngressSink(),
|
||||
new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc)));
|
||||
Reporter = new RecordingEmergencyReporter(Sequence);
|
||||
Dispatcher = new RecordingDispatcher(config, Logger, post, Sequence);
|
||||
}
|
||||
|
||||
public void Handle(string reason = "session gone: http 410") =>
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.HandleSessionGone(
|
||||
reason, Reporter, Dispatcher, Shutdown, ShutdownComplete, Logger);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Shutdown.Dispose();
|
||||
ShutdownComplete.Dispose();
|
||||
Tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reports_once_with_410_then_dispatches_forced_terminate_tagged_session_gone()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
|
||||
rig.Handle();
|
||||
|
||||
Assert.Equal(new[] { "report", "dispatch" }, rig.Sequence);
|
||||
|
||||
var report = Assert.Single(rig.Reporter.Reports);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType);
|
||||
Assert.Equal(410, report.HttpStatusCode);
|
||||
Assert.Contains("410", report.Message);
|
||||
|
||||
var action = Assert.Single(rig.Dispatcher.Dispatched);
|
||||
Assert.Equal(ServerActionTypes.TerminateSession, action.Type);
|
||||
Assert.Contains("410", action.Reason);
|
||||
Assert.NotNull(action.Params);
|
||||
var parameters = action.Params!;
|
||||
Assert.Equal("true", parameters["forceSelfDestruct"]);
|
||||
Assert.Equal("0", parameters["gracePeriodSeconds"]);
|
||||
Assert.Equal(TerminationOrigins.SessionGone, parameters["origin"]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, false)]
|
||||
[InlineData(false, true)]
|
||||
public void Does_nothing_when_a_shutdown_is_already_in_progress(bool shutdownSet, bool shutdownCompleteSet)
|
||||
{
|
||||
using var rig = new Rig();
|
||||
if (shutdownSet) rig.Shutdown.Set();
|
||||
if (shutdownCompleteSet) rig.ShutdownComplete.Set();
|
||||
|
||||
rig.Handle();
|
||||
|
||||
Assert.Empty(rig.Reporter.Reports);
|
||||
Assert.Empty(rig.Dispatcher.Dispatched);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReadOrigin_returns_the_origin_param_or_null()
|
||||
{
|
||||
Assert.Equal(TerminationOrigins.SessionGone,
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = TerminationOrigins.SessionGone }));
|
||||
Assert.Equal(TerminationOrigins.KillSignal,
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = TerminationOrigins.KillSignal, ["forceSelfDestruct"] = "true" }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["adminOutcome"] = "Failed" }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = string.Empty }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(null!));
|
||||
}
|
||||
}
|
||||
}
|
||||
+59
@@ -310,6 +310,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
new EnrollmentTerminatedEventArgs(
|
||||
reason, outcome, stage.ToString(), at ?? EndUtc);
|
||||
|
||||
/// <summary>
|
||||
/// The args <c>ServerControlPlane.OnTerminateRequested</c> synthesises for a
|
||||
/// <c>terminate_session</c> action tagged with <paramref name="origin"/>.
|
||||
/// </summary>
|
||||
private static EnrollmentTerminatedEventArgs ServerArgs(string origin) =>
|
||||
new EnrollmentTerminatedEventArgs(
|
||||
EnrollmentTerminationReason.DecisionTerminalStage,
|
||||
EnrollmentTerminationOutcome.Failed,
|
||||
SessionStage.Failed.ToString(),
|
||||
EndUtc,
|
||||
details: "Server-requested termination",
|
||||
origin: origin);
|
||||
|
||||
[Fact]
|
||||
public void Handle_writes_final_status_json_in_state_directory()
|
||||
{
|
||||
@@ -458,6 +471,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.Equal(0, rig.CleanupService.Invocations);
|
||||
}
|
||||
|
||||
// ============================================================= origin=session_gone (HTTP 410)
|
||||
|
||||
[Fact]
|
||||
public void Handle_session_gone_skips_spool_drain_and_diagnostics_but_still_cleans_up()
|
||||
{
|
||||
// The backend no longer knows the session: every drain would only burn its budget on
|
||||
// 410s and a diagnostics package would describe a deleted session. Marker, cleanup
|
||||
// and the (forced) self-destruct still run.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build();
|
||||
rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5);
|
||||
var spoolPolls = 0;
|
||||
rig.PendingItemCountAccessor = () => { spoolPolls++; return 5; }; // never drains — a poll would burn the budget
|
||||
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true);
|
||||
|
||||
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.SessionGone));
|
||||
|
||||
Assert.Equal(0, spoolPolls);
|
||||
Assert.Equal(0, rig.DiagnosticsUploads);
|
||||
Assert.DoesNotContain(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
|
||||
Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes);
|
||||
Assert.True(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
|
||||
Assert.Equal(1, rig.CleanupService.Invocations);
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_kill_signal_origin_still_drains_spool_and_uploads_diagnostics()
|
||||
{
|
||||
// The administrator kill signal keeps its full sequence — only session_gone skips.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build();
|
||||
rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5);
|
||||
var spoolPolls = 0;
|
||||
rig.PendingItemCountAccessor = () => { spoolPolls++; return 0; };
|
||||
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true);
|
||||
|
||||
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.KillSignal));
|
||||
|
||||
Assert.True(spoolPolls > 0);
|
||||
Assert.Equal(1, rig.DiagnosticsUploads);
|
||||
Assert.Contains(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
|
||||
Assert.Equal(1, rig.CleanupService.Invocations);
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_signals_shutdown_exactly_once_even_on_error()
|
||||
{
|
||||
|
||||
+35
@@ -275,9 +275,44 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.False(result.IsTransient);
|
||||
Assert.False(result.IsSessionGone);
|
||||
Assert.Contains(((int)status).ToString(), result.ErrorReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Gone_410_maps_to_SessionGone()
|
||||
{
|
||||
// 410 = the backend no longer knows this session. Not transient (no retry), not auth,
|
||||
// not poison, not a size problem — a distinct terminal kind the drain reacts to.
|
||||
using var rig = new Rig();
|
||||
rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "session unknown");
|
||||
|
||||
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.True(result.IsSessionGone);
|
||||
Assert.False(result.IsTransient);
|
||||
Assert.False(result.IsAuthFailure);
|
||||
Assert.False(result.IsPoison);
|
||||
Assert.False(result.RequiresSplit);
|
||||
Assert.Contains("410", result.ErrorReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Gone_410_with_poison_body_is_still_SessionGone_never_poison()
|
||||
{
|
||||
// A poison body on 410 must not be honoured: dropping items + emitting
|
||||
// telemetry_upload_poisoned would only produce the next 410 (emit/reject loop).
|
||||
using var rig = new Rig();
|
||||
rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "{\"poison\":true,\"rejectedRowKeys\":[\"rk1\"],\"reason\":\"x\"}");
|
||||
|
||||
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.True(result.IsSessionGone);
|
||||
Assert.False(result.IsPoison);
|
||||
Assert.Null(result.PoisonRowKeys);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.RequestTimeout)] // 408
|
||||
[InlineData((HttpStatusCode)429)] // TooManyRequests
|
||||
|
||||
+6
@@ -68,6 +68,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeBackendTelemetryUploader QueueSessionGone(string reason = "session gone: http 410")
|
||||
{
|
||||
_script.Enqueue(_ => UploadResult.SessionGone(reason));
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeBackendTelemetryUploader QueueThrow(Exception ex)
|
||||
{
|
||||
_script.Enqueue(_ => throw ex);
|
||||
|
||||
@@ -10,10 +10,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
public sealed class TelemetrySpoolTests
|
||||
{
|
||||
private static TelemetryItemDraft EventDraft(string rowKey = "row-1") =>
|
||||
private const string OwnKey = "tenant_session";
|
||||
private const string ForeignKey = "tenant_old-session";
|
||||
|
||||
private static TelemetryItemDraft EventDraft(string rowKey = "row-1") => EventDraft(rowKey, OwnKey);
|
||||
|
||||
private static TelemetryItemDraft EventDraft(string rowKey, string partitionKey) =>
|
||||
new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "tenant_session",
|
||||
partitionKey: partitionKey,
|
||||
rowKey: rowKey,
|
||||
payloadJson: "{\"EventType\":\"agent_started\"}",
|
||||
isSessionScoped: true);
|
||||
@@ -195,6 +200,86 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Equal(1, next.TelemetryItemId);
|
||||
}
|
||||
|
||||
// ============================================================ Startup partition-key filter
|
||||
|
||||
[Fact]
|
||||
public void Ctor_drops_leading_pending_items_of_another_session_and_persists_cursor()
|
||||
{
|
||||
// Same spool directory, new session id: the previous session's unsent items would only
|
||||
// draw a 410 that must not end the new session. They are dropped on startup; the own
|
||||
// item behind them stays pending.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
s1.Enqueue(EventDraft("own", OwnKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(1, s2.LastUploadedItemId);
|
||||
Assert.Equal(2, s2.LastAssignedItemId);
|
||||
Assert.Equal(1, s2.PendingItemCount);
|
||||
Assert.Equal("own", Assert.Single(s2.Peek(10)).RowKey);
|
||||
|
||||
// Cursor persisted: a later start without the filter sees the same tail with the same bytes.
|
||||
var s3 = new TelemetrySpool(tmp.Path, Clock());
|
||||
Assert.Equal(1, s3.LastUploadedItemId);
|
||||
Assert.Equal(1, s3.PendingItemCount);
|
||||
Assert.Equal(s3.PendingBytes, s2.PendingBytes);
|
||||
|
||||
var next = s2.Enqueue(EventDraft("next", OwnKey));
|
||||
Assert.Equal(3, next.TelemetryItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_drops_nothing_when_a_foreign_item_follows_an_own_item()
|
||||
{
|
||||
// Impossible by construction (one process = one session); if it happens the order is
|
||||
// not trustworthy, so the filter keeps everything instead of guessing.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("own", OwnKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(3, s2.PendingItemCount);
|
||||
Assert.Equal(s1.PendingBytes, s2.PendingBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_keeps_pending_items_that_carry_the_expected_key()
|
||||
{
|
||||
// WhiteGlove Part 2 reuses the Part-1 session id: its leftovers carry the expected key.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("a", OwnKey));
|
||||
s1.Enqueue(EventDraft("b", OwnKey));
|
||||
s1.Enqueue(EventDraft("c", OwnKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(3, s2.PendingItemCount);
|
||||
Assert.Equal(s1.PendingBytes, s2.PendingBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_without_expected_key_keeps_foreign_items()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock());
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(2, s2.PendingItemCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Enqueue_with_RequiresImmediateFlush_raises_ImmediateFlushRequested()
|
||||
{
|
||||
|
||||
+118
-2
@@ -10,10 +10,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
public sealed class TelemetryUploadOrchestratorTests
|
||||
{
|
||||
private static TelemetryItemDraft Draft(string rowKey) =>
|
||||
private const string OwnKey = "tenant_session";
|
||||
|
||||
private static TelemetryItemDraft Draft(string rowKey) => Draft(rowKey, OwnKey);
|
||||
|
||||
private static TelemetryItemDraft Draft(string rowKey, string partitionKey) =>
|
||||
new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "tenant_session",
|
||||
partitionKey: partitionKey,
|
||||
rowKey: rowKey,
|
||||
payloadJson: $"{{\"rk\":\"{rowKey}\"}}",
|
||||
isSessionScoped: true);
|
||||
@@ -157,6 +161,118 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Equal(1, blockedCount);
|
||||
}
|
||||
|
||||
// ============================================================ HTTP 410 — session gone
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_for_current_session_stops_drain_retains_cursor_and_raises_once()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("a"));
|
||||
spool.Enqueue(Draft("b"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
var gone = new System.Collections.Generic.List<string>();
|
||||
int blockedCount = 0;
|
||||
PoisonReport? poison = null;
|
||||
sut.SessionGone += (_, reason) => gone.Add(reason);
|
||||
sut.UploadBlocked += (_, __) => blockedCount++;
|
||||
sut.BatchPoisoned += (_, p) => poison = p;
|
||||
|
||||
var first = await sut.DrainAllAsync();
|
||||
|
||||
Assert.False(first.Success);
|
||||
Assert.Equal(1, first.FailedBatches);
|
||||
Assert.Equal(0, first.UploadedItems);
|
||||
Assert.Equal(1, uploader.CallCount); // no retries — nothing can make a 410 succeed
|
||||
Assert.Equal(-1, sut.LastUploadedItemId); // retained — items stay on disk for the diag ZIP
|
||||
Assert.Single(gone);
|
||||
Assert.Contains("410", gone[0]);
|
||||
Assert.Equal(0, blockedCount); // not a "blocked" pipeline
|
||||
Assert.Null(poison);
|
||||
|
||||
// One-shot: the next drain hits the 410 again but does not raise again.
|
||||
var second = await sut.DrainAllAsync();
|
||||
Assert.False(second.Success);
|
||||
Assert.Equal(2, uploader.CallCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
Assert.Single(gone);
|
||||
Assert.Equal(0, blockedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_for_leading_foreign_items_drops_only_them_and_continues()
|
||||
{
|
||||
// Leftovers of a previous session id at the head of the spool (defence in depth — the
|
||||
// spool filters these at startup). A 410 for them must not end the current session:
|
||||
// drop exactly the leading foreign run, then carry on with the current session's items.
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("old-a", "tenant_old-session"));
|
||||
spool.Enqueue(Draft("old-b", "tenant_old-session"));
|
||||
spool.Enqueue(Draft("new", OwnKey));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueOk();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
int goneCount = 0, blockedCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
sut.UploadBlocked += (_, __) => blockedCount++;
|
||||
|
||||
var result = await sut.DrainAllAsync();
|
||||
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal(0, result.FailedBatches);
|
||||
Assert.Equal(1, result.UploadedItems);
|
||||
Assert.Equal(2, uploader.CallCount);
|
||||
Assert.Equal(3, uploader.Received[0].Count); // first batch carried all three
|
||||
Assert.Equal("new", Assert.Single(uploader.Received[1]).RowKey); // retry carried only the own item
|
||||
Assert.Equal(2, sut.LastUploadedItemId);
|
||||
Assert.Equal(0, goneCount);
|
||||
Assert.Equal(0, blockedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_is_not_latched_before_a_handler_is_subscribed()
|
||||
{
|
||||
// The host subscribes after EnrollmentOrchestrator.Start; a 410 on the very first drain
|
||||
// must still reach it on the next drain instead of being swallowed by the one-shot guard.
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("only"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
|
||||
await sut.DrainAllAsync(); // nobody listening yet
|
||||
|
||||
int goneCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
await sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, goneCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_without_a_current_key_treats_every_batch_as_own()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("x", "tenant_whatever"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock());
|
||||
int goneCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
|
||||
var result = await sut.DrainAllAsync();
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.Equal(1, goneCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TooLarge_splits_batch_and_uploads_everything()
|
||||
{
|
||||
|
||||
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
@@ -14,6 +15,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// - Same error key (ErrorType + HTTP status code) is only ever sent once per session
|
||||
/// - At most <see cref="MaxReportsPerSession"/> emergency reports total per session
|
||||
/// - At most one report every <see cref="MinIntervalMinutes"/> minutes regardless of error type
|
||||
/// (the terminal <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/> report is exempt)
|
||||
///
|
||||
/// A failure in this reporter is silently swallowed — it must never cascade into the main loop.
|
||||
/// </summary>
|
||||
@@ -32,6 +34,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
private readonly string _tenantId;
|
||||
private readonly string _agentVersion;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly IClock _clock;
|
||||
|
||||
private readonly HashSet<string> _reportedKeys = new HashSet<string>();
|
||||
private readonly object _antiFloodLock = new object();
|
||||
@@ -43,13 +46,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
string agentVersion,
|
||||
AgentLogger logger)
|
||||
AgentLogger logger,
|
||||
IClock clock = null)
|
||||
{
|
||||
_apiClient = apiClient;
|
||||
_sessionId = sessionId;
|
||||
_tenantId = tenantId;
|
||||
_agentVersion = agentVersion;
|
||||
_logger = logger;
|
||||
_clock = clock ?? SystemClock.Instance;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -100,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
// All anti-flood checks are protected by a lock to prevent race conditions when
|
||||
// TrySendAsync is called concurrently (fire-and-forget from multiple threads).
|
||||
var key = $"{errorType}:{httpStatusCode}";
|
||||
var now = _clock.UtcNow;
|
||||
int currentReportCount;
|
||||
DateTime? previousLastReportTime;
|
||||
|
||||
@@ -117,8 +123,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
return;
|
||||
}
|
||||
|
||||
if (_lastReportTime != null &&
|
||||
(DateTime.UtcNow - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
|
||||
// The session-unknown report is terminal and sent at most once per process (the
|
||||
// termination follows immediately): a routine report minutes earlier must not cost
|
||||
// the last word about the session. Only the cooldown is waived; key dedup and the
|
||||
// per-session maximum still apply.
|
||||
var exemptFromCooldown = errorType == AgentErrorType.TelemetryRejectedSessionUnknown;
|
||||
if (!exemptFromCooldown && _lastReportTime != null &&
|
||||
(now - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
|
||||
{
|
||||
_logger?.Debug($"[EmergencyChannel] Suppressed (cooldown active, last report at {_lastReportTime.Value:HH:mm:ss}): {key}");
|
||||
return;
|
||||
@@ -127,7 +138,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
previousLastReportTime = _lastReportTime;
|
||||
_reportedKeys.Add(key);
|
||||
_reportCount++;
|
||||
_lastReportTime = DateTime.UtcNow;
|
||||
_lastReportTime = now;
|
||||
currentReportCount = _reportCount;
|
||||
}
|
||||
|
||||
@@ -145,7 +156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
SessionAgeHours = sessionAgeHours,
|
||||
PriorRegistrationFailure = priorRegistrationFailure,
|
||||
AgentVersion = _agentVersion,
|
||||
Timestamp = DateTime.UtcNow
|
||||
Timestamp = now
|
||||
};
|
||||
|
||||
var delivered = false;
|
||||
|
||||
@@ -371,8 +371,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
// 3) Telemetry-Transport. Batch size flows from AgentConfiguration.MaxBatchSize
|
||||
// via Program.cs (P1 fix: previously the remote-config knob was merged but
|
||||
// never applied, so tenants saw drainInterval/MaxBatchSize have no effect).
|
||||
_spool = new TelemetrySpool(_transportDirectory, _clock, _logger);
|
||||
_transport = new TelemetryUploadOrchestrator(_spool, _uploader, _clock, batchSize: _uploadBatchSize);
|
||||
// The partition key names the registered session: the spool drops leftovers of a
|
||||
// previous session id on startup, and the orchestrator tells a 410 for this session
|
||||
// (SessionGone → host terminates) from a 410 for such leftovers (dropped).
|
||||
var partitionKey = TelemetryPartitionKey.ForSession(_tenantId, _sessionId);
|
||||
_spool = new TelemetrySpool(_transportDirectory, _clock, _logger, expectedPartitionKey: partitionKey);
|
||||
_transport = new TelemetryUploadOrchestrator(
|
||||
_spool, _uploader, _clock, batchSize: _uploadBatchSize, currentPartitionKey: partitionKey, logger: _logger);
|
||||
|
||||
// TRACE-H1 — surface skipped oversized items + blocked-pipeline conditions as timeline
|
||||
// events. Raised after the drain guard releases, so posting back through the ingress
|
||||
|
||||
+25
-1
@@ -73,13 +73,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// </summary>
|
||||
public string? StopReason { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <c>origin</c> param of a server-requested <c>terminate_session</c> action
|
||||
/// (<see cref="TerminationOrigins"/>); null for every termination the agent decided
|
||||
/// itself. <see cref="TerminationOrigins.SessionGone"/> makes the termination handler
|
||||
/// skip the spool drain and the diagnostics upload — the backend no longer knows the
|
||||
/// session, so both could only fail.
|
||||
/// </summary>
|
||||
public string? Origin { get; }
|
||||
|
||||
public EnrollmentTerminatedEventArgs(
|
||||
EnrollmentTerminationReason reason,
|
||||
EnrollmentTerminationOutcome outcome,
|
||||
string? stageName,
|
||||
DateTime terminatedAtUtc,
|
||||
string? details = null,
|
||||
string? stopReason = null)
|
||||
string? stopReason = null,
|
||||
string? origin = null)
|
||||
{
|
||||
Reason = reason;
|
||||
Outcome = outcome;
|
||||
@@ -87,6 +97,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
TerminatedAtUtc = terminatedAtUtc;
|
||||
Details = details;
|
||||
StopReason = stopReason;
|
||||
Origin = origin;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Values of the <c>origin</c> param on a synthesised <c>terminate_session</c>
|
||||
/// <c>ServerAction</c>, carried into <see cref="EnrollmentTerminatedEventArgs.Origin"/>.
|
||||
/// </summary>
|
||||
public static class TerminationOrigins
|
||||
{
|
||||
/// <summary>Administrator kill signal (<c>DeviceKillSignal</c> on a 2xx telemetry response).</summary>
|
||||
public const string KillSignal = "kill_signal";
|
||||
|
||||
/// <summary>The backend answered HTTP 410 for the current session's telemetry.</summary>
|
||||
public const string SessionGone = "session_gone";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Events
|
||||
_sequenceCounter = sequenceCounter ?? throw new ArgumentNullException(nameof(sequenceCounter));
|
||||
_sessionId = sessionId;
|
||||
_tenantId = tenantId;
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
_traceEventsEnabled = traceEventsEnabled;
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Signals
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId));
|
||||
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Transitions
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId));
|
||||
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+29
-6
@@ -306,7 +306,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
});
|
||||
|
||||
DelayLateEventGrace();
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
// Option 2 (WG Part 1 graceful-exit hardening, 2026-04-30): write the
|
||||
// clean-exit marker BEFORE _signalShutdown returns control to the main
|
||||
@@ -329,7 +329,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
|
||||
RunUploadDiagnosticsWithEvents(args);
|
||||
WriteEnrollmentCompleteMarker(args);
|
||||
RunStandaloneRebootIfRequested();
|
||||
RunStandaloneRebootIfRequested(args);
|
||||
RunSelfDestructIfAppropriate(args);
|
||||
// Option 2 (same hardening) — covers the standard Completed / Failed terminal
|
||||
// path too. Cleanup PowerShell does not touch this marker, so writing it before
|
||||
@@ -979,6 +979,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// </param>
|
||||
private void RunUploadDiagnosticsWithEvents(EnrollmentTerminatedEventArgs args, string suffixOverride = null)
|
||||
{
|
||||
if (IsSessionGone(args))
|
||||
{
|
||||
// The backend answered 410 for this session: the package would describe a session
|
||||
// that no longer exists, and its diagnostics_* events could never be accepted.
|
||||
_logger.Info("EnrollmentTerminationHandler: diagnostics upload skipped — backend no longer knows this session.");
|
||||
return;
|
||||
}
|
||||
|
||||
var mode = _configuration.DiagnosticsUploadMode ?? "Off";
|
||||
if (!_configuration.DiagnosticsUploadEnabled || string.Equals(mode, "Off", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
@@ -1124,7 +1132,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// but enables <c>RebootOnComplete</c>, the agent's final act is <c>shutdown.exe /r</c>
|
||||
/// with the configured delay, giving the user a visible countdown.
|
||||
/// </summary>
|
||||
private void RunStandaloneRebootIfRequested()
|
||||
private void RunStandaloneRebootIfRequested(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
if (_configuration.SelfDestructOnComplete) return;
|
||||
if (!_configuration.RebootOnComplete) return;
|
||||
@@ -1148,7 +1156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
ImmediateUpload = true,
|
||||
});
|
||||
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -1185,7 +1193,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
// just above. The WG-Part-1 and standalone-reboot paths already drain first; this
|
||||
// closes the gap on the self-destruct path. Bounded + best-effort (never blocks
|
||||
// termination on a wedged backend).
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
var service = _cleanupServiceFactory();
|
||||
service.ExecuteSelfDestruct();
|
||||
@@ -1318,8 +1326,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
}
|
||||
}
|
||||
|
||||
private void DrainSpool()
|
||||
/// <summary>
|
||||
/// <see cref="TerminationOrigins.SessionGone"/>: the backend no longer knows the session.
|
||||
/// Every drain would only collect more 410s against a bounded budget, and a diagnostics
|
||||
/// package would describe a deleted session — both are skipped; marker, cleanup and
|
||||
/// self-destruct run as usual.
|
||||
/// </summary>
|
||||
private static bool IsSessionGone(EnrollmentTerminatedEventArgs args) =>
|
||||
string.Equals(args.Origin, TerminationOrigins.SessionGone, StringComparison.Ordinal);
|
||||
|
||||
private void DrainSpool(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
if (IsSessionGone(args))
|
||||
{
|
||||
_logger.Info("EnrollmentTerminationHandler: spool drain skipped — backend no longer knows this session.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Block briefly so pending events can land before the next destructive step
|
||||
// (shutdown.exe, self-destruct). Two phases share the same bounded budget:
|
||||
//
|
||||
|
||||
+10
@@ -165,6 +165,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
_logger?.Warning($"BackendTelemetryUploader: ingest TRANSIENT (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
}
|
||||
else if (result.IsSessionGone)
|
||||
{
|
||||
_logger?.Error($"BackendTelemetryUploader: ingest rejected — backend no longer knows this session (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger?.Error($"BackendTelemetryUploader: ingest PERMANENT fail (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
@@ -208,6 +212,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
|
||||
case HttpStatusCode.RequestEntityTooLarge: // 413 — the data is fine, only the batch
|
||||
return UploadResult.TooLarge(shortReason); // size is wrong → split & retry (P1).
|
||||
|
||||
case HttpStatusCode.Gone: // 410 — the backend no longer knows this session.
|
||||
// Decided BEFORE the poison-body check: a poison body on 410 would make the
|
||||
// drain drop items and emit telemetry_upload_poisoned, which the backend
|
||||
// rejects with 410 again — an emit/reject loop for a session that is over.
|
||||
return UploadResult.SessionGone($"session gone: {shortReason}");
|
||||
}
|
||||
|
||||
if (statusCode >= 500 && statusCode <= 599) return UploadResult.Transient(shortReason);
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
#nullable enable
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
/// <summary>
|
||||
/// The one composition of a session's Azure-Table partition key. The three emitters stamp
|
||||
/// it on every <see cref="TelemetryItem"/>; the spool and the upload orchestrator compare
|
||||
/// against it to tell the current session's items from leftovers of a previous session id.
|
||||
/// A drift between producer and comparer would silently drop or retain the wrong items.
|
||||
/// </summary>
|
||||
public static class TelemetryPartitionKey
|
||||
{
|
||||
public static string ForSession(string tenantId, string sessionId) => $"{tenantId}_{sessionId}";
|
||||
}
|
||||
}
|
||||
@@ -74,7 +74,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
public int ByteLength { get; }
|
||||
}
|
||||
|
||||
public TelemetrySpool(string directoryPath, IClock clock, Logging.AgentLogger? logger = null)
|
||||
/// <param name="expectedPartitionKey">
|
||||
/// Partition key of the session this process serves (<see cref="TelemetryPartitionKey.ForSession"/>).
|
||||
/// When set, pending items of another session id found at the head of the spool are dropped
|
||||
/// on startup — see <see cref="DropLeadingForeignPending"/>. Null disables the filter.
|
||||
/// </param>
|
||||
public TelemetrySpool(
|
||||
string directoryPath,
|
||||
IClock clock,
|
||||
Logging.AgentLogger? logger = null,
|
||||
string? expectedPartitionKey = null)
|
||||
{
|
||||
if (string.IsNullOrEmpty(directoryPath))
|
||||
{
|
||||
@@ -96,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// count as "pending" and need to be rehydrated into _pending.
|
||||
_lastUploadedItemId = _cursor.Load();
|
||||
_lastAssignedItemId = ScanSpoolAndRehydratePending();
|
||||
DropLeadingForeignPending(expectedPartitionKey);
|
||||
_peakPendingCount = _pending.Count;
|
||||
}
|
||||
|
||||
@@ -312,5 +322,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
}
|
||||
return highest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Startup filter: drops the LEADING run of pending items whose partition key is not
|
||||
/// <paramref name="expectedPartitionKey"/>, advancing and persisting the cursor past them.
|
||||
/// Such items are leftovers of a previous session id in the same spool directory; the
|
||||
/// backend answers 410 for them, and that 410 must never end the session this process
|
||||
/// serves. One process serves one session, so a foreign item AFTER an own item cannot
|
||||
/// occur — if it does, the order is not trustworthy and nothing is dropped. WhiteGlove
|
||||
/// Part 2 reuses the Part-1 session id, so its leftovers carry the expected key and stay.
|
||||
/// Called once from the ctor, after rehydration.
|
||||
/// </summary>
|
||||
private void DropLeadingForeignPending(string? expectedPartitionKey)
|
||||
{
|
||||
if (string.IsNullOrEmpty(expectedPartitionKey) || _pending.Count == 0) return;
|
||||
|
||||
var leadingForeign = 0;
|
||||
long lastForeignId = -1;
|
||||
var seenOwn = false;
|
||||
foreach (var entry in _pending)
|
||||
{
|
||||
var isForeign = !string.Equals(entry.Item.PartitionKey, expectedPartitionKey, StringComparison.Ordinal);
|
||||
if (!isForeign)
|
||||
{
|
||||
seenOwn = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!seenOwn)
|
||||
{
|
||||
leadingForeign++;
|
||||
lastForeignId = entry.Item.TelemetryItemId;
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger?.Error(
|
||||
$"TelemetrySpool: pending itemId={entry.Item.TelemetryItemId} carries a foreign partition key after items of the current session — spool order is inconsistent; keeping all {_pending.Count} pending item(s).");
|
||||
return;
|
||||
}
|
||||
|
||||
if (leadingForeign == 0) return;
|
||||
|
||||
try
|
||||
{
|
||||
_cursor.Save(lastForeignId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Keep the items rather than risk a cursor that disagrees with disk; the drain
|
||||
// loop drops the same leading run on its first 410 (defence in depth).
|
||||
_logger?.Warning($"TelemetrySpool: could not persist the cursor past {leadingForeign} foreign pending item(s): {ex.Message}");
|
||||
return;
|
||||
}
|
||||
|
||||
_lastUploadedItemId = lastForeignId;
|
||||
for (var i = 0; i < leadingForeign; i++)
|
||||
{
|
||||
_pendingBytes -= _pending.First!.Value.ByteLength;
|
||||
_pending.RemoveFirst();
|
||||
}
|
||||
|
||||
_logger?.Warning(
|
||||
$"TelemetrySpool: dropped {leadingForeign} pending item(s) of a previous session id (cursor advanced to {lastForeignId}, {_pending.Count} pending item(s) of the current session kept).");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+81
-1
@@ -45,12 +45,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// telemetry_upload_blocked exactly once per process rather than on every 30 s drain.
|
||||
private bool _uploadBlockedReported;
|
||||
|
||||
// Partition key of the session this process serves; null = every batch counts as own.
|
||||
private readonly string? _currentPartitionKey;
|
||||
private readonly Logging.AgentLogger? _logger;
|
||||
|
||||
// One-shot guard for SessionGone. Latched only when a handler is subscribed: the host
|
||||
// wires the handler after EnrollmentOrchestrator.Start, and a 410 on the very first
|
||||
// drain must not be swallowed before anyone listens.
|
||||
private bool _sessionGoneReported;
|
||||
|
||||
/// <param name="currentPartitionKey">
|
||||
/// <see cref="TelemetryPartitionKey.ForSession"/> of the registered session. A 410 on a
|
||||
/// batch that starts with this key ends the session (<see cref="SessionGone"/>); a 410 on
|
||||
/// a batch that starts with any other key only drops those leftover items.
|
||||
/// </param>
|
||||
public TelemetryUploadOrchestrator(
|
||||
ITelemetrySpool spool,
|
||||
IBackendTelemetryUploader uploader,
|
||||
IClock clock,
|
||||
int batchSize = 100,
|
||||
IReadOnlyList<TimeSpan>? retryBackoffs = null)
|
||||
IReadOnlyList<TimeSpan>? retryBackoffs = null,
|
||||
string? currentPartitionKey = null,
|
||||
Logging.AgentLogger? logger = null)
|
||||
{
|
||||
_spool = spool ?? throw new ArgumentNullException(nameof(spool));
|
||||
_uploader = uploader ?? throw new ArgumentNullException(nameof(uploader));
|
||||
@@ -61,6 +77,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
_effectiveBatchSize = batchSize;
|
||||
|
||||
_retryBackoffs = retryBackoffs ?? DefaultRetryBackoffs;
|
||||
_currentPartitionKey = currentPartitionKey;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -97,6 +115,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// </summary>
|
||||
public event EventHandler<string>? UploadBlocked;
|
||||
|
||||
/// <summary>
|
||||
/// Raised once per process (after the drain guard releases) when the backend answered
|
||||
/// HTTP 410 for a batch of the CURRENT session: the session is over server-side (never
|
||||
/// registered, deleted, or being deleted) and no retry can succeed. The batch is retained
|
||||
/// (cursor stays) and the drain stops; the host reports once over the emergency channel
|
||||
/// and terminates the agent with self-destruct. Carries the backend error reason.
|
||||
/// </summary>
|
||||
public event EventHandler<string>? SessionGone;
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> when the last successful upload returned <see cref="UploadResult.DeviceBlocked"/>
|
||||
/// and the paused-until window has not expired. Drain is a no-op in this state.
|
||||
@@ -124,6 +151,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
List<UploadResult>? pendingSignals = null;
|
||||
List<PoisonReport>? pendingPoison = null;
|
||||
string? pendingBlockedReason = null;
|
||||
string? pendingSessionGoneReason = null;
|
||||
DrainResult result;
|
||||
|
||||
try
|
||||
@@ -247,6 +275,35 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
new PoisonReport(1, oversizeId, "oversize: " + outcome.ErrorReason, PoisonKind.Oversize));
|
||||
break;
|
||||
}
|
||||
else if (outcome.IsSessionGone)
|
||||
{
|
||||
// HTTP 410 — the backend no longer knows the session these items belong to.
|
||||
var foreignRun = CountLeadingForeignItems(batch);
|
||||
if (foreignRun > 0)
|
||||
{
|
||||
// Leftovers of a previous session id (the spool drops these at startup;
|
||||
// this is defence in depth). Drop only that leading run and carry on: a
|
||||
// legitimate new session must never be ended because of old items.
|
||||
var throughId = batch[foreignRun - 1].TelemetryItemId;
|
||||
_spool.MarkUploaded(throughId);
|
||||
_logger?.Warning(
|
||||
$"TelemetryUploadOrchestrator: dropped {foreignRun} item(s) of a previous session id the backend rejected with 410 (cursor advanced to {throughId}).");
|
||||
continue;
|
||||
}
|
||||
|
||||
// The current session is over server-side. RETAIN (cursor stays, the items
|
||||
// remain on disk for the diagnostics ZIP), stop this drain, and surface it
|
||||
// once — not as telemetry_upload_blocked, which would only queue another
|
||||
// item behind a batch that can never be accepted.
|
||||
failedBatches++;
|
||||
lastError = outcome.ErrorReason;
|
||||
if (!_sessionGoneReported && SessionGone != null)
|
||||
{
|
||||
_sessionGoneReported = true;
|
||||
pendingSessionGoneReason = outcome.ErrorReason;
|
||||
}
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
// P1 — RETAIN. Transient (retry next drain), auth-permanent (uploader already
|
||||
@@ -301,9 +358,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
if (pendingSessionGoneReason != null)
|
||||
{
|
||||
try { SessionGone?.Invoke(this, pendingSessionGoneReason); }
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Number of items at the head of <paramref name="batch"/> whose partition key is not the
|
||||
/// current session's. 0 when no current key is configured (every batch counts as own).
|
||||
/// </summary>
|
||||
private int CountLeadingForeignItems(IReadOnlyList<TelemetryItem> batch)
|
||||
{
|
||||
if (_currentPartitionKey == null) return 0;
|
||||
|
||||
var count = 0;
|
||||
while (count < batch.Count
|
||||
&& !string.Equals(batch[count].PartitionKey, _currentPartitionKey, StringComparison.Ordinal))
|
||||
{
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private static bool CarriesSignal(UploadResult outcome) =>
|
||||
outcome.DeviceBlocked
|
||||
|| outcome.DeviceKillSignal
|
||||
|
||||
@@ -30,7 +30,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
bool isAuthFailure = false,
|
||||
bool requiresSplit = false,
|
||||
bool isPoison = false,
|
||||
IReadOnlyList<string>? poisonRowKeys = null)
|
||||
IReadOnlyList<string>? poisonRowKeys = null,
|
||||
bool isSessionGone = false)
|
||||
{
|
||||
Success = success;
|
||||
ErrorReason = errorReason;
|
||||
@@ -44,6 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
RequiresSplit = requiresSplit;
|
||||
IsPoison = isPoison;
|
||||
PoisonRowKeys = poisonRowKeys;
|
||||
IsSessionGone = isSessionGone;
|
||||
}
|
||||
|
||||
public bool Success { get; }
|
||||
@@ -81,6 +83,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// <summary>The RowKeys the backend named as poison (item-level). Non-empty when <see cref="IsPoison"/>.</summary>
|
||||
public IReadOnlyList<string>? PoisonRowKeys { get; }
|
||||
|
||||
/// <summary>
|
||||
/// HTTP 410: the backend no longer knows the session the batch belongs to (never
|
||||
/// registered, deleted, or being deleted). Not transient, not auth, not poison — retrying
|
||||
/// can never succeed. The orchestrator stops the drain (cursor stays) and raises
|
||||
/// <see cref="TelemetryUploadOrchestrator.SessionGone"/> when the batch belongs to the
|
||||
/// current session, or drops the batch's leading items when they are leftovers of a
|
||||
/// previous session id.
|
||||
/// </summary>
|
||||
public bool IsSessionGone { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Non-terminal quarantine signalled by the backend. The agent should stop draining
|
||||
/// uploads until <see cref="UnblockAt"/> (if set) elapses — the session stays alive so
|
||||
@@ -165,5 +177,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
isPoison: true, poisonRowKeys: rejectedRowKeys ?? throw new ArgumentNullException(nameof(rejectedRowKeys)));
|
||||
|
||||
/// <summary>
|
||||
/// HTTP 410 — the backend no longer knows the session. Flagged <see cref="IsSessionGone"/>;
|
||||
/// never retried, never split, never treated as poison.
|
||||
/// </summary>
|
||||
public static UploadResult SessionGone(string reason) =>
|
||||
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
isSessionGone: true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,6 +439,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
lifecyclePost,
|
||||
() => terminationHandler,
|
||||
agentConfig,
|
||||
auth.EmergencyReporter,
|
||||
shutdown,
|
||||
shutdownComplete,
|
||||
logger);
|
||||
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Termination;
|
||||
using AutopilotMonitor.Shared;
|
||||
@@ -103,6 +105,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// dispatcher because these are real actions, not admin-button echoes.</item>
|
||||
/// <item><b>DeviceBlocked</b> — non-terminal upload pause. Transport already reacted in
|
||||
/// <c>TelemetryUploadOrchestrator.ApplyControlSignals</c>; here we only log.</item>
|
||||
/// <item><b>SessionGone</b> (<see cref="TelemetryUploadOrchestrator.SessionGone"/>, HTTP 410
|
||||
/// on the current session's telemetry) — one emergency report, then the kill-signal
|
||||
/// termination path tagged <c>origin=session_gone</c>. See <see cref="HandleSessionGone"/>.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// <b>Shutdown race guard</b>: if the agent has already finished its own termination
|
||||
@@ -117,9 +122,14 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
InformationalEventPost post,
|
||||
Func<EnrollmentTerminationHandler> terminationHandlerAccessor,
|
||||
AgentConfiguration agentConfig,
|
||||
EmergencyReporter emergencyReporter,
|
||||
ManualResetEventSlim shutdown,
|
||||
ManualResetEventSlim shutdownComplete,
|
||||
AgentLogger logger)
|
||||
{
|
||||
orchestrator.Transport.SessionGone += (sender, reason) =>
|
||||
HandleSessionGone(reason, emergencyReporter, dispatcher, shutdown, shutdownComplete, logger);
|
||||
|
||||
orchestrator.Transport.ServerResponseReceived += (sender, upload) =>
|
||||
{
|
||||
if (shutdownComplete.IsSet)
|
||||
@@ -140,7 +150,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{
|
||||
{ "forceSelfDestruct", "true" },
|
||||
{ "gracePeriodSeconds", "0" },
|
||||
{ "origin", "kill_signal" },
|
||||
{ "origin", TerminationOrigins.KillSignal },
|
||||
},
|
||||
};
|
||||
try { dispatcher.DispatchAsync(new List<ServerAction> { killAction }).GetAwaiter().GetResult(); }
|
||||
@@ -181,6 +191,76 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
};
|
||||
}
|
||||
|
||||
// Bounded blocking send for the one session-gone report: the terminate that follows ends
|
||||
// the process and the cleanup script deletes the agent, so a fire-and-forget send would
|
||||
// be abandoned mid-flight. Retries stay inside one anti-flood reservation (ONE report).
|
||||
private const int SessionGoneReportAttempts = 2;
|
||||
private static readonly TimeSpan SessionGoneReportTimeout = TimeSpan.FromSeconds(15);
|
||||
private static readonly TimeSpan SessionGoneReportRetryDelay = TimeSpan.FromSeconds(3);
|
||||
private static readonly TimeSpan SessionGoneReportBudget = TimeSpan.FromSeconds(40);
|
||||
|
||||
/// <summary>
|
||||
/// The backend answered HTTP 410 for the current session's telemetry: the session is over
|
||||
/// server-side. (a) Send ONE <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/>
|
||||
/// report over the emergency channel — the telemetry channel is exactly what was refused,
|
||||
/// and the backend accepts this report for unknown sessions. (b) Terminate through the same
|
||||
/// synthesised <c>terminate_session</c> path as the administrator kill signal (forced
|
||||
/// self-destruct, no grace), tagged <see cref="TerminationOrigins.SessionGone"/> so the
|
||||
/// termination handler skips the spool drain and the diagnostics upload. A shutdown that is
|
||||
/// already in progress owns the outcome — then nothing happens here.
|
||||
/// </summary>
|
||||
internal static void HandleSessionGone(
|
||||
string reason,
|
||||
EmergencyReporter emergencyReporter,
|
||||
ServerActionDispatcher dispatcher,
|
||||
ManualResetEventSlim shutdown,
|
||||
ManualResetEventSlim shutdownComplete,
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (shutdownComplete.IsSet || shutdown.IsSet)
|
||||
{
|
||||
logger.Debug("ServerControlPlane: session gone (HTTP 410) while shutdown is already in progress — ignoring.");
|
||||
return;
|
||||
}
|
||||
|
||||
logger.Error($"Backend no longer knows this session (HTTP 410: {reason}) — reporting once over the emergency channel, then terminating with self-destruct.");
|
||||
|
||||
if (emergencyReporter != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
emergencyReporter
|
||||
.TrySendAsync(
|
||||
AgentErrorType.TelemetryRejectedSessionUnknown,
|
||||
$"Telemetry upload rejected with HTTP 410 — backend no longer knows this session ({reason}). Agent terminating with self-destruct.",
|
||||
httpStatusCode: (int)HttpStatusCode.Gone,
|
||||
attempts: SessionGoneReportAttempts,
|
||||
perAttemptTimeout: SessionGoneReportTimeout,
|
||||
retryDelay: SessionGoneReportRetryDelay)
|
||||
.Wait(SessionGoneReportBudget);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Debug($"Session-gone report send failed (best-effort): {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
var terminateAction = new ServerAction
|
||||
{
|
||||
Type = ServerActionTypes.TerminateSession,
|
||||
Reason = "Backend no longer knows this session (HTTP 410)",
|
||||
QueuedAt = DateTime.UtcNow,
|
||||
Params = new Dictionary<string, string>
|
||||
{
|
||||
{ "forceSelfDestruct", "true" },
|
||||
{ "gracePeriodSeconds", "0" },
|
||||
{ "origin", TerminationOrigins.SessionGone },
|
||||
},
|
||||
};
|
||||
try { dispatcher.DispatchAsync(new List<ServerAction> { terminateAction }).GetAwaiter().GetResult(); }
|
||||
catch (Exception ex) { logger.Error("ServerActionDispatcher.DispatchAsync (session gone) threw during server-response wiring.", ex); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Soft admin-override handling. Portal Mark-Succeeded / Mark-Failed is an echo of an
|
||||
/// admin button click, not a ServerAction: it bypasses the dispatcher (no
|
||||
@@ -281,6 +361,17 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
: EnrollmentTerminationOutcome.Failed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The <c>origin</c> param of a synthesised <c>terminate_session</c> action
|
||||
/// (<see cref="TerminationOrigins"/>), or null when absent — it travels into
|
||||
/// <see cref="EnrollmentTerminatedEventArgs.Origin"/>.
|
||||
/// </summary>
|
||||
internal static string ReadOrigin(IReadOnlyDictionary<string, string> parameters)
|
||||
{
|
||||
if (parameters == null) return null;
|
||||
return parameters.TryGetValue("origin", out var origin) && !string.IsNullOrEmpty(origin) ? origin : null;
|
||||
}
|
||||
|
||||
// ============================================================ Private terminate-callback
|
||||
|
||||
/// <summary>
|
||||
@@ -326,8 +417,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// before, masquerading every admin override as an error in SummaryDialog +
|
||||
// firing spurious diagnostics uploads).
|
||||
var mappedOutcome = MapAdminOutcome(action?.Params);
|
||||
var origin = ReadOrigin(action?.Params);
|
||||
|
||||
logger.Warning($"ServerAction terminate_session received (ruleId={action?.RuleId}, reason={action?.Reason}, outcome={mappedOutcome}) — invoking termination handler.");
|
||||
logger.Warning($"ServerAction terminate_session received (ruleId={action?.RuleId}, reason={action?.Reason}, outcome={mappedOutcome}, origin={origin ?? "none"}) — invoking termination handler.");
|
||||
// Synthesise a Terminated event as if the kernel fired it.
|
||||
terminationHandler.Handle(
|
||||
sender: null,
|
||||
@@ -336,7 +428,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
outcome: mappedOutcome,
|
||||
stageName: orchestrator.CurrentState?.Stage.ToString(),
|
||||
terminatedAtUtc: DateTime.UtcNow,
|
||||
details: $"Server-requested termination: ruleId={action?.RuleId}, reason={action?.Reason}"));
|
||||
details: $"Server-requested termination: ruleId={action?.RuleId}, reason={action?.Reason}",
|
||||
origin: origin));
|
||||
|
||||
// Plan §6.2 synchronous-shutdown — block the ingest dispatcher thread until the
|
||||
// main-thread cleanup (orchestrator.Stop, client disposal) has fully run. This
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Analyze;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Vulnerability;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The two queue handlers derive rows for a session (rule results, reports, inventory counters,
|
||||
/// CveIndex). For a session the backend does not know, or one a cascade is tearing down, they
|
||||
/// must stop before the first read of session data — nothing derived may ever outlive the
|
||||
/// session's cascade. Both handlers are driven up to that gate with every later dependency
|
||||
/// unavailable, so a regression that moves the gate shows up as a null dereference or an
|
||||
/// unexpected mock call.
|
||||
/// </summary>
|
||||
public class DerivedWriterLivenessTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
|
||||
private static SessionDeletionGuard GuardFor(TableEntity? row)
|
||||
{
|
||||
var reader = new Mock<ISessionDeletionInventoryReader>();
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(row);
|
||||
return new SessionDeletionGuard(reader.Object, NullLogger<SessionDeletionGuard>.Instance);
|
||||
}
|
||||
|
||||
private static Mock<AdminConfigurationService> AdminConfigWithCorrelationEnabled()
|
||||
{
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions()));
|
||||
adminConfig.Setup(a => a.GetConfigurationAsync())
|
||||
.ReturnsAsync(new AdminConfiguration { VulnerabilityCorrelationEnabled = true });
|
||||
return adminConfig;
|
||||
}
|
||||
|
||||
public static IEnumerable<object?[]> NotLiveRows()
|
||||
{
|
||||
yield return new object?[] { null };
|
||||
yield return new object?[] { new TableEntity(TenantId, SessionId) { ["DeletionState"] = SessionDeletionState.Running } };
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(NotLiveRows))]
|
||||
public async Task VulnerabilityCorrelate_skips_a_missing_or_locked_session_before_loading_inventory(TableEntity? row)
|
||||
{
|
||||
var inventoryLoader = new Mock<IVulnerabilityInventoryLoader>(MockBehavior.Strict);
|
||||
var vulnRepo = new Mock<IVulnerabilityRepository>(MockBehavior.Strict);
|
||||
var handler = new VulnerabilityCorrelateHandler(
|
||||
AdminConfigWithCorrelationEnabled().Object,
|
||||
vulnerabilityCorrelation: null!,
|
||||
vulnRepo.Object,
|
||||
Mock.Of<ISessionRepository>(),
|
||||
inventoryLoader.Object,
|
||||
Mock.Of<IAnalyzeOnEnrollmentEndProducer>(),
|
||||
signalRNotification: null!,
|
||||
GuardFor(row),
|
||||
NullLogger<VulnerabilityCorrelateHandler>.Instance);
|
||||
|
||||
await handler.HandleAsync(new VulnerabilityCorrelateEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, Reason = VulnerabilityCorrelateHandler.ReasonShutdownInventory, EnqueuedAt = DateTime.UtcNow,
|
||||
});
|
||||
|
||||
inventoryLoader.VerifyNoOtherCalls();
|
||||
vulnRepo.VerifyNoOtherCalls();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(NotLiveRows))]
|
||||
public async Task Analyze_skips_a_missing_or_locked_session_before_the_rule_engine_runs(TableEntity? row)
|
||||
{
|
||||
var ruleRepo = new Mock<IRuleRepository>(MockBehavior.Strict);
|
||||
var metricsRepo = new Mock<IMetricsRepository>(MockBehavior.Strict);
|
||||
var handler = new AnalyzeOnEnrollmentEndHandler(
|
||||
ruleService: null!,
|
||||
ruleRepo.Object,
|
||||
Mock.Of<ISessionRepository>(),
|
||||
metricsRepo.Object,
|
||||
signalRNotification: null!,
|
||||
configService: null!,
|
||||
channelDispatcher: null!,
|
||||
GuardFor(row),
|
||||
NullLogger<AnalyzeOnEnrollmentEndHandler>.Instance);
|
||||
|
||||
await handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, Reason = AnalyzeOnEnrollmentEndHandler.ReasonEnrollmentComplete, EnqueuedAt = DateTime.UtcNow,
|
||||
});
|
||||
|
||||
ruleRepo.VerifyNoOtherCalls();
|
||||
metricsRepo.VerifyNoOtherCalls();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_guard_read_error_propagates_so_the_queue_retries_instead_of_skipping()
|
||||
{
|
||||
var reader = new Mock<ISessionDeletionInventoryReader>();
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
|
||||
var handler = new AnalyzeOnEnrollmentEndHandler(
|
||||
ruleService: null!, Mock.Of<IRuleRepository>(), Mock.Of<ISessionRepository>(), Mock.Of<IMetricsRepository>(),
|
||||
signalRNotification: null!, configService: null!, channelDispatcher: null!,
|
||||
new SessionDeletionGuard(reader.Object, NullLogger<SessionDeletionGuard>.Instance),
|
||||
NullLogger<AnalyzeOnEnrollmentEndHandler>.Instance);
|
||||
|
||||
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, Reason = AnalyzeOnEnrollmentEndHandler.ReasonEnrollmentComplete, EnqueuedAt = DateTime.UtcNow,
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using System.Threading.Tasks;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Functions.Ingest;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
@@ -155,6 +156,27 @@ public class IngestTelemetryAuthPathTests
|
||||
Assert.Equal(SessionStatus.Succeeded, SessionRowProjections.TryReadStatus(returned));
|
||||
}
|
||||
|
||||
// ============================================================ Session unknown → 410
|
||||
|
||||
[Fact]
|
||||
public async Task SessionUnknown_guard_returns_null_that_Run_maps_to_410_session_unknown()
|
||||
{
|
||||
// Run: a null guard row (no Sessions row, no active tombstone) is refused with 410 via
|
||||
// WriteSessionUnknownAsync and stamped IngestRefusal=session_unknown; the cascade lock keeps
|
||||
// its own value so the two 410 reasons stay apart on the request row.
|
||||
var guard = NewGuard(out var reader);
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((TableEntity?)null);
|
||||
reader.Setup(r => r.GetActiveSessionTombstoneAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((TableEntity?)null);
|
||||
|
||||
var row = await guard.EnsureWritableAndGetRowAsync(TenantId, SessionId, callerContext: "V2.IngestTelemetry");
|
||||
|
||||
Assert.Null(row);
|
||||
Assert.NotEqual(RequestRowMarkers.IngestRefusal.SessionUnknown, RequestRowMarkers.IngestRefusal.CascadeLocked);
|
||||
Assert.Equal("IngestRefusal", RequestRowMarkers.IngestRefusalKey);
|
||||
}
|
||||
|
||||
// ============================================================ TryReadStatus (unique helper)
|
||||
|
||||
[Theory]
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
public class MaintenanceStepTimerTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Records_every_step_in_order_and_returns_the_body_result()
|
||||
{
|
||||
var timer = new MaintenanceStepTimer();
|
||||
|
||||
await timer.RunAsync("first", () => Task.CompletedTask);
|
||||
var value = await timer.RunAsync<int>("second", () => Task.FromResult(42));
|
||||
|
||||
Assert.Equal(42, value);
|
||||
Assert.Equal(new[] { "first", "second" }, timer.Steps.Keys);
|
||||
Assert.All(timer.Steps.Values, ms => Assert.True(ms >= 0));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_throwing_step_is_still_recorded_and_the_exception_propagates()
|
||||
{
|
||||
var timer = new MaintenanceStepTimer();
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => timer.RunAsync("boom", () => throw new InvalidOperationException("x")));
|
||||
|
||||
Assert.True(timer.Steps.ContainsKey("boom"));
|
||||
}
|
||||
}
|
||||
@@ -121,13 +121,17 @@ public class OffboardingSessionEnumeratorTests
|
||||
public Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException();
|
||||
public Task<List<string>> GetAllTenantIdsAsync() => throw new NotImplementedException();
|
||||
public Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<int> BackfillSessionIndexAsync() => throw new NotImplementedException();
|
||||
public Task<int> CleanupGhostSessionIndexEntriesAsync() => throw new NotImplementedException();
|
||||
public Task<bool> IsSessionIndexEmptyAsync() => throw new NotImplementedException();
|
||||
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => throw new NotImplementedException();
|
||||
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<List<(string TenantId, string SessionId)>> GetAllSessionTombstoneKeysAsync() => throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the per-table filter of the orphan-session sweep to the writer's key shape. A filter
|
||||
/// that names the wrong side (tenant where the session belongs, or vice versa) would either
|
||||
/// miss the session's rows or touch another session's — both silently.
|
||||
/// </summary>
|
||||
public class OrphanSessionRowFiltersTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
|
||||
[Fact]
|
||||
public void Every_filter_table_has_a_filter_naming_both_ids()
|
||||
{
|
||||
foreach (var table in OrphanSessionRowFilters.Tables)
|
||||
{
|
||||
var filter = OrphanSessionRowFilters.For(table, TenantId, SessionId);
|
||||
Assert.Contains(TenantId, filter);
|
||||
Assert.Contains(SessionId, filter);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(Constants.TableNames.Events)]
|
||||
[InlineData(Constants.TableNames.RuleResults)]
|
||||
[InlineData(Constants.TableNames.Signals)]
|
||||
[InlineData(Constants.TableNames.DecisionTransitions)]
|
||||
[InlineData(Constants.TableNames.VulnerabilityReports)]
|
||||
public void Session_partition_tables_filter_on_the_composite_partition_key(string table)
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq '{TenantId}_{SessionId}'", OrphanSessionRowFilters.For(table, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(Constants.TableNames.DeviceSnapshot)]
|
||||
[InlineData(Constants.TableNames.SessionTimeBreakdowns)]
|
||||
public void One_row_per_session_tables_filter_on_tenant_partition_and_session_row_key(string table)
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq '{TenantId}' and RowKey eq '{SessionId}'", OrphanSessionRowFilters.For(table, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AppInstallSummaries_filter_on_the_SessionId_column()
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq '{TenantId}' and SessionId eq '{SessionId}'",
|
||||
OrphanSessionRowFilters.For(Constants.TableNames.AppInstallSummaries, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EventTypeIndex_filter_scans_the_tenant_range_and_matches_the_SessionId_column()
|
||||
{
|
||||
// RowKey is {invertedTicks}_{sessionId}; without a Sessions row the ticks are not reconstructible.
|
||||
Assert.Equal($"PartitionKey ge '{TenantId}_' and PartitionKey lt '{TenantId}_~' and SessionId eq '{SessionId}'",
|
||||
OrphanSessionRowFilters.For(Constants.TableNames.EventTypeIndex, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CveIndex_filter_scans_the_tenant_range_and_matches_the_session_row_key()
|
||||
{
|
||||
Assert.Equal($"PartitionKey ge '{TenantId}_' and PartitionKey lt '{TenantId}_~' and RowKey eq '{SessionId}'",
|
||||
OrphanSessionRowFilters.For(Constants.TableNames.CveIndex, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SessionAnnotations_filter_names_every_lane()
|
||||
{
|
||||
var filter = OrphanSessionRowFilters.For(Constants.TableNames.SessionAnnotations, TenantId, SessionId);
|
||||
Assert.StartsWith($"PartitionKey eq '{TenantId}' and (", filter);
|
||||
foreach (var lane in AnnotationLanes.All)
|
||||
Assert.Contains($"RowKey eq '{SessionId}_{lane}'", filter);
|
||||
Assert.Equal(AnnotationLanes.All.Length - 1, filter.Split(" or ").Length - 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SessionTenantLookup_filter_pins_the_row_to_this_tenant()
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq '{SessionId}' and RowKey eq 'tenant' and TenantId eq '{TenantId}'",
|
||||
OrphanSessionRowFilters.For(Constants.TableNames.SessionTenantLookup, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_table_throws_instead_of_guessing()
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => OrphanSessionRowFilters.For(Constants.TableNames.Sessions, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-guid", SessionId)]
|
||||
[InlineData(TenantId, "'; drop")]
|
||||
public void Non_guid_ids_throw(string tenantId, string sessionId)
|
||||
{
|
||||
Assert.ThrowsAny<ArgumentException>(() => OrphanSessionRowFilters.For(Constants.TableNames.Events, tenantId, sessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Swept_tables_are_the_filter_tables_plus_inventory_pair_plus_handle()
|
||||
{
|
||||
var expected = OrphanSessionRowFilters.Tables
|
||||
.Concat(new[]
|
||||
{
|
||||
Constants.TableNames.SessionInventoryContributions,
|
||||
Constants.TableNames.SoftwareInventory,
|
||||
Constants.TableNames.EventSessionIndex,
|
||||
})
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
Assert.Equal(expected, OrphanSessionSweeper.SweptTables.ToHashSet(StringComparer.Ordinal));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Behaviour of the orphan-session sweep: which handles become candidates, the re-checks right
|
||||
/// before the deletes, the inventory claim-then-decrement order, the handle as the last delete,
|
||||
/// the cap/budget, and the weekly read-only reconciliation. Storage is mocked at the repository
|
||||
/// seams; the ordered action log is the evidence for every ordering invariant.
|
||||
/// </summary>
|
||||
public class OrphanSessionSweeperTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
private const string SessionB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
|
||||
private const string SessionC = "cccccccc-cccc-4ccc-8ccc-cccccccccccc";
|
||||
|
||||
// A Wednesday: the weekly reconciliation is not due unless forced or overdue.
|
||||
private static readonly DateTime Now = new DateTime(2026, 10, 14, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public readonly Mock<IMaintenanceRepository> Maint = new();
|
||||
public readonly Mock<IVulnerabilityRepository> Vuln = new();
|
||||
public readonly Mock<ISessionDeletionInventoryReader> Reader = new();
|
||||
public readonly Mock<AdminConfigurationService> AdminConfig;
|
||||
public readonly Mock<OrphanSweepLockStore> LockStore;
|
||||
public readonly List<OpsEventEntry> Ops = new();
|
||||
public readonly List<string> Log = new();
|
||||
|
||||
public readonly HashSet<(string TenantId, string SessionId)> SessionKeys = new();
|
||||
public readonly List<OrphanSessionHandle> Handles = new();
|
||||
public readonly Dictionary<string, OrphanSessionHandle?> LiveHandles = new(StringComparer.Ordinal);
|
||||
public readonly HashSet<string> RowsPresent = new(StringComparer.Ordinal);
|
||||
public readonly HashSet<string> Tombstoned = new(StringComparer.Ordinal);
|
||||
public readonly Dictionary<string, List<DeletionDecrementKey>> Inventory = new(StringComparer.Ordinal);
|
||||
public readonly HashSet<(string TenantId, string SessionId)> IndexKeys = new();
|
||||
public readonly Dictionary<string, int> RowsPerTable = new(StringComparer.Ordinal);
|
||||
public readonly AdminConfiguration Config = new AdminConfiguration();
|
||||
|
||||
public DateTime Clock = Now;
|
||||
public TimeSpan Budget = TimeSpan.FromMinutes(5);
|
||||
public bool StampFails;
|
||||
public bool HandleDeleteFails;
|
||||
public bool DecrementFails;
|
||||
public bool LeaseHeld;
|
||||
public string? FailingTable;
|
||||
|
||||
public Harness()
|
||||
{
|
||||
Maint.Setup(m => m.GetSessionKeysAsync(It.IsAny<CancellationToken>()))
|
||||
.Callback(() => Log.Add("sessions"))
|
||||
.ReturnsAsync(() => new HashSet<(string, string)>(SessionKeys));
|
||||
Maint.Setup(m => m.GetEventSessionIndexHandlesAsync(It.IsAny<CancellationToken>()))
|
||||
.Callback(() => Log.Add("handles"))
|
||||
.ReturnsAsync(() => Handles.ToList());
|
||||
Maint.Setup(m => m.GetEventSessionIndexHandleAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string s, CancellationToken _) =>
|
||||
LiveHandles.TryGetValue(s, out var live) ? live : Handles.FirstOrDefault(h => h.SessionId == s));
|
||||
Maint.Setup(m => m.StampHandleInventoryDecrementedAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string, string, CancellationToken>((_, s, _, _) => Log.Add($"stamp:{s}"))
|
||||
.ReturnsAsync(() => StampFails ? null : "etag-stamped");
|
||||
Maint.Setup(m => m.DeleteOrphanSessionRowsAsync(It.IsAny<string>(), TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns((string table, string _, string s, CancellationToken _) =>
|
||||
{
|
||||
Log.Add($"delete:{table}:{s}");
|
||||
if (string.Equals(table, FailingTable, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("storage said no");
|
||||
return Task.FromResult(RowsPerTable.TryGetValue(table, out var n) ? n : 1);
|
||||
});
|
||||
Maint.Setup(m => m.DeleteEventSessionIndexHandleAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string, string, CancellationToken>((_, s, _, _) => Log.Add($"handle:{s}"))
|
||||
.ReturnsAsync(() => !HandleDeleteFails);
|
||||
Maint.Setup(m => m.GetEventTypeIndexSessionKeysAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => new HashSet<(string, string)>(IndexKeys));
|
||||
|
||||
Vuln.Setup(v => v.GetSessionInventoryContributionsAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string s, CancellationToken _) =>
|
||||
Inventory.TryGetValue(s, out var keys) ? (IReadOnlyList<DeletionDecrementKey>?)keys : null);
|
||||
Vuln.Setup(v => v.DecrementSoftwareInventoryEntryAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns((string _, string vendor, string name, string version, CancellationToken _) =>
|
||||
{
|
||||
Log.Add($"decrement:{name}");
|
||||
if (DecrementFails) throw new InvalidOperationException("counter busy");
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
Vuln.Setup(v => v.DeleteSessionInventoryContributionsAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string, CancellationToken>((_, s, _) => Log.Add($"sideRow:{s}"))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
Reader.Setup(r => r.GetSessionRowAsync(TenantId, It.IsAny<string>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string s, IEnumerable<string> _, CancellationToken _) =>
|
||||
RowsPresent.Contains(s) ? new TableEntity(TenantId, s) : null);
|
||||
Reader.Setup(r => r.GetActiveSessionTombstoneAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string s, CancellationToken _) =>
|
||||
Tombstoned.Contains(s) ? new TableEntity(TenantId, s) : null);
|
||||
|
||||
AdminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions()));
|
||||
AdminConfig.Setup(a => a.GetConfigurationAsync()).ReturnsAsync(() => Config);
|
||||
AdminConfig.Setup(a => a.UpdateAsync(It.IsAny<Func<AdminConfiguration, string?>>(), It.IsAny<string>(), It.IsAny<string?>()))
|
||||
.Returns((Func<AdminConfiguration, string?> mutate, string _, string? _) =>
|
||||
{
|
||||
mutate(Config);
|
||||
return Task.FromResult(new AdminConfigurationUpdateResult { Before = Config, After = Config });
|
||||
});
|
||||
|
||||
LockStore = new Mock<OrphanSweepLockStore>(new SessionDeletionMaintenanceFunctionTests.MaintenanceBlobStub());
|
||||
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(() => LeaseHeld
|
||||
? throw new LeaseHeldException("held", new InvalidOperationException("lease"))
|
||||
: Task.FromResult(new Mock<Azure.Storage.Blobs.Specialized.BlobLeaseClient>().Object));
|
||||
}
|
||||
|
||||
public OrphanSessionHandle AddHandle(string sessionId, TimeSpan ingestAge, TimeSpan? writtenAge = null, string etag = "etag-1", bool decremented = false)
|
||||
{
|
||||
var handle = new OrphanSessionHandle
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = sessionId,
|
||||
LastIngestAt = Now - ingestAge,
|
||||
WrittenAt = Now - (writtenAge ?? TimeSpan.FromDays(1)),
|
||||
ETag = etag,
|
||||
InventoryDecremented = decremented,
|
||||
};
|
||||
Handles.Add(handle);
|
||||
return handle;
|
||||
}
|
||||
|
||||
public OrphanSessionSweeper Build()
|
||||
{
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => { lock (Ops) Ops.Add(e); })
|
||||
.Returns(Task.CompletedTask);
|
||||
var ops = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance,
|
||||
TestNotifications.InertOpsAlertDispatch(AdminConfig.Object));
|
||||
return new OrphanSessionSweeper(Maint.Object, Vuln.Object, Reader.Object, AdminConfig.Object, LockStore.Object, ops,
|
||||
NullLogger<OrphanSessionSweeper>.Instance, () => Clock, Budget);
|
||||
}
|
||||
|
||||
public Task<OrphanSweepRunResult> RunCoreAsync(bool forceDeepCheck = false) => Build().RunCoreAsync(forceDeepCheck, CancellationToken.None);
|
||||
}
|
||||
|
||||
private static DeletionDecrementKey Key(string name) => new DeletionDecrementKey { Vendor = "v", Name = name, Version = "1" };
|
||||
|
||||
// ============================================================ candidates
|
||||
|
||||
[Fact]
|
||||
public async Task Candidates_are_handles_without_a_session_key_outside_both_grace_windows()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3)); // orphan, old
|
||||
h.AddHandle(SessionB, TimeSpan.FromDays(3)); // has a row
|
||||
h.SessionKeys.Add((TenantId, SessionB));
|
||||
h.AddHandle(SessionC, TimeSpan.FromHours(1)); // ingest grace
|
||||
h.AddHandle("dddddddd-dddd-4ddd-8ddd-dddddddddddd", TimeSpan.FromDays(3), writtenAge: TimeSpan.FromMinutes(30)); // restore grace
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.Candidates);
|
||||
Assert.Equal(SessionA, Assert.Single(result.Cleaned).SessionId);
|
||||
Assert.Equal(4, result.HandlesScanned);
|
||||
Assert.Equal(1, result.SessionKeys);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sessions_are_drained_before_handles()
|
||||
{
|
||||
var h = new Harness();
|
||||
await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(new[] { "sessions", "handles" }, h.Log.Take(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Legacy_handles_are_flagged_and_come_first()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.AddHandle(SessionB, Now - OrphanSessionSweeper.LegacyHandleLastIngestAt);
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(new[] { SessionB, SessionA }, result.Cleaned.Select(c => c.SessionId));
|
||||
Assert.True(result.Cleaned[0].Legacy);
|
||||
Assert.False(result.Cleaned[1].Legacy);
|
||||
}
|
||||
|
||||
// ============================================================ re-checks
|
||||
|
||||
[Fact]
|
||||
public async Task Handle_gone_at_recheck_is_skipped_silently()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.LiveHandles[SessionA] = null;
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.SkippedHandleGone);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handle_changed_since_the_scan_is_skipped()
|
||||
{
|
||||
var h = new Harness();
|
||||
var scanned = h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.LiveHandles[SessionA] = new OrphanSessionHandle
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionA, LastIngestAt = scanned.LastIngestAt, WrittenAt = scanned.WrittenAt, ETag = "etag-later",
|
||||
};
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.SkippedHandleChanged);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Row_present_at_recheck_skips_everything()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.RowsPresent.Add(SessionA);
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.SkippedRowReturned);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal) || l.StartsWith("stamp:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tombstone_counts_as_afterCascade_and_the_session_is_still_cleaned()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Tombstoned.Add(SessionA);
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.AfterCascade);
|
||||
Assert.True(Assert.Single(result.Cleaned).AfterCascade);
|
||||
}
|
||||
|
||||
// ============================================================ inventory + ordering
|
||||
|
||||
[Fact]
|
||||
public async Task Inventory_is_claimed_then_decremented_then_the_side_row_goes_before_the_tables()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1"), Key("app-2") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
var stamp = h.Log.IndexOf($"stamp:{SessionA}");
|
||||
var firstDecrement = h.Log.IndexOf("decrement:app-1");
|
||||
var sideRow = h.Log.IndexOf($"sideRow:{SessionA}");
|
||||
var firstTable = h.Log.FindIndex(l => l.StartsWith("delete:", StringComparison.Ordinal));
|
||||
var handle = h.Log.IndexOf($"handle:{SessionA}");
|
||||
Assert.True(0 <= stamp && stamp < firstDecrement && firstDecrement < sideRow && sideRow < firstTable && firstTable < handle,
|
||||
string.Join(" | ", h.Log));
|
||||
|
||||
var cleaned = Assert.Single(result.Cleaned);
|
||||
Assert.Equal(2, cleaned.InventoryKeysDecremented);
|
||||
Assert.Equal(2, result.TotalsByTable[Constants.TableNames.SoftwareInventory]);
|
||||
Assert.Equal(1, result.TotalsByTable[Constants.TableNames.SessionInventoryContributions]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_rerun_with_the_stamp_set_skips_the_decrements_but_still_removes_the_side_row()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3), decremented: true);
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("stamp:", StringComparison.Ordinal) || l.StartsWith("decrement:", StringComparison.Ordinal));
|
||||
Assert.Contains($"sideRow:{SessionA}", h.Log);
|
||||
Assert.Single(result.Cleaned);
|
||||
Assert.Equal(0, result.TotalsByTable[Constants.TableNames.SoftwareInventory]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failed_stamp_aborts_the_candidate_before_any_decrement_or_delete()
|
||||
{
|
||||
var h = new Harness { StampFails = true };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.SkippedHandleChanged);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("decrement:", StringComparison.Ordinal) || l.StartsWith("delete:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failed_decrement_is_counted_and_the_session_is_still_cleaned()
|
||||
{
|
||||
var h = new Harness { DecrementFails = true };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1"), Key("app-2") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(2, result.InventoryDecrementsFailed);
|
||||
Assert.Single(result.Cleaned);
|
||||
Assert.Contains($"handle:{SessionA}", h.Log);
|
||||
}
|
||||
|
||||
// ============================================================ the handle last
|
||||
|
||||
[Fact]
|
||||
public async Task Every_filter_table_is_cleared_and_the_handle_is_deleted_last_with_the_stamped_etag()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
foreach (var table in OrphanSessionRowFilters.Tables)
|
||||
Assert.Contains($"delete:{table}:{SessionA}", h.Log);
|
||||
Assert.Equal($"handle:{SessionA}", h.Log.Last());
|
||||
h.Maint.Verify(m => m.DeleteEventSessionIndexHandleAsync(TenantId, SessionA, "etag-stamped", It.IsAny<CancellationToken>()), Times.Once);
|
||||
Assert.Equal(1, result.TotalsByTable[Constants.TableNames.EventSessionIndex]);
|
||||
Assert.Equal(OrphanSessionRowFilters.Tables.Length, Assert.Single(result.Cleaned).TotalRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_handle_rewritten_before_the_final_delete_is_kept_and_reported()
|
||||
{
|
||||
var h = new Harness { HandleDeleteFails = true };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
var cleaned = Assert.Single(result.Cleaned);
|
||||
Assert.True(cleaned.HandleKept);
|
||||
Assert.Equal(0, result.TotalsByTable[Constants.TableNames.EventSessionIndex]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_table_leaves_the_handle_and_the_run_continues_with_the_next_candidate()
|
||||
{
|
||||
var h = new Harness { FailingTable = Constants.TableNames.Signals };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(4));
|
||||
h.AddHandle(SessionB, TimeSpan.FromDays(3));
|
||||
// Only session A's Signals delete fails: the harness throws for the table regardless of
|
||||
// session, so flip the failure off once A has been attempted.
|
||||
h.Maint.Setup(m => m.DeleteOrphanSessionRowsAsync(Constants.TableNames.Signals, TenantId, SessionB, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(1);
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.FailedSessions);
|
||||
Assert.DoesNotContain($"handle:{SessionA}", h.Log);
|
||||
Assert.Contains($"handle:{SessionB}", h.Log);
|
||||
Assert.Equal(SessionB, Assert.Single(result.Cleaned).SessionId);
|
||||
}
|
||||
|
||||
// ============================================================ cap + budget
|
||||
|
||||
[Fact]
|
||||
public async Task The_cap_leaves_the_rest_for_the_next_run()
|
||||
{
|
||||
var h = new Harness();
|
||||
for (var i = 0; i < OrphanSessionSweeper.MaxCandidatesPerRun + 3; i++)
|
||||
h.AddHandle(Guid.NewGuid().ToString(), TimeSpan.FromDays(2) + TimeSpan.FromMinutes(i));
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(OrphanSessionSweeper.MaxCandidatesPerRun, result.Cleaned.Count);
|
||||
Assert.Equal(3, result.Remaining);
|
||||
Assert.False(result.BudgetExhausted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_exhausted_budget_stops_before_the_next_candidate()
|
||||
{
|
||||
var h = new Harness { Budget = TimeSpan.Zero };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.AddHandle(SessionB, TimeSpan.FromDays(3));
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.True(result.BudgetExhausted);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.Equal(2, result.Remaining);
|
||||
}
|
||||
|
||||
// ============================================================ weekly reconciliation
|
||||
|
||||
public static IEnumerable<object?[]> DeepCheckCases()
|
||||
{
|
||||
var sunday0845 = new DateTime(2026, 10, 18, 8, 45, 0, DateTimeKind.Utc);
|
||||
var sunday0445 = new DateTime(2026, 10, 18, 4, 45, 0, DateTimeKind.Utc);
|
||||
var tuesday = new DateTime(2026, 10, 20, 12, 45, 0, DateTimeKind.Utc);
|
||||
yield return new object?[] { null, sunday0845, true }; // never ran: first Sunday window
|
||||
yield return new object?[] { null, tuesday, false }; // never ran: wait for Sunday
|
||||
yield return new object?[] { sunday0845.AddDays(-7), sunday0845, true }; // weekly
|
||||
yield return new object?[] { sunday0845.AddDays(-7), sunday0445, false }; // Sunday, but before 08:00
|
||||
yield return new object?[] { sunday0845.AddDays(-3), sunday0845, false }; // ran this week already
|
||||
yield return new object?[] { tuesday.AddDays(-9), tuesday, true }; // missed Sunday: overdue
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(DeepCheckCases))]
|
||||
public void Deep_check_is_due_on_Sunday_mornings_or_when_overdue(DateTime? lastRun, DateTime now, bool due)
|
||||
{
|
||||
Assert.Equal(due, OrphanSessionSweeper.IsDeepCheckDue(lastRun, now));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_reports_only_sessions_without_a_row_and_without_a_handle_and_stamps_the_marker()
|
||||
{
|
||||
var h = new Harness();
|
||||
const string noHandleNoRow = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee";
|
||||
const string noHandleButRow = "ffffffff-ffff-4fff-8fff-ffffffffffff";
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3)); // in the sweep's own pipeline
|
||||
h.SessionKeys.Add((TenantId, SessionB)); // live
|
||||
h.RowsPresent.Add(noHandleButRow); // registered after the drain
|
||||
h.IndexKeys.UnionWith(new[] { (TenantId, SessionA), (TenantId, SessionB), (TenantId, noHandleNoRow), (TenantId, noHandleButRow) });
|
||||
|
||||
var result = await h.RunCoreAsync(forceDeepCheck: true);
|
||||
|
||||
var reconcile = Assert.IsType<OrphanReconcileResult>(result.Reconcile);
|
||||
Assert.Equal(4, reconcile.IndexSessions);
|
||||
Assert.Equal(1, reconcile.ResidueSessions);
|
||||
Assert.Equal((TenantId, noHandleNoRow), Assert.Single(reconcile.Sample));
|
||||
var evt = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
|
||||
Assert.Equal(OpsEventSeverity.Warning, evt.Severity);
|
||||
Assert.Equal(Now.ToString("o"), h.Config.OrphanDeepCheckLastRunUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_without_residue_is_Info_and_is_not_due_again_mid_week()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionKeys.Add((TenantId, SessionB));
|
||||
h.IndexKeys.Add((TenantId, SessionB));
|
||||
|
||||
var first = await h.RunCoreAsync(forceDeepCheck: true);
|
||||
var evt = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
|
||||
Assert.Equal(OpsEventSeverity.Info, evt.Severity);
|
||||
Assert.Equal(0, first.Reconcile!.ResidueSessions);
|
||||
|
||||
var second = await h.RunCoreAsync();
|
||||
Assert.Null(second.Reconcile);
|
||||
}
|
||||
|
||||
// ============================================================ RunAsync: lease + ops events
|
||||
|
||||
[Fact]
|
||||
public async Task A_timer_run_writes_the_heartbeat_and_EventsCleaned_only_when_something_was_cleaned()
|
||||
{
|
||||
var quiet = new Harness();
|
||||
Assert.True(await quiet.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
|
||||
Assert.Single(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanSweepCompleted && e.Severity == OpsEventSeverity.Info);
|
||||
Assert.DoesNotContain(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanEventsCleaned);
|
||||
Assert.DoesNotContain(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
|
||||
|
||||
var busy = new Harness();
|
||||
busy.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
Assert.True(await busy.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
|
||||
var cleaned = Assert.Single(busy.Ops, e => e.EventType == OpsEventTypes.OrphanEventsCleaned);
|
||||
Assert.Equal(OpsEventSeverity.Warning, cleaned.Severity);
|
||||
Assert.Contains(SessionA, cleaned.Details ?? string.Empty);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_manual_run_always_includes_the_reconciliation()
|
||||
{
|
||||
var h = new Harness();
|
||||
Assert.True(await h.Build().RunAsync("alice@contoso.com", CancellationToken.None));
|
||||
Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_held_lease_records_SkippedLocked_and_touches_nothing()
|
||||
{
|
||||
var h = new Harness { LeaseHeld = true };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
|
||||
Assert.False(await h.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
|
||||
|
||||
Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepSkippedLocked);
|
||||
Assert.Empty(h.Log);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_scan_records_OrphanSweepFailed()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.Maint.Setup(m => m.GetSessionKeysAsync(It.IsAny<CancellationToken>())).ThrowsAsync(new InvalidOperationException("storage down"));
|
||||
|
||||
Assert.True(await h.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
|
||||
|
||||
var failed = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepFailed);
|
||||
Assert.Equal(OpsEventSeverity.Error, failed.Severity);
|
||||
Assert.DoesNotContain(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepCompleted);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Shape of the ops events the orphan sweep and the maintenance run write: the dashboards and the
|
||||
/// health pass read these details by name.
|
||||
/// </summary>
|
||||
public class OrphanSweepOpsEventTests
|
||||
{
|
||||
private static (OpsEventService Service, List<OpsEventEntry> Saved) Rig()
|
||||
{
|
||||
var saved = new List<OpsEventEntry>();
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => { lock (saved) saved.Add(e); })
|
||||
.Returns(Task.CompletedTask);
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions())) { CallBase = false };
|
||||
var alertDispatch = TestNotifications.InertOpsAlertDispatch(adminConfig.Object);
|
||||
return (new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch), saved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MaintenanceCompleted_carries_the_step_timings()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var steps = new Dictionary<string, long> { ["stalledSessions"] = 1200, ["aggregateMetrics"] = 90000 };
|
||||
|
||||
await service.RecordMaintenanceCompletedAsync(123456, "Timer", stepsMs: steps);
|
||||
|
||||
var evt = Assert.Single(saved);
|
||||
Assert.Equal(OpsEventTypes.MaintenanceCompleted, evt.EventType);
|
||||
Assert.Contains("\"stepsMs\"", evt.Details);
|
||||
Assert.Contains("\"aggregateMetrics\":90000", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphanEventsCleaned_is_a_Warning_with_per_table_totals_and_the_legacy_count()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var result = new OrphanSweepRunResult { Remaining = 7 };
|
||||
var legacy = new OrphanSweepCleanedSession { TenantId = "t", SessionId = "s-legacy", Legacy = true, InventoryKeysDecremented = 3, HadInventorySideRow = true };
|
||||
legacy.RowsByTable[Constants.TableNames.Events] = 18;
|
||||
legacy.RowsByTable[Constants.TableNames.EventTypeIndex] = 13;
|
||||
result.Add(legacy);
|
||||
var fresh = new OrphanSweepCleanedSession { TenantId = "t", SessionId = "s-fresh" };
|
||||
fresh.RowsByTable[Constants.TableNames.Events] = 1;
|
||||
result.Add(fresh);
|
||||
|
||||
await service.RecordOrphanEventsCleanedAsync(result);
|
||||
|
||||
var evt = Assert.Single(saved);
|
||||
Assert.Equal(OpsEventTypes.OrphanEventsCleaned, evt.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Warning, evt.Severity);
|
||||
Assert.Contains("\"orphanSessions\":2", evt.Details);
|
||||
Assert.Contains("\"totalEventsDeleted\":19", evt.Details);
|
||||
Assert.Contains("\"legacySessions\":1", evt.Details);
|
||||
Assert.Contains("\"remaining\":7", evt.Details);
|
||||
Assert.Contains($"\"{Constants.TableNames.SoftwareInventory}\":3", evt.Details);
|
||||
Assert.Contains($"\"{Constants.TableNames.EventSessionIndex}\":2", evt.Details);
|
||||
Assert.Contains("\"legacy\":true", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphanSweepCompleted_is_the_Info_heartbeat_with_the_counters()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var result = new OrphanSweepRunResult
|
||||
{
|
||||
HandlesScanned = 16784, SessionKeys = 16700, Candidates = 3, Remaining = 1, SkippedHandleGone = 2, DurationMs = 4200,
|
||||
Reconcile = new OrphanReconcileResult { IndexSessions = 16000, ResidueSessions = 0, DurationMs = 180000 },
|
||||
};
|
||||
|
||||
await service.RecordOrphanSweepCompletedAsync(result, "Timer");
|
||||
|
||||
var evt = Assert.Single(saved);
|
||||
Assert.Equal(OpsEventTypes.OrphanSweepCompleted, evt.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Info, evt.Severity);
|
||||
Assert.Contains("\"handlesScanned\":16784", evt.Details);
|
||||
Assert.Contains("\"skippedHandleGone\":2", evt.Details);
|
||||
Assert.Contains("\"residueSessions\":0", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphanReconcileCompleted_flips_to_Warning_with_a_sample_when_residue_exists()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var clean = new OrphanReconcileResult { IndexSessions = 10, ResidueSessions = 0 };
|
||||
var dirty = new OrphanReconcileResult { IndexSessions = 10, ResidueSessions = 2, Sample = { ("t1", "s1"), ("t1", "s2") } };
|
||||
|
||||
await service.RecordOrphanReconcileCompletedAsync(clean);
|
||||
await service.RecordOrphanReconcileCompletedAsync(dirty);
|
||||
|
||||
Assert.Equal(OpsEventSeverity.Info, saved[0].Severity);
|
||||
Assert.Equal(OpsEventSeverity.Warning, saved[1].Severity);
|
||||
Assert.Contains("\"sessionId\":\"s2\"", saved[1].Details);
|
||||
Assert.Contains("\"sampleTruncated\":false", saved[1].Details);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Data.Tables;
|
||||
@@ -133,6 +134,60 @@ public class SessionDeletionGuardTests
|
||||
Assert.Equal(SessionId, ex.SessionId);
|
||||
}
|
||||
|
||||
// ============================================================ GetLivenessAsync (derived writers) ====
|
||||
|
||||
[Fact]
|
||||
public async Task GetLivenessAsync_reports_Missing_when_the_row_does_not_exist()
|
||||
{
|
||||
// Unlike the ingest contract, a derived writer (queue handler, rescan) stops here: nothing
|
||||
// may be derived for a session the backend does not know.
|
||||
var guard = NewGuard(out var reader);
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((TableEntity?)null);
|
||||
|
||||
Assert.Equal(SessionLiveness.Missing, await guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityCorrelate"));
|
||||
reader.Verify(r => r.GetActiveSessionTombstoneAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(SessionDeletionState.Preparing)]
|
||||
[InlineData(SessionDeletionState.Queued)]
|
||||
[InlineData(SessionDeletionState.Running)]
|
||||
[InlineData(SessionDeletionState.Poisoned)]
|
||||
public async Task GetLivenessAsync_reports_Locked_for_every_lock_state(string state)
|
||||
{
|
||||
var guard = NewGuard(out var reader);
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableEntity(TenantId, SessionId) { ["DeletionState"] = state });
|
||||
|
||||
Assert.Equal(SessionLiveness.Locked, await guard.GetLivenessAsync(TenantId, SessionId, "AnalyzeOnEnrollmentEnd"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("None")]
|
||||
public async Task GetLivenessAsync_reports_Live_for_an_unlocked_row(string? state)
|
||||
{
|
||||
var guard = NewGuard(out var reader);
|
||||
var row = new TableEntity(TenantId, SessionId);
|
||||
if (state != null) row["DeletionState"] = state;
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(row);
|
||||
|
||||
Assert.Equal(SessionLiveness.Live, await guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityRescan"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetLivenessAsync_propagates_a_read_error()
|
||||
{
|
||||
// A storage failure must surface to the queue worker (retry/poison), never read as "missing".
|
||||
var guard = NewGuard(out var reader);
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
|
||||
|
||||
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityCorrelate"));
|
||||
}
|
||||
|
||||
// ============================================================ Codex F3: tombstone-marker disambiguation ====
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -81,6 +81,32 @@ public class TableLifecycleBucketTests
|
||||
Constants.TableNames.DelegationInvitations,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The orphan-session sweep is the cascade for sessions without a Sessions row, so it must
|
||||
/// clear exactly the tables the cascade manifest clears — minus the tombstone pair, which does
|
||||
/// not exist for such a session. A new manifest step without a sweep entry (or the reverse)
|
||||
/// fails here until someone decides the table's orphan lifecycle.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OrphanSweep_ClearsEveryPerSessionCascadeTable_ExceptTheTombstonePair()
|
||||
{
|
||||
var cascade = await PerSessionCascadeTablesAsync();
|
||||
cascade.Remove(Constants.TableNames.Sessions);
|
||||
cascade.Remove(Constants.TableNames.SessionsIndex);
|
||||
|
||||
var swept = AutopilotMonitor.Functions.Services.Maintenance.OrphanSessionSweeper.SweptTables.ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
var missingFromSweep = cascade.Except(swept, StringComparer.Ordinal).OrderBy(t => t, StringComparer.Ordinal).ToList();
|
||||
var sweptButNotCascaded = swept.Except(cascade, StringComparer.Ordinal).OrderBy(t => t, StringComparer.Ordinal).ToList();
|
||||
|
||||
Assert.True(missingFromSweep.Count == 0,
|
||||
"Per-session cascade tables the orphan sweep does not clear — add them to OrphanSessionRowFilters.Tables "
|
||||
+ "(with the writer's key shape) or handle them in OrphanSessionSweeper:\n - " + string.Join("\n - ", missingFromSweep));
|
||||
Assert.True(sweptButNotCascaded.Count == 0,
|
||||
"Orphan sweep clears tables the cascade manifest does not know — add a manifest step first:\n - "
|
||||
+ string.Join("\n - ", sweptButNotCascaded));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TableNamesConstants_SetEquals_TableNamesAll()
|
||||
{
|
||||
|
||||
+2
@@ -1019,6 +1019,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{ "MsrcLastSyncUtc", config.MsrcLastSyncUtc },
|
||||
{ "NvdCacheLastRefreshUtc", config.NvdCacheLastRefreshUtc },
|
||||
{ "EpssLastSyncUtc", config.EpssLastSyncUtc },
|
||||
{ "OrphanDeepCheckLastRunUtc", config.OrphanDeepCheckLastRunUtc },
|
||||
{ "MaxDiagnosticsDownloadSizeMB", config.MaxDiagnosticsDownloadSizeMB },
|
||||
{ "DiagnosticsDownloadTimeoutSeconds", config.DiagnosticsDownloadTimeoutSeconds },
|
||||
{ "OpsEventRetentionDays", config.OpsEventRetentionDays },
|
||||
@@ -1094,6 +1095,7 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
MsrcLastSyncUtc = entity.GetString("MsrcLastSyncUtc"),
|
||||
NvdCacheLastRefreshUtc = entity.GetString("NvdCacheLastRefreshUtc"),
|
||||
EpssLastSyncUtc = entity.GetString("EpssLastSyncUtc"),
|
||||
OrphanDeepCheckLastRunUtc = entity.GetString("OrphanDeepCheckLastRunUtc"),
|
||||
MaxDiagnosticsDownloadSizeMB = entity.GetInt32("MaxDiagnosticsDownloadSizeMB") ?? 500,
|
||||
DiagnosticsDownloadTimeoutSeconds = entity.GetInt32("DiagnosticsDownloadTimeoutSeconds") ?? 120,
|
||||
OpsEventRetentionDays = entity.GetInt32("OpsEventRetentionDays") ?? 90,
|
||||
|
||||
+19
-7
@@ -87,9 +87,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<List<string>> GetAllTenantIdsAsync()
|
||||
=> _storage.GetAllTenantIdsAsync();
|
||||
|
||||
public Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId)
|
||||
=> _storage.DeleteSessionEventsAsync(tenantId, sessionId);
|
||||
|
||||
public Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId)
|
||||
=> _storage.DeleteSessionRuleResultsAsync(tenantId, sessionId);
|
||||
|
||||
@@ -102,11 +99,26 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<bool> IsSessionIndexEmptyAsync()
|
||||
=> _storage.IsSessionIndexEmptyAsync();
|
||||
|
||||
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod)
|
||||
=> _storage.GetOrphanedEventSessionsAsync(gracePeriod);
|
||||
public Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(CancellationToken ct = default)
|
||||
=> _storage.GetSessionKeysAsync(ct);
|
||||
|
||||
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId)
|
||||
=> _storage.DeleteEventSessionIndexEntryAsync(tenantId, sessionId);
|
||||
public Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(CancellationToken ct = default)
|
||||
=> _storage.GetEventSessionIndexHandlesAsync(ct);
|
||||
|
||||
public Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default)
|
||||
=> _storage.GetEventSessionIndexHandleAsync(tenantId, sessionId, ct);
|
||||
|
||||
public Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default)
|
||||
=> _storage.StampHandleInventoryDecrementedAsync(tenantId, sessionId, etag, ct);
|
||||
|
||||
public Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default)
|
||||
=> _storage.DeleteEventSessionIndexHandleAsync(tenantId, sessionId, etag, ct);
|
||||
|
||||
public Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default)
|
||||
=> _storage.DeleteOrphanSessionRowsAsync(table, tenantId, sessionId, ct);
|
||||
|
||||
public Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default)
|
||||
=> _storage.GetEventTypeIndexSessionKeysAsync(ct);
|
||||
|
||||
public Task<List<(string TenantId, string SessionId)>> GetAllSessionTombstoneKeysAsync()
|
||||
=> _storage.GetAllSessionTombstoneKeysAsync();
|
||||
|
||||
+1
@@ -55,6 +55,7 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
"LatestAgentV2Version", "LatestAgentV2Sha256", "LatestAgentV2ExeSha256", "LatestBootstrapV2ScriptVersion",
|
||||
"PlanTierDefinitionsJson",
|
||||
"VulnerabilityDataLastSyncUtc", "MsrcLastSyncUtc", "NvdCacheLastRefreshUtc", "EpssLastSyncUtc",
|
||||
"OrphanDeepCheckLastRunUtc",
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -228,12 +228,14 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
return AsOutput(await WriteDeviceBlockedAsync(req, sessionVerdict));
|
||||
}
|
||||
|
||||
// Cascade-delete guard: refuse the batch with 410 Gone when a V2 cascade owns the
|
||||
// Sessions row (states Preparing/Queued/Running/Poisoned). Without this check, the
|
||||
// hot-path writers below (StoreEventsBatchAsync, signal/transition StoreBatchAsync,
|
||||
// UpdateSessionImeAgentVersionAsync upsert, …) would land rows past the lock and
|
||||
// leave orphan data the manifest cannot describe. One read per batch; absent
|
||||
// Sessions row → silent pass (caller handles session-not-found in its own write).
|
||||
// Session guard: refuse the batch with 410 Gone when a V2 cascade owns the Sessions
|
||||
// row (states Preparing/Queued/Running/Poisoned) OR when there is no Sessions row at
|
||||
// all. Both mean "this session is over server-side": the hot-path writers below
|
||||
// (StoreEventsBatchAsync, signal/transition StoreBatchAsync, the index upserts, …)
|
||||
// would otherwise land rows no cascade can ever reach, because a cascade needs the
|
||||
// Sessions row. Registration always precedes the first upload (and re-creates an
|
||||
// absent row), so a live session never sees this. One read per batch; the row's
|
||||
// IngestRefusal marker tells the two 410 reasons apart on the request row.
|
||||
Azure.Data.Tables.TableEntity? guardSessionRow;
|
||||
try
|
||||
{
|
||||
@@ -241,12 +243,22 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
}
|
||||
catch (SessionDeletionLockedException locked)
|
||||
{
|
||||
RequestRowMarkers.Stamp(req, RequestRowMarkers.IngestRefusalKey, RequestRowMarkers.IngestRefusal.CascadeLocked);
|
||||
_logger.LogInformation(
|
||||
"IngestTelemetry: refused batch — cascade in flight tenant={Tenant} session={Session} state={State} manifestId={ManifestId}",
|
||||
bodyTenantId, sessionId, locked.CurrentState, locked.ManifestId);
|
||||
return AsOutput(await WriteSessionLockedAsync(req, locked));
|
||||
}
|
||||
|
||||
if (guardSessionRow == null)
|
||||
{
|
||||
RequestRowMarkers.Stamp(req, RequestRowMarkers.IngestRefusalKey, RequestRowMarkers.IngestRefusal.SessionUnknown);
|
||||
_logger.LogInformation(
|
||||
"IngestTelemetry: refused batch — session unknown tenant={Tenant} session={Session}",
|
||||
bodyTenantId, sessionId);
|
||||
return AsOutput(await WriteSessionUnknownAsync(req));
|
||||
}
|
||||
|
||||
// The header serial is caller-declared; the Sessions row carries the serial the
|
||||
// session was registered under. When they differ, the ROW's serial is held against
|
||||
// the block list too — a mid-session header change must not dodge a block placed
|
||||
@@ -415,6 +427,19 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
ProcessedAt = DateTime.UtcNow,
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
/// 410 for a session without a Sessions row: same status and body shape as the cascade lock,
|
||||
/// because the agent's reaction is the same (stop, clean up). Never a poison body — the
|
||||
/// agent would drop the items and re-emit a marker event into the same dead session.
|
||||
/// </summary>
|
||||
private static Task<HttpResponseData> WriteSessionUnknownAsync(HttpRequestData req)
|
||||
=> req.JsonAsync(HttpStatusCode.Gone, new IngestEventsResponse
|
||||
{
|
||||
Success = false,
|
||||
Message = "Session is unknown to the backend (never registered, or already deleted); further telemetry will be rejected.",
|
||||
ProcessedAt = DateTime.UtcNow,
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
/// 422 + poison body: the backend end of the agent's item-level poison protocol
|
||||
/// (<c>BackendTelemetryUploader.TryReadPoisonSignalAsync</c>). Logged as a Warning with
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Timer for the orphan-session sweep (<see cref="OrphanSessionSweeper"/>): rows of sessions
|
||||
/// that have no Sessions row — which no cascade can ever reach — are found through their
|
||||
/// EventSessionIndex handle and removed from every session table. Its own timer, lease and
|
||||
/// ops events: the sweep needs a slower cadence than the 2h <c>Maintenance</c> run, must never
|
||||
/// race the retention cascade's 00:00/12:00 tick, and its cost (two key drains instead of one
|
||||
/// Sessions point-read per handle) should stay visible on its own heartbeat.
|
||||
/// </summary>
|
||||
public sealed class OrphanSessionSweepFunction
|
||||
{
|
||||
// Minute 45 of every fourth hour: no other timer fires at :45 (Maintenance :00 of even
|
||||
// hours, SessionSweep :30, WhatsNew :20, backup watchdog :00/:30, retention 00:00/12:00).
|
||||
// const per repo convention (see SessionSweepFunction.cs).
|
||||
private const string Cron = "0 45 */4 * * *";
|
||||
|
||||
private readonly OrphanSessionSweeper _sweeper;
|
||||
private readonly ILogger<OrphanSessionSweepFunction> _logger;
|
||||
|
||||
public OrphanSessionSweepFunction(OrphanSessionSweeper sweeper, ILogger<OrphanSessionSweepFunction> logger)
|
||||
{
|
||||
_sweeper = sweeper;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[Function("OrphanSessionSweep")]
|
||||
public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("OrphanSessionSweep timer trigger fired");
|
||||
// Lease, run and every ops event (Completed / Failed / SkippedLocked) live in the sweeper,
|
||||
// shared with the manual maintenance run.
|
||||
await _sweeper.RunAsync("Timer", cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
+13
-1
@@ -24,19 +24,22 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly VulnerabilityCorrelationService _correlationService;
|
||||
private readonly IVulnerabilityInventoryLoader _inventoryLoader;
|
||||
private readonly Services.Deletion.SessionDeletionGuard _deletionGuard;
|
||||
|
||||
public GetVulnerabilityReportFunction(
|
||||
ILogger<GetVulnerabilityReportFunction> logger,
|
||||
IVulnerabilityRepository vulnRepo,
|
||||
ISessionRepository sessionRepo,
|
||||
VulnerabilityCorrelationService correlationService,
|
||||
IVulnerabilityInventoryLoader inventoryLoader)
|
||||
IVulnerabilityInventoryLoader inventoryLoader,
|
||||
Services.Deletion.SessionDeletionGuard deletionGuard)
|
||||
{
|
||||
_logger = logger;
|
||||
_vulnRepo = vulnRepo;
|
||||
_sessionRepo = sessionRepo;
|
||||
_correlationService = correlationService;
|
||||
_inventoryLoader = inventoryLoader;
|
||||
_deletionGuard = deletionGuard;
|
||||
}
|
||||
|
||||
[Function("GetVulnerabilityReport")]
|
||||
@@ -65,6 +68,15 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
|
||||
if (rescan)
|
||||
{
|
||||
// A re-scan derives rows (report, CveIndex, inventory counters): only for a live
|
||||
// session, like the queue handler — never for one the backend does not know or a
|
||||
// cascade is tearing down.
|
||||
var liveness = await _deletionGuard.GetLivenessAsync(effectiveTenantId, sessionId, "VulnerabilityRescan");
|
||||
if (liveness == Services.Deletion.SessionLiveness.Missing)
|
||||
return await req.NotFoundAsync("Session not found.");
|
||||
if (liveness == Services.Deletion.SessionLiveness.Locked)
|
||||
return await req.ConflictAsync("Session is being deleted; a re-scan is not possible.");
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Vulnerability re-scan requested for session {SessionId} by {User} (query: {Query})",
|
||||
|
||||
@@ -198,6 +198,10 @@ public class RequestTelemetryMiddleware : IFunctionsWorkerMiddleware
|
||||
if (context.Items.TryGetValue(RequestRowMarkers.DeviceValidationKey, out var deviceValidation) && deviceValidation is string deviceValidationOutcome)
|
||||
requestTelemetry.Properties[RequestRowMarkers.DeviceValidationKey] = deviceValidationOutcome;
|
||||
|
||||
// INGEST-REFUSAL — set by IngestTelemetry on its 410 paths (cascade lock vs unknown session).
|
||||
if (context.Items.TryGetValue(RequestRowMarkers.IngestRefusalKey, out var ingestRefusal) && ingestRefusal is string ingestRefusalReason)
|
||||
requestTelemetry.Properties[RequestRowMarkers.IngestRefusalKey] = ingestRefusalReason;
|
||||
|
||||
// CERT-DEVICE-BINDING — the Intune device lookup behind the certificate (admitting or
|
||||
// observing), set by SecurityValidator whenever a lookup result exists for the request.
|
||||
foreach (var key in RequestRowMarkers.CertDeviceBindingKeys)
|
||||
|
||||
@@ -294,6 +294,10 @@ builder.Services.AddHttpClient<GitHubRuleRepository>()
|
||||
// BackgroundService worker that runs a queued manual run.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunLockStore>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.MaintenanceRunGate>();
|
||||
// Orphan-session sweep: own 4h timer (OrphanSessionSweepFunction), own lease, also run by the
|
||||
// manual maintenance trigger. The cascade for sessions that have no Sessions row.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.OrphanSweepLockStore>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Maintenance.OrphanSessionSweeper>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Services.Maintenance.IMaintenanceTriggerProducer,
|
||||
AutopilotMonitor.Functions.Services.Maintenance.AzureQueueMaintenanceTriggerProducer>();
|
||||
|
||||
@@ -41,6 +41,22 @@ namespace AutopilotMonitor.Functions.Security
|
||||
public static string DeviceValidationValue(ValidatorType validatedBy)
|
||||
=> validatedBy == ValidatorType.Unknown ? DeviceValidation.None : validatedBy.ToString();
|
||||
|
||||
/// <summary>
|
||||
/// Why a telemetry batch was refused with 410: the session is locked by a cascade delete,
|
||||
/// or the backend has no Sessions row for it. Both share the status so the agent treats
|
||||
/// them alike; the row keeps them apart.
|
||||
/// </summary>
|
||||
public const string IngestRefusalKey = "IngestRefusal";
|
||||
|
||||
/// <summary>Closed set of <see cref="IngestRefusalKey"/> values.</summary>
|
||||
public static class IngestRefusal
|
||||
{
|
||||
/// <summary>The Sessions row is in a cascade lock state (or freshly tombstoned).</summary>
|
||||
public const string CascadeLocked = "cascade_locked";
|
||||
/// <summary>No Sessions row and no tombstone: never registered, or deleted longer ago than the marker lives.</summary>
|
||||
public const string SessionUnknown = "session_unknown";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Outcome of the Intune device lookup behind the client certificate
|
||||
/// (<see cref="IntuneDeviceBindingOutcome"/> name). Absent when no lookup result exists for
|
||||
|
||||
+17
@@ -50,6 +50,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
|
||||
private readonly SignalRNotificationService _signalRNotification;
|
||||
private readonly TenantConfigurationService _configService;
|
||||
private readonly Notifications.NotificationChannelDispatcher _channelDispatcher;
|
||||
private readonly Deletion.SessionDeletionGuard _deletionGuard;
|
||||
private readonly ILogger<AnalyzeOnEnrollmentEndHandler> _logger;
|
||||
|
||||
public const string ReasonEnrollmentComplete = "enrollment_complete";
|
||||
@@ -66,6 +67,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
|
||||
SignalRNotificationService signalRNotification,
|
||||
TenantConfigurationService configService,
|
||||
Notifications.NotificationChannelDispatcher channelDispatcher,
|
||||
Deletion.SessionDeletionGuard deletionGuard,
|
||||
ILogger<AnalyzeOnEnrollmentEndHandler> logger)
|
||||
{
|
||||
_ruleService = ruleService;
|
||||
@@ -75,6 +77,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
|
||||
_signalRNotification = signalRNotification;
|
||||
_configService = configService;
|
||||
_channelDispatcher = channelDispatcher;
|
||||
_deletionGuard = deletionGuard;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -94,6 +97,20 @@ namespace AutopilotMonitor.Functions.Services.Analyze
|
||||
}
|
||||
|
||||
var sessionPrefix = $"[Session: {envelope.SessionId.Substring(0, Math.Min(8, envelope.SessionId.Length))}]";
|
||||
|
||||
// Rule results, stats and channel alerts only for a live session: a session the backend
|
||||
// does not know, or one a cascade is tearing down, gets nothing derived (no cascade could
|
||||
// reach the rows afterwards, and an alert would name a session that does not exist).
|
||||
var liveness = await _deletionGuard.GetLivenessAsync(
|
||||
envelope.TenantId, envelope.SessionId, "AnalyzeOnEnrollmentEnd", cancellationToken).ConfigureAwait(false);
|
||||
if (liveness != Deletion.SessionLiveness.Live)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"{Prefix} Analyze skipped — session {Liveness} (reason={Reason})",
|
||||
sessionPrefix, liveness, envelope.Reason);
|
||||
return;
|
||||
}
|
||||
|
||||
var isVulnerabilityRerun = string.Equals(
|
||||
envelope.Reason, ReasonVulnerabilityCorrelated, StringComparison.OrdinalIgnoreCase);
|
||||
var isWhitegloveSealed = string.Equals(
|
||||
|
||||
@@ -8,6 +8,17 @@ using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>Outcome of <see cref="SessionDeletionGuard.GetLivenessAsync"/>.</summary>
|
||||
public enum SessionLiveness
|
||||
{
|
||||
/// <summary>Sessions row present and not in a cascade lock state.</summary>
|
||||
Live,
|
||||
/// <summary>No Sessions row: never registered, or deleted.</summary>
|
||||
Missing,
|
||||
/// <summary>Sessions row present but owned by a cascade (Preparing / Queued / Running / Poisoned).</summary>
|
||||
Locked,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Single chokepoint for the cascade-delete writer-block invariant (Plan §1 P7 / §5 PR3).
|
||||
/// Two APIs split for hot-path I/O cost:
|
||||
@@ -88,6 +99,35 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
/// <summary>Columns the guard reads off the row itself; a projected read adds them to the caller's list.</summary>
|
||||
public static readonly string[] LockColumns = { "DeletionState", "PendingDeletionManifestId" };
|
||||
|
||||
/// <summary>
|
||||
/// For writers that DERIVE rows for a session (queue handlers, the manual rescan). Unlike
|
||||
/// the registration/ingest contract, a missing Sessions row is a stop here, not a pass:
|
||||
/// a derived row for a session the backend does not know can never be reached by a
|
||||
/// cascade and would only feed the orphan sweep. Read errors propagate (queue retry).
|
||||
/// </summary>
|
||||
public async Task<SessionLiveness> GetLivenessAsync(string tenantId, string sessionId, string callerContext, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var sessionRow = await _reader.GetSessionRowAsync(tenantId, sessionId, LockColumns, cancellationToken);
|
||||
if (sessionRow == null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SessionDeletionGuard: no Sessions row — derived write skipped: tenant={TenantId} session={SessionId} caller={Caller}",
|
||||
tenantId, sessionId, callerContext);
|
||||
return SessionLiveness.Missing;
|
||||
}
|
||||
|
||||
var state = sessionRow.GetString("DeletionState");
|
||||
if (SessionDeletionState.IsLocked(state))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionGuard: session locked — derived write skipped: tenant={TenantId} session={SessionId} state={State} caller={Caller}",
|
||||
tenantId, sessionId, state, callerContext);
|
||||
return SessionLiveness.Locked;
|
||||
}
|
||||
|
||||
return SessionLiveness.Live;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="EnsureWritableAndGetRowAsync(string, string, string, CancellationToken)"/> with the
|
||||
/// row projected to <paramref name="select"/> plus <see cref="LockColumns"/> — the agent hot
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-step wall-clock of one maintenance run, reported as <c>stepsMs</c> on the
|
||||
/// <c>MaintenanceCompleted</c> ops event. Worker-side timing logs never reach App Insights,
|
||||
/// and the run's composition is what decides which steps leave the 2h timer next.
|
||||
/// A step that throws is still recorded (its time counts), then the exception propagates.
|
||||
/// </summary>
|
||||
public sealed class MaintenanceStepTimer
|
||||
{
|
||||
private readonly Dictionary<string, long> _steps = new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Durations in declaration order (ms, rounded).</summary>
|
||||
public IReadOnlyDictionary<string, long> Steps => _steps;
|
||||
|
||||
public async Task RunAsync(string step, Func<Task> body)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
await body().ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_steps[step] = sw.ElapsedMilliseconds;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<T> RunAsync<T>(string step, Func<Task<T>> body)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
return await body().ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_steps[step] = sw.ElapsedMilliseconds;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// The OData filter that selects every row of ONE session in each table the orphan sweep
|
||||
/// clears — the key shape of the table's writer, not of the cascade manifest (a session
|
||||
/// without a Sessions row has no manifest). Pure, so the shapes are unit-tested against the
|
||||
/// writers once instead of trusted per call.
|
||||
/// </summary>
|
||||
public static class OrphanSessionRowFilters
|
||||
{
|
||||
/// <summary>
|
||||
/// Tables cleared through a filter, in the order the sweep runs them. The handle
|
||||
/// (EventSessionIndex) and the inventory pair (SessionInventoryContributions + the
|
||||
/// SoftwareInventory decrements) are handled by the sweeper itself and are not listed here.
|
||||
/// </summary>
|
||||
public static readonly string[] Tables =
|
||||
{
|
||||
Constants.TableNames.Events,
|
||||
Constants.TableNames.RuleResults,
|
||||
Constants.TableNames.AppInstallSummaries,
|
||||
Constants.TableNames.VulnerabilityReports,
|
||||
Constants.TableNames.DeviceSnapshot,
|
||||
Constants.TableNames.Signals,
|
||||
Constants.TableNames.DecisionTransitions,
|
||||
Constants.TableNames.EventTypeIndex,
|
||||
Constants.TableNames.CveIndex,
|
||||
Constants.TableNames.SessionTimeBreakdowns,
|
||||
Constants.TableNames.SessionAnnotations,
|
||||
Constants.TableNames.SessionTenantLookup,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Filter for <paramref name="table"/>; throws for a table that has no declared shape so a
|
||||
/// new session table can never be swept with a guessed filter.
|
||||
/// </summary>
|
||||
public static string For(string table, string tenantId, string sessionId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
|
||||
var t = ODataSanitizer.EscapeValue(tenantId);
|
||||
var s = ODataSanitizer.EscapeValue(sessionId);
|
||||
|
||||
switch (table)
|
||||
{
|
||||
// Partition per session: {tenantId}_{sessionId}.
|
||||
case Constants.TableNames.Events:
|
||||
case Constants.TableNames.RuleResults:
|
||||
case Constants.TableNames.Signals:
|
||||
case Constants.TableNames.DecisionTransitions:
|
||||
case Constants.TableNames.VulnerabilityReports:
|
||||
return $"PartitionKey eq '{t}_{s}'";
|
||||
|
||||
// One row per session under the tenant partition.
|
||||
case Constants.TableNames.DeviceSnapshot:
|
||||
case Constants.TableNames.SessionTimeBreakdowns:
|
||||
return $"PartitionKey eq '{t}' and RowKey eq '{s}'";
|
||||
|
||||
// Tenant partition, session in a column.
|
||||
case Constants.TableNames.AppInstallSummaries:
|
||||
return $"PartitionKey eq '{t}' and SessionId eq '{s}'";
|
||||
|
||||
// PK {tenantId}_{eventType}, RK {invertedTicks}_{sessionId}: the RowKey of a session
|
||||
// without a Sessions row is not reconstructible (one row per batch), so the session
|
||||
// is found by its column across the tenant's partitions. Tenant ids are GUIDs, so the
|
||||
// '_'..'_~' range cannot reach another tenant.
|
||||
case Constants.TableNames.EventTypeIndex:
|
||||
return $"PartitionKey ge '{t}_' and PartitionKey lt '{t}_~' and SessionId eq '{s}'";
|
||||
|
||||
// PK {tenantId}_{cveId}, RK sessionId.
|
||||
case Constants.TableNames.CveIndex:
|
||||
return $"PartitionKey ge '{t}_' and PartitionKey lt '{t}_~' and RowKey eq '{s}'";
|
||||
|
||||
// PK tenantId, RK {sessionId}_{lane}; the lanes are a closed set.
|
||||
case Constants.TableNames.SessionAnnotations:
|
||||
var lanes = string.Join(" or ", AnnotationLanes.All.Select(lane => $"RowKey eq '{s}_{lane}'"));
|
||||
return $"PartitionKey eq '{t}' and ({lanes})";
|
||||
|
||||
// PK sessionId, RK 'tenant'; the TenantId column pins the row to this tenant's session.
|
||||
case Constants.TableNames.SessionTenantLookup:
|
||||
return $"PartitionKey eq '{s}' and RowKey eq 'tenant' and TenantId eq '{t}'";
|
||||
|
||||
default:
|
||||
throw new ArgumentException($"No orphan-sweep filter is declared for table '{table}'.", nameof(table));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// The cascade for sessions that have no Sessions row. A cascade needs the row (CAS lock,
|
||||
/// manifest, tombstone); rows a writer leaves behind for a session the backend does not know
|
||||
/// are therefore reachable only through the discovery handle every events write creates
|
||||
/// (EventSessionIndex, PK tenantId / RK sessionId). The sweep diffs those handles against the
|
||||
/// Sessions keys, re-checks every candidate moments before touching it, clears every session
|
||||
/// table the cascade manifest knows (ratchet in <c>TableLifecycleBucketTests</c>), and deletes
|
||||
/// the handle LAST — a failure before that point leaves the session discoverable for the next
|
||||
/// run; every delete is idempotent.
|
||||
/// <para>
|
||||
/// Once a week (Sunday morning UTC, or on any manual maintenance run) it additionally
|
||||
/// reconciles EventTypeIndex against the Sessions keys, read-only: the proof that nothing
|
||||
/// without a handle is left behind.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class OrphanSessionSweeper
|
||||
{
|
||||
/// <summary>A handle younger than this may belong to a session whose registration is still in flight.</summary>
|
||||
public static readonly TimeSpan IngestGrace = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>
|
||||
/// A restore re-inserts the handle (old LastIngestAt, fresh write time) and the Sessions row
|
||||
/// LAST; a handle written this recently is left alone until the restore has either finished
|
||||
/// or failed.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan HandleWriteGrace = TimeSpan.FromHours(2);
|
||||
|
||||
public const int MaxCandidatesPerRun = 50;
|
||||
public static readonly TimeSpan RunBudget = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>Sunday at or after this hour (UTC) is the weekly reconciliation window — the quiet part of the week.</summary>
|
||||
public const int DeepCheckSundayHourUtc = 8;
|
||||
public static readonly TimeSpan DeepCheckInterval = TimeSpan.FromDays(6);
|
||||
/// <summary>A missed Sunday (deploy, outage) does not postpone the reconciliation past this age.</summary>
|
||||
public static readonly TimeSpan DeepCheckOverdue = TimeSpan.FromDays(8);
|
||||
public const int DeepCheckSampleSize = 20;
|
||||
|
||||
/// <summary>LastIngestAt of a handle written by hand for a legacy orphan: no ingest ever happened at that time.</summary>
|
||||
public static readonly DateTime LegacyHandleLastIngestAt = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
public const string Actor = "System.OrphanSweep";
|
||||
public const string TimerTrigger = "Timer";
|
||||
|
||||
/// <summary>
|
||||
/// Every table the sweep clears: the filter tables, the inventory pair (side row + the
|
||||
/// SoftwareInventory decrements) and the handle. Pinned against the cascade manifest.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<string> SweptTables = OrphanSessionRowFilters.Tables
|
||||
.Concat(new[]
|
||||
{
|
||||
Constants.TableNames.SessionInventoryContributions,
|
||||
Constants.TableNames.SoftwareInventory,
|
||||
Constants.TableNames.EventSessionIndex,
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly IVulnerabilityRepository _vulnRepo;
|
||||
private readonly ISessionDeletionInventoryReader _reader;
|
||||
private readonly AdminConfigurationService _adminConfig;
|
||||
private readonly OrphanSweepLockStore _lockStore;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ILogger<OrphanSessionSweeper> _logger;
|
||||
private readonly Func<DateTime> _clock;
|
||||
private readonly TimeSpan _runBudget;
|
||||
|
||||
public OrphanSessionSweeper(
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
IVulnerabilityRepository vulnRepo,
|
||||
ISessionDeletionInventoryReader reader,
|
||||
AdminConfigurationService adminConfig,
|
||||
OrphanSweepLockStore lockStore,
|
||||
OpsEventService opsEvents,
|
||||
ILogger<OrphanSessionSweeper> logger)
|
||||
: this(maintenanceRepo, vulnRepo, reader, adminConfig, lockStore, opsEvents, logger, () => DateTime.UtcNow, RunBudget)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Test seam: injectable clock and run budget.</summary>
|
||||
internal OrphanSessionSweeper(
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
IVulnerabilityRepository vulnRepo,
|
||||
ISessionDeletionInventoryReader reader,
|
||||
AdminConfigurationService adminConfig,
|
||||
OrphanSweepLockStore lockStore,
|
||||
OpsEventService opsEvents,
|
||||
ILogger<OrphanSessionSweeper> logger,
|
||||
Func<DateTime> clock,
|
||||
TimeSpan runBudget)
|
||||
{
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_vulnRepo = vulnRepo;
|
||||
_reader = reader;
|
||||
_adminConfig = adminConfig;
|
||||
_lockStore = lockStore;
|
||||
_opsEvents = opsEvents;
|
||||
_logger = logger;
|
||||
_clock = clock;
|
||||
_runBudget = runBudget;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One run under the sweep lease: timer and manual maintenance enter here. Returns false
|
||||
/// when another run held the lease. A manual run always includes the reconciliation.
|
||||
/// Every outcome is an ops event; nothing propagates to the caller.
|
||||
/// </summary>
|
||||
public async Task<bool> RunAsync(string triggeredBy, CancellationToken cancellationToken)
|
||||
{
|
||||
BlobLeaseClient lease;
|
||||
try
|
||||
{
|
||||
lease = await _lockStore.AcquireLeaseAsync(ct: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseHeldException)
|
||||
{
|
||||
_logger.LogWarning("OrphanSweep: lease held by another run — skipping (triggeredBy={TriggeredBy})", triggeredBy);
|
||||
await _opsEvents.RecordOrphanSweepSkippedLockedAsync(triggeredBy).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
using var runCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
var leaseHolder = new MaintenanceLeaseHolder(lease, runCts, _logger);
|
||||
try
|
||||
{
|
||||
OrphanSweepRunResult result;
|
||||
try
|
||||
{
|
||||
result = await RunCoreAsync(forceDeepCheck: triggeredBy != TimerTrigger, runCts.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "OrphanSweep: run failed (triggeredBy={TriggeredBy})", triggeredBy);
|
||||
await _opsEvents.RecordOrphanSweepFailedAsync(ex.Message, triggeredBy).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
|
||||
await _opsEvents.RecordOrphanSweepCompletedAsync(result, triggeredBy).ConfigureAwait(false);
|
||||
if (result.Cleaned.Count > 0)
|
||||
await _opsEvents.RecordOrphanEventsCleanedAsync(result).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (leaseHolder.RenewalFailureReason != null)
|
||||
_logger.LogWarning("OrphanSweep: the lease could not be renewed during the run ({Reason})", leaseHolder.RenewalFailureReason);
|
||||
await leaseHolder.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The run without lease and ops events (unit-testable).</summary>
|
||||
internal async Task<OrphanSweepRunResult> RunCoreAsync(bool forceDeepCheck, CancellationToken ct)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
var now = _clock();
|
||||
var result = new OrphanSweepRunResult();
|
||||
|
||||
// Sessions first: a cascade that finishes between the two drains removes the handle
|
||||
// (step 6) before the row (step 18), so this order cannot produce a candidate the
|
||||
// cascade is still working on.
|
||||
var sessionKeys = await _maintenanceRepo.GetSessionKeysAsync(ct).ConfigureAwait(false);
|
||||
var handles = await _maintenanceRepo.GetEventSessionIndexHandlesAsync(ct).ConfigureAwait(false);
|
||||
result.SessionKeys = sessionKeys.Count;
|
||||
result.HandlesScanned = handles.Count;
|
||||
|
||||
var candidates = handles
|
||||
.Where(h => !sessionKeys.Contains((h.TenantId, h.SessionId)))
|
||||
.Where(h => h.LastIngestAt <= now - IngestGrace && h.WrittenAt <= now - HandleWriteGrace)
|
||||
.OrderBy(h => h.LastIngestAt)
|
||||
.ToList();
|
||||
result.Candidates = candidates.Count;
|
||||
|
||||
var processed = 0;
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
if (processed >= MaxCandidatesPerRun) break;
|
||||
if (sw.Elapsed >= _runBudget)
|
||||
{
|
||||
result.BudgetExhausted = true;
|
||||
break;
|
||||
}
|
||||
ct.ThrowIfCancellationRequested();
|
||||
processed++;
|
||||
try
|
||||
{
|
||||
await CleanCandidateAsync(candidate, result, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.FailedSessions++;
|
||||
_logger.LogError(ex,
|
||||
"OrphanSweep: cleaning tenant={TenantId} session={SessionId} failed — the handle stays for the next run",
|
||||
candidate.TenantId, candidate.SessionId);
|
||||
}
|
||||
}
|
||||
result.Remaining = candidates.Count - processed;
|
||||
|
||||
if (forceDeepCheck || IsDeepCheckDue(await ReadDeepCheckLastRunAsync().ConfigureAwait(false), now))
|
||||
result.Reconcile = await ReconcileAsync(sessionKeys, handles, now, ct).ConfigureAwait(false);
|
||||
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task CleanCandidateAsync(OrphanSessionHandle candidate, OrphanSweepRunResult result, CancellationToken ct)
|
||||
{
|
||||
var tenantId = candidate.TenantId;
|
||||
var sessionId = candidate.SessionId;
|
||||
|
||||
// Re-checks moments before the deletes — the scan above is minutes old.
|
||||
var handle = await _maintenanceRepo.GetEventSessionIndexHandleAsync(tenantId, sessionId, ct).ConfigureAwait(false);
|
||||
if (handle == null)
|
||||
{
|
||||
result.SkippedHandleGone++; // a cascade got there first
|
||||
return;
|
||||
}
|
||||
if (!string.Equals(handle.ETag, candidate.ETag, StringComparison.Ordinal))
|
||||
{
|
||||
result.SkippedHandleChanged++; // a later write; its own grace starts over
|
||||
return;
|
||||
}
|
||||
var row = await _reader.GetSessionRowAsync(tenantId, sessionId, SessionDeletionGuard.LockColumns, ct).ConfigureAwait(false);
|
||||
if (row != null)
|
||||
{
|
||||
result.SkippedRowReturned++; // registered or restored meanwhile
|
||||
return;
|
||||
}
|
||||
var afterCascade = await _reader.GetActiveSessionTombstoneAsync(tenantId, sessionId, ct).ConfigureAwait(false) != null;
|
||||
if (afterCascade) result.AfterCascade++;
|
||||
|
||||
var cleaned = new OrphanSweepCleanedSession
|
||||
{
|
||||
TenantId = tenantId,
|
||||
SessionId = sessionId,
|
||||
Legacy = candidate.LastIngestAt <= LegacyHandleLastIngestAt,
|
||||
AfterCascade = afterCascade,
|
||||
};
|
||||
var etag = handle.ETag;
|
||||
|
||||
// Inventory first, with the handle as the progress record. The stamp is the claim: a
|
||||
// crash after it loses at most this session's decrements (the counts stay high, as they
|
||||
// did before the sweep existed), and a rerun can never decrement twice.
|
||||
if (!handle.InventoryDecremented)
|
||||
{
|
||||
var stamped = await _maintenanceRepo.StampHandleInventoryDecrementedAsync(tenantId, sessionId, etag, ct).ConfigureAwait(false);
|
||||
if (stamped == null)
|
||||
{
|
||||
result.SkippedHandleChanged++;
|
||||
return;
|
||||
}
|
||||
etag = stamped;
|
||||
|
||||
var keys = await _vulnRepo.GetSessionInventoryContributionsAsync(tenantId, sessionId, ct).ConfigureAwait(false);
|
||||
if (keys != null)
|
||||
{
|
||||
cleaned.HadInventorySideRow = true;
|
||||
foreach (var key in keys)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _vulnRepo.DecrementSoftwareInventoryEntryAsync(tenantId, key.Vendor, key.Name, key.Version, ct).ConfigureAwait(false);
|
||||
cleaned.InventoryKeysDecremented++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.InventoryDecrementsFailed++;
|
||||
_logger.LogWarning(ex,
|
||||
"OrphanSweep: inventory decrement failed tenant={TenantId} session={SessionId} key={Vendor}/{Name}/{Version}",
|
||||
tenantId, sessionId, key.Vendor, key.Name, key.Version);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
await _vulnRepo.DeleteSessionInventoryContributionsAsync(tenantId, sessionId, ct).ConfigureAwait(false);
|
||||
|
||||
foreach (var table in OrphanSessionRowFilters.Tables)
|
||||
{
|
||||
var rows = await _maintenanceRepo.DeleteOrphanSessionRowsAsync(table, tenantId, sessionId, ct).ConfigureAwait(false);
|
||||
cleaned.RowsByTable[table] = rows;
|
||||
}
|
||||
|
||||
// The handle last, and only while unchanged: a batch that re-wrote it meanwhile has
|
||||
// rows the next run must still find.
|
||||
var handleDeleted = await _maintenanceRepo.DeleteEventSessionIndexHandleAsync(tenantId, sessionId, etag, ct).ConfigureAwait(false);
|
||||
cleaned.HandleKept = !handleDeleted;
|
||||
|
||||
result.Add(cleaned);
|
||||
_logger.LogInformation(
|
||||
"OrphanSweep: cleaned tenant={TenantId} session={SessionId} rows={Rows} inventoryKeys={Keys} legacy={Legacy} afterCascade={AfterCascade} handleKept={HandleKept}",
|
||||
tenantId, sessionId, cleaned.TotalRows, cleaned.InventoryKeysDecremented, cleaned.Legacy, cleaned.AfterCascade, cleaned.HandleKept);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sunday at/after 08:00 UTC once the last run is at least six days old; a run older than
|
||||
/// eight days is due at the next tick regardless. Never run before: the first Sunday.
|
||||
/// </summary>
|
||||
internal static bool IsDeepCheckDue(DateTime? lastRunUtc, DateTime nowUtc)
|
||||
{
|
||||
var inSundayWindow = nowUtc.DayOfWeek == DayOfWeek.Sunday && nowUtc.Hour >= DeepCheckSundayHourUtc;
|
||||
if (lastRunUtc == null) return inSundayWindow;
|
||||
var age = nowUtc - lastRunUtc.Value;
|
||||
if (age >= DeepCheckOverdue) return true;
|
||||
return inSundayWindow && age >= DeepCheckInterval;
|
||||
}
|
||||
|
||||
private async Task<DateTime?> ReadDeepCheckLastRunAsync()
|
||||
{
|
||||
var config = await _adminConfig.GetConfigurationAsync().ConfigureAwait(false);
|
||||
return DateTime.TryParse(config?.OrphanDeepCheckLastRunUtc, CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var parsed)
|
||||
? parsed
|
||||
: (DateTime?)null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Read-only: every (tenant, session) in EventTypeIndex that has neither a Sessions row nor
|
||||
/// a handle (handles are the sweep's own pipeline) is residue — confirmed against the live
|
||||
/// Sessions row, because the drains are minutes old. Residue is reported, never cleaned
|
||||
/// here: the operator gives it a handle, and the normal path with all its checks does the rest.
|
||||
/// </summary>
|
||||
private async Task<OrphanReconcileResult> ReconcileAsync(
|
||||
HashSet<(string TenantId, string SessionId)> sessionKeys,
|
||||
IReadOnlyList<OrphanSessionHandle> handles,
|
||||
DateTime now,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
var indexKeys = await _maintenanceRepo.GetEventTypeIndexSessionKeysAsync(ct).ConfigureAwait(false);
|
||||
var handleKeys = new HashSet<(string TenantId, string SessionId)>(handles.Select(h => (h.TenantId, h.SessionId)));
|
||||
|
||||
var residue = new List<(string TenantId, string SessionId)>();
|
||||
foreach (var key in indexKeys)
|
||||
{
|
||||
if (sessionKeys.Contains(key) || handleKeys.Contains(key)) continue;
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var row = await _reader.GetSessionRowAsync(key.TenantId, key.SessionId, SessionDeletionGuard.LockColumns, ct).ConfigureAwait(false);
|
||||
if (row != null) continue;
|
||||
residue.Add(key);
|
||||
}
|
||||
|
||||
var result = new OrphanReconcileResult
|
||||
{
|
||||
IndexSessions = indexKeys.Count,
|
||||
ResidueSessions = residue.Count,
|
||||
Sample = residue.Take(DeepCheckSampleSize).ToList(),
|
||||
DurationMs = sw.ElapsedMilliseconds,
|
||||
};
|
||||
await _opsEvents.RecordOrphanReconcileCompletedAsync(result).ConfigureAwait(false);
|
||||
|
||||
var stamp = now.ToString("o", CultureInfo.InvariantCulture);
|
||||
await _adminConfig.UpdateAsync(c => { c.OrphanDeepCheckLastRunUtc = stamp; return null; }, Actor, "orphan-sweep").ConfigureAwait(false);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One session the sweep cleaned in a run.</summary>
|
||||
public sealed class OrphanSweepCleanedSession
|
||||
{
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
public string SessionId { get; set; } = string.Empty;
|
||||
/// <summary>The handle was written by hand for a legacy orphan (LastIngestAt 2000-01-01).</summary>
|
||||
public bool Legacy { get; set; }
|
||||
/// <summary>A tombstone marker existed: a cascade ran, rows were written after it.</summary>
|
||||
public bool AfterCascade { get; set; }
|
||||
/// <summary>The handle changed between the deletes and the final delete; the next run sees the session again.</summary>
|
||||
public bool HandleKept { get; set; }
|
||||
public bool HadInventorySideRow { get; set; }
|
||||
public int InventoryKeysDecremented { get; set; }
|
||||
public Dictionary<string, int> RowsByTable { get; } = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
public int TotalRows => RowsByTable.Values.Sum();
|
||||
}
|
||||
|
||||
/// <summary>Outcome of the weekly reconciliation.</summary>
|
||||
public sealed class OrphanReconcileResult
|
||||
{
|
||||
public int IndexSessions { get; set; }
|
||||
public int ResidueSessions { get; set; }
|
||||
public List<(string TenantId, string SessionId)> Sample { get; set; } = new List<(string TenantId, string SessionId)>();
|
||||
public long DurationMs { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Counters of one sweep run; the Completed ops event carries them verbatim.</summary>
|
||||
public sealed class OrphanSweepRunResult
|
||||
{
|
||||
public int HandlesScanned { get; set; }
|
||||
public int SessionKeys { get; set; }
|
||||
public int Candidates { get; set; }
|
||||
public List<OrphanSweepCleanedSession> Cleaned { get; } = new List<OrphanSweepCleanedSession>();
|
||||
public int SkippedRowReturned { get; set; }
|
||||
public int SkippedHandleGone { get; set; }
|
||||
public int SkippedHandleChanged { get; set; }
|
||||
public int AfterCascade { get; set; }
|
||||
public int Remaining { get; set; }
|
||||
public int FailedSessions { get; set; }
|
||||
public int InventoryDecrementsFailed { get; set; }
|
||||
public bool BudgetExhausted { get; set; }
|
||||
public long DurationMs { get; set; }
|
||||
public Dictionary<string, int> TotalsByTable { get; } = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
public OrphanReconcileResult? Reconcile { get; set; }
|
||||
|
||||
public void Add(OrphanSweepCleanedSession cleaned)
|
||||
{
|
||||
Cleaned.Add(cleaned);
|
||||
foreach (var kv in cleaned.RowsByTable)
|
||||
TotalsByTable[kv.Key] = TotalsByTable.TryGetValue(kv.Key, out var n) ? n + kv.Value : kv.Value;
|
||||
Bump(Constants.TableNames.SessionInventoryContributions, cleaned.HadInventorySideRow ? 1 : 0);
|
||||
Bump(Constants.TableNames.SoftwareInventory, cleaned.InventoryKeysDecremented);
|
||||
Bump(Constants.TableNames.EventSessionIndex, cleaned.HandleKept ? 0 : 1);
|
||||
}
|
||||
|
||||
private void Bump(string table, int by)
|
||||
=> TotalsByTable[table] = TotalsByTable.TryGetValue(table, out var n) ? n + by : by;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using AutopilotMonitor.Functions.Services.Locking;
|
||||
using AutopilotMonitor.Shared;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Blob-lease lock for the orphan-session sweep: the 4h timer, the manual maintenance run and
|
||||
/// a second host instance enter through it. Its own sentinel — the sweep has no reason to
|
||||
/// exclude the platform maintenance run or the retention cascade, and they none to exclude it.
|
||||
/// </summary>
|
||||
public class OrphanSweepLockStore : BlobLeaseLockStore
|
||||
{
|
||||
/// <summary>Sentinel blob path. <c>_lock/…</c> does not parse as a manifest blob name, so the manifest-TTL sweep skips it.</summary>
|
||||
public const string LockBlobName = "_lock/orphan-session-sweep.lock";
|
||||
|
||||
public OrphanSweepLockStore(BlobStorageService blobs)
|
||||
: base(blobs, Constants.BlobContainers.DeletionManifests, LockBlobName, "orphan session sweep")
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1217,80 +1217,6 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_logger.LogWarning(ex, "Failed to cleanup old verdict-calibration aggregates");
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// Detects and cleans up orphaned events — events stored in the Events table
|
||||
/// whose session no longer exists in the Sessions table.
|
||||
/// Uses the EventSessionIndex side-table for efficient detection (no full Events scan).
|
||||
/// Grace period: 24 hours to protect against register+ingest race conditions.
|
||||
/// </summary>
|
||||
private async Task CleanupOrphanedEventsAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Starting orphaned events cleanup...");
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
var orphans = await _maintenanceRepo.GetOrphanedEventSessionsAsync(TimeSpan.FromHours(24));
|
||||
|
||||
if (orphans.Count == 0)
|
||||
{
|
||||
_logger.LogInformation("No orphaned events found");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogWarning("Found {Count} orphaned event sessions, cleaning up...", orphans.Count);
|
||||
|
||||
int totalEventsDeleted = 0;
|
||||
int sessionsCleanedUp = 0;
|
||||
// Per-orphan breakdown for the OpsEvent so the dashboard shows which tenant/session
|
||||
// was cleaned (the worker LogInformation below does not reach App Insights).
|
||||
var cleanedOrphans = new List<OrphanedEventSession>();
|
||||
|
||||
foreach (var orphan in orphans)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Ordering invariant: the index entry is removed ONLY after the event
|
||||
// delete completed cleanly (DeleteSessionEventsAsync throws on failure).
|
||||
// Orphan detection scans only the index — deleting it first would make
|
||||
// surviving event rows permanently undiscoverable.
|
||||
var deletedEvents = await _maintenanceRepo.DeleteSessionEventsAsync(orphan.TenantId, orphan.SessionId);
|
||||
await _maintenanceRepo.DeleteEventSessionIndexEntryAsync(orphan.TenantId, orphan.SessionId);
|
||||
|
||||
totalEventsDeleted += deletedEvents;
|
||||
sessionsCleanedUp++;
|
||||
// Report the actual rows deleted (may differ from the index's recorded count).
|
||||
cleanedOrphans.Add(new OrphanedEventSession
|
||||
{
|
||||
TenantId = orphan.TenantId,
|
||||
SessionId = orphan.SessionId,
|
||||
LastIngestAt = orphan.LastIngestAt,
|
||||
EventCount = deletedEvents
|
||||
});
|
||||
|
||||
_logger.LogInformation(
|
||||
"Cleaned orphan: TenantId={TenantId}, SessionId={SessionId}, Events={Events}",
|
||||
orphan.TenantId, orphan.SessionId, deletedEvents);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to cleanup orphan: TenantId={TenantId}, SessionId={SessionId}",
|
||||
orphan.TenantId, orphan.SessionId);
|
||||
}
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
_logger.LogInformation(
|
||||
"Orphaned events cleanup completed in {Ms}ms: {Sessions} sessions, {Events} events deleted",
|
||||
sw.ElapsedMilliseconds, sessionsCleanedUp, totalEventsDeleted);
|
||||
|
||||
await _opsEventService.RecordOrphanEventsCleanedAsync(sessionsCleanedUp, totalEventsDeleted, cleanedOrphans);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Orphaned events cleanup failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Monitoring;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
@@ -56,6 +57,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private readonly PreviewWhitelistService _previewWhitelistService;
|
||||
private readonly IStorageInitializer _storageInitializer;
|
||||
private readonly Maintenance.MaintenanceRunGate _runGate;
|
||||
private readonly Maintenance.OrphanSessionSweeper _orphanSweeper;
|
||||
private readonly ILogger<MaintenanceService> _logger;
|
||||
|
||||
private const string PlatformStatsAliasFileName = "platform-stats.json";
|
||||
@@ -93,6 +95,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
PreviewWhitelistService previewWhitelistService,
|
||||
IStorageInitializer storageInitializer,
|
||||
Maintenance.MaintenanceRunGate runGate,
|
||||
Maintenance.OrphanSessionSweeper orphanSweeper,
|
||||
ILogger<MaintenanceService> logger)
|
||||
{
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
@@ -124,6 +127,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_previewWhitelistService = previewWhitelistService;
|
||||
_storageInitializer = storageInitializer;
|
||||
_runGate = runGate;
|
||||
_orphanSweeper = orphanSweeper;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -150,53 +154,56 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_logger.LogInformation($"Daily maintenance started at {DateTime.UtcNow}");
|
||||
var maintenanceStart = Stopwatch.StartNew();
|
||||
|
||||
// Every step is timed into the Completed event (stepsMs): the run's composition is
|
||||
// what decides which steps leave this timer next, and worker logs never show it.
|
||||
var steps = new Maintenance.MaintenanceStepTimer();
|
||||
try
|
||||
{
|
||||
// Startup only point-reads the table schema sentinel (see
|
||||
// TableStorageService.InitializeTablesAsync); this daily full pass is what
|
||||
// recreates a table that was deleted out-of-band. Cheap (idempotent creates).
|
||||
await EnsureAllTablesAsync();
|
||||
await MarkStalledSessionsAsTimedOutAsync();
|
||||
await DetectExcessiveEventSessionsAsync();
|
||||
await AggregateMetricsWithCatchUpAsync();
|
||||
await steps.RunAsync("ensureTables", async () => { await EnsureAllTablesAsync(); });
|
||||
await steps.RunAsync("stalledSessions", async () => { await MarkStalledSessionsAsTimedOutAsync(); });
|
||||
await steps.RunAsync("excessiveEvents", async () => { await DetectExcessiveEventSessionsAsync(); });
|
||||
await steps.RunAsync("aggregateMetrics", async () => { await AggregateMetricsWithCatchUpAsync(); });
|
||||
// F3 PR6: rule-frequency regression radar, right after the rule-stats
|
||||
// aggregation so the window rows are fresh. Anchored on YESTERDAY — whole
|
||||
// days only, a partial today would understate the window rate. Fail-soft.
|
||||
await RunRuleRegressionRadarAsync(DateTime.UtcNow.Date.AddDays(-1));
|
||||
await steps.RunAsync("ruleRegressionRadar", async () => { await RunRuleRegressionRadarAsync(DateTime.UtcNow.Date.AddDays(-1)); });
|
||||
// App-version duration regression radar: same episode/tracker pattern over the
|
||||
// install summaries (trailing 35d horizon loaded internally). Fail-soft.
|
||||
await RunAppVersionRegressionRadarAsync();
|
||||
await steps.RunAsync("appVersionRadar", async () => { await RunAppVersionRegressionRadarAsync(); });
|
||||
// ONE projected cross-tenant window scan feeds every rolling sweep below (the
|
||||
// StartedAt-only filter is a full-table drain in Table Storage — four of them per
|
||||
// tick was four times the same read). Each sweep slices its own window.
|
||||
var sweepWindow = await LoadSweepWindowSessionsAsync();
|
||||
var sweepWindow = await steps.RunAsync<List<SessionSummary>>("sweepWindow", () => LoadSweepWindowSessionsAsync());
|
||||
// F1 PR2: rolling 30d breakdown backfill + daily attribution aggregates. Owns
|
||||
// its own window (NOT the snapshot-gated catch-up above) so late-terminating
|
||||
// sessions still reach their StartedAt-date's aggregate. Fail-soft internally.
|
||||
await SweepTimeAttributionAsync(sweepWindow);
|
||||
await steps.RunAsync("timeAttribution", async () => { await SweepTimeAttributionAsync(sweepWindow); });
|
||||
// F2 PR4: device-history chain heal (incl. deleted-session ref cleanup) + daily
|
||||
// FTR aggregates over the same rolling window. Fail-soft internally.
|
||||
await SweepDeviceJourneysAsync(sweepWindow);
|
||||
await steps.RunAsync("deviceJourneys", async () => { await SweepDeviceJourneysAsync(sweepWindow); });
|
||||
// Verdict calibration: per-verdict-path daily buckets over the same rolling
|
||||
// window, AFTER the device-journey sweep so the re-enrollment proxy reads
|
||||
// freshly merged chains. Fail-soft internally.
|
||||
await SweepVerdictCalibrationAsync(sweepWindow);
|
||||
await steps.RunAsync("verdictCalibration", async () => { await SweepVerdictCalibrationAsync(sweepWindow); });
|
||||
// Verdict-calibration drift radar over the rows the sweep just refreshed; anchored
|
||||
// on yesterday like the rule radar (whole days only). Fail-soft.
|
||||
await RunVerdictCalibrationRadarAsync(DateTime.UtcNow.Date.AddDays(-1), sweepWindow);
|
||||
await steps.RunAsync("verdictCalibrationRadar", async () => { await RunVerdictCalibrationRadarAsync(DateTime.UtcNow.Date.AddDays(-1), sweepWindow); });
|
||||
// Plan §5 PR6 / §16 R14: session retention fanout extracted out of the 2h timer
|
||||
// into the dedicated 12h SessionDeletionMaintenanceFunction so cascade-lifecycle
|
||||
// work has independent cadence + kill-switch + OpsEvent watchdogs. The non-session
|
||||
// tail of the old CleanupOldDataAsync (UserUsageLog + RuleStats) stays here.
|
||||
await CleanupOldUsageDataAsync();
|
||||
await CleanupOldDistressReportsAsync();
|
||||
await CleanupOldOpsEventsAsync();
|
||||
await CleanupUnboundedTablesAsync();
|
||||
await CleanupOrphanedEventsAsync();
|
||||
await CheckAgentBlobStorageAsync();
|
||||
await CheckEmbeddedCertExpiryAsync();
|
||||
await CheckPoisonQueueBacklogAsync();
|
||||
await RecomputePlatformStatsAsync();
|
||||
// tail of the old CleanupOldDataAsync (UserUsageLog + RuleStats) stays here. The
|
||||
// orphan-session sweep left this timer too (OrphanSessionSweepFunction, every 4h).
|
||||
await steps.RunAsync("usageData", async () => { await CleanupOldUsageDataAsync(); });
|
||||
await steps.RunAsync("distressReports", async () => { await CleanupOldDistressReportsAsync(); });
|
||||
await steps.RunAsync("opsEvents", async () => { await CleanupOldOpsEventsAsync(); });
|
||||
await steps.RunAsync("unboundedTables", async () => { await CleanupUnboundedTablesAsync(); });
|
||||
await steps.RunAsync("agentBlobStorage", async () => { await CheckAgentBlobStorageAsync(); });
|
||||
await steps.RunAsync("embeddedCertExpiry", async () => { await CheckEmbeddedCertExpiryAsync(); });
|
||||
await steps.RunAsync("poisonQueues", async () => { await CheckPoisonQueueBacklogAsync(); });
|
||||
await steps.RunAsync("platformStats", async () => { await RecomputePlatformStatsAsync(); });
|
||||
|
||||
// Backfill and repair tasks run only via manual trigger (RunManualAsync)
|
||||
// to keep the timer-triggered path lightweight. See RunManualAsync for:
|
||||
@@ -205,7 +212,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
maintenanceStart.Stop();
|
||||
_logger.LogInformation($"Daily maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
|
||||
await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer");
|
||||
await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer", stepsMs: steps.Steps);
|
||||
|
||||
// Soft watchdog: the run completed, but if it is climbing toward the host's 60min
|
||||
// functionTimeout (e.g. a large first-time retention backlog deleting row-by-row),
|
||||
@@ -316,6 +323,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
await CleanupUnboundedTablesAsync();
|
||||
result.DataCleanupExecuted = true;
|
||||
|
||||
// The orphan-session sweep has its own timer and lease; a manual run includes it
|
||||
// (with the reconciliation) so an operator can drive it from the same button.
|
||||
await _orphanSweeper.RunAsync(triggeredBy, CancellationToken.None);
|
||||
|
||||
// --- Backfill & repair tasks (manual-only, not in timer path) ---
|
||||
|
||||
// Safety net: backfill any sessions missing from SessionsIndex
|
||||
|
||||
@@ -84,15 +84,17 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <paramref name="report"/> is the run report of a manual run: the trigger answers 202
|
||||
/// before the run starts, so this event is where the operator reads what the run did.
|
||||
/// </summary>
|
||||
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null)
|
||||
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null,
|
||||
IReadOnlyDictionary<string, long>? stepsMs = null)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info,
|
||||
$"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})",
|
||||
null, triggeredBy,
|
||||
report == null
|
||||
? (object)new { durationMs }
|
||||
? (object)new { durationMs, stepsMs }
|
||||
: new
|
||||
{
|
||||
durationMs,
|
||||
stepsMs,
|
||||
aggregatedDate = report.AggregatedDate,
|
||||
stalledSessionsChecked = report.StalledSessionsChecked,
|
||||
dataCleanupExecuted = report.DataCleanupExecuted,
|
||||
@@ -138,36 +140,113 @@ namespace AutopilotMonitor.Functions.Services
|
||||
$"Cleaned up {deletedCount} ops events older than {retentionDays} days",
|
||||
null, "System.Maintenance", new { deletedCount, retentionDays });
|
||||
|
||||
// ── Orphan session sweep (OrphanSessionSweeper) ─────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Records the orphan-cleanup result. <paramref name="cleanedOrphans"/> carries the
|
||||
/// per-session breakdown (which tenant + session lost lingering events and how many),
|
||||
/// so the ops dashboard shows *what* was cleaned, not just a count. The list is capped
|
||||
/// to keep the OpsEvents Table row under the per-property (32 KB) / entity (~1 MB) limits;
|
||||
/// the full count always survives in <c>orphanSessions</c> and a <c>detailsTruncated</c>
|
||||
/// flag flags any clipping. Note: an "orphan" is a session row that no longer exists while
|
||||
/// its events lingered past the 24h grace — the tenant itself usually still exists.
|
||||
/// Heartbeat of every sweep run: scan sizes, candidates, what was cleaned, skipped and
|
||||
/// left for the next run, the per-table totals and — when it ran — the reconciliation.
|
||||
/// </summary>
|
||||
public Task RecordOrphanEventsCleanedAsync(int orphanSessions, int totalEventsDeleted,
|
||||
IReadOnlyList<OrphanedEventSession>? cleanedOrphans = null)
|
||||
public Task RecordOrphanSweepCompletedAsync(Maintenance.OrphanSweepRunResult result, string triggeredBy)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepCompleted, OpsEventSeverity.Info,
|
||||
$"Orphan session sweep completed in {result.DurationMs}ms — handles={result.HandlesScanned} sessions={result.SessionKeys} "
|
||||
+ $"candidates={result.Candidates} cleaned={result.Cleaned.Count} remaining={result.Remaining} failed={result.FailedSessions}",
|
||||
null, Maintenance.OrphanSessionSweeper.Actor,
|
||||
new
|
||||
{
|
||||
triggeredBy,
|
||||
durationMs = result.DurationMs,
|
||||
handlesScanned = result.HandlesScanned,
|
||||
sessionKeys = result.SessionKeys,
|
||||
candidates = result.Candidates,
|
||||
cleaned = result.Cleaned.Count,
|
||||
remaining = result.Remaining,
|
||||
failedSessions = result.FailedSessions,
|
||||
skippedRowReturned = result.SkippedRowReturned,
|
||||
skippedHandleGone = result.SkippedHandleGone,
|
||||
skippedHandleChanged = result.SkippedHandleChanged,
|
||||
afterCascade = result.AfterCascade,
|
||||
inventoryDecrementsFailed = result.InventoryDecrementsFailed,
|
||||
budgetExhausted = result.BudgetExhausted,
|
||||
totalsByTable = result.TotalsByTable,
|
||||
reconcile = result.Reconcile == null ? null : new
|
||||
{
|
||||
indexSessions = result.Reconcile.IndexSessions,
|
||||
residueSessions = result.Reconcile.ResidueSessions,
|
||||
durationMs = result.Reconcile.DurationMs,
|
||||
},
|
||||
});
|
||||
|
||||
public Task RecordOrphanSweepFailedAsync(string error, string triggeredBy)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepFailed, OpsEventSeverity.Error,
|
||||
$"Orphan session sweep failed: {error}",
|
||||
null, Maintenance.OrphanSessionSweeper.Actor, new { error, triggeredBy });
|
||||
|
||||
public Task RecordOrphanSweepSkippedLockedAsync(string triggeredBy)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanSweepSkippedLocked, OpsEventSeverity.Warning,
|
||||
$"Orphan session sweep skipped — another run holds the lease (triggered by {triggeredBy})",
|
||||
null, Maintenance.OrphanSessionSweeper.Actor, new { reason = "lease held by another run", triggeredBy });
|
||||
|
||||
/// <summary>
|
||||
/// Weekly read-only reconciliation of EventTypeIndex against the Sessions keys. Info at
|
||||
/// zero residue; Warning with a capped sample otherwise — residue means rows exist for a
|
||||
/// session that has neither a Sessions row nor a handle, which the sweep cannot reach on its own.
|
||||
/// </summary>
|
||||
public Task RecordOrphanReconcileCompletedAsync(Maintenance.OrphanReconcileResult result)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanReconcileCompleted,
|
||||
result.ResidueSessions == 0 ? OpsEventSeverity.Info : OpsEventSeverity.Warning,
|
||||
result.ResidueSessions == 0
|
||||
? $"Orphan reconciliation: no residue across {result.IndexSessions} indexed sessions ({result.DurationMs}ms)"
|
||||
: $"Orphan reconciliation: {result.ResidueSessions} indexed sessions without a Sessions row or a handle ({result.DurationMs}ms)",
|
||||
null, Maintenance.OrphanSessionSweeper.Actor,
|
||||
new
|
||||
{
|
||||
indexSessions = result.IndexSessions,
|
||||
residueSessions = result.ResidueSessions,
|
||||
sample = result.Sample.Select(k => new { tenantId = k.TenantId, sessionId = k.SessionId }).ToList(),
|
||||
sampleTruncated = result.ResidueSessions > result.Sample.Count,
|
||||
durationMs = result.DurationMs,
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
/// Rows of sessions without a Sessions row were removed (Warning: after the ingest refusal
|
||||
/// every real hit is a writer outside the registration path or a restore residue; legacy
|
||||
/// handles are expected while the backlog drains). <c>totalEventsDeleted</c> keeps its old
|
||||
/// meaning (Events rows) next to the per-table totals. The per-session list is capped to
|
||||
/// keep the row inside the Table Storage property limits; the counts are never clipped.
|
||||
/// </summary>
|
||||
public Task RecordOrphanEventsCleanedAsync(Maintenance.OrphanSweepRunResult result)
|
||||
{
|
||||
const int maxDetailRows = 50;
|
||||
|
||||
var orphanList = cleanedOrphans ?? Array.Empty<OrphanedEventSession>();
|
||||
var detail = orphanList
|
||||
.OrderByDescending(o => o.EventCount)
|
||||
var detail = result.Cleaned
|
||||
.OrderByDescending(c => c.TotalRows)
|
||||
.Take(maxDetailRows)
|
||||
.Select(o => new { tenantId = o.TenantId, sessionId = o.SessionId, eventCount = o.EventCount })
|
||||
.Select(c => new
|
||||
{
|
||||
tenantId = c.TenantId,
|
||||
sessionId = c.SessionId,
|
||||
rows = c.TotalRows,
|
||||
inventoryKeys = c.InventoryKeysDecremented,
|
||||
legacy = c.Legacy,
|
||||
afterCascade = c.AfterCascade,
|
||||
handleKept = c.HandleKept,
|
||||
})
|
||||
.ToList();
|
||||
result.TotalsByTable.TryGetValue(Shared.Constants.TableNames.Events, out var eventsDeleted);
|
||||
var legacy = result.Cleaned.Count(c => c.Legacy);
|
||||
|
||||
return WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanEventsCleaned, OpsEventSeverity.Warning,
|
||||
$"Cleaned {totalEventsDeleted} orphaned events across {orphanSessions} sessions",
|
||||
null, "System.Maintenance",
|
||||
$"Cleaned {result.TotalsByTable.Values.Sum()} rows of {result.Cleaned.Count} orphan sessions ({legacy} legacy handles)",
|
||||
null, Maintenance.OrphanSessionSweeper.Actor,
|
||||
new
|
||||
{
|
||||
orphanSessions,
|
||||
totalEventsDeleted,
|
||||
orphanSessions = result.Cleaned.Count,
|
||||
totalEventsDeleted = eventsDeleted,
|
||||
legacySessions = legacy,
|
||||
remaining = result.Remaining,
|
||||
totalsByTable = result.TotalsByTable,
|
||||
orphans = detail,
|
||||
detailsTruncated = orphanList.Count > maxDetailRows
|
||||
detailsTruncated = result.Cleaned.Count > maxDetailRows,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -935,23 +935,6 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
// ===== DELETION HELPERS =====
|
||||
|
||||
/// <summary>
|
||||
/// Deletes all events for a session from storage
|
||||
/// </summary>
|
||||
public async Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
|
||||
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Events);
|
||||
var partitionKey = $"{tenantId}_{sessionId}";
|
||||
var filter = $"PartitionKey eq '{partitionKey}'";
|
||||
var deleted = await DeleteByFilterInBatchesAsync(tableClient, filter, $"events for session {sessionId}");
|
||||
if (deleted > 0)
|
||||
_logger.LogInformation($"Deleted {deleted} events for session {sessionId}");
|
||||
return deleted;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes all rule results for a session
|
||||
/// </summary>
|
||||
@@ -1188,81 +1171,5 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== ORPHAN EVENT DETECTION =====
|
||||
|
||||
/// <summary>
|
||||
/// Scans EventSessionIndex, checks each entry against the Sessions table,
|
||||
/// and returns entries where no session exists and LastIngestAt is older than the grace period.
|
||||
/// </summary>
|
||||
public async Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod)
|
||||
{
|
||||
var orphans = new List<OrphanedEventSession>();
|
||||
var cutoff = DateTime.UtcNow - gracePeriod;
|
||||
|
||||
try
|
||||
{
|
||||
var indexClient = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
|
||||
var sessionsClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
|
||||
|
||||
await foreach (var entity in indexClient.QueryAsync<TableEntity>())
|
||||
{
|
||||
var tenantId = entity.PartitionKey;
|
||||
var sessionId = entity.RowKey;
|
||||
var lastIngestAt = entity.GetDateTimeOffset("LastIngestAt")?.UtcDateTime ?? DateTime.MinValue;
|
||||
var eventCount = entity.GetInt32("EventCount") ?? 0;
|
||||
|
||||
// Grace period: skip recent entries (race condition protection)
|
||||
if (lastIngestAt > cutoff)
|
||||
continue;
|
||||
|
||||
// Check if session exists
|
||||
try
|
||||
{
|
||||
var session = await sessionsClient.GetEntityIfExistsAsync<TableEntity>(tenantId, sessionId, select: new[] { "PartitionKey" });
|
||||
if (!session.HasValue)
|
||||
{
|
||||
orphans.Add(new OrphanedEventSession
|
||||
{
|
||||
TenantId = tenantId,
|
||||
SessionId = sessionId,
|
||||
LastIngestAt = lastIngestAt,
|
||||
EventCount = eventCount
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (RequestFailedException ex)
|
||||
{
|
||||
// GetEntityIfExistsAsync does NOT throw on 404 (handled via HasValue above),
|
||||
// so this only fires on transient errors (429/500/503). Treating those as
|
||||
// "orphan" would delete events of live sessions — skip the entry instead.
|
||||
_logger.LogWarning(ex, "Transient error checking session existence for {TenantId}/{SessionId} (status {Status}); skipping orphan classification this run", tenantId, sessionId, ex.Status);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to scan EventSessionIndex for orphans");
|
||||
}
|
||||
|
||||
return orphans;
|
||||
}
|
||||
|
||||
public async Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var indexClient = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
|
||||
await indexClient.DeleteEntityAsync(tenantId, sessionId);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// Already deleted, ignore
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete EventSessionIndex entry for {TenantId}/{SessionId}", tenantId, sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Storage side of the orphan-session sweep (<see cref="Maintenance.OrphanSessionSweeper"/>):
|
||||
/// the two key drains the diff runs on, the EventSessionIndex handle as the sweep's progress
|
||||
/// row, and the per-table delete by the writer's key shape.
|
||||
/// </summary>
|
||||
public partial class TableStorageService
|
||||
{
|
||||
private const string HandleInventoryDecrementedColumn = "InventoryDecremented";
|
||||
|
||||
// Timestamp must be named explicitly in a projection; ETag always travels with the row.
|
||||
private static readonly string[] HandleColumns =
|
||||
{ "PartitionKey", "RowKey", "Timestamp", "LastIngestAt", HandleInventoryDecrementedColumn };
|
||||
|
||||
public async Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(CancellationToken ct = default)
|
||||
{
|
||||
var client = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
|
||||
var keys = new HashSet<(string TenantId, string SessionId)>();
|
||||
await foreach (var row in client.QueryAsync<TableEntity>(
|
||||
maxPerPage: 1000, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
|
||||
{
|
||||
keys.Add((row.PartitionKey, row.RowKey));
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(CancellationToken ct = default)
|
||||
{
|
||||
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
|
||||
var handles = new List<OrphanSessionHandle>();
|
||||
await foreach (var row in client.QueryAsync<TableEntity>(maxPerPage: 1000, select: HandleColumns, cancellationToken: ct))
|
||||
{
|
||||
handles.Add(ToHandle(row));
|
||||
}
|
||||
return handles;
|
||||
}
|
||||
|
||||
public async Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
|
||||
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
|
||||
var row = await client.GetEntityIfExistsAsync<TableEntity>(tenantId, sessionId, select: HandleColumns, cancellationToken: ct);
|
||||
return row.HasValue ? ToHandle(row.Value!) : null;
|
||||
}
|
||||
|
||||
public async Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
|
||||
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
|
||||
var stamp = new TableEntity(tenantId, sessionId) { [HandleInventoryDecrementedColumn] = true };
|
||||
try
|
||||
{
|
||||
var response = await client.UpdateEntityAsync(stamp, new ETag(etag), TableUpdateMode.Merge, ct);
|
||||
return response.Headers.ETag?.ToString();
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412 || ex.Status == 404)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
|
||||
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventSessionIndex);
|
||||
try
|
||||
{
|
||||
await client.DeleteEntityAsync(tenantId, sessionId, new ETag(etag), ct);
|
||||
return true;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412 || ex.Status == 404)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default)
|
||||
{
|
||||
var filter = Maintenance.OrphanSessionRowFilters.For(table, tenantId, sessionId);
|
||||
var client = _tableServiceClient.GetTableClient(table);
|
||||
var keys = new List<(string Pk, string Rk)>();
|
||||
await foreach (var row in client.QueryAsync<TableEntity>(
|
||||
filter: filter, maxPerPage: 1000, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
|
||||
{
|
||||
keys.Add((row.PartitionKey, row.RowKey));
|
||||
}
|
||||
if (keys.Count == 0) return 0;
|
||||
|
||||
// Groups by partition and falls back to per-row deletes on a 404 inside a batch, so a
|
||||
// rerun after a partial failure is idempotent; any other failure propagates.
|
||||
await DeleteByExactKeysInBatchesAsync(table, keys, ct);
|
||||
return keys.Count;
|
||||
}
|
||||
|
||||
public async Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default)
|
||||
{
|
||||
var client = _tableServiceClient.GetTableClient(Constants.TableNames.EventTypeIndex);
|
||||
var keys = new HashSet<(string TenantId, string SessionId)>();
|
||||
await foreach (var row in client.QueryAsync<TableEntity>(
|
||||
maxPerPage: 1000, select: new[] { "PartitionKey", "SessionId", "TenantId" }, cancellationToken: ct))
|
||||
{
|
||||
var sessionId = row.GetString("SessionId");
|
||||
if (string.IsNullOrEmpty(sessionId)) continue; // the readers skip such rows as well
|
||||
|
||||
var tenantId = row.GetString("TenantId");
|
||||
if (string.IsNullOrEmpty(tenantId))
|
||||
{
|
||||
// Rows older than the TenantId column: PartitionKey is {tenantId}_{eventType} and a
|
||||
// tenant id (a GUID) never contains an underscore.
|
||||
var cut = row.PartitionKey.IndexOf('_');
|
||||
if (cut <= 0) continue;
|
||||
tenantId = row.PartitionKey.Substring(0, cut);
|
||||
}
|
||||
keys.Add((tenantId!, sessionId!));
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
private static OrphanSessionHandle ToHandle(TableEntity row) => new OrphanSessionHandle
|
||||
{
|
||||
TenantId = row.PartitionKey,
|
||||
SessionId = row.RowKey,
|
||||
LastIngestAt = row.GetDateTimeOffset("LastIngestAt")?.UtcDateTime ?? DateTime.MinValue,
|
||||
WrittenAt = row.Timestamp?.UtcDateTime ?? DateTime.MinValue,
|
||||
ETag = row.ETag.ToString(),
|
||||
InventoryDecremented = row.GetBoolean(HandleInventoryDecrementedColumn) ?? false,
|
||||
};
|
||||
}
|
||||
}
|
||||
+16
@@ -57,6 +57,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
private readonly IVulnerabilityInventoryLoader _inventoryLoader;
|
||||
private readonly IAnalyzeOnEnrollmentEndProducer _analyzeProducer;
|
||||
private readonly SignalRNotificationService _signalRNotification;
|
||||
private readonly Deletion.SessionDeletionGuard _deletionGuard;
|
||||
private readonly ILogger<VulnerabilityCorrelateHandler> _logger;
|
||||
|
||||
public VulnerabilityCorrelateHandler(
|
||||
@@ -67,6 +68,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
IVulnerabilityInventoryLoader inventoryLoader,
|
||||
IAnalyzeOnEnrollmentEndProducer analyzeProducer,
|
||||
SignalRNotificationService signalRNotification,
|
||||
Deletion.SessionDeletionGuard deletionGuard,
|
||||
ILogger<VulnerabilityCorrelateHandler> logger)
|
||||
{
|
||||
_adminConfigService = adminConfigService;
|
||||
@@ -76,6 +78,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
_inventoryLoader = inventoryLoader;
|
||||
_analyzeProducer = analyzeProducer;
|
||||
_signalRNotification = signalRNotification;
|
||||
_deletionGuard = deletionGuard;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -107,6 +110,19 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
return;
|
||||
}
|
||||
|
||||
// Derived rows only for a live session: a session the backend does not know, or one a
|
||||
// cascade is tearing down, must not gain an inventory side-row, CveIndex rows or a
|
||||
// report — no cascade could reach them afterwards.
|
||||
var liveness = await _deletionGuard.GetLivenessAsync(
|
||||
envelope.TenantId, envelope.SessionId, "VulnerabilityCorrelate", cancellationToken).ConfigureAwait(false);
|
||||
if (liveness != Deletion.SessionLiveness.Live)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"{Prefix} VulnerabilityCorrelate skipped — session {Liveness} (reason={Reason})",
|
||||
sessionPrefix, liveness, envelope.Reason);
|
||||
return;
|
||||
}
|
||||
|
||||
// Filter inventory by the envelope's WhiteGlovePart so a Part-2 correlation
|
||||
// does not pick up Part-1 inventory items (which would mis-tag findings as
|
||||
// user_enrollment) and a delayed Part-1 retry does not later overwrite a
|
||||
|
||||
+4
-3
@@ -271,9 +271,10 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
// Persist software inventory via the contributions side-row (Plan §17). Awaited inside
|
||||
// the invocation: a detached task can be cut off by a scale-in after the side-row has
|
||||
// already promised the key set, leaving the counters short with no trace. The method
|
||||
// never throws, so the report — the primary product — is stored regardless. PR3 wires
|
||||
// SessionDeletionGuard above this layer (at VulnerabilityCorrelateHandler) so a session
|
||||
// locked by a cascade never reaches this code path.
|
||||
// never throws, so the report — the primary product — is stored regardless. Both callers
|
||||
// (VulnerabilityCorrelateHandler, GetVulnerabilityReportFunction's rescan) check
|
||||
// SessionDeletionGuard.GetLivenessAsync first, so a session the backend does not know
|
||||
// or a cascade is tearing down never reaches this write.
|
||||
await ApplyInventoryContributionsAsync(tenantId, sessionId, inventoryItems, cpeMappingsDict);
|
||||
|
||||
// Summary log for production monitoring
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* search_sessions_by_event reads the EventTypeIndex, and the backend drops index rows whose
|
||||
* session no longer exists (deleted, or not yet removed by the orphan sweep). A page can therefore
|
||||
* be empty and still carry a nextLink, so the tool must go through the same auto-exhausting
|
||||
* scan as search_sessions instead of a single fetch. The scan seam is the observable here.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
||||
|
||||
const { scanMock } = vi.hoisted(() => ({ scanMock: vi.fn() }));
|
||||
|
||||
vi.mock('../client.js', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('../client.js')>();
|
||||
return { ...actual, scanWithTimeoutFallback: scanMock };
|
||||
});
|
||||
|
||||
import { registerSessionTools } from '../tools/sessions.js';
|
||||
|
||||
type Handler = (args: Record<string, unknown>) => Promise<{ content: Array<{ text: string }>; isError?: boolean }>;
|
||||
|
||||
function searchSessionsByEvent(): Handler {
|
||||
const handlers: Record<string, Handler> = {};
|
||||
const fake = { registerTool: (name: string, _def: unknown, handler: Handler) => { handlers[name] = handler; } };
|
||||
registerSessionTools(fake as never, true);
|
||||
return handlers.search_sessions_by_event;
|
||||
}
|
||||
|
||||
const PAGE = { count: 1, sessions: [{ sessionId: 's1', tenantId: 't1' }], scannedPages: 3 };
|
||||
|
||||
describe('search_sessions_by_event — auto-exhausts empty-but-continuable pages', () => {
|
||||
beforeEach(() => {
|
||||
scanMock.mockReset();
|
||||
scanMock.mockResolvedValue(PAGE);
|
||||
});
|
||||
|
||||
it('reaches the backend through the scanning fetch, not a single apiFetch', async () => {
|
||||
const result = await searchSessionsByEvent()({ eventType: 'app_install_failed' });
|
||||
|
||||
expect(result.isError).toBeFalsy();
|
||||
expect(scanMock).toHaveBeenCalledTimes(1);
|
||||
// Global vs tenant route follows the request's routing context, not the registration flag.
|
||||
const [path, basePath] = scanMock.mock.calls[0] as [string, string];
|
||||
expect(basePath).toMatch(/^\/api\/(global\/)?search\/sessions-by-event$/);
|
||||
expect(path.startsWith(`${basePath}?`)).toBe(true);
|
||||
expect(path).toContain('eventType=app_install_failed');
|
||||
expect(result.content[0].text).toContain('"scannedPages":3');
|
||||
});
|
||||
|
||||
it('keeps the nextLink cursor when continuing', async () => {
|
||||
const next = '/api/search/sessions-by-event?eventType=app_install_failed&pageSize=50&continuation=c1';
|
||||
await searchSessionsByEvent()({ eventType: 'app_install_failed', tenantId: 't1', continuation: next });
|
||||
|
||||
const [path] = scanMock.mock.calls[0] as [string, string];
|
||||
expect(path).toContain('continuation=c1');
|
||||
expect(path).toContain('eventType=app_install_failed');
|
||||
});
|
||||
});
|
||||
@@ -200,6 +200,8 @@ export interface AdminConfiguration {
|
||||
nvdCacheLastRefreshUtc: string;
|
||||
/** Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss. */
|
||||
epssLastSyncUtc: string;
|
||||
/** Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal. */
|
||||
orphanDeepCheckLastRunUtc: string;
|
||||
}
|
||||
|
||||
/** The immutable Entra identity behind a cross-tenant-role UPN: the HOME tenant the UPN was granted for and, once known, the user's object id. A role row (GlobalAdmins / DelegatedAdmins / TenantGroupAssignments) confers nothing unless the caller's validated JWT matches this binding. */
|
||||
|
||||
@@ -70,6 +70,10 @@ export const OPS_EVENT_TYPES = [
|
||||
"SessionSweepFailed",
|
||||
"OpsEventCleanup",
|
||||
"OrphanEventsCleaned",
|
||||
"OrphanSweepCompleted",
|
||||
"OrphanSweepFailed",
|
||||
"OrphanSweepSkippedLocked",
|
||||
"OrphanReconcileCompleted",
|
||||
"SessionDeletionMaintenanceStarted",
|
||||
"SessionDeletionMaintenanceBudgetExceeded",
|
||||
"SessionDeletionMaintenanceSkippedLocked",
|
||||
|
||||
@@ -236,9 +236,11 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated:
|
||||
const { eventType, tenantId: rawTenantId, continuation } = args;
|
||||
const pageSize = pageSizeForCall(args.pageSize, continuation, DEFAULT_FIRST_PAGE_SIZE);
|
||||
const tenantId = enforceDelegatedTenantForPage(rawTenantId, continuation);
|
||||
// eventType is the sole filter and is applied server-side (EventTypeIndex OData),
|
||||
// so an empty page never carries a nextLink — no auto-exhaust needed. An empty result
|
||||
// for an uncatalogued type says so, so a typo does not read as "no such sessions".
|
||||
// eventType is applied server-side (EventTypeIndex OData), but the backend drops index
|
||||
// rows whose session no longer exists (deleted, or not yet removed by the orphan sweep),
|
||||
// so a page can be empty and still carry a nextLink — auto-exhaust forward like
|
||||
// search_sessions. An empty result for an uncatalogued type says so, so a typo does not
|
||||
// read as "no such sessions".
|
||||
const basePath = pickGlobalOrTenantPath('/api/global/search/sessions-by-event', '/api/search/sessions-by-event', tenantId);
|
||||
const path = followNextLink(
|
||||
basePath,
|
||||
@@ -246,7 +248,7 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated:
|
||||
continuation,
|
||||
{ pageSize },
|
||||
);
|
||||
const data = await apiFetch<SessionListResponse>(path);
|
||||
const data = await scanWithTimeoutFallback(path, basePath, effectivePageSize(pageSize, continuation));
|
||||
return toolResultText(withEventTypeNote(data, eventType, !continuation), MAX_RESULT_SIZE_CHARS.indexSessions);
|
||||
} catch (error: unknown) {
|
||||
return toolError('search_sessions_by_event', args, error);
|
||||
|
||||
@@ -130,7 +130,6 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
Task<List<string>> GetAllTenantIdsAsync();
|
||||
|
||||
// --- Cleanup ---
|
||||
Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId);
|
||||
Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId);
|
||||
|
||||
// --- Index Maintenance ---
|
||||
@@ -138,13 +137,51 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
Task<int> CleanupGhostSessionIndexEntriesAsync();
|
||||
Task<bool> IsSessionIndexEmptyAsync();
|
||||
|
||||
// --- Orphan Event Detection ---
|
||||
// --- Orphan session sweep (rows of sessions that have no Sessions row) ---
|
||||
|
||||
/// <summary>
|
||||
/// Returns EventSessionIndex entries whose session no longer exists in the Sessions table
|
||||
/// and whose last ingest is older than the grace period.
|
||||
/// Every (tenantId, sessionId) key of the Sessions table, projected to the keys (one range
|
||||
/// request per 1000 rows). The sweep diffs the EventSessionIndex handles against this set
|
||||
/// instead of point-reading one Sessions row per handle.
|
||||
/// </summary>
|
||||
Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod);
|
||||
Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId);
|
||||
Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(System.Threading.CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Every EventSessionIndex handle with the columns the sweep decides on: last ingest, the
|
||||
/// row's own write time, ETag and the inventory progress stamp.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(System.Threading.CancellationToken ct = default);
|
||||
|
||||
/// <summary>Point-reads one handle; null when it no longer exists (a cascade took it).</summary>
|
||||
Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, System.Threading.CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Marks the handle's software-inventory decrements as applied (ETag-conditional merge).
|
||||
/// Returns the handle's new ETag, or null when the row changed or vanished since
|
||||
/// <paramref name="etag"/> was read.
|
||||
/// </summary>
|
||||
Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, System.Threading.CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes the handle only while it still carries <paramref name="etag"/>: false when a
|
||||
/// later write changed it (412) or it is already gone (404).
|
||||
/// </summary>
|
||||
Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, System.Threading.CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes every row of <paramref name="table"/> that belongs to the session, by that table's
|
||||
/// own key shape. Fail-loud on query or delete errors; rows already missing count as done.
|
||||
/// Returns the number of rows found.
|
||||
/// </summary>
|
||||
Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, System.Threading.CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Distinct (tenantId, sessionId) pairs present in EventTypeIndex — the input of the weekly
|
||||
/// reconciliation (a full projected drain, roughly 65 rows per session). The tenant comes
|
||||
/// from the TenantId column, falling back to the PartitionKey prefix for rows written
|
||||
/// before that column existed.
|
||||
/// </summary>
|
||||
Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(System.Threading.CancellationToken ct = default);
|
||||
|
||||
// --- Session Tombstones (F2 device-journey sweep input) ---
|
||||
/// <summary>
|
||||
@@ -167,12 +204,31 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
IAsyncEnumerable<string> EnumerateSessionsForOffboardingAsync(string tenantId, System.Threading.CancellationToken ct = default);
|
||||
}
|
||||
|
||||
public class OrphanedEventSession
|
||||
/// <summary>
|
||||
/// One EventSessionIndex row as the orphan sweep sees it — the discovery handle that every
|
||||
/// events write creates (PK tenantId, RK sessionId).
|
||||
/// </summary>
|
||||
public sealed class OrphanSessionHandle
|
||||
{
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
public string SessionId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Server time of the last ingest that touched the handle.</summary>
|
||||
public DateTime LastIngestAt { get; set; }
|
||||
public int EventCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// System write time of the row. A restore re-inserts the handle with its old
|
||||
/// LastIngestAt but a fresh write time, so the sweep waits on this one too.
|
||||
/// </summary>
|
||||
public DateTime WrittenAt { get; set; }
|
||||
|
||||
public string ETag { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// True once the sweep applied the session's software-inventory decrements. Progress
|
||||
/// marker across runs: a rerun after a crash skips the decrements and only removes the rows.
|
||||
/// </summary>
|
||||
public bool InventoryDecremented { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -38,6 +38,14 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
public const string SessionSweepFailed = "SessionSweepFailed";
|
||||
public const string OpsEventCleanup = "OpsEventCleanup";
|
||||
public const string OrphanEventsCleaned = "OrphanEventsCleaned";
|
||||
/// <summary>Every orphan-session sweep run (Info heartbeat): scanned handles, candidates, cleaned, remaining, per-table totals.</summary>
|
||||
public const string OrphanSweepCompleted = "OrphanSweepCompleted";
|
||||
/// <summary>The sweep run itself failed before or while enumerating (Error); per-candidate failures stay inside Completed.</summary>
|
||||
public const string OrphanSweepFailed = "OrphanSweepFailed";
|
||||
/// <summary>Another sweep run (timer, manual maintenance or a second host instance) held the sweep lease (Warning).</summary>
|
||||
public const string OrphanSweepSkippedLocked = "OrphanSweepSkippedLocked";
|
||||
/// <summary>Weekly read-only reconciliation of EventTypeIndex against the Sessions keys: Info at zero residue, Warning with a sample otherwise.</summary>
|
||||
public const string OrphanReconcileCompleted = "OrphanReconcileCompleted";
|
||||
public const string SessionDeletionMaintenanceStarted = "SessionDeletionMaintenanceStarted";
|
||||
public const string SessionDeletionMaintenanceBudgetExceeded = "SessionDeletionMaintenanceBudgetExceeded";
|
||||
public const string SessionDeletionMaintenanceSkippedLocked = "SessionDeletionMaintenanceSkippedLocked";
|
||||
@@ -184,7 +192,7 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
public static readonly IReadOnlyList<string> All = new[]
|
||||
{
|
||||
ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles,
|
||||
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed,
|
||||
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, OrphanSweepCompleted, OrphanSweepFailed, OrphanSweepSkippedLocked, OrphanReconcileCompleted, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed,
|
||||
DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen,
|
||||
OffboardingFeedbackReceived, FeedbackReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled,
|
||||
SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected,
|
||||
|
||||
@@ -640,6 +640,13 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public string EpssLastSyncUtc { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only
|
||||
/// EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself,
|
||||
/// never patched from the portal.
|
||||
/// </summary>
|
||||
public string OrphanDeepCheckLastRunUtc { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// Returns a shallow copy with all secret-bearing string fields replaced by
|
||||
/// <see cref="Constants.RedactedSecretPlaceholder"/> (empty values are left empty). Used when
|
||||
|
||||
@@ -38,6 +38,15 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// out the silence grace. See tasks/enrollment-status-reclassification.md.
|
||||
/// </summary>
|
||||
SessionAgeEmergencyBreak = 4,
|
||||
|
||||
/// <summary>
|
||||
/// POST /api/agent/telemetry answered 410 Gone for the agent's own session: the backend
|
||||
/// no longer knows it (never registered, deleted, or being deleted). The agent stops
|
||||
/// uploading and terminates with self-destruct; this single report is the last thing the
|
||||
/// device says about the session. Sent over the emergency channel because the telemetry
|
||||
/// channel is exactly what was refused.
|
||||
/// </summary>
|
||||
TelemetryRejectedSessionUnknown = 5,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -88,8 +88,16 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
// PR2: GA restored a single row from a backup. Warning severity so operators
|
||||
// can wire a Telegram rule and see the audit in near-real-time.
|
||||
"BackupRowRestored",
|
||||
// Orphan cleanup sweep (Warning): events whose parent session no longer exists were
|
||||
// deleted during maintenance aggregation. Backend helper RecordOrphanEventsCleanedAsync.
|
||||
// Orphan-session sweep (OrphanSessionSweepFunction, every 4h at :45) — dual-register per
|
||||
// memory feedback_ops_event_types_dual_register. Backend helpers RecordOrphanSweep{Completed,
|
||||
// Failed,SkippedLocked}Async, RecordOrphanReconcileCompletedAsync, RecordOrphanEventsCleanedAsync.
|
||||
// Completed is the 4h heartbeat (Info); EventsCleaned (Warning) fires only when rows of a
|
||||
// session without a Sessions row were removed; ReconcileCompleted is the weekly read-only
|
||||
// EventTypeIndex check (Info at zero residue, Warning otherwise).
|
||||
"OrphanSweepCompleted",
|
||||
"OrphanSweepFailed",
|
||||
"OrphanSweepSkippedLocked",
|
||||
"OrphanReconcileCompleted",
|
||||
"OrphanEventsCleaned",
|
||||
],
|
||||
// SLA breach evaluation (SlaBreachEvaluationService). Dual-register per memory
|
||||
|
||||
@@ -61,7 +61,8 @@ export const SHARED_MANIFEST = {
|
||||
"vulnerabilityDataLastSyncUtc",
|
||||
"msrcLastSyncUtc",
|
||||
"nvdCacheLastRefreshUtc",
|
||||
"epssLastSyncUtc"
|
||||
"epssLastSyncUtc",
|
||||
"orphanDeepCheckLastRunUtc"
|
||||
],
|
||||
"defaults": {
|
||||
"partitionKey": "GlobalConfig",
|
||||
@@ -118,7 +119,8 @@ export const SHARED_MANIFEST = {
|
||||
"vulnerabilityDataLastSyncUtc": null,
|
||||
"msrcLastSyncUtc": null,
|
||||
"nvdCacheLastRefreshUtc": null,
|
||||
"epssLastSyncUtc": null
|
||||
"epssLastSyncUtc": null,
|
||||
"orphanDeepCheckLastRunUtc": null
|
||||
}
|
||||
},
|
||||
"tenantConfiguration": {
|
||||
@@ -604,6 +606,10 @@ export const SHARED_MANIFEST = {
|
||||
"SessionSweepFailed",
|
||||
"OpsEventCleanup",
|
||||
"OrphanEventsCleaned",
|
||||
"OrphanSweepCompleted",
|
||||
"OrphanSweepFailed",
|
||||
"OrphanSweepSkippedLocked",
|
||||
"OrphanReconcileCompleted",
|
||||
"SessionDeletionMaintenanceStarted",
|
||||
"SessionDeletionMaintenanceBudgetExceeded",
|
||||
"SessionDeletionMaintenanceSkippedLocked",
|
||||
|
||||
@@ -57,7 +57,8 @@
|
||||
"vulnerabilityDataLastSyncUtc",
|
||||
"msrcLastSyncUtc",
|
||||
"nvdCacheLastRefreshUtc",
|
||||
"epssLastSyncUtc"
|
||||
"epssLastSyncUtc",
|
||||
"orphanDeepCheckLastRunUtc"
|
||||
],
|
||||
"defaults": {
|
||||
"partitionKey": "GlobalConfig",
|
||||
@@ -114,7 +115,8 @@
|
||||
"vulnerabilityDataLastSyncUtc": null,
|
||||
"msrcLastSyncUtc": null,
|
||||
"nvdCacheLastRefreshUtc": null,
|
||||
"epssLastSyncUtc": null
|
||||
"epssLastSyncUtc": null,
|
||||
"orphanDeepCheckLastRunUtc": null
|
||||
}
|
||||
},
|
||||
"tenantConfiguration": {
|
||||
@@ -600,6 +602,10 @@
|
||||
"SessionSweepFailed",
|
||||
"OpsEventCleanup",
|
||||
"OrphanEventsCleaned",
|
||||
"OrphanSweepCompleted",
|
||||
"OrphanSweepFailed",
|
||||
"OrphanSweepSkippedLocked",
|
||||
"OrphanReconcileCompleted",
|
||||
"SessionDeletionMaintenanceStarted",
|
||||
"SessionDeletionMaintenanceBudgetExceeded",
|
||||
"SessionDeletionMaintenanceSkippedLocked",
|
||||
@@ -1750,6 +1756,14 @@
|
||||
"name": "string"
|
||||
},
|
||||
"doc": "Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss."
|
||||
},
|
||||
{
|
||||
"name": "orphanDeepCheckLastRunUtc",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"doc": "Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal."
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -200,6 +200,8 @@ export interface AdminConfiguration {
|
||||
nvdCacheLastRefreshUtc: string;
|
||||
/** Last successful FIRST EPSS re-score of the cached CVEs (UTC ISO 8601). Updated by VulnerabilityDataSyncFunction (daily timer) and the manual /api/vulnerability/sync-epss. */
|
||||
epssLastSyncUtc: string;
|
||||
/** Last run of the weekly orphan-session reconciliation (UTC ISO 8601): the read-only EventTypeIndex-against-Sessions pass of the orphan sweep. Written by the sweep itself, never patched from the portal. */
|
||||
orphanDeepCheckLastRunUtc: string;
|
||||
}
|
||||
|
||||
/** The immutable Entra identity behind a cross-tenant-role UPN: the HOME tenant the UPN was granted for and, once known, the user's object id. A role row (GlobalAdmins / DelegatedAdmins / TenantGroupAssignments) confers nothing unless the caller's validated JWT matches this binding. */
|
||||
|
||||
Reference in New Issue
Block a user