From 5dbc44c4ecd0924c00c35ba72ee3f2f6d0af2b7c Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Tue, 12 May 2026 23:44:20 +0200 Subject: [PATCH] feat(backend): cascade-delete worker + restore + codex-fixes (PR4/7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR4 ships the cascade-delete executor for the safe-cascade-delete plan: the SessionDeletionWorker (BackgroundService queue poll-loop) plus the SessionDeletionHandler (per-step execution, live verification, tombstone). Worker is NOT registered in Program.cs — that lights up in PR5. Three logical phases land together as one commit per Oliver's call: PR4 — Worker + Handler + CascadeVerificationService - SessionDeletionWorker: BatchSize=1, kill-switch entry guard, heartbeat task extending visibility every 60s during long handler runs, poison after max-dequeue - SessionDeletionHandler: download manifest+progress, CAS Queued→Running (with Running→Running resume), per-step dispatch by Class, AGGREGATE decrement, live verification, ordered tombstone (SessionsIndex first, Sessions last) - CascadeVerificationService: per-Class live residual scans (CveIndex is manifest-key-only per §12-Q8; AGGREGATE+FINAL skipped) - NotifySessionDeletedAsync added to ISignalRNotificationService - BlobStorageService: DownloadDeletionProgressAsync + ETag-CAS UpdateDeletionProgressAsync PR4b — Restore endpoint - POST /api/admin/sessions/{sessionId}/restore (GA-only via Policy Catalog) - SessionRestoreService: dispatch table per §13.4, full + partial-poisoned- recovery modes - TableStorageService.Restore.cs: RestoreRowsByExactKeysInBatchesAsync + ConvertFromPropValue + ConvertDumpToEntity (Sessions-row override forces DeletionState=None on restore so the row isn't permanently locked) - RestoreSoftwareInventoryEntryByKeyAsync helper - Worker MoveToPoisonAsync now CAS-transitions Running→Poisoned (E1 fix) so restore can dispatch into partial-restore mode PR4c — Codex review fixes (6 findings) - F1: AGGREGATE per-key idempotency via DeletionProgress.AggregateDecrementsApplied (persist-first/decrement-second bounds drift to +1 per crash vs unbounded -N) - F2: TombstoneStarted flag closes the gap where sessions=null+completedAt=null blocked restore after a tombstone crash - F3a: Sessions row inserted LAST in full restore (was first → trapped retry with RejectAlreadyAtOriginalState) - F3b: RestoreMode.Full now 409-ignores (was throw-on-409); converges with Partial mode, makes retry-after-partial-failure idempotent - F4: Partial-restore per-key re-increment idempotency via DeletionProgress.RestoreReIncrementsApplied - F5: Poison transition pre-checks PendingDeletionManifestId so stale envelopes can't flip a fresh active cascade to Poisoned - F6: DownloadDeletionManifestWithShaAsync exposes snapshot SHA; Handler + Restore both verify against progress.SnapshotSha256 (snapshot↔progress binding per plan §3) DeletionProgress schema gains 3 additive optional fields (AggregateDecrementsApplied, RestoreReIncrementsApplied, TombstoneStarted) with safe defaults so PR1-PR4 progress blobs deserialize cleanly. Tests: 1457/1457 backend xUnit facts green (1391 pre-PR4 + 66 new across PR4, PR4b, PR4c). Full solution build green. Program.cs has zero SessionDeletionWorker references (PR5 territory stays untouched). Co-Authored-By: Claude Opus 4.7 (1M context) --- ...BlobStorageServiceDeletionProgressTests.cs | 174 +++++ .../CascadeVerificationServiceTests.cs | 399 ++++++++++ .../FakeSignalRNotificationService.cs | 7 + .../RestoreSessionFunctionTests.cs | 55 ++ .../SessionDeletionHandlerTests.cs | 667 ++++++++++++++++ .../SessionDeletionPr4cCodexFixesTests.cs | 736 ++++++++++++++++++ .../SessionDeletionWorkerTests.cs | 438 +++++++++++ .../SessionRestoreServiceTests.cs | 509 ++++++++++++ .../TableStoragePropValueConversionTests.cs | 177 +++++ .../TableStorageRestoreBatchTests.cs | 220 ++++++ .../Sessions/RestoreSessionFunction.cs | 180 +++++ .../AutopilotMonitor.Functions/Program.cs | 4 + .../Security/EndpointAccessPolicyCatalog.cs | 1 + .../Services/BlobStorageService.cs | 92 ++- .../Deletion/CascadeVerificationService.cs | 233 ++++++ .../Deletion/SessionDeletionHandler.cs | 617 +++++++++++++++ .../Deletion/SessionDeletionWorker.cs | 502 ++++++++++++ .../Deletion/SessionRestoreService.cs | 586 ++++++++++++++ .../Services/ISignalRNotificationService.cs | 9 + .../Services/SignalRNotificationService.cs | 31 + .../Services/TableStorageService.Deletion.cs | 4 +- .../Services/TableStorageService.Inventory.cs | 87 ++- .../Services/TableStorageService.Restore.cs | 246 ++++++ .../Models/Deletion/DeletionProgress.cs | 37 + 24 files changed, 6006 insertions(+), 5 deletions(-) create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/BlobStorageServiceDeletionProgressTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/CascadeVerificationServiceTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/RestoreSessionFunctionTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionWorkerTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/SessionRestoreServiceTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/TableStoragePropValueConversionTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/TableStorageRestoreBatchTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RestoreSessionFunction.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Deletion/CascadeVerificationService.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionWorker.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRestoreService.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Restore.cs diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/BlobStorageServiceDeletionProgressTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/BlobStorageServiceDeletionProgressTests.cs new file mode 100644 index 00000000..27b41578 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/BlobStorageServiceDeletionProgressTests.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Tests for the PR4 cascade-deletion progress-blob Download + CAS-Update helpers on +/// . Same fake-subclass pattern as +/// BlobStorageServiceDeletionManifestTests: override the blob-IO seams, capture bytes +/// + options, replay on download. No Azurite. +/// +public class BlobStorageServiceDeletionProgressTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210"; + private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111"; + + [Fact] + public async Task Download_round_trips_progress_with_etag() + { + var sut = new FakeBlobStorageService(); + // Seed the fake with an initial progress blob written via the existing helper. + var initialEtag = await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256); + Assert.False(string.IsNullOrEmpty(initialEtag)); + + var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId); + + Assert.Equal(Sha256, progress.SnapshotSha256); + Assert.Empty(progress.CompletedSteps); + Assert.False(progress.VerificationDone); + Assert.Null(progress.CompletedAt); + Assert.False(string.IsNullOrEmpty(etag)); + // ETag from a fresh download must match what UploadInitial returned. + Assert.Equal(initialEtag, etag); + } + + [Fact] + public async Task Update_with_matching_etag_writes_new_state_and_returns_fresh_etag() + { + var sut = new FakeBlobStorageService(); + var initialEtag = await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256); + var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId); + + progress.CompletedSteps.Add(1); + progress.CompletedSteps.Add(2); + var newEtag = await sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, etag); + + Assert.False(string.IsNullOrEmpty(newEtag)); + Assert.NotEqual(initialEtag, newEtag); + + var (refetched, _) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId); + Assert.Equal(new HashSet { 1, 2 }, refetched.CompletedSteps); + } + + [Fact] + public async Task Update_with_mismatched_etag_throws_412_RequestFailedException() + { + var sut = new FakeBlobStorageService(); + await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256); + var (progress, _) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId); + + progress.CompletedSteps.Add(1); + + var ex = await Assert.ThrowsAsync( + () => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, ifMatchEtag: "\"0xSTALE_ETAG\"")); + Assert.Equal(412, ex.Status); + } + + [Fact] + public async Task Update_writes_IfMatch_condition_on_the_upload_options() + { + var sut = new FakeBlobStorageService(); + await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256); + var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId); + + await sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, etag); + + Assert.NotNull(sut.LastWrittenProgressOptions); + Assert.NotNull(sut.LastWrittenProgressOptions!.Conditions); + // PR4 §12-Q10 contract: every update is an ETag-CAS via IfMatch (not IfNoneMatch). + Assert.NotNull(sut.LastWrittenProgressOptions.Conditions!.IfMatch); + Assert.Equal(new ETag(etag), sut.LastWrittenProgressOptions.Conditions.IfMatch); + // Content type stays application/json — no gzip on the small progress blob. + Assert.Equal("application/json", sut.LastWrittenProgressOptions.HttpHeaders!.ContentType); + } + + [Fact] + public async Task Update_throws_on_missing_required_fields() + { + var sut = new FakeBlobStorageService(); + + await Assert.ThrowsAsync( + () => sut.UpdateDeletionProgressAsync("", SessionId, ManifestId, new DeletionProgress(), "etag")); + await Assert.ThrowsAsync( + () => sut.UpdateDeletionProgressAsync(TenantId, "", ManifestId, new DeletionProgress(), "etag")); + await Assert.ThrowsAsync( + () => sut.UpdateDeletionProgressAsync(TenantId, SessionId, "", new DeletionProgress(), "etag")); + await Assert.ThrowsAsync( + () => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, null!, "etag")); + await Assert.ThrowsAsync( + () => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, new DeletionProgress(), "")); + } + + [Fact] + public async Task Download_throws_on_missing_required_fields() + { + var sut = new FakeBlobStorageService(); + + await Assert.ThrowsAsync( + () => sut.DownloadDeletionProgressAsync("", SessionId, ManifestId)); + await Assert.ThrowsAsync( + () => sut.DownloadDeletionProgressAsync(TenantId, "", ManifestId)); + await Assert.ThrowsAsync( + () => sut.DownloadDeletionProgressAsync(TenantId, SessionId, "")); + } + + // ---------------------------------------------------------------- Test fixture ---- + + /// + /// Captures the progress-blob bytes + options written by Upload/Update and replays them on + /// Download. Each write returns a fresh ETag; an Update with a stale ETag throws 412 so the + /// Handler's bounded-retry loop can be exercised against this fake. + /// + private sealed class FakeBlobStorageService : BlobStorageService + { + private byte[]? _lastBytes; + private BlobUploadOptions? _lastOptions; + private ETag _currentEtag; + private int _etagCounter; + + public BlobUploadOptions? LastWrittenProgressOptions => _lastOptions; + + public FakeBlobStorageService() + : base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger.Instance, usesManagedIdentity: false) + { + } + + protected internal override Task WriteDeletionProgressBlobAsync( + string blobName, byte[] payload, BlobUploadOptions options, CancellationToken cancellationToken) + { + if (options.Conditions?.IfMatch != null + && _currentEtag != default + && options.Conditions.IfMatch != _currentEtag) + { + throw new RequestFailedException(412, "ConditionNotMet", "ConditionNotMet", innerException: null); + } + _lastBytes = payload; + _lastOptions = options; + _currentEtag = new ETag($"\"0xFAKE_ETAG_{++_etagCounter}\""); + return Task.FromResult(_currentEtag); + } + + protected internal override Task<(byte[] Payload, ETag ETag)> ReadDeletionProgressBlobAsync( + string blobName, CancellationToken cancellationToken) + { + if (_lastBytes == null) + { + throw new InvalidOperationException( + "Test must call UploadInitialDeletionProgressAsync before DownloadDeletionProgressAsync."); + } + return Task.FromResult((_lastBytes, _currentEtag)); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/CascadeVerificationServiceTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/CascadeVerificationServiceTests.cs new file mode 100644 index 00000000..5a5fd0bd --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/CascadeVerificationServiceTests.cs @@ -0,0 +1,399 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Azure.Data.Tables; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Per-class behaviour tests for . Verifies that each +/// step class drives the right inventory-reader call shape (filter for scan-based classes, +/// direct GetEntity for manifest-key-only classes), that AGGREGATE + FINAL are skipped, and +/// that the residual-sample cap is honoured. +/// +public class CascadeVerificationServiceTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + + [Fact] + public async Task Verify_returns_clean_when_all_tables_empty() + { + var reader = new FakeReader(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = BuildManifestWithAllClasses(); + + var result = await sut.VerifyAsync(manifest); + + Assert.True(result.IsClean); + Assert.Empty(result.Residuals); + } + + [Fact] + public async Task PkBySession_uses_partition_filter() + { + var reader = new FakeReader(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 }, + }, + }; + + await sut.VerifyAsync(manifest); + + var call = Assert.Single(reader.QueryCalls); + Assert.Equal("Events", call.Table); + Assert.Equal($"PartitionKey eq '{TenantId}_{SessionId}'", call.Filter); + } + + [Fact] + public async Task PropTenantPk_uses_partition_and_sessionid_filter() + { + var reader = new FakeReader(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep { Order = 3, Table = "AppInstallSummaries", Class = DeletionStepClass.PropTenantPk, RowCount = 0 }, + }, + }; + + await sut.VerifyAsync(manifest); + + var call = Assert.Single(reader.QueryCalls); + Assert.Equal($"PartitionKey eq '{TenantId}' and SessionId eq '{SessionId}'", call.Filter); + } + + [Fact] + public async Task PkRkExact_uses_direct_GetEntity_per_manifest_row() + { + var reader = new FakeReader(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep + { + Order = 5, Table = "DeviceSnapshot", Class = DeletionStepClass.PkRkExact, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } }, + }, + }, + }; + + await sut.VerifyAsync(manifest); + + Assert.Empty(reader.QueryCalls); + var get = Assert.Single(reader.GetCalls); + Assert.Equal("DeviceSnapshot", get.Table); + Assert.Equal(TenantId, get.Pk); + Assert.Equal(SessionId, get.Rk); + } + + [Fact] + public async Task DiscriminatorPkRkExact_uses_manifest_key_GetEntity_not_live_scan() + { + // Plan §12-Q8: CveIndex is the deliberate exception. Manifest-key only — verifier MUST NOT + // do a tenant-prefix scan on this class. + var reader = new FakeReader(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep + { + Order = 10, Table = "CveIndex", Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 2, + Rows = new List + { + new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId }, + new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0002", Rk = SessionId }, + }, + }, + }, + }; + + await sut.VerifyAsync(manifest); + + Assert.Empty(reader.QueryCalls); // no tenant-prefix scan + Assert.Equal(2, reader.GetCalls.Count); + Assert.All(reader.GetCalls, c => Assert.Equal("CveIndex", c.Table)); + } + + [Fact] + public async Task DiscriminatorPkRkSuffix_uses_tenant_prefix_scan_with_client_side_suffix_filter() + { + var reader = new FakeReader(); + // Seed with a row matching the suffix and one that doesn't — client-side filter must drop the non-match. + reader.QueryResults["EventTypeIndex"] = new List + { + BuildEntity($"{TenantId}_install_failed", $"6299999999999999_{SessionId}"), // matches + BuildEntity($"{TenantId}_install_failed", "6299999999999999_other-session"), // does not match + }; + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep { Order = 9, Table = "EventTypeIndex", Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 0 }, + }, + }; + + var result = await sut.VerifyAsync(manifest); + + var call = Assert.Single(reader.QueryCalls); + Assert.Contains($"PartitionKey ge '{TenantId}_'", call.Filter); + Assert.Contains($"PartitionKey lt '{TenantId}_~'", call.Filter); + // Suffix-matching row is the only residual reported. + Assert.False(result.IsClean); + Assert.Single(result.Residuals); + Assert.EndsWith($"_{SessionId}", result.Residuals[0].Rk); + } + + [Fact] + public async Task DiscriminatorPkProp_uses_tenant_prefix_and_sessionid_filter() + { + var reader = new FakeReader(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep { Order = 11, Table = "SessionsByTerminal", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0 }, + }, + }; + + await sut.VerifyAsync(manifest); + + var call = Assert.Single(reader.QueryCalls); + Assert.Contains($"PartitionKey ge '{TenantId}_'", call.Filter); + Assert.Contains($"PartitionKey lt '{TenantId}_~'", call.Filter); + Assert.Contains($"SessionId eq '{SessionId}'", call.Filter); + } + + [Fact] + public async Task Aggregate_step_is_skipped_without_verification_call() + { + var reader = new FakeReader(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep + { + Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement, + Class = DeletionStepClass.Aggregate, RowCount = 3, + Decrements = new List + { + new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" }, + }, + }, + }, + }; + + var result = await sut.VerifyAsync(manifest); + + Assert.True(result.IsClean); + Assert.Empty(reader.QueryCalls); + Assert.Empty(reader.GetCalls); + } + + [Fact] + public async Task Final_step_is_skipped_without_verification_call() + { + var reader = new FakeReader(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep + { + Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2, + Rows = new List + { + new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" }, + new DeletionRowDump { Pk = TenantId, Rk = SessionId }, + }, + }, + }, + }; + + var result = await sut.VerifyAsync(manifest); + + Assert.True(result.IsClean); + Assert.Empty(reader.QueryCalls); + Assert.Empty(reader.GetCalls); + } + + [Fact] + public async Task Single_ghost_row_in_PkBySession_returns_not_clean_with_residual() + { + var reader = new FakeReader(); + reader.QueryResults["Events"] = new List + { + BuildEntity($"{TenantId}_{SessionId}", "ghost-event-rk"), + }; + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 }, + }, + }; + + var result = await sut.VerifyAsync(manifest); + + Assert.False(result.IsClean); + var residual = Assert.Single(result.Residuals); + Assert.Equal("Events", residual.Table); + Assert.Equal($"{TenantId}_{SessionId}", residual.Pk); + Assert.Equal("ghost-event-rk", residual.Rk); + } + + [Fact] + public async Task Residual_sample_is_capped_at_MaxResidualSampleSize() + { + var reader = new FakeReader(); + reader.QueryResults["Events"] = Enumerable.Range(0, 200) + .Select(i => BuildEntity($"{TenantId}_{SessionId}", $"rk-{i:D3}")) + .ToList(); + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 }, + }, + }; + + var result = await sut.VerifyAsync(manifest); + + Assert.False(result.IsClean); + Assert.Equal(CascadeVerificationService.MaxResidualSampleSize, result.Residuals.Count); + } + + [Fact] + public async Task First_table_failure_short_circuits_remaining_steps() + { + var reader = new FakeReader(); + reader.QueryResults["Events"] = new List + { + BuildEntity($"{TenantId}_{SessionId}", "ghost-rk"), + }; + // RuleResults would have ghosts too, but the verifier must NOT enumerate it after the + // first-table failure — the short-circuit keeps verification cost bounded. + reader.QueryResults["RuleResults"] = new List + { + BuildEntity($"{TenantId}_{SessionId}", "should-not-be-seen"), + }; + var sut = new CascadeVerificationService(reader, NullLogger.Instance); + var manifest = new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 }, + new DeletionStep { Order = 2, Table = "RuleResults", Class = DeletionStepClass.PkBySession, RowCount = 0 }, + }, + }; + + var result = await sut.VerifyAsync(manifest); + + Assert.False(result.IsClean); + // Only Events was queried; RuleResults short-circuited. + Assert.Single(reader.QueryCalls); + Assert.Equal("Events", reader.QueryCalls[0].Table); + } + + // ---------------------------------------------------------------- Helpers ---- + + private static TableEntity BuildEntity(string pk, string rk) => new TableEntity(pk, rk); + + private static DeletionManifest BuildManifestWithAllClasses() => new DeletionManifest + { + TenantId = TenantId, + SessionId = SessionId, + Steps = new List + { + new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 }, + new DeletionStep { Order = 3, Table = "AppInstallSummaries", Class = DeletionStepClass.PropTenantPk, RowCount = 0 }, + new DeletionStep { Order = 5, Table = "DeviceSnapshot", Class = DeletionStepClass.PkRkExact, RowCount = 0 }, + new DeletionStep { Order = 9, Table = "EventTypeIndex", Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 0 }, + new DeletionStep { Order = 10, Table = "CveIndex", Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 0 }, + new DeletionStep { Order = 11, Table = "SessionsByTerminal", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0 }, + new DeletionStep { Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement, Class = DeletionStepClass.Aggregate, RowCount = 0 }, + new DeletionStep { Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 0 }, + }, + }; + + private sealed class FakeReader : ISessionDeletionInventoryReader + { + public List<(string Table, string Filter)> QueryCalls { get; } = new List<(string, string)>(); + public List<(string Table, string Pk, string Rk)> GetCalls { get; } = new List<(string, string, string)>(); + public Dictionary> QueryResults { get; } = new Dictionary>(StringComparer.Ordinal); + public Dictionary<(string Table, string Pk, string Rk), TableEntity> GetResults { get; } + = new Dictionary<(string, string, string), TableEntity>(); + + public Task GetSessionRowAsync(string tenantId, string sessionId, CancellationToken ct = default) + => Task.FromResult(null); + + public Task GetSessionsIndexRowAsync(string tenantId, string indexRowKey, CancellationToken ct = default) + => Task.FromResult(null); + + public async IAsyncEnumerable QueryAsync( + string tableName, string filter, [EnumeratorCancellation] CancellationToken ct = default) + { + QueryCalls.Add((tableName, filter)); + if (QueryResults.TryGetValue(tableName, out var rows)) + { + foreach (var row in rows) + { + await Task.Yield(); + yield return row; + } + } + } + + public Task GetEntityOrNullAsync(string tableName, string partitionKey, string rowKey, CancellationToken ct = default) + { + GetCalls.Add((tableName, partitionKey, rowKey)); + GetResults.TryGetValue((tableName, partitionKey, rowKey), out var entity); + return Task.FromResult(entity); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/FakeSignalRNotificationService.cs b/src/Backend/AutopilotMonitor.Functions.Tests/FakeSignalRNotificationService.cs index bc3be033..fc833a7f 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/FakeSignalRNotificationService.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/FakeSignalRNotificationService.cs @@ -20,6 +20,7 @@ public class FakeSignalRNotificationService : ISignalRNotificationService public int GlobalDismissAllCount { get; private set; } public List<(string TenantId, string SessionId, int Count)> RuleResultsCalls { get; } = new(); public List<(string TenantId, string SessionId, string Risk)> VulnReportCalls { get; } = new(); + public List<(string TenantId, string SessionId)> SessionDeletedCalls { get; } = new(); public Task NotifyRuleResultsAvailableAsync(string tenantId, string sessionId, int resultCount) { @@ -33,6 +34,12 @@ public class FakeSignalRNotificationService : ISignalRNotificationService return Task.CompletedTask; } + public Task NotifySessionDeletedAsync(string tenantId, string sessionId) + { + SessionDeletedCalls.Add((tenantId, sessionId)); + return Task.CompletedTask; + } + public Task SendTenantNotificationAsync(string tenantId, NotificationAudience audience, object dto) { TenantSends.Add((tenantId, audience, dto)); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RestoreSessionFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RestoreSessionFunctionTests.cs new file mode 100644 index 00000000..412ba0c7 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RestoreSessionFunctionTests.cs @@ -0,0 +1,55 @@ +using System.Linq; +using System.Net; +using AutopilotMonitor.Functions.Functions.Sessions; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Functions.Services.Deletion; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Thin HTTP-layer tests for (PR4b). The bulk of behaviour +/// lives in (covered by SessionRestoreServiceTests); +/// this file only verifies the layer-specific pieces: status mapping, route registration in +/// the policy catalog (GA-only enforcement), and body shape. +/// +public class RestoreSessionFunctionTests +{ + [Theory] + [InlineData(SessionRestoreOutcome.Restored, HttpStatusCode.OK)] + [InlineData(SessionRestoreOutcome.DryRunOk, HttpStatusCode.OK)] + [InlineData(SessionRestoreOutcome.RejectManifestNotFound, HttpStatusCode.NotFound)] + [InlineData(SessionRestoreOutcome.RejectAlreadyAtOriginalState, HttpStatusCode.Conflict)] + [InlineData(SessionRestoreOutcome.RejectActiveCascade, HttpStatusCode.Conflict)] + [InlineData(SessionRestoreOutcome.RejectManifestIdMismatch, HttpStatusCode.Conflict)] + [InlineData(SessionRestoreOutcome.RejectCorruptState, HttpStatusCode.Conflict)] + [InlineData(SessionRestoreOutcome.RejectManifestCorruption, HttpStatusCode.Conflict)] + [InlineData(SessionRestoreOutcome.RejectCasConflictOnClear, HttpStatusCode.Conflict)] + public void MapOutcomeToStatus_returns_expected_status(SessionRestoreOutcome outcome, HttpStatusCode expected) + { + Assert.Equal(expected, RestoreSessionFunction.MapOutcomeToStatus(outcome)); + } + + [Fact] + public void Restore_route_is_registered_in_policy_catalog_as_GlobalAdminOnly() + { + // Per memory feedback_route_policy_catalog — every HTTP route MUST be registered in + // EndpointAccessPolicyCatalog. An unregistered route fail-closes to 403. This test + // guards the restore endpoint's GA-only enforcement. + var entry = EndpointAccessPolicyCatalog.FindPolicy("POST", "admin/sessions/some-id/restore"); + + Assert.NotNull(entry); + Assert.Equal(EndpointPolicy.GlobalAdminOnly, entry!.Policy); + } + + [Fact] + public void Restore_route_is_NOT_registered_for_non_POST_methods() + { + // GET to the restore route should not match the POST registration. + var getEntry = EndpointAccessPolicyCatalog.FindPolicy("GET", "admin/sessions/some-id/restore"); + Assert.Null(getEntry); + + var putEntry = EndpointAccessPolicyCatalog.FindPolicy("PUT", "admin/sessions/some-id/restore"); + Assert.Null(putEntry); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs new file mode 100644 index 00000000..5e5bfc50 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs @@ -0,0 +1,667 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// End-to-end Handler facts for plan §5 PR4. Moq-based, no Azurite. Every TableStorageService / +/// BlobStorageService / CascadeVerificationService dependency is mocked through the existing +/// virtual seams; the Handler exercises the full §3 schema flow without touching real storage. +/// +public class SessionDeletionHandlerTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210"; + private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111"; + + // ============================================================ §5 PR4 plan tests ==== + + [Fact] + public async Task Cascade_deletes_full_session_round_trip() + { + var harness = new Harness(); + harness.SetHappyPath(); + var manifest = harness.SetFullSessionManifest(); + + await harness.Sut.HandleAsync(harness.Envelope); + + // All 6 table-step deletes invoked with the manifest's keys. + foreach (var step in manifest.Steps.Where(s => s.Class != DeletionStepClass.Aggregate && s.Class != DeletionStepClass.Final && s.Rows.Count > 0)) + { + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + step.Table!, It.Is>(keys => keys.Count == step.Rows.Count), It.IsAny()), + Times.AtLeastOnce, $"step {step.Order} ({step.Table}) was not deleted"); + } + // AGGREGATE decrement: 3 keys → 3 calls. + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Exactly(3)); + // Tombstone: 2 deletes (Index + Sessions). + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.SessionsIndex, It.IsAny>(), It.IsAny()), + Times.AtLeastOnce); + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.Sessions, It.IsAny>(), It.IsAny()), + Times.AtLeastOnce); + + Assert.Single(harness.SignalR.SessionDeletedCalls); + Assert.Equal((TenantId, SessionId), harness.SignalR.SessionDeletedCalls[0]); + + // Final audit asserts both step-completed entries AND deletion_completed. + Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_completed"); + Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_step_completed"); + } + + [Fact] + public async Task Cascade_includes_CveIndex_step() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + + await harness.Sut.HandleAsync(harness.Envelope); + + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.CveIndex, + It.Is>(keys => + keys.Count == 1 && keys[0].Pk == $"{TenantId}_CVE-2024-0001" && keys[0].Rk == SessionId), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Cascade_is_idempotent_when_re_run_after_completion() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + // Progress shows CompletedAt already set — handler must early-return. + harness.ProgressFake.CompletedAt = DateTime.UtcNow; + + await harness.Sut.HandleAsync(harness.Envelope); + + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), It.IsAny()), + Times.Never); + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Assert.Empty(harness.SignalR.SessionDeletedCalls); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_completed"); + } + + [Fact] + public async Task Cascade_resumes_after_partial_failure_by_skipping_completed_steps() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + // Steps 1–8 already complete on prior pickup; cascade resumes from step 9 onward. + harness.ProgressFake.CompletedSteps.UnionWith(new[] { 1, 2, 3, 4, 5, 6, 7, 8 }); + + await harness.Sut.HandleAsync(harness.Envelope); + + // Steps 1–8 must NOT be re-issued. + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.Events, It.IsAny>(), It.IsAny()), + Times.Never); + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.RuleResults, It.IsAny>(), It.IsAny()), + Times.Never); + // Step 9 onward DOES get issued. + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.EventTypeIndex, It.IsAny>(), It.IsAny()), + Times.AtLeastOnce); + } + + [Fact] + public async Task Cascade_corrupted_snapshot_hash_mismatch_throws_so_worker_poisons() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + // BlobStorageService throws InvalidDataException on SHA mismatch (existing helper contract). + // PR4c F6: Handler uses the with-Sha variant; mock it to throw on the same condition. + harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ThrowsAsync(new System.IO.InvalidDataException("snapshot SHA mismatch")); + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + + // No deletes, no signalR, no completion audit — the handler must throw before touching anything. + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), It.IsAny()), + Times.Never); + Assert.Empty(harness.SignalR.SessionDeletedCalls); + } + + [Fact] + public async Task Cascade_throws_on_DISCRIMINATOR_step_with_zero_rows_but_nonzero_count() + { + var harness = new Harness(); + harness.SetHappyPath(); + var manifest = harness.SetSingleStepManifest(new DeletionStep + { + Order = 11, Table = "SessionsByTerminal", + Class = DeletionStepClass.DiscriminatorPkProp, + RowCount = 5, + Rows = new List(), // empty despite RowCount = 5 + }); + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + "SessionsByTerminal", It.IsAny>(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Cascade_does_not_tombstone_on_live_verification_failure() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + // Verifier reports residuals → handler aborts before tombstone. + harness.Verifier.Setup(v => v.VerifyAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CascadeVerificationResult( + isClean: false, + residuals: new List { new CascadeResidualKey("Events", "ghost-pk", "ghost-rk") })); + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + + // The two FINAL-step deletes (SessionsIndex + Sessions) MUST NOT have fired. + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.SessionsIndex, It.IsAny>(), It.IsAny()), + Times.Never); + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.Sessions, It.IsAny>(), It.IsAny()), + Times.Never); + Assert.Empty(harness.SignalR.SessionDeletedCalls); + Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_verification_failed"); + } + + [Fact] + public async Task Cascade_decrements_software_inventory_via_contributions_row() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + + await harness.Sut.HandleAsync(harness.Envelope); + + // The 3 keys in the AGGREGATE step → 3 decrement calls. + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, "Microsoft", "Office", "16.0", It.IsAny()), Times.Once); + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, "Adobe", "Acrobat", "23.1", It.IsAny()), Times.Once); + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, "Mozilla", "Firefox", "120.0", It.IsAny()), Times.Once); + } + + [Fact] + public async Task Cascade_skips_software_inventory_when_no_contributions_row() + { + var harness = new Harness(); + harness.SetHappyPath(); + // Manifest WITHOUT the side-row step 16 + 17 (pre-side-row session). + harness.SetMinimalManifestNoInventory(); + + await harness.Sut.HandleAsync(harness.Envelope); + + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Cascade_aggregate_step_runs_before_side_row_delete() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + + var callOrder = new List(); + harness.Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback(() => callOrder.Add("decrement")) + .Returns(Task.CompletedTask); + harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.SessionInventoryContributions, It.IsAny>(), It.IsAny())) + .Callback(() => callOrder.Add("delete-side-row")) + .ReturnsAsync(new DeletionBatchResult(1, 1, 0)); + + await harness.Sut.HandleAsync(harness.Envelope); + + // All decrement calls must happen before the side-row delete. + var firstSideRow = callOrder.IndexOf("delete-side-row"); + Assert.True(firstSideRow >= 0, "side-row delete was never issued"); + Assert.True(callOrder.Take(firstSideRow).All(c => c == "decrement"), + "decrements must complete before the SessionInventoryContributions side-row is deleted (§17.5)"); + } + + [Fact] + public async Task Cascade_does_not_touch_diagnostics_blob_even_when_recorded_in_manifest() + { + var harness = new Harness(); + harness.SetHappyPath(); + var manifest = harness.SetFullSessionManifest(); + manifest.DiagnosticsBlobName = "diagnostics-uploads/contoso/session-abc/diag.zip"; + + await harness.Sut.HandleAsync(harness.Envelope); + + // The Blob mock surfaces ONLY deletion-manifest container operations; no other blob calls. + // Assertion: no Verify on diagnostics blob methods (we never set them up, so nothing fires). + // Stronger: assert no Blob mock method other than the manifest helpers was ever invoked. + // We do this by verifying the only Setup'd methods received calls — anything else would + // have thrown via Moq's strict-mode... but we're using loose mode. Instead: assert no + // Storage call mentions the diag blob name. + harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync( + It.Is(table => table.Contains("diagnostics", StringComparison.OrdinalIgnoreCase)), + It.IsAny>(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Cascade_emits_signalr_notification_on_completion() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + + await harness.Sut.HandleAsync(harness.Envelope); + + var call = Assert.Single(harness.SignalR.SessionDeletedCalls); + Assert.Equal(TenantId, call.TenantId); + Assert.Equal(SessionId, call.SessionId); + } + + [Fact] + public async Task Cascade_deletes_sessionsIndex_before_sessions_row() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + + var deleteOrder = new List(); + harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.SessionsIndex, It.IsAny>(), It.IsAny())) + .Callback(() => deleteOrder.Add(Constants.TableNames.SessionsIndex)) + .ReturnsAsync(new DeletionBatchResult(1, 1, 0)); + harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.Sessions, It.IsAny>(), It.IsAny())) + .Callback(() => deleteOrder.Add(Constants.TableNames.Sessions)) + .ReturnsAsync(new DeletionBatchResult(1, 1, 0)); + + await harness.Sut.HandleAsync(harness.Envelope); + + var indexIdx = deleteOrder.IndexOf(Constants.TableNames.SessionsIndex); + var sessIdx = deleteOrder.IndexOf(Constants.TableNames.Sessions); + Assert.True(indexIdx >= 0, "SessionsIndex was not deleted"); + Assert.True(sessIdx >= 0, "Sessions was not deleted"); + Assert.True(indexIdx < sessIdx, "SessionsIndex must be deleted before Sessions"); + } + + // ============================================================ State-machine + resume tests ==== + + [Fact] + public async Task Handler_throws_when_state_is_None_with_matching_manifestId() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + harness.SessionRow["DeletionState"] = SessionDeletionState.None; + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + } + + [Fact] + public async Task Handler_throws_when_PendingDeletionManifestId_does_not_match_envelope() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + harness.SessionRow["PendingDeletionManifestId"] = "DIFFERENT-MANIFEST-ID"; + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + } + + [Fact] + public async Task Handler_throws_when_sessions_row_missing_at_pickup() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + // Override Sessions read to return null. + harness.Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync((TableEntity?)null); + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + } + + [Fact] + public async Task Handler_resumes_when_DeletionState_already_Running_with_matching_manifestId() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + harness.SessionRow["DeletionState"] = SessionDeletionState.Running; + // Track CAS calls — Running → Running is NOT a valid transition; handler must NOT CAS. + harness.Storage.Setup(s => s.CasSetSessionDeletionStateAsync( + It.IsAny(), It.IsAny(), + SessionDeletionState.Queued, SessionDeletionState.Running, + It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("CAS Queued→Running should not be issued on resume")); + + // Should not throw — handler proceeds with cascade. + await harness.Sut.HandleAsync(harness.Envelope); + + Assert.Single(harness.SignalR.SessionDeletedCalls); + } + + [Fact] + public async Task Handler_throws_when_state_is_Poisoned() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + harness.SessionRow["DeletionState"] = SessionDeletionState.Poisoned; + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + } + + [Fact] + public async Task Handler_audits_deletion_step_failed_when_table_delete_throws() + { + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + // First delete throws — Handler must audit and rethrow. + harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.Events, It.IsAny>(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(503, "ServiceUnavailable")); + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + + Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_step_failed"); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_completed"); + Assert.Empty(harness.SignalR.SessionDeletedCalls); + } + + // ============================================================ Harness ==== + + private sealed class Harness + { + public Mock Storage { get; } + public Mock Blob { get; } + public Mock Verifier { get; } + public Mock Maintenance { get; } + public FakeSignalRNotificationService SignalR { get; } + public SessionDeletionHandler Sut { get; } + + public SessionDeletionEnvelope Envelope { get; } + public TableEntity SessionRow { get; } + public DeletionProgress ProgressFake { get; } + + public List AuditCalls { get; } = new List(); + + public Harness() + { + Storage = new Mock( + Mock.Of(), + NullLogger.Instance); + + Blob = new Mock( + new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, + false); + + Verifier = new Mock( + Mock.Of(), + NullLogger.Instance); + + Maintenance = new Mock(); + Maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .Returns?>( + (tenantId, action, entityType, entityId, performedBy, details) => + { + AuditCalls.Add(new AuditEntry(tenantId, action, entityType, entityId, performedBy, details)); + return Task.FromResult(true); + }); + + SignalR = new FakeSignalRNotificationService(); + + Envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, + SessionId = SessionId, + ManifestId = ManifestId, + Reason = "admin_delete", + EnqueuedAt = DateTime.UtcNow, + }; + + SessionRow = new TableEntity(TenantId, SessionId) + { + ["DeletionState"] = SessionDeletionState.Queued, + ["PendingDeletionManifestId"] = ManifestId, + }; + + ProgressFake = new DeletionProgress + { + SnapshotSha256 = Sha256, + CompletedSteps = new HashSet(), + VerificationDone = false, + CompletedAt = null, + }; + + Sut = new SessionDeletionHandler( + Storage.Object, Blob.Object, Verifier.Object, + Maintenance.Object, SignalR, + NullLogger.Instance); + } + + public void SetHappyPath() + { + // Sessions row read (state-machine). + Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync(() => SessionRow); + + // CAS Queued → Running succeeds. + Storage.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Queued, SessionDeletionState.Running, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.Running, + CurrentManifestId = ManifestId, + }); + + // Per-table deletes succeed with token counts. + Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), It.IsAny())) + .ReturnsAsync((string _, IReadOnlyList<(string, string)> keys, CancellationToken _) => + new DeletionBatchResult(keys.Count, keys.Count, 0)); + + Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // Blob: progress download returns the live ProgressFake snapshot + a stable ETag. + Blob.Setup(b => b.DownloadDeletionProgressAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync(() => (CloneProgress(ProgressFake), "\"0xFAKE_ETAG\"")); + + // Update mutates the ProgressFake in place so each subsequent download sees the latest state. + Blob.Setup(b => b.UpdateDeletionProgressAsync( + TenantId, SessionId, ManifestId, It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, string _, string _, DeletionProgress p, string _, CancellationToken _) => + { + ProgressFake.CompletedSteps = new HashSet(p.CompletedSteps); + ProgressFake.VerificationDone = p.VerificationDone; + ProgressFake.CompletedAt = p.CompletedAt; + // PR4c F1 + F2: propagate the new per-key + tombstone fields too so the harness + // mirrors live behaviour for tests that inspect them post-Handle. + ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null + ? null : new HashSet(p.AggregateDecrementsApplied, StringComparer.Ordinal); + ProgressFake.RestoreReIncrementsApplied = p.RestoreReIncrementsApplied == null + ? null : new HashSet(p.RestoreReIncrementsApplied, StringComparer.Ordinal); + ProgressFake.TombstoneStarted = p.TombstoneStarted; + return "\"0xFAKE_ETAG_2\""; + }); + + // Verifier reports clean by default. + Verifier.Setup(v => v.VerifyAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CascadeVerificationResult(true, new List())); + } + + // PR4c F6: Handler uses DownloadDeletionManifestWithShaAsync; return a stable sha hex + // string that matches the harness's progress.SnapshotSha256 so the binding check passes. + private const string FakeManifestSha = "1111111111111111111111111111111111111111111111111111111111111111"; + + public DeletionManifest SetFullSessionManifest() + { + var manifest = BuildFullManifest(); + Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync((manifest, FakeManifestSha)); + return manifest; + } + + public DeletionManifest SetMinimalManifestNoInventory() + { + // Same as full but omit the AGGREGATE step 16 and side-row step 17. + var manifest = BuildFullManifest(); + manifest.Steps = manifest.Steps + .Where(s => s.Class != DeletionStepClass.Aggregate + && !(s.Class == DeletionStepClass.PkRkExact && s.Table == Constants.TableNames.SessionInventoryContributions)) + .ToList(); + Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync((manifest, FakeManifestSha)); + return manifest; + } + + public DeletionManifest SetSingleStepManifest(DeletionStep step) + { + var manifest = new DeletionManifest + { + ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId, + CreatedAt = DateTime.UtcNow, + Steps = new List + { + step, + new DeletionStep + { + Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2, + Rows = new List + { + new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" }, + new DeletionRowDump { Pk = TenantId, Rk = SessionId }, + }, + }, + }, + }; + Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync((manifest, FakeManifestSha)); + return manifest; + } + + private static DeletionManifest BuildFullManifest() => new DeletionManifest + { + ManifestId = ManifestId, + TenantId = TenantId, + SessionId = SessionId, + CreatedAt = new DateTime(2026, 5, 12, 10, 0, 0, DateTimeKind.Utc), + CreatedBy = new DeletionActor { Type = "admin", Actor = "alice@example.com" }, + Reason = "admin_delete", + RetentionContext = new DeletionRetentionContext { TenantRetentionDays = 90 }, + SchemaHash = "sha256:test", + Steps = new List + { + new DeletionStep { Order = 1, Table = Constants.TableNames.Events, Class = DeletionStepClass.PkBySession, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "evt-1" } } }, + new DeletionStep { Order = 2, Table = Constants.TableNames.RuleResults, Class = DeletionStepClass.PkBySession, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "rule-1" } } }, + new DeletionStep { Order = 3, Table = Constants.TableNames.AppInstallSummaries, Class = DeletionStepClass.PropTenantPk, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = "app-1" } } }, + new DeletionStep { Order = 4, Table = Constants.TableNames.VulnerabilityReports, Class = DeletionStepClass.PkRkExact, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "report" } } }, + new DeletionStep { Order = 5, Table = Constants.TableNames.DeviceSnapshot, Class = DeletionStepClass.PkRkExact, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } }, + new DeletionStep { Order = 6, Table = Constants.TableNames.EventSessionIndex, Class = DeletionStepClass.PkRkExact, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } }, + new DeletionStep { Order = 7, Table = Constants.TableNames.Signals, Class = DeletionStepClass.PkBySession, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "sig-1" } } }, + new DeletionStep { Order = 8, Table = Constants.TableNames.DecisionTransitions, Class = DeletionStepClass.PkBySession, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "trans-1" } } }, + new DeletionStep { Order = 9, Table = Constants.TableNames.EventTypeIndex, Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_install_failed", Rk = $"6299999999999999_{SessionId}" } } }, + new DeletionStep { Order = 10, Table = Constants.TableNames.CveIndex, Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId } } }, + new DeletionStep { Order = 11, Table = Constants.TableNames.SessionsByTerminal, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_failed", Rk = "rk-1" } } }, + new DeletionStep { Order = 12, Table = Constants.TableNames.SessionsByStage, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_AccountSetup", Rk = "rk-2" } } }, + new DeletionStep { Order = 13, Table = Constants.TableNames.DeadEndsByReason, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0, + Rows = new List() }, + new DeletionStep { Order = 14, Table = Constants.TableNames.ClassifierVerdictsByIdLevel, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_clf_warn", Rk = "rk-3" } } }, + new DeletionStep { Order = 15, Table = Constants.TableNames.SignalsByKind, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_hello", Rk = "rk-4" } } }, + new DeletionStep + { + Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement, + Class = DeletionStepClass.Aggregate, RowCount = 3, + Decrements = new List + { + new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" }, + new DeletionDecrementKey { Vendor = "Adobe", Name = "Acrobat", Version = "23.1" }, + new DeletionDecrementKey { Vendor = "Mozilla", Name = "Firefox", Version = "120.0" }, + }, + }, + new DeletionStep { Order = 17, Table = Constants.TableNames.SessionInventoryContributions, Class = DeletionStepClass.PkRkExact, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } }, + new DeletionStep + { + Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2, + Rows = new List + { + new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" }, + new DeletionRowDump { Pk = TenantId, Rk = SessionId }, + }, + }, + }, + PreflightCounts = new Dictionary(), + }; + + private static DeletionProgress CloneProgress(DeletionProgress source) => new DeletionProgress + { + SnapshotSha256 = source.SnapshotSha256, + CompletedSteps = new HashSet(source.CompletedSteps), + VerificationDone = source.VerificationDone, + CompletedAt = source.CompletedAt, + }; + } + + private sealed record AuditEntry( + string TenantId, string Action, string EntityType, string EntityId, string PerformedBy, + Dictionary? Details); +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs new file mode 100644 index 00000000..a32a54c4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs @@ -0,0 +1,736 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Codex review fixes for PR4 + PR4b — six findings, all High/Medium severity. Tests here are +/// organized one section per finding (F1 → F6). Where a finding is best exercised via the +/// existing test harnesses (Handler / RestoreService / Worker), the new fact extends that file; +/// where the assertion is layer-pure (e.g. BlobStorage SHA round-trip), it lives here as a +/// focused unit test. +/// +public class SessionDeletionPr4cCodexFixesTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210"; + private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111"; + + // ============================================================ F1: AGGREGATE per-key idempotency ==== + + [Fact] + public async Task F1_Cascade_aggregate_skips_already_applied_decrement_keys_on_retry() + { + var harness = new HandlerHarness(); + harness.SetHappyPath(); + var manifest = harness.SetFullSessionManifest(); + // Simulate a prior partial run: pretend keys for "Microsoft:Office:16.0" and + // "Adobe:Acrobat:23.1" were already decremented, but the AGGREGATE step never marked + // complete. On retry the Handler must skip those two and only decrement Firefox. + harness.ProgressFake.AggregateDecrementsApplied = new HashSet(StringComparer.Ordinal) + { + "Microsoft:Office:16.0", + "Adobe:Acrobat:23.1", + }; + + await harness.Sut.HandleAsync(harness.Envelope); + + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, "Microsoft", "Office", "16.0", It.IsAny()), Times.Never); + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, "Adobe", "Acrobat", "23.1", It.IsAny()), Times.Never); + harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, "Mozilla", "Firefox", "120.0", It.IsAny()), Times.Once); + } + + [Fact] + public async Task F1_Cascade_aggregate_persists_per_key_progress_before_each_decrement() + { + // Ordering invariant: persist-first / decrement-second bounds drift to +1 per crash. + var harness = new HandlerHarness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + + var callOrder = new List(); + harness.Blob.Setup(b => b.UpdateDeletionProgressAsync( + TenantId, SessionId, ManifestId, + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((_, _, _, p, _, _) => + { + // Capture progress state mutations that include AggregateDecrementsApplied changes + // so we can correlate them with the decrement order. + var snapshot = p.AggregateDecrementsApplied == null + ? "init" + : string.Join(",", p.AggregateDecrementsApplied); + callOrder.Add($"persist:{snapshot}"); + harness.ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null + ? null : new HashSet(p.AggregateDecrementsApplied, StringComparer.Ordinal); + harness.ProgressFake.CompletedSteps = new HashSet(p.CompletedSteps); + harness.ProgressFake.VerificationDone = p.VerificationDone; + harness.ProgressFake.CompletedAt = p.CompletedAt; + harness.ProgressFake.TombstoneStarted = p.TombstoneStarted; + return Task.FromResult("\"0xFAKE_ETAG\""); + }); + harness.Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync( + TenantId, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((string _, string vendor, string name, string ver, CancellationToken _) => + callOrder.Add($"decrement:{vendor}:{name}:{ver}")) + .Returns(Task.CompletedTask); + + await harness.Sut.HandleAsync(harness.Envelope); + + // Each "decrement:X" must be preceded by a "persist:…X…" entry. We verify the pattern by + // checking that for every decrement, the most-recent prior persist contains the composite. + for (var i = 0; i < callOrder.Count; i++) + { + if (!callOrder[i].StartsWith("decrement:", StringComparison.Ordinal)) continue; + var composite = callOrder[i].Substring("decrement:".Length); + var priorPersists = callOrder.Take(i).Where(c => c.StartsWith("persist:", StringComparison.Ordinal)).ToList(); + Assert.NotEmpty(priorPersists); + Assert.Contains(priorPersists, p => p.Contains(composite, StringComparison.Ordinal)); + } + } + + // ============================================================ F2: TombstoneStarted gap closure ==== + + [Fact] + public async Task F2_Cascade_persists_TombstoneStarted_before_first_tombstone_delete() + { + var harness = new HandlerHarness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + + var callOrder = new List(); + harness.Blob.Setup(b => b.UpdateDeletionProgressAsync( + TenantId, SessionId, ManifestId, + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((_, _, _, p, _, _) => + { + if (p.TombstoneStarted && !harness.ProgressFake.TombstoneStarted) + callOrder.Add("persist:tombstone-started"); + harness.ProgressFake.CompletedSteps = new HashSet(p.CompletedSteps); + harness.ProgressFake.VerificationDone = p.VerificationDone; + harness.ProgressFake.CompletedAt = p.CompletedAt; + harness.ProgressFake.TombstoneStarted = p.TombstoneStarted; + harness.ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null + ? null : new HashSet(p.AggregateDecrementsApplied, StringComparer.Ordinal); + return Task.FromResult("\"0xFAKE_ETAG\""); + }); + harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + It.Is(t => t == Constants.TableNames.SessionsIndex || t == Constants.TableNames.Sessions), + It.IsAny>(), It.IsAny())) + .Callback((string t, IReadOnlyList<(string, string)> _, CancellationToken _) => + callOrder.Add($"delete:{t}")) + .ReturnsAsync(new DeletionBatchResult(1, 1, 0)); + + await harness.Sut.HandleAsync(harness.Envelope); + + var tombstonePersistIdx = callOrder.IndexOf("persist:tombstone-started"); + var firstTombstoneDeleteIdx = callOrder.FindIndex(c => c.StartsWith("delete:Sessions", StringComparison.Ordinal)); + Assert.True(tombstonePersistIdx >= 0, "TombstoneStarted persist was not observed"); + Assert.True(firstTombstoneDeleteIdx >= 0, "Tombstone delete was not observed"); + Assert.True(tombstonePersistIdx < firstTombstoneDeleteIdx, + "TombstoneStarted=true must be persisted BEFORE any tombstone-row delete (Codex F2 gap closure)."); + } + + [Fact] + public async Task F2_Restore_full_dispatches_on_TombstoneStarted_gap_when_sessions_null_and_completedAt_null() + { + // The exact gap scenario: cascade wrote TombstoneStarted=true, deleted Sessions row, but + // crashed before writing CompletedAt. Restore must dispatch as Full (not corrupt-state). + var harness = new RestoreHarness(); + harness.SetPoisonedCascade(); // gives us a manifest + harness.SessionRow = null; // tombstone removed it + harness.Progress.CompletedAt = null; + harness.Progress.TombstoneStarted = true; // PR4c F2: gap signal + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome); + Assert.Equal("full", result.Mode); + } + + // ============================================================ F3: Full restore retry safety ==== + + [Fact] + public async Task F3_Restore_full_inserts_sessions_row_last_not_first() + { + var harness = new RestoreHarness(); + harness.SetCompletedCascade(); + + var insertOrder = new List(); + harness.Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), + It.IsAny(), It.IsAny())) + .Callback((string table, IReadOnlyList _, RestoreMode _, CancellationToken _) => + insertOrder.Add(table)) + .ReturnsAsync((string _, IReadOnlyList rows, RestoreMode _, CancellationToken _) => + new RestoreBatchResult(rows.Count, rows.Count, 0)); + + await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + // Sessions row must be the very LAST insert. + var lastInsert = insertOrder.LastOrDefault(); + Assert.Equal(Constants.TableNames.Sessions, lastInsert); + // SessionsIndex must come BEFORE Sessions. + var sessionsIdx = insertOrder.LastIndexOf(Constants.TableNames.Sessions); + var sessionsIndexIdx = insertOrder.LastIndexOf(Constants.TableNames.SessionsIndex); + Assert.True(sessionsIndexIdx < sessionsIdx, + "SessionsIndex must be restored before Sessions (PR4c F3a — Sessions LAST)."); + } + + [Fact] + public async Task F3_Restore_full_409_now_counts_as_skipped_no_throw() + { + // Simulates retry-after-partial-failure: some rows already inserted by the previous + // attempt. PR4c F3b: Full mode now 409-ignores like Partial; the result reports them + // in RowsSkippedByTable, no exception. + var harness = new RestoreHarness(); + harness.SetCompletedCascade(); + harness.Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync( + Constants.TableNames.Events, It.IsAny>(), + RestoreMode.Full, It.IsAny())) + .ReturnsAsync(new RestoreBatchResult(attempted: 1, restored: 0, skipped: 1)); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome); + Assert.Equal(1, result.RowsSkippedByTable[Constants.TableNames.Events]); + } + + // ============================================================ F4: Partial restore per-key idempotency ==== + + [Fact] + public async Task F4_Restore_partial_skips_already_applied_re_increment_keys_on_retry() + { + var harness = new RestoreHarness(); + harness.SetPoisonedCascade(); + // Simulate a prior partial restore that already re-incremented Office + Acrobat but + // crashed before the CAS Poisoned → None (the exact Codex F4 scenario). + var aggregateOrder = harness.Manifest.Steps.First(s => s.Class == DeletionStepClass.Aggregate).Order; + harness.Progress.CompletedSteps.Add(aggregateOrder); + harness.Progress.RestoreReIncrementsApplied = new HashSet(StringComparer.Ordinal) + { + "Microsoft:Office:16.0", + "Adobe:Acrobat:23.1", + }; + + await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + TenantId, It.Is(k => k.Vendor == "Microsoft" && k.Name == "Office"), + It.IsAny()), Times.Never); + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + TenantId, It.Is(k => k.Vendor == "Adobe" && k.Name == "Acrobat"), + It.IsAny()), Times.Never); + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + TenantId, It.Is(k => k.Vendor == "Mozilla" && k.Name == "Firefox"), + It.IsAny()), Times.Once); + } + + // ============================================================ F5: Stale poison envelope ==== + + [Fact] + public async Task F5_Worker_skips_poison_state_transition_when_envelope_manifestId_does_not_match_current_pending() + { + var harness = new WorkerHarness(); + var staleEnvelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, + ManifestId = "STALE-OLD-MANIFEST-ID", // ≠ row.PendingDeletionManifestId + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(staleEnvelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1); + // Default GetSessionRowAsync setup returns PendingDeletionManifestId = ManifestId + // (a fresh active cascade); the envelope's STALE-OLD-MANIFEST-ID must NOT touch state. + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + // CAS must NOT have been issued — fresh cascade preserved. + harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + It.IsAny(), SessionDeletionState.Poisoned, + It.IsAny(), It.IsAny()), + Times.Never); + // Poison-queue + audit still fire so the stale message is off the main queue. + harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + } + + // ============================================================ F6: SHA binding ==== + + [Fact] + public async Task F6_DownloadDeletionManifestWithShaAsync_returns_hex_sha_matching_blob_metadata() + { + // Layer-pure: build a manifest, upload through the fake, download via with-Sha variant, + // verify the returned SHA equals the metadata SHA. + var fake = new FakeBlobService(); + var manifest = new DeletionManifest + { + ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId, + Reason = "admin_delete", SchemaHash = "sha256:test", + }; + + var pointer = await fake.UploadDeletionManifestAsync(manifest); + var (downloaded, sha) = await fake.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId); + + Assert.Equal(manifest.ManifestId, downloaded.ManifestId); + Assert.Equal(pointer.SnapshotSha256, sha); + Assert.Matches("^[0-9a-f]{64}$", sha); + } + + [Fact] + public async Task F6_Cascade_throws_when_progress_snapshotSha256_does_not_match_manifest_sha() + { + var harness = new HandlerHarness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + // Force a mismatch: progress carries one SHA, manifest download returns a different one. + harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync((new DeletionManifest + { + ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId, + Steps = new List(), + }, "deadbeef" + new string('0', 56))); + harness.ProgressFake.SnapshotSha256 = Sha256; // different value + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + } + + [Fact] + public async Task F6_Restore_rejects_when_progress_snapshotSha256_does_not_match_manifest_sha() + { + var harness = new RestoreHarness(); + harness.SetCompletedCascade(); + // Force the SHA binding to mismatch. + harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync((harness.Manifest, "deadbeef" + new string('0', 56))); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.RejectManifestCorruption, result.Outcome); + Assert.Contains("SHA binding", result.Message ?? string.Empty); + } + + // ============================================================ DeletionProgress back-compat ==== + + [Fact] + public void DeletionProgress_defaults_keep_back_compat_for_PR1_PR4_progress_blobs() + { + // New fields all default to safe values so PR1-PR4 progress blobs (no new fields) + // deserialize cleanly without surprises. + var progress = new DeletionProgress(); + + Assert.Null(progress.AggregateDecrementsApplied); + Assert.Null(progress.RestoreReIncrementsApplied); + Assert.False(progress.TombstoneStarted); + } + + // ============================================================ Test harnesses ==== + // These mirror the per-test-file harnesses in SessionDeletionHandlerTests.cs and + // SessionRestoreServiceTests.cs, kept private so PR4c facts don't depend on the + // other files' internals. + + private const string FakeManifestSha = Sha256; + + private sealed class HandlerHarness + { + public Mock Storage { get; } + public Mock Blob { get; } + public Mock Verifier { get; } + public Mock Maintenance { get; } + public FakeSignalRNotificationService SignalR { get; } + public SessionDeletionHandler Sut { get; } + + public SessionDeletionEnvelope Envelope { get; } + public TableEntity SessionRow { get; set; } + public DeletionProgress ProgressFake { get; } + + public List AuditCalls { get; } = new List(); + + public HandlerHarness() + { + Storage = new Mock(Mock.Of(), NullLogger.Instance); + Blob = new Mock( + new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, false); + Verifier = new Mock( + Mock.Of(), + NullLogger.Instance); + Maintenance = new Mock(); + Maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .Returns?>( + (t, a, e, eid, p, d) => { AuditCalls.Add(new AuditEntry(t, a, e, eid, p, d)); return Task.FromResult(true); }); + + SignalR = new FakeSignalRNotificationService(); + Envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + SessionRow = new TableEntity(TenantId, SessionId) + { + ["DeletionState"] = SessionDeletionState.Queued, + ["PendingDeletionManifestId"] = ManifestId, + }; + ProgressFake = new DeletionProgress + { + SnapshotSha256 = Sha256, + CompletedSteps = new HashSet(), + VerificationDone = false, + CompletedAt = null, + }; + + Sut = new SessionDeletionHandler( + Storage.Object, Blob.Object, Verifier.Object, + Maintenance.Object, SignalR, + NullLogger.Instance); + } + + public void SetHappyPath() + { + Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync(() => SessionRow); + Storage.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Queued, SessionDeletionState.Running, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.Running, CurrentManifestId = ManifestId, + }); + Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), It.IsAny())) + .ReturnsAsync((string _, IReadOnlyList<(string, string)> keys, CancellationToken _) => + new DeletionBatchResult(keys.Count, keys.Count, 0)); + Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + Blob.Setup(b => b.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync(() => (CloneProgress(ProgressFake), "\"0xFAKE_ETAG\"")); + Blob.Setup(b => b.UpdateDeletionProgressAsync( + TenantId, SessionId, ManifestId, It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, string _, string _, DeletionProgress p, string _, CancellationToken _) => + { + ProgressFake.CompletedSteps = new HashSet(p.CompletedSteps); + ProgressFake.VerificationDone = p.VerificationDone; + ProgressFake.CompletedAt = p.CompletedAt; + ProgressFake.TombstoneStarted = p.TombstoneStarted; + ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null + ? null : new HashSet(p.AggregateDecrementsApplied, StringComparer.Ordinal); + return "\"0xFAKE_ETAG_2\""; + }); + Verifier.Setup(v => v.VerifyAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CascadeVerificationResult(true, new List())); + } + + public DeletionManifest SetFullSessionManifest() + { + var manifest = BuildFullManifest(); + Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync((manifest, FakeManifestSha)); + return manifest; + } + + private static DeletionProgress CloneProgress(DeletionProgress s) => new DeletionProgress + { + SnapshotSha256 = s.SnapshotSha256, + CompletedSteps = new HashSet(s.CompletedSteps), + VerificationDone = s.VerificationDone, + CompletedAt = s.CompletedAt, + TombstoneStarted = s.TombstoneStarted, + AggregateDecrementsApplied = s.AggregateDecrementsApplied == null + ? null : new HashSet(s.AggregateDecrementsApplied, StringComparer.Ordinal), + RestoreReIncrementsApplied = s.RestoreReIncrementsApplied == null + ? null : new HashSet(s.RestoreReIncrementsApplied, StringComparer.Ordinal), + }; + } + + private sealed class RestoreHarness + { + public Mock Storage { get; } + public Mock Blob { get; } + public Mock Maintenance { get; } + public SessionRestoreService Sut { get; } + public TableEntity? SessionRow { get; set; } + public DeletionProgress Progress { get; set; } = new DeletionProgress(); + public DeletionManifest Manifest { get; set; } = new DeletionManifest(); + + public RestoreHarness() + { + Storage = new Mock(Mock.Of(), NullLogger.Instance); + Blob = new Mock( + new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, false); + Maintenance = new Mock(); + Maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .ReturnsAsync(true); + + Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync(() => SessionRow); + Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync(() => (Manifest, Sha256)); + Blob.Setup(b => b.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync(() => (Progress, "\"0xETAG\"")); + Blob.Setup(b => b.UpdateDeletionProgressAsync( + TenantId, SessionId, ManifestId, + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync("\"0xETAG-NEW\""); + Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), + It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, IReadOnlyList rows, RestoreMode _, CancellationToken _) => + new RestoreBatchResult(rows.Count, rows.Count, 0)); + Storage.Setup(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + Storage.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, SessionDeletionState.Poisoned, SessionDeletionState.None, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.None, + }); + + Sut = new SessionRestoreService(Storage.Object, Blob.Object, Maintenance.Object, + NullLogger.Instance); + } + + public void SetCompletedCascade() + { + SessionRow = null; + Progress = new DeletionProgress + { + SnapshotSha256 = Sha256, + CompletedSteps = new HashSet(Enumerable.Range(1, 18)), + VerificationDone = true, + CompletedAt = DateTime.UtcNow, + TombstoneStarted = true, + }; + Manifest = BuildFullManifest(); + } + + public void SetPoisonedCascade() + { + SessionRow = new TableEntity(TenantId, SessionId) + { + ["DeletionState"] = SessionDeletionState.Poisoned, + ["PendingDeletionManifestId"] = ManifestId, + }; + Progress = new DeletionProgress + { + SnapshotSha256 = Sha256, + CompletedSteps = new HashSet { 1, 2, 3 }, + VerificationDone = false, + CompletedAt = null, + }; + Manifest = BuildFullManifest(); + } + } + + private sealed class WorkerHarness + { + public Mock MainQueue { get; } + public Mock PoisonQueue { get; } + public Mock StorageMock { get; } + public Mock AdminConfig { get; } + public Mock Maintenance { get; } + public SessionDeletionWorker Sut { get; } + private readonly Queue _pendingMessages = new Queue(); + + public WorkerHarness() + { + MainQueue = new Mock(); + PoisonQueue = new Mock(); + MainQueue.Setup(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny())) + .ReturnsAsync((Response?)null); + PoisonQueue.Setup(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny())) + .ReturnsAsync((Response?)null); + + MainQueue.Setup(q => q.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((max, _, _) => + { + var batch = new List(); + while (batch.Count < max && _pendingMessages.Count > 0) batch.Add(_pendingMessages.Dequeue()); + return Task.FromResult(Response.FromValue(batch.ToArray(), new Mock().Object)); + }); + MainQueue.Setup(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new Mock().Object); + PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny())) + .Returns((_, _) => + { + var receipt = QueuesModelFactory.SendReceipt("p", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(7), "pr", DateTimeOffset.UtcNow); + return Task.FromResult(Response.FromValue(receipt, new Mock().Object)); + }); + + StorageMock = new Mock(Mock.Of(), NullLogger.Instance); + // PR4c F5: default GetSessionRowAsync returns a row whose PendingDeletionManifestId + // matches the constant ManifestId, simulating a FRESH ACTIVE cascade. The stale- + // envelope test then sends an envelope with a DIFFERENT manifestId. + StorageMock.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync(() => new TableEntity(TenantId, SessionId) + { + ["DeletionState"] = SessionDeletionState.Running, + ["PendingDeletionManifestId"] = ManifestId, // fresh cascade + }); + StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync( + It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.Poisoned, + }); + + var blobMock = new Mock( + new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, false); + var verifierMock = new Mock( + Mock.Of(), + NullLogger.Instance); + var handlerMock = new Mock( + StorageMock.Object, blobMock.Object, verifierMock.Object, + Mock.Of(), + new FakeSignalRNotificationService(), + NullLogger.Instance); + handlerMock.Setup(h => h.HandleAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + AdminConfig = new Mock( + Mock.Of(), + NullLogger.Instance, + new MemoryCache(new MemoryCacheOptions())); + AdminConfig.Setup(a => a.GetConfigurationAsync()) + .ReturnsAsync(new AdminConfiguration { SessionDeletionKillSwitch = false }); + + Maintenance = new Mock(); + Maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .ReturnsAsync(true); + + Sut = new SessionDeletionWorker( + MainQueue.Object, PoisonQueue.Object, + handlerMock.Object, StorageMock.Object, + AdminConfig.Object, Maintenance.Object, + NullLogger.Instance, + heartbeatInterval: TimeSpan.FromMilliseconds(200), + pollInterval: TimeSpan.FromMilliseconds(50)); + } + + public void EnqueueMessage(string body, int dequeueCount) + { + var msg = QueuesModelFactory.QueueMessage( + messageId: "msg-" + Guid.NewGuid().ToString("N"), + popReceipt: "pop-" + Guid.NewGuid().ToString("N"), + body: new BinaryData(body), + dequeueCount: dequeueCount); + _pendingMessages.Enqueue(msg); + } + + public async Task RunForAsync(TimeSpan duration) + { + using var cts = new CancellationTokenSource(duration); + try { await Sut.StartAsync(cts.Token); } catch (OperationCanceledException) { } + try { await Task.Delay(duration, cts.Token); } catch (OperationCanceledException) { } + try { await Sut.StopAsync(CancellationToken.None); } catch (OperationCanceledException) { } + } + } + + /// + /// Fake that captures the manifest upload and replays it on download, mirroring the + /// BlobStorageServiceDeletionManifestTests pattern. + /// + private sealed class FakeBlobService : BlobStorageService + { + private byte[]? _bytes; + private System.Collections.Generic.IDictionary? _metadata; + + public FakeBlobService() + : base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger.Instance, false) { } + + protected internal override Task WriteDeletionManifestBlobAsync( + string blobName, byte[] gzipped, Azure.Storage.Blobs.Models.BlobUploadOptions options, CancellationToken ct) + { + _bytes = gzipped; + _metadata = options.Metadata; + return Task.CompletedTask; + } + + protected internal override Task<(byte[] Gzipped, System.Collections.Generic.IDictionary? Metadata)> + ReadDeletionManifestBlobAsync(string blobName, CancellationToken ct) + => Task.FromResult((_bytes!, _metadata)); + } + + // ============================================================ Shared helpers ==== + + private static DeletionManifest BuildFullManifest() => new DeletionManifest + { + ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId, + Reason = "admin_delete", + Steps = new List + { + new DeletionStep { Order = 1, Table = Constants.TableNames.Events, Class = DeletionStepClass.PkBySession, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "evt-1" } } }, + new DeletionStep + { + Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement, + Class = DeletionStepClass.Aggregate, RowCount = 3, + Decrements = new List + { + new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" }, + new DeletionDecrementKey { Vendor = "Adobe", Name = "Acrobat", Version = "23.1" }, + new DeletionDecrementKey { Vendor = "Mozilla", Name = "Firefox", Version = "120.0" }, + }, + }, + new DeletionStep + { + Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2, + Rows = new List + { + new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" }, + new DeletionRowDump { Pk = TenantId, Rk = SessionId }, + }, + }, + }, + }; + + private sealed record AuditEntry( + string TenantId, string Action, string EntityType, string EntityId, string PerformedBy, + Dictionary? Details); +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionWorkerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionWorkerTests.cs new file mode 100644 index 00000000..e52d0fda --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionWorkerTests.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Worker-level tests for plan §5 PR4. Wraps the BackgroundService poll-loop in a +/// CancellationTokenSource and exercises the four explicit behaviours: +/// kill-switch entry guard, poison after max-dequeue (+ audit), malformed-envelope drop, +/// heartbeat extends visibility on long handler runs. +/// +public class SessionDeletionWorkerTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210"; + + [Fact] + public async Task Worker_does_not_receive_messages_while_kill_switch_active() + { + var harness = new Harness(); + harness.SetKillSwitch(true); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + harness.MainQueue.Verify(q => q.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + harness.HandlerMock.Verify(h => h.HandleAsync( + It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Worker_moves_message_to_poison_after_max_dequeue_and_audits() + { + var harness = new Harness(); + var envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + // Poison queue received the body and main queue deleted the original. + harness.PoisonQueue.Verify(q => q.SendMessageAsync( + It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + harness.MainQueue.Verify(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + // Audit deletion_poisoned written with the parsed envelope's manifestId. + Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_poisoned" + && a.Details != null && a.Details["manifestId"] == ManifestId); + } + + [Fact] + public async Task Worker_transitions_state_to_Poisoned_before_sending_to_poison_queue() + { + // PR4b E1 fix: MoveToPoisonAsync must CAS Sessions.DeletionState: Running → Poisoned + // BEFORE sending to the poison queue. Without this transition, the restore endpoint + // (PR4b) cannot dispatch into partial-restore mode (it keys off DeletionState=Poisoned). + var harness = new Harness(); + var envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1); + + // Track call order: the CAS must come BEFORE the poison-queue send. + var callOrder = new List(); + harness.StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Running, SessionDeletionState.Poisoned, + It.IsAny(), It.IsAny())) + .Callback(() => callOrder.Add("cas-running-to-poisoned")) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.Poisoned, + }); + harness.PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny())) + .Callback(() => callOrder.Add("poison-queue-send")) + .Returns((body, _) => + { + var receipt = QueuesModelFactory.SendReceipt("poison-msg", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(7), "poison-pop", DateTimeOffset.UtcNow); + return Task.FromResult(Response.FromValue(receipt, new Mock().Object)); + }); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + var casIdx = callOrder.IndexOf("cas-running-to-poisoned"); + var poisonIdx = callOrder.IndexOf("poison-queue-send"); + Assert.True(casIdx >= 0, "CAS Running→Poisoned was not issued"); + Assert.True(poisonIdx >= 0, "Poison-queue send was not issued"); + Assert.True(casIdx < poisonIdx, + "CAS Running→Poisoned must precede the poison-queue send so the restore endpoint can dispatch."); + } + + [Fact] + public async Task Worker_falls_back_to_Queued_to_Poisoned_when_Running_CAS_misses() + { + // Rare case: cascade poisoned before the worker ever successfully transitioned Queued→Running + // (handler always threw at the state-acquire step). In that case Running→Poisoned hits + // WrongState and the fallback Queued→Poisoned must succeed. + var harness = new Harness(); + var envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1); + + // First attempt: Running→Poisoned WrongState (current=Queued). + harness.StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Running, SessionDeletionState.Poisoned, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.WrongState, + CurrentState = SessionDeletionState.Queued, + }); + // Fallback attempt: Queued→Poisoned Updated. + harness.StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Queued, SessionDeletionState.Poisoned, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.Poisoned, + }); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + // Both CAS attempts fired, and the poison-queue send still completed. + harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Running, SessionDeletionState.Poisoned, + It.IsAny(), It.IsAny()), + Times.Once); + harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Queued, SessionDeletionState.Poisoned, + It.IsAny(), It.IsAny()), + Times.Once); + harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + } + + [Fact] + public async Task Worker_still_poisons_message_when_state_CAS_persistently_fails() + { + // Best-effort contract: if both CAS attempts fail (concurrent writer chaos), the + // poison-queue move + audit must STILL complete so the operator has an observable + // trail. Restore endpoint may reject with 409 "active_cascade" until the operator + // manually intervenes — but the cascade message itself is off the main queue. + var harness = new Harness(); + var envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1); + + harness.StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + It.IsAny(), SessionDeletionState.Poisoned, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.WrongState, + CurrentState = "Unknown", + }); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_poisoned"); + } + + [Fact] + public async Task Worker_drops_malformed_envelope_without_invoking_handler() + { + var harness = new Harness(); + harness.EnqueueMessage("{ \"this is not valid JSON for an envelope: garbage", dequeueCount: 1); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + harness.HandlerMock.Verify(h => h.HandleAsync( + It.IsAny(), It.IsAny()), + Times.Never); + // Malformed messages are removed from the main queue (otherwise they'd loop forever). + harness.MainQueue.Verify(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + // Poison queue should NOT receive a malformed body — those are dropped, not poisoned. + harness.PoisonQueue.Verify(q => q.SendMessageAsync( + It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Worker_heartbeat_extends_visibility_while_handler_runs() + { + var harness = new Harness(heartbeatInterval: TimeSpan.FromMilliseconds(50)); + var envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: 1); + + // Handler hangs for 350ms so the heartbeat task ticks at least 5×. + harness.HandlerMock.Setup(h => h.HandleAsync( + It.IsAny(), It.IsAny())) + .Returns(async (SessionDeletionEnvelope _, CancellationToken ct) => + { + await Task.Delay(TimeSpan.FromMilliseconds(350), ct); + }); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(800)); + + // Heartbeat called at least 3× while the handler was busy (350ms / 50ms ≈ 7, minus a few + // due to scheduling jitter). One call is enough to prove the heartbeat task is wired. + harness.MainQueue.Verify(q => q.UpdateMessageAsync( + It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), + Times.AtLeast(3)); + } + + [Fact] + public async Task Worker_drops_envelope_missing_required_fields_without_invoking_handler() + { + var harness = new Harness(); + var envelope = new SessionDeletionEnvelope + { + // ManifestId deliberately empty — required-fields guard must catch this. + TenantId = TenantId, SessionId = SessionId, ManifestId = string.Empty, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: 1); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + harness.HandlerMock.Verify(h => h.HandleAsync( + It.IsAny(), It.IsAny()), + Times.Never); + harness.MainQueue.Verify(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + } + + // ============================================================ Harness ==== + + private sealed class Harness + { + public Mock MainQueue { get; } + public Mock PoisonQueue { get; } + public Mock HandlerMock { get; } + public Mock AdminConfig { get; } + public Mock Maintenance { get; } + public List AuditCalls { get; } = new List(); + public SessionDeletionWorker Sut { get; } + + private readonly Queue _pendingMessages = new Queue(); + + public Harness(TimeSpan? heartbeatInterval = null) + { + MainQueue = new Mock(); + PoisonQueue = new Mock(); + + MainQueue.Setup(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny())) + .ReturnsAsync((Response?)null); + PoisonQueue.Setup(q => q.CreateIfNotExistsAsync( + It.IsAny>(), It.IsAny())) + .ReturnsAsync((Response?)null); + + MainQueue.Setup(q => q.ReceiveMessagesAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((maxMessages, _, _) => + { + var batch = new List(); + while (batch.Count < maxMessages && _pendingMessages.Count > 0) + { + batch.Add(_pendingMessages.Dequeue()); + } + var response = QueuesModelFactory.QueueMessage( + messageId: "msg-batch", popReceipt: "pop-batch", + body: new BinaryData(string.Empty), dequeueCount: 0); + return Task.FromResult(Response.FromValue(batch.ToArray(), new Mock().Object)); + }); + + MainQueue.Setup(q => q.DeleteMessageAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new Mock().Object); + + MainQueue.Setup(q => q.UpdateMessageAsync( + It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((id, pr, body, vis, ct) => + { + var updated = QueuesModelFactory.UpdateReceipt( + "pop-extended-" + Guid.NewGuid().ToString("N"), + DateTimeOffset.UtcNow.Add(vis)); + return Task.FromResult(Response.FromValue(updated, new Mock().Object)); + }); + + PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny())) + .Returns((body, _) => + { + var receipt = QueuesModelFactory.SendReceipt("poison-msg", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(7), "poison-pop", DateTimeOffset.UtcNow); + return Task.FromResult(Response.FromValue(receipt, new Mock().Object)); + }); + + // Handler — virtual HandleAsync; default behaviour is no-op (handler succeeded). + // Moq needs real ctor args for the proxy; the worker only calls HandleAsync so the + // inner deps stay untouched. + var storageMock = new Mock(Mock.Of(), NullLogger.Instance); + // PR4c F5: Worker now reads the Sessions row to verify PendingDeletionManifestId + // matches the envelope's manifestId before issuing the poison CAS. Default mock + // returns a row whose pending matches the test's ManifestId so the existing PR4b + // tests (which exercise the post-pre-check CAS paths) keep working. + storageMock.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync(() => new TableEntity(TenantId, SessionId) + { + ["DeletionState"] = SessionDeletionState.Running, + ["PendingDeletionManifestId"] = ManifestId, + }); + // Default: poison-state CAS succeeds. Individual tests override to verify the E1 fix. + storageMock.Setup(s => s.CasSetSessionDeletionStateAsync( + It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.Poisoned, + }); + var blobMock = new Mock( + new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, false); + var verifierMock = new Mock( + Mock.Of(), + NullLogger.Instance); + HandlerMock = new Mock( + storageMock.Object, blobMock.Object, verifierMock.Object, + Mock.Of(), + new FakeSignalRNotificationService(), + NullLogger.Instance); + HandlerMock.Setup(h => h.HandleAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + AdminConfig = new Mock( + Mock.Of(), + NullLogger.Instance, + new MemoryCache(new MemoryCacheOptions())); + AdminConfig.Setup(a => a.GetConfigurationAsync()) + .ReturnsAsync(new AdminConfiguration { SessionDeletionKillSwitch = false }); + + Maintenance = new Mock(); + Maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .Returns?>( + (tenantId, action, entityType, entityId, performedBy, details) => + { + AuditCalls.Add(new AuditEntry(tenantId, action, entityType, entityId, performedBy, details)); + return Task.FromResult(true); + }); + + Sut = new SessionDeletionWorker( + MainQueue.Object, PoisonQueue.Object, + HandlerMock.Object, storageMock.Object, + AdminConfig.Object, Maintenance.Object, + NullLogger.Instance, + heartbeatInterval: heartbeatInterval ?? TimeSpan.FromMilliseconds(200), + pollInterval: TimeSpan.FromMilliseconds(50)); + + StorageMock = storageMock; + } + + public Mock StorageMock { get; private set; } = null!; + + public void SetKillSwitch(bool active) + { + AdminConfig.Setup(a => a.GetConfigurationAsync()) + .ReturnsAsync(new AdminConfiguration { SessionDeletionKillSwitch = active }); + } + + public void EnqueueMessage(string body, int dequeueCount) + { + var msg = QueuesModelFactory.QueueMessage( + messageId: "msg-" + Guid.NewGuid().ToString("N"), + popReceipt: "pop-" + Guid.NewGuid().ToString("N"), + body: new BinaryData(body), + dequeueCount: dequeueCount); + _pendingMessages.Enqueue(msg); + } + + public async Task RunForAsync(TimeSpan duration) + { + using var cts = new CancellationTokenSource(duration); + try { await Sut.StartAsync(cts.Token); } + catch (OperationCanceledException) { /* expected */ } + try { await Task.Delay(duration, cts.Token); } + catch (OperationCanceledException) { /* expected on timeout */ } + try { await Sut.StopAsync(CancellationToken.None); } + catch (OperationCanceledException) { /* expected */ } + } + } + + private sealed record AuditEntry( + string TenantId, string Action, string EntityType, string EntityId, string PerformedBy, + Dictionary? Details); +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionRestoreServiceTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionRestoreServiceTests.cs new file mode 100644 index 00000000..b7c517a9 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionRestoreServiceTests.cs @@ -0,0 +1,509 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// End-to-end Service facts for plan §13 (PR4b). All 14 plan-listed tests + dispatch-table +/// edge cases. Moq-based, no Azurite — matches PR1–PR4 convention. +/// +public class SessionRestoreServiceTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210"; + private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111"; + + // ============================================================ Full restore (8 facts) ==== + + [Fact] + public async Task Restore_full_round_trips_session_after_completed_cascade() + { + var harness = new Harness(); + harness.SetCompletedCascade(); // sessionRow=null + progress.CompletedAt set + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome); + Assert.Equal("full", result.Mode); + // Sessions row inserted (manifest.FINAL.rows[1] in builder order). + harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync( + Constants.TableNames.Sessions, It.IsAny>(), RestoreMode.Full, It.IsAny()), + Times.AtLeastOnce); + // SessionsIndex inserted. + harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync( + Constants.TableNames.SessionsIndex, It.IsAny>(), RestoreMode.Full, It.IsAny()), + Times.AtLeastOnce); + // Inventory re-increments fired. + Assert.Equal(3, result.InventoryReIncrements); + } + + [Fact] + public async Task Restore_full_re_increments_software_inventory_via_contributions() + { + var harness = new Harness(); + harness.SetCompletedCascade(); + + await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + TenantId, + It.Is(k => k.Vendor == "Microsoft" && k.Name == "Office" && k.Version == "16.0"), + It.IsAny()), + Times.Once); + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + TenantId, + It.Is(k => k.Vendor == "Adobe" && k.Name == "Acrobat" && k.Version == "23.1"), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Restore_full_skips_software_inventory_when_no_contributions_row_in_snapshot() + { + var harness = new Harness(); + harness.SetCompletedCascade(includeInventorySteps: false); // pre-side-row session + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome); + Assert.Equal(0, result.InventoryReIncrements); + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Restore_full_rejects_when_session_already_exists_with_state_None() + { + var harness = new Harness(); + harness.SetCompletedCascade(); + harness.SessionRow = MakeSessionRow(SessionDeletionState.None, pendingManifestId: null); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.RejectAlreadyAtOriginalState, result.Outcome); + harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Restore_full_rejects_on_snapshot_hash_mismatch() + { + var harness = new Harness(); + harness.SetCompletedCascade(); + harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny())) + .ThrowsAsync(new InvalidDataException("Snapshot SHA-256 mismatch")); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.RejectManifestCorruption, result.Outcome); + Assert.Empty(result.RowsRestoredByTable); + } + + [Fact] + public async Task Restore_full_audits_manifest_download_before_any_read() + { + var harness = new Harness(); + harness.SetCompletedCascade(); + // Force a post-download reject path so we can verify the audit fired BEFORE the reject. + harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny())) + .ThrowsAsync(new InvalidDataException("corruption")); + + await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_manifest_downloaded" + && a.Details != null && a.Details["reason"] == "restore_attempt"); + } + + [Fact] + public async Task Restore_full_does_not_touch_SessionReports_or_session_reports_container() + { + var harness = new Harness(); + harness.SetCompletedCascade(); + + await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + // The harness's RestoreRowsByExactKeysInBatchesAsync mock records the tableName. We + // assert it was never invoked for SessionReports. + harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync( + "SessionReports", It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Restore_rejects_manifest_not_found_with_404_outcome() + { + var harness = new Harness(); + harness.SetCompletedCascade(); + harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny())) + .ThrowsAsync(new RequestFailedException(404, "BlobNotFound")); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.RejectManifestNotFound, result.Outcome); + } + + [Fact] + public async Task Restore_rejects_corrupt_state_when_sessions_row_missing_but_no_completedAt() + { + var harness = new Harness(); + harness.SetPoisonedCascade(); + harness.SessionRow = null; // Sessions row gone but progress NOT completed → corruption + harness.Progress.CompletedAt = null; + // PR4c F2: TombstoneStarted=false (default) — distinguishes "Sessions row removed by + // something outside the cascade" (true corruption) from the new gap-recovery case where + // TombstoneStarted=true allows full-restore to proceed. + harness.Progress.TombstoneStarted = false; + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.RejectCorruptState, result.Outcome); + } + + [Fact] + public async Task Restore_dryRun_writes_no_data_but_reports_what_would_restore() + { + var harness = new Harness(); + harness.SetCompletedCascade(); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: true, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.DryRunOk, result.Outcome); + Assert.Equal("dryRun", result.Mode); + Assert.NotEmpty(result.WouldRestoreByTable); + // No live writes. + harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Never); + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + // dryRun does NOT emit deletion_restored (only the download audit). + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_restored"); + } + + // ============================================================ Partial restore (6 facts) ==== + + [Fact] + public async Task Restore_partial_recovers_session_poisoned_mid_cascade() + { + var harness = new Harness(); + harness.SetPoisonedCascade(); + harness.Progress.CompletedSteps.UnionWith(new[] { 1, 2, 3, 4, 5, 6, 7 }); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome); + Assert.Equal("partial", result.Mode); + // Partial mode used for at least one restore call. + harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), RestoreMode.Partial, It.IsAny()), + Times.AtLeastOnce); + // CAS Poisoned→None fired at end. + harness.Storage.Verify(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Poisoned, SessionDeletionState.None, + It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Restore_partial_skips_inventory_re_increment_when_decrement_never_ran() + { + var harness = new Harness(); + harness.SetPoisonedCascade(); + // CompletedSteps does NOT include the AGGREGATE step's order (16). + harness.Progress.CompletedSteps.Clear(); + harness.Progress.CompletedSteps.Add(1); // poisoned right after step 1 + + await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Restore_partial_re_increments_inventory_when_decrement_already_ran() + { + var harness = new Harness(); + harness.SetPoisonedCascade(); + // CompletedSteps includes the AGGREGATE step's order (16). + harness.Progress.CompletedSteps.UnionWith(new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }); + + await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + TenantId, It.IsAny(), It.IsAny()), + Times.Exactly(3)); + } + + [Theory] + [InlineData(SessionDeletionState.Running)] + [InlineData(SessionDeletionState.Preparing)] + [InlineData(SessionDeletionState.Queued)] + public async Task Restore_partial_rejects_when_state_is_not_Poisoned(string state) + { + var harness = new Harness(); + harness.SetPoisonedCascade(); + harness.SessionRow = MakeSessionRow(state, pendingManifestId: ManifestId); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.RejectActiveCascade, result.Outcome); + Assert.Equal(state, result.CurrentState); + } + + [Fact] + public async Task Restore_partial_rejects_manifestid_mismatch() + { + var harness = new Harness(); + harness.SetPoisonedCascade(); + harness.SessionRow = MakeSessionRow(SessionDeletionState.Poisoned, pendingManifestId: "DIFFERENT-MANIFEST"); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.RejectManifestIdMismatch, result.Outcome); + Assert.Equal("DIFFERENT-MANIFEST", result.PendingManifestId); + } + + [Fact] + public async Task Restore_partial_returns_RejectCasConflictOnClear_when_final_CAS_fails() + { + var harness = new Harness(); + harness.SetPoisonedCascade(); + harness.Storage.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Poisoned, SessionDeletionState.None, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.EtagConflict, + CurrentState = SessionDeletionState.Poisoned, + }); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.RejectCasConflictOnClear, result.Outcome); + } + + [Fact] + public async Task Restore_audits_deletion_restored_on_success_with_full_details() + { + var harness = new Harness(); + harness.SetCompletedCascade(); + + await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); + + var audit = Assert.Single(harness.AuditCalls, a => a.Action == "deletion_restored"); + Assert.NotNull(audit.Details); + Assert.Equal(ManifestId, audit.Details!["manifestId"]); + Assert.Equal("full", audit.Details["mode"]); + Assert.True(audit.Details.ContainsKey("rowsRestoredByTable")); + Assert.True(audit.Details.ContainsKey("durationMs")); + } + + // ============================================================ Harness ==== + + private static TableEntity MakeSessionRow(string state, string? pendingManifestId) + { + var entity = new TableEntity(TenantId, SessionId) + { + ["DeletionState"] = state, + }; + if (pendingManifestId != null) + { + entity["PendingDeletionManifestId"] = pendingManifestId; + } + return entity; + } + + private sealed class Harness + { + public Mock Storage { get; } + public Mock Blob { get; } + public Mock Maintenance { get; } + public SessionRestoreService Sut { get; } + public List AuditCalls { get; } = new List(); + + // Per-test state — modify via Set*() methods. + public TableEntity? SessionRow { get; set; } + public DeletionProgress Progress { get; set; } = new DeletionProgress(); + public DeletionManifest Manifest { get; set; } = new DeletionManifest(); + + public Harness() + { + Storage = new Mock( + Mock.Of(), + NullLogger.Instance); + + Blob = new Mock( + new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, false); + + Maintenance = new Mock(); + Maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .Returns?>( + (tenantId, action, entityType, entityId, performedBy, details) => + { + AuditCalls.Add(new AuditEntry(tenantId, action, entityType, entityId, performedBy, details)); + return Task.FromResult(true); + }); + + // Defaults for the live-storage interactions. + Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync(() => SessionRow); + + // PR4c F6: RestoreService uses DownloadDeletionManifestWithShaAsync; the harness's + // Progress.SnapshotSha256 (set to the same Sha256 constant) matches the fake SHA so + // the snapshot↔progress binding check passes on the happy path. + Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync(() => (Manifest, Sha256)); + Blob.Setup(b => b.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync(() => (Progress, "\"0xETAG\"")); + + // PR4c F4: RestoreService now persists per-key re-increment progress via ETag-CAS. + // The harness default lets UpdateDeletionProgressAsync succeed unconditionally so + // tests focus on behaviour, not on simulating the bounded-retry loop. + Blob.Setup(b => b.UpdateDeletionProgressAsync( + TenantId, SessionId, ManifestId, + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync("\"0xETAG-NEW\""); + + Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), + It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, IReadOnlyList rows, RestoreMode _, CancellationToken _) => + new RestoreBatchResult(rows.Count, rows.Count, 0)); + + Storage.Setup(s => s.RestoreSoftwareInventoryEntryByKeyAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + Storage.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Poisoned, SessionDeletionState.None, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.None, + }); + + Sut = new SessionRestoreService( + Storage.Object, Blob.Object, Maintenance.Object, + NullLogger.Instance); + } + + public void SetCompletedCascade(bool includeInventorySteps = true) + { + SessionRow = null; + Progress = new DeletionProgress + { + SnapshotSha256 = Sha256, + CompletedSteps = new HashSet(Enumerable.Range(1, 18)), + VerificationDone = true, + CompletedAt = DateTime.UtcNow, + }; + Manifest = BuildFullManifest(includeInventorySteps); + } + + public void SetPoisonedCascade(bool includeInventorySteps = true) + { + SessionRow = MakeSessionRow(SessionDeletionState.Poisoned, pendingManifestId: ManifestId); + Progress = new DeletionProgress + { + SnapshotSha256 = Sha256, + CompletedSteps = new HashSet { 1, 2, 3 }, + VerificationDone = false, + CompletedAt = null, + }; + Manifest = BuildFullManifest(includeInventorySteps); + } + } + + private static DeletionManifest BuildFullManifest(bool includeInventorySteps) + { + var steps = new List + { + new DeletionStep { Order = 1, Table = Constants.TableNames.Events, Class = DeletionStepClass.PkBySession, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "evt-1" } } }, + new DeletionStep { Order = 2, Table = Constants.TableNames.RuleResults, Class = DeletionStepClass.PkBySession, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "rule-1" } } }, + new DeletionStep { Order = 5, Table = Constants.TableNames.DeviceSnapshot, Class = DeletionStepClass.PkRkExact, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } }, + new DeletionStep { Order = 10, Table = Constants.TableNames.CveIndex, Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId } } }, + }; + + if (includeInventorySteps) + { + steps.Add(new DeletionStep + { + Order = 16, + Step = DeletionStepNames.SoftwareInventoryDecrement, + Class = DeletionStepClass.Aggregate, + RowCount = 3, + Decrements = new List + { + new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" }, + new DeletionDecrementKey { Vendor = "Adobe", Name = "Acrobat", Version = "23.1" }, + new DeletionDecrementKey { Vendor = "Mozilla", Name = "Firefox", Version = "120.0" }, + }, + }); + steps.Add(new DeletionStep + { + Order = 17, + Table = Constants.TableNames.SessionInventoryContributions, + Class = DeletionStepClass.PkRkExact, + RowCount = 1, + Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } }, + }); + } + + steps.Add(new DeletionStep + { + Order = 18, + Step = DeletionStepNames.Tombstone, + Class = DeletionStepClass.Final, + RowCount = 2, + Rows = new List + { + new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" }, // SessionsIndex + new DeletionRowDump { Pk = TenantId, Rk = SessionId }, // Sessions + }, + }); + + return new DeletionManifest + { + ManifestId = ManifestId, + TenantId = TenantId, + SessionId = SessionId, + Steps = steps, + PreflightCounts = new Dictionary(), + }; + } + + private sealed record AuditEntry( + string TenantId, string Action, string EntityType, string EntityId, string PerformedBy, + Dictionary? Details); +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TableStoragePropValueConversionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TableStoragePropValueConversionTests.cs new file mode 100644 index 00000000..505eb242 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TableStoragePropValueConversionTests.cs @@ -0,0 +1,177 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models.Deletion; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Decode-direction tests for + +/// (PR4b). Pairs with the existing +/// DeletionManifestSerializationTests which covers the encode direction. Together they +/// establish the byte-faithful EDM round-trip the manifest-as-backup guarantee depends on +/// (plan §13 / §3 "types preserved"). +/// +public class TableStoragePropValueConversionTests +{ + [Fact] + public void ConvertFromPropValue_round_trips_String() + { + var prop = MakeProp(DeletionPropEdmType.String, "\"hello world\""); + Assert.Equal("hello world", TableStorageService.ConvertFromPropValue(prop)); + } + + [Fact] + public void ConvertFromPropValue_round_trips_Boolean_true_and_false() + { + Assert.Equal(true, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Boolean, "true"))); + Assert.Equal(false, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Boolean, "false"))); + } + + [Fact] + public void ConvertFromPropValue_round_trips_Int32() + { + Assert.Equal(42, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int32, "42"))); + Assert.Equal(-1, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int32, "-1"))); + Assert.Equal(int.MaxValue, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int32, int.MaxValue.ToString(CultureInfo.InvariantCulture)))); + } + + [Fact] + public void ConvertFromPropValue_round_trips_Int64() + { + Assert.Equal(123456789012L, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int64, "123456789012"))); + Assert.Equal(long.MaxValue, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int64, long.MaxValue.ToString(CultureInfo.InvariantCulture)))); + } + + [Fact] + public void ConvertFromPropValue_round_trips_Double() + { + Assert.Equal(3.14, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Double, "3.14"))); + Assert.Equal(0.0, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Double, "0"))); + } + + [Fact] + public void ConvertFromPropValue_round_trips_DateTime_as_UTC() + { + // Encode side stamps with "o" round-trip format and forces UTC kind. + var json = JsonSerializer.Serialize(new DateTime(2026, 5, 12, 10, 0, 0, DateTimeKind.Utc).ToString("o", CultureInfo.InvariantCulture)); + var result = TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.DateTime, json)); + var dt = Assert.IsType(result); + Assert.Equal(new DateTime(2026, 5, 12, 10, 0, 0, DateTimeKind.Utc), dt.ToUniversalTime()); + Assert.Equal(DateTimeKind.Utc, dt.Kind); + } + + [Fact] + public void ConvertFromPropValue_round_trips_Guid_in_D_format() + { + var guid = new Guid("33333333-3333-3333-3333-333333333333"); + var json = JsonSerializer.Serialize(guid.ToString("D")); + var result = TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Guid, json)); + Assert.Equal(guid, Assert.IsType(result)); + } + + [Fact] + public void ConvertFromPropValue_round_trips_Binary_as_Base64() + { + var bytes = new byte[] { 0x01, 0x02, 0x03, 0xff }; + var json = JsonSerializer.Serialize(Convert.ToBase64String(bytes)); + var result = TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Binary, json)); + var decoded = Assert.IsType(result); + Assert.Equal(bytes, decoded); + } + + [Fact] + public void ConvertFromPropValue_returns_null_for_JsonNull_value() + { + // Encode path stamps EdmType=String + JsonElement(null) for .NET null values. + var prop = MakeProp(DeletionPropEdmType.String, "null"); + Assert.Null(TableStorageService.ConvertFromPropValue(prop)); + } + + [Fact] + public void ConvertFromPropValue_throws_on_unknown_EdmType() + { + var prop = new DeletionPropValue + { + EdmType = "ThisEdmTypeDoesNotExist", + Value = ParseJson("\"x\""), + }; + Assert.Throws(() => TableStorageService.ConvertFromPropValue(prop)); + } + + [Fact] + public void ConvertDumpToEntity_overrides_DeletionState_to_None_for_Sessions_row() + { + // Snapshot was captured after CAS-Preparing — without the override the restored row + // would carry DeletionState=Preparing + the old manifestId, locking the session forever. + var dump = new DeletionRowDump + { + Pk = "tenant-1", + Rk = "session-1", + Etag = "\"0xETAG\"", + Props = new Dictionary(StringComparer.Ordinal) + { + ["DeletionState"] = MakeProp(DeletionPropEdmType.String, "\"Preparing\""), + ["PendingDeletionManifestId"] = MakeProp(DeletionPropEdmType.String, "\"some-old-manifest\""), + ["CurrentPhase"] = MakeProp(DeletionPropEdmType.String, "\"AccountSetup\""), + }, + }; + + var entity = TableStorageService.ConvertDumpToEntity(dump, Constants.TableNames.Sessions); + + Assert.Equal(SessionDeletionState.None, entity.GetString("DeletionState")); + Assert.Null(entity["PendingDeletionManifestId"]); + // Other props preserved. + Assert.Equal("AccountSetup", entity.GetString("CurrentPhase")); + } + + [Fact] + public void ConvertDumpToEntity_does_NOT_override_DeletionState_for_other_tables() + { + // SessionsIndex (or any other table) shouldn't have DeletionState forced. + var dump = new DeletionRowDump + { + Pk = "tenant-1", + Rk = "rk-1", + Props = new Dictionary(StringComparer.Ordinal) + { + ["SomeColumn"] = MakeProp(DeletionPropEdmType.String, "\"keep-me\""), + }, + }; + + var entity = TableStorageService.ConvertDumpToEntity(dump, Constants.TableNames.SessionsIndex); + + Assert.Equal("keep-me", entity.GetString("SomeColumn")); + Assert.False(entity.ContainsKey("DeletionState")); // Not added by override + } + + [Fact] + public void ConvertDumpToEntity_preserves_pk_and_rk() + { + var dump = new DeletionRowDump + { + Pk = "tenant-foo", + Rk = "session-bar", + Props = new Dictionary(), + }; + var entity = TableStorageService.ConvertDumpToEntity(dump, "AnyTable"); + Assert.Equal("tenant-foo", entity.PartitionKey); + Assert.Equal("session-bar", entity.RowKey); + } + + private static DeletionPropValue MakeProp(string edmType, string jsonValue) => new DeletionPropValue + { + EdmType = edmType, + Value = ParseJson(jsonValue), + }; + + private static JsonElement ParseJson(string json) + { + using var doc = JsonDocument.Parse(json); + return doc.RootElement.Clone(); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TableStorageRestoreBatchTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TableStorageRestoreBatchTests.cs new file mode 100644 index 00000000..07767438 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TableStorageRestoreBatchTests.cs @@ -0,0 +1,220 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// SDK-mocked behaviour tests for +/// (PR4b). Mirror of : PK-grouped batches, 100-action +/// chunks, per-row fallback when batch surfaces 409. Verifies the Full vs Partial 409 semantics +/// asymmetry that distinguishes "row already there is a corruption signal" (Full) from "row +/// already there is expected" (Partial). +/// +public class TableStorageRestoreBatchTests +{ + private const string TableName = Constants.TableNames.Events; + + [Fact] + public async Task Restore_returns_empty_result_for_empty_row_list() + { + var harness = new Harness(); + var result = await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(TableName, new List(), RestoreMode.Full); + + Assert.Equal(0, result.Attempted); + Assert.Equal(0, result.Restored); + Assert.Equal(0, result.Skipped); + Assert.Empty(harness.SubmittedBatches); + } + + [Fact] + public async Task Restore_groups_rows_by_partition_so_one_batch_per_pk() + { + var harness = new Harness(); + var rows = new[] + { + MakeDump("pkA", "rk1"), MakeDump("pkA", "rk2"), + MakeDump("pkB", "rk3"), MakeDump("pkB", "rk4"), MakeDump("pkB", "rk5"), + MakeDump("pkA", "rk6"), + }.ToList(); + + var result = await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(TableName, rows, RestoreMode.Full); + + Assert.Equal(6, result.Attempted); + Assert.Equal(6, result.Restored); + Assert.Equal(0, result.Skipped); + // Two batches: pkA (3 rows) + pkB (3 rows). + Assert.Equal(2, harness.SubmittedBatches.Count); + Assert.All(harness.SubmittedBatches, batch => + { + var pks = batch.Select(a => a.Entity.PartitionKey).Distinct().ToList(); + Assert.Single(pks); // batch is partition-scoped + }); + } + + [Fact] + public async Task Restore_uses_Add_action_so_409_surfaces_for_existing_rows() + { + var harness = new Harness(); + await harness.Sut.RestoreRowsByExactKeysInBatchesAsync( + TableName, new[] { MakeDump("pkA", "rk_0") }, RestoreMode.Full); + + var batch = harness.SubmittedBatches.Single(); + Assert.Equal(TableTransactionActionType.Add, batch.Single().ActionType); + } + + [Fact] + public async Task Restore_Full_mode_now_counts_409_as_skipped_like_Partial() + { + // PR4c F3b: Full mode used to throw InvalidDataException on per-row 409 (as a "corruption + // signal"). That intent was hypothetical (Session GUID reuse is unlikely) and blocked the + // operationally-real case of retrying a partial-failed restore. Both modes now share + // 409-ignore semantics; RowsSkippedByTable surfaces the count for operator inspection. + var harness = new Harness + { + BatchBehavior = _ => throw new RequestFailedException(409, "EntityAlreadyExists"), + PerRowAddBehavior = _ => throw new RequestFailedException(409, "EntityAlreadyExists"), + }; + + var result = await harness.Sut.RestoreRowsByExactKeysInBatchesAsync( + TableName, new[] { MakeDump("pkA", "rk_x") }, RestoreMode.Full); + + Assert.Equal(1, result.Attempted); + Assert.Equal(0, result.Restored); + Assert.Equal(1, result.Skipped); + } + + [Fact] + public async Task Restore_Partial_mode_counts_409_as_skipped_not_thrown() + { + var harness = new Harness + { + BatchBehavior = _ => throw new RequestFailedException(409, "EntityAlreadyExists"), + PerRowAddBehavior = rk => rk == "rk_already_there" + ? throw new RequestFailedException(409, "EntityAlreadyExists") + : (Response)new Mock().Object, + }; + + var result = await harness.Sut.RestoreRowsByExactKeysInBatchesAsync( + TableName, + new[] { MakeDump("pkA", "rk_already_there"), MakeDump("pkA", "rk_new") }, + RestoreMode.Partial); + + Assert.Equal(2, result.Attempted); + Assert.Equal(1, result.Restored); + Assert.Equal(1, result.Skipped); + } + + [Fact] + public async Task Restore_propagates_non_409_non_400_storage_errors() + { + var harness = new Harness + { + BatchBehavior = _ => throw new RequestFailedException(503, "ServiceUnavailable"), + }; + + await Assert.ThrowsAsync(() => + harness.Sut.RestoreRowsByExactKeysInBatchesAsync( + TableName, new[] { MakeDump("pkA", "rk_0") }, RestoreMode.Full)); + } + + [Fact] + public async Task Restore_throws_on_null_args() + { + var harness = new Harness(); + await Assert.ThrowsAsync(() => + harness.Sut.RestoreRowsByExactKeysInBatchesAsync("", new List(), RestoreMode.Full)); + await Assert.ThrowsAsync(() => + harness.Sut.RestoreRowsByExactKeysInBatchesAsync(TableName, null!, RestoreMode.Full)); + } + + [Fact] + public async Task Restore_preserves_dumps_props_via_ConvertDumpToEntity() + { + var harness = new Harness(); + var dump = new DeletionRowDump + { + Pk = "pkA", + Rk = "rk_0", + Props = new Dictionary(StringComparer.Ordinal) + { + ["MyStringCol"] = MakeProp(DeletionPropEdmType.String, "\"abc\""), + ["MyIntCol"] = MakeProp(DeletionPropEdmType.Int32, "42"), + }, + }; + + await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(TableName, new[] { dump }, RestoreMode.Full); + + var inserted = Assert.IsType(harness.SubmittedBatches.Single().Single().Entity); + Assert.Equal("abc", inserted.GetString("MyStringCol")); + Assert.Equal(42, inserted.GetInt32("MyIntCol")); + } + + // ---------------------------------------------------------------- Helpers ---- + + private static DeletionRowDump MakeDump(string pk, string rk) => new DeletionRowDump + { + Pk = pk, + Rk = rk, + Props = new Dictionary(), + }; + + private static DeletionPropValue MakeProp(string edmType, string jsonValue) + { + using var doc = JsonDocument.Parse(jsonValue); + return new DeletionPropValue { EdmType = edmType, Value = doc.RootElement.Clone() }; + } + + private sealed class Harness + { + public List> SubmittedBatches { get; } = new List>(); + public List<(string Pk, string Rk)> PerRowAdds { get; } = new List<(string, string)>(); + + public Func, Response>>? BatchBehavior { get; set; } + public Func? PerRowAddBehavior { get; set; } + + public TableStorageService Sut { get; } + + public Harness() + { + var mockTableClient = new Mock(); + + mockTableClient + .Setup(c => c.SubmitTransactionAsync(It.IsAny>(), It.IsAny())) + .Returns, CancellationToken>((actions, _) => + { + var snapshot = actions.ToList(); + SubmittedBatches.Add(snapshot); + if (BatchBehavior != null) return Task.FromResult(BatchBehavior(snapshot)); + var inner = (IReadOnlyList)snapshot.Select(_ => new Mock().Object).ToList(); + return Task.FromResult(Response.FromValue(inner, new Mock().Object)); + }); + + mockTableClient + .Setup(c => c.AddEntityAsync(It.IsAny(), It.IsAny())) + .Returns((entity, _) => + { + PerRowAdds.Add((entity.PartitionKey, entity.RowKey)); + if (PerRowAddBehavior != null) return Task.FromResult(PerRowAddBehavior(entity.RowKey)); + return Task.FromResult(new Mock().Object); + }); + + var mockServiceClient = new Mock(); + mockServiceClient.Setup(s => s.GetTableClient(It.IsAny())).Returns(mockTableClient.Object); + + Sut = new TableStorageService(mockServiceClient.Object, NullLogger.Instance); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RestoreSessionFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RestoreSessionFunction.cs new file mode 100644 index 00000000..a2829440 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RestoreSessionFunction.cs @@ -0,0 +1,180 @@ +using System; +using System.Net; +using System.Text.Json; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared.DataAccess; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Sessions +{ + /// + /// POST /api/admin/sessions/{sessionId}/restore — GA-only cascade-delete restore + /// endpoint (plan §13, PR4b). Body: { "manifestId": "...", "dryRun": false }. + /// Dispatches into which auto-selects full vs + /// partial-poisoned-recovery mode based on the (Sessions row state, progress.CompletedAt) + /// tuple. Status mapping: + /// + /// 200 OK — restore (or dry-run) succeeded, body carries the row counts. + /// 400 Bad Request — missing/invalid body, missing manifestId. + /// 404 Not Found — manifest blob missing (GC'd past 33-day window) or session unknown. + /// 409 Conflict — every Reject* outcome (state mismatch, manifest corruption, already at None, manifestId mismatch, CAS conflict on clear). + /// + /// Route registration in EndpointAccessPolicyCatalog with + /// EndpointPolicy.GlobalAdminOnly is what makes this GA-only; unregistered routes + /// fail-closed → 403 (per memory feedback_route_policy_catalog). + /// + public class RestoreSessionFunction + { + private readonly ILogger _logger; + private readonly SessionRestoreService _restoreService; + private readonly ISessionRepository _sessionRepo; + + public RestoreSessionFunction( + ILogger logger, + SessionRestoreService restoreService, + ISessionRepository sessionRepo) + { + _logger = logger; + _restoreService = restoreService; + _sessionRepo = sessionRepo; + } + + [Function("RestoreSession")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "admin/sessions/{sessionId}/restore")] HttpRequestData req, + string sessionId) + { + if (string.IsNullOrWhiteSpace(sessionId)) + { + return await WriteJsonAsync(req, HttpStatusCode.BadRequest, new { success = false, message = "sessionId is required" }); + } + + // Parse body. + RestoreRequestBody? body; + try + { + body = await JsonSerializer.DeserializeAsync( + req.Body, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "RestoreSession: malformed body for session {SessionId}", sessionId); + return await WriteJsonAsync(req, HttpStatusCode.BadRequest, + new { success = false, message = "Request body is not valid JSON. Expected: { \"manifestId\": \"...\", \"dryRun\": false }" }); + } + if (body == null || string.IsNullOrWhiteSpace(body.ManifestId)) + { + return await WriteJsonAsync(req, HttpStatusCode.BadRequest, + new { success = false, message = "manifestId is required in the request body." }); + } + + // Tenant resolution: prefer JWT TargetTenantId, fall back to SessionsIndex lookup + // (Global Admin can target any session by id alone). Mirrors the preview endpoint. + var requestCtx = req.GetRequestContext(); + var actorEmail = TenantHelper.GetUserIdentifier(req); + var tenantId = requestCtx.TargetTenantId; + if (string.IsNullOrEmpty(tenantId) || string.Equals(tenantId, "global", StringComparison.OrdinalIgnoreCase)) + { + tenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId); + } + if (string.IsNullOrEmpty(tenantId)) + { + // Sessions row is gone — but a full restore IS legal in that case. We need a way + // to determine the tenantId without the SessionsIndex (which is also gone). + // The manifest blob path encodes tenantId, but the operator only gives us + // sessionId + manifestId. Require the operator to scope the call by providing + // the tenantId in the request body (or via the X-Target-Tenant header). + if (string.IsNullOrWhiteSpace(body.TenantId)) + { + return await WriteJsonAsync(req, HttpStatusCode.NotFound, new + { + success = false, + message = $"Session {sessionId} not found in any active SessionsIndex entry. " + + "If the cascade completed (Sessions row gone), pass the tenantId explicitly in the request body." + }); + } + tenantId = body.TenantId; + } + + _logger.LogInformation( + "RestoreSession: tenant={TenantId} session={SessionId} manifestId={ManifestId} dryRun={DryRun} actor={Actor}", + tenantId, sessionId, body.ManifestId, body.DryRun, actorEmail); + + SessionRestoreResult result; + try + { + result = await _restoreService.RestoreAsync( + tenantId, sessionId, body.ManifestId, + body.DryRun, actorEmail); + } + catch (Exception ex) + { + _logger.LogError(ex, + "RestoreSession: unhandled exception for tenant={TenantId} session={SessionId} manifestId={ManifestId}", + tenantId, sessionId, body.ManifestId); + return await WriteJsonAsync(req, HttpStatusCode.InternalServerError, + new { success = false, message = "Internal error during restore — see audit log + telemetry for details.", exceptionType = ex.GetType().Name }); + } + + return await WriteResultAsync(req, result); + } + + /// + /// Pure status-mapping function for the restore outcome → HTTP status code translation. + /// Extracted as static + public so unit tests can verify it without spinning up the full + /// HttpRequestData mocking machinery. + /// + public static HttpStatusCode MapOutcomeToStatus(SessionRestoreOutcome outcome) => outcome switch + { + SessionRestoreOutcome.Restored => HttpStatusCode.OK, + SessionRestoreOutcome.DryRunOk => HttpStatusCode.OK, + SessionRestoreOutcome.RejectManifestNotFound => HttpStatusCode.NotFound, + _ => HttpStatusCode.Conflict, + }; + + private static async Task WriteResultAsync(HttpRequestData req, SessionRestoreResult result) + { + var status = MapOutcomeToStatus(result.Outcome); + + var body = new + { + success = status == HttpStatusCode.OK, + outcome = result.Outcome.ToString(), + mode = result.Mode, + message = result.Message, + currentState = result.CurrentState, + pendingManifestId = result.PendingManifestId, + rowsRestoredByTable = result.RowsRestoredByTable, + rowsSkippedByTable = result.RowsSkippedByTable, + wouldRestoreByTable = result.WouldRestoreByTable, + inventoryReIncrements = result.InventoryReIncrements, + durationMs = result.DurationMs, + }; + + return await WriteJsonAsync(req, status, body); + } + + private static async Task WriteJsonAsync(HttpRequestData req, HttpStatusCode status, object body) + { + var response = req.CreateResponse(status); + await response.WriteAsJsonAsync(body); + return response; + } + + private sealed class RestoreRequestBody + { + public string ManifestId { get; set; } = string.Empty; + public bool DryRun { get; set; } + /// + /// Optional: when the Sessions row is already gone (full-restore case after a completed + /// cascade), there's no SessionsIndex entry to look up the tenant from. Operators + /// should provide tenantId explicitly in this scenario. + /// + public string? TenantId { get; set; } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index d8dc93a5..2e82626b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -136,6 +136,10 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); +// PR4b: cascade-delete restore endpoint (full + partial-poisoned-recovery). Consumed by +// RestoreSessionFunction; GA-only via EndpointAccessPolicyCatalog. No hosted-service registration +// needed — the function is HTTP-triggered. +builder.Services.AddSingleton(); builder.Services.AddHostedService(); // Initialize all tables at startup // Data Access Layer — repository interfaces backed by Table Storage. diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index d3815cbf..dc8adbfd 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -214,6 +214,7 @@ public static class EndpointAccessPolicyCatalog new("GET", "sessions/{sessionId}/decision-graph", EndpointPolicy.GlobalAdminOnly), new("GET", "sessions/{sessionId}/reducer-verification", EndpointPolicy.GlobalAdminOnly), new("GET", "admin/sessions/{sessionId}/delete/preview", EndpointPolicy.GlobalAdminOnly), + new("POST", "admin/sessions/{sessionId}/restore", EndpointPolicy.GlobalAdminOnly), new("GET", "global/raw/sessions", EndpointPolicy.GlobalAdminOnly), new("GET", "global/raw/events", EndpointPolicy.GlobalAdminOnly), new("GET", "global/raw/events/search", EndpointPolicy.GlobalAdminOnly), diff --git a/src/Backend/AutopilotMonitor.Functions/Services/BlobStorageService.cs b/src/Backend/AutopilotMonitor.Functions/Services/BlobStorageService.cs index f2792115..b7ee0068 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/BlobStorageService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/BlobStorageService.cs @@ -191,7 +191,22 @@ namespace AutopilotMonitor.Functions.Services /// restore semantics rely on this); rethrows the underlying /// on 404 or other storage error. /// - public async Task DownloadDeletionManifestAsync( + public virtual async Task DownloadDeletionManifestAsync( + string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default) + { + var (manifest, _) = await DownloadDeletionManifestWithShaAsync(tenantId, sessionId, manifestId, cancellationToken).ConfigureAwait(false); + return manifest; + } + + /// + /// PR4c F6: variant of that also returns the + /// verified snapshot SHA-256 (hex string, lowercase) so the caller can enforce the + /// snapshot↔progress binding (plan §3): the caller compares this against + /// and refuses to proceed on mismatch. + /// Existing behaviour is preserved — the helper still throws + /// on blob-metadata vs payload SHA mismatch. + /// + public virtual async Task<(DeletionManifest Manifest, string Sha256Hex)> DownloadDeletionManifestWithShaAsync( string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default) { if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId)); @@ -238,7 +253,7 @@ namespace AutopilotMonitor.Functions.Services { throw new InvalidDataException($"Deletion manifest blob {blobName} deserialized to null."); } - return manifest; + return (manifest, actualSha256); } /// @@ -327,6 +342,79 @@ namespace AutopilotMonitor.Functions.Services return response.Value.ETag; } + /// + /// Downloads the cascade-deletion progress blob and returns the parsed + /// alongside the current ETag. The ETag must be passed + /// back to for the ETag-CAS that prevents two + /// parallel worker invocations (e.g. queue re-delivery before visibility expires) from + /// clobbering each other's progress writes (plan §3 + §12-Q10). Fails loud on 404 — the + /// progress blob is uploaded by the producer before the queue message is sent, so its + /// absence at worker pickup time is a corruption signal. + /// + public virtual async Task<(DeletionProgress Progress, string ETag)> DownloadDeletionProgressAsync( + string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId)); + if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("sessionId is required", nameof(sessionId)); + if (string.IsNullOrEmpty(manifestId)) throw new ArgumentException("manifestId is required", nameof(manifestId)); + + var blobName = BuildProgressBlobName(tenantId, sessionId, manifestId); + var (payload, etag) = await ReadDeletionProgressBlobAsync(blobName, cancellationToken); + + var progress = JsonSerializer.Deserialize(payload, DeletionManifestJson.SerializerOptions); + if (progress == null) + { + throw new InvalidDataException($"Deletion progress blob {blobName} deserialized to null."); + } + return (progress, etag.ToString()); + } + + /// + /// Test seam: reads the progress blob bytes + current ETag. Production override calls + /// Azure Blob Storage; tests subclass to return canned bytes without spinning up the SDK. + /// + protected internal virtual async Task<(byte[] Payload, ETag ETag)> ReadDeletionProgressBlobAsync( + string blobName, CancellationToken cancellationToken) + { + var containerClient = _blobServiceClient.GetBlobContainerClient(DeletionManifestsContainer); + var blobClient = containerClient.GetBlobClient(blobName); + using var downloaded = new MemoryStream(); + var response = await blobClient.DownloadToAsync(downloaded, cancellationToken); + var properties = await blobClient.GetPropertiesAsync(cancellationToken: cancellationToken); + return (downloaded.ToArray(), properties.Value.ETag); + } + + /// + /// Writes the progress blob with If-Match=ifMatchEtag — single CAS attempt. On + /// ETag mismatch (412 Precondition Failed) the underlying + /// propagates unchanged so the caller can implement + /// the §12-Q10 bounded-retry loop (re-download progress, check whether the target step + /// is already in — concurrent-winner case + /// — or retry the write). Returns the freshly-assigned ETag on success. + /// + public virtual async Task UpdateDeletionProgressAsync( + string tenantId, string sessionId, string manifestId, + DeletionProgress progress, string ifMatchEtag, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId)); + if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("sessionId is required", nameof(sessionId)); + if (string.IsNullOrEmpty(manifestId)) throw new ArgumentException("manifestId is required", nameof(manifestId)); + if (progress == null) throw new ArgumentNullException(nameof(progress)); + if (string.IsNullOrEmpty(ifMatchEtag)) throw new ArgumentException("ifMatchEtag is required", nameof(ifMatchEtag)); + + var json = JsonSerializer.SerializeToUtf8Bytes(progress, DeletionManifestJson.SerializerOptions); + var blobName = BuildProgressBlobName(tenantId, sessionId, manifestId); + var options = new BlobUploadOptions + { + HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" }, + Conditions = new BlobRequestConditions { IfMatch = new ETag(ifMatchEtag) }, + }; + + var etag = await WriteDeletionProgressBlobAsync(blobName, json, options, cancellationToken); + return etag.ToString(); + } + /// /// Lightweight existence check for a cascade-deletion snapshot blob. Used by the /// producer's resume-from-Preparing path to distinguish "the prior producer crashed diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/CascadeVerificationService.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/CascadeVerificationService.cs new file mode 100644 index 00000000..57eb0922 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/CascadeVerificationService.cs @@ -0,0 +1,233 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Deletion +{ + /// + /// Live verification pass between the table-step phase and the FINAL tombstone phase of the + /// cascade worker (plan §1 P4 + §3 verification table). For every cascade-targeted table the + /// service issues a LIVE query against current data — not a manifest-key existence check — so + /// it catches "ghost rows" written by paths that bypassed the writer-block guard within the + /// T2-T4 race window (§16 R13). A single hit on any table aborts the cascade: caller audits + /// deletion_verification_failed with a capped residual-key sample and throws so the + /// worker poisons after max-dequeue; operator then runs §13 restore-from-poisoned. + /// + /// CveIndex is the deliberate exception (§12-Q8): tenant-wide live scan would be cost-prohibitive + /// on a per-CVE-partition projection, so the verification falls back to per-manifest-key + /// GetEntity checks. Safety is preserved upstream by the SessionDeletionGuard + /// wired into every CveIndex writer (§5 PR3). + /// + /// + /// is excluded — SoftwareInventory counters are + /// statistical (approximate by design; clamp ≥ 0) and a live residual check would always + /// over-report. is excluded — it IS the tombstone the + /// verification gate sits in front of. + /// + /// + public class CascadeVerificationService + { + private readonly ISessionDeletionInventoryReader _reader; + private readonly ILogger _logger; + + // The verification residual sample passed to the audit log is bounded so a pathological + // ghost-row scenario doesn't blow up the AuditLogs payload. Operator sees the first + // sample; the precise total is recorded as a separate count field. + internal const int MaxResidualSampleSize = 50; + + public CascadeVerificationService(ISessionDeletionInventoryReader reader, ILogger logger) + { + _reader = reader; + _logger = logger; + } + + /// + /// Runs the verification pass over every non-FINAL, non-AGGREGATE step in the manifest. + /// Returns the outcome plus the residual sample to attach to the audit log. The caller + /// (Handler) decides whether to throw on a non-clean outcome. + /// + public virtual async Task VerifyAsync(DeletionManifest manifest, CancellationToken ct = default) + { + if (manifest == null) throw new ArgumentNullException(nameof(manifest)); + + var residuals = new List(); + var safeTenantId = ODataSanitizer.EscapeValue(manifest.TenantId); + var safeSessionId = ODataSanitizer.EscapeValue(manifest.SessionId); + + foreach (var step in manifest.Steps) + { + ct.ThrowIfCancellationRequested(); + + if (step.Class == DeletionStepClass.Aggregate || step.Class == DeletionStepClass.Final) + { + continue; + } + + if (string.IsNullOrEmpty(step.Table)) + { + // Synthetic non-AGGREGATE step is a corruption signal — every table-targeted + // step must carry a Table name. Defensive: caller already validates the + // manifest shape but the verifier must not silently skip. + throw new InvalidOperationException( + $"Manifest step Order={step.Order} Class={step.Class} has no Table — verification cannot proceed."); + } + + var found = await VerifyStepAsync(step, safeTenantId, safeSessionId, manifest.SessionId, ct); + if (found.Count == 0) continue; + + _logger.LogWarning( + "Cascade verification found {Count} residual rows in table {Table} for tenant={TenantId} session={SessionId} manifestId={ManifestId}", + found.Count, step.Table, manifest.TenantId, manifest.SessionId, manifest.ManifestId); + + foreach (var key in found) + { + if (residuals.Count >= MaxResidualSampleSize) break; + residuals.Add(key); + } + + // First-table-failure short-circuit: once we know the verification gate must reject + // tombstone, the audit sample is informational; we do not need to enumerate every + // remaining table. The Handler's poison + restore-from-poisoned path is the + // recovery mechanism, not a per-table inventory. + return new CascadeVerificationResult(isClean: false, residuals: residuals); + } + + return new CascadeVerificationResult(isClean: true, residuals: residuals); + } + + private async Task> VerifyStepAsync( + DeletionStep step, string safeTenantId, string safeSessionId, string sessionId, CancellationToken ct) + { + var found = new List(); + var table = step.Table!; + + switch (step.Class) + { + case DeletionStepClass.PkBySession: + { + var filter = $"PartitionKey eq '{safeTenantId}_{safeSessionId}'"; + await foreach (var entity in _reader.QueryAsync(table, filter, ct)) + { + found.Add(new CascadeResidualKey(table, entity.PartitionKey ?? string.Empty, entity.RowKey ?? string.Empty)); + if (found.Count >= MaxResidualSampleSize) break; + } + break; + } + + case DeletionStepClass.PropTenantPk: + { + var filter = $"PartitionKey eq '{safeTenantId}' and SessionId eq '{safeSessionId}'"; + await foreach (var entity in _reader.QueryAsync(table, filter, ct)) + { + found.Add(new CascadeResidualKey(table, entity.PartitionKey ?? string.Empty, entity.RowKey ?? string.Empty)); + if (found.Count >= MaxResidualSampleSize) break; + } + break; + } + + case DeletionStepClass.PkRkExact: + { + // The manifest stamped (PK, RK) at preflight; verification is the same lookup. + foreach (var row in step.Rows) + { + ct.ThrowIfCancellationRequested(); + var entity = await _reader.GetEntityOrNullAsync(table, row.Pk, row.Rk, ct); + if (entity != null) + { + found.Add(new CascadeResidualKey(table, row.Pk, row.Rk)); + if (found.Count >= MaxResidualSampleSize) break; + } + } + break; + } + + case DeletionStepClass.DiscriminatorPkRkSuffix: + { + var filter = $"PartitionKey ge '{safeTenantId}_' and PartitionKey lt '{safeTenantId}_~'"; + var suffix = $"_{sessionId}"; + await foreach (var entity in _reader.QueryAsync(table, filter, ct)) + { + if (entity.RowKey != null && entity.RowKey.EndsWith(suffix, StringComparison.Ordinal)) + { + found.Add(new CascadeResidualKey(table, entity.PartitionKey ?? string.Empty, entity.RowKey)); + if (found.Count >= MaxResidualSampleSize) break; + } + } + break; + } + + case DeletionStepClass.DiscriminatorPkRkExact: + { + // §12-Q8: manifest-key only — direct GetEntity per (PK, RK). + foreach (var row in step.Rows) + { + ct.ThrowIfCancellationRequested(); + var entity = await _reader.GetEntityOrNullAsync(table, row.Pk, row.Rk, ct); + if (entity != null) + { + found.Add(new CascadeResidualKey(table, row.Pk, row.Rk)); + if (found.Count >= MaxResidualSampleSize) break; + } + } + break; + } + + case DeletionStepClass.DiscriminatorPkProp: + { + var filter = $"PartitionKey ge '{safeTenantId}_' and PartitionKey lt '{safeTenantId}_~' and SessionId eq '{safeSessionId}'"; + await foreach (var entity in _reader.QueryAsync(table, filter, ct)) + { + found.Add(new CascadeResidualKey(table, entity.PartitionKey ?? string.Empty, entity.RowKey ?? string.Empty)); + if (found.Count >= MaxResidualSampleSize) break; + } + break; + } + + default: + throw new InvalidOperationException( + $"Manifest step Order={step.Order} has unsupported verification class '{step.Class}' (expected one of the table-targeted classes)."); + } + + return found; + } + } + + /// + /// Outcome of a call. When + /// is false, carries the first up to + /// MaxResidualSampleSize ghost rows for the audit log. + /// + public sealed class CascadeVerificationResult + { + public bool IsClean { get; } + public IReadOnlyList Residuals { get; } + + public CascadeVerificationResult(bool isClean, IReadOnlyList residuals) + { + IsClean = isClean; + Residuals = residuals; + } + } + + /// + /// One residual key found during verification. Carried in the + /// deletion_verification_failed audit entry's details Dictionary for operator triage. + /// + public sealed class CascadeResidualKey + { + public string Table { get; } + public string Pk { get; } + public string Rk { get; } + + public CascadeResidualKey(string table, string pk, string rk) + { + Table = table; + Pk = pk; + Rk = rk; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs new file mode 100644 index 00000000..575a11fc --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs @@ -0,0 +1,617 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Deletion +{ + /// + /// Per-envelope cascade executor (plan §5 PR4). Consumed by + /// ; pure DI, no queue knowledge. Runs the §3 schema: + /// + /// Download snapshot + progress; refuse on hash mismatch or already-completed. + /// Inspect Sessions row state (Plan §16 R13): Queued → CAS to Running; + /// Running with matching ManifestId → resume (no CAS); anything else with + /// matching ManifestId → poison; mismatched ManifestId → poison. + /// Iterate manifest steps in order, skipping those already in + /// . Dispatch by + /// ; AGGREGATE runs the per-key decrement; + /// FINAL is deferred until after live verification. + /// Live verification pass (). One residual + /// row aborts the cascade — audit deletion_verification_failed and throw so + /// the worker poisons after max-dequeue. + /// Tombstone: delete SessionsIndex first (UI listings update immediately), + /// then Sessions (row removal IS the completion signal — Plan §1 P7 + §16 R2). + /// Audit deletion_completed + SignalR sessionDeleted. + /// + /// All progress-blob writes are ETag-CAS with a bounded retry loop per §12-Q10 + /// (10 attempts or 60s wall-clock); on a 412 the loop re-reads the progress blob and + /// treats the mutation as already-applied if a concurrent worker already wrote it. + /// + public class SessionDeletionHandler + { + // §12-Q10: 10 attempts OR 60s wall-clock, whichever first → throw so the worker poisons. + internal const int ProgressCasMaxAttempts = 10; + internal static readonly TimeSpan ProgressCasMaxWallClock = TimeSpan.FromSeconds(60); + + // Cap on the per-table audit-detail `residualKeys` sample — paired with + // CascadeVerificationService.MaxResidualSampleSize. The Dictionary entry + // holds a small JSON array of "{table}/{pk}/{rk}" strings. + internal const int AuditResidualSampleSize = 50; + + private readonly TableStorageService _storage; + private readonly BlobStorageService _blob; + private readonly CascadeVerificationService _verifier; + private readonly IMaintenanceRepository _maintenanceRepo; + private readonly ISignalRNotificationService _signalR; + private readonly ILogger _logger; + + public SessionDeletionHandler( + TableStorageService storage, + BlobStorageService blob, + CascadeVerificationService verifier, + IMaintenanceRepository maintenanceRepo, + ISignalRNotificationService signalR, + ILogger logger) + { + _storage = storage; + _blob = blob; + _verifier = verifier; + _maintenanceRepo = maintenanceRepo; + _signalR = signalR; + _logger = logger; + } + + public virtual async Task HandleAsync(SessionDeletionEnvelope envelope, CancellationToken cancellationToken = default) + { + if (envelope == null) throw new ArgumentNullException(nameof(envelope)); + if (string.IsNullOrEmpty(envelope.TenantId)) throw new ArgumentException("envelope.TenantId is required", nameof(envelope)); + if (string.IsNullOrEmpty(envelope.SessionId)) throw new ArgumentException("envelope.SessionId is required", nameof(envelope)); + if (string.IsNullOrEmpty(envelope.ManifestId)) throw new ArgumentException("envelope.ManifestId is required", nameof(envelope)); + + var sw = Stopwatch.StartNew(); + var tenantId = envelope.TenantId; + var sessionId = envelope.SessionId; + var manifestId = envelope.ManifestId; + + // (1) Download the snapshot. SHA-256 is verified inside DownloadDeletionManifestWithShaAsync; + // any InvalidDataException propagates and the worker will poison after max-dequeue. + // PR4c F6: we use the with-Sha variant so we can additionally verify the + // snapshot↔progress binding once both blobs are loaded. + var (manifest, snapshotSha) = await _blob.DownloadDeletionManifestWithShaAsync(tenantId, sessionId, manifestId, cancellationToken); + if (!string.Equals(manifest.TenantId, tenantId, StringComparison.Ordinal) + || !string.Equals(manifest.SessionId, sessionId, StringComparison.Ordinal) + || !string.Equals(manifest.ManifestId, manifestId, StringComparison.Ordinal)) + { + throw new System.IO.InvalidDataException( + $"Manifest envelope mismatch: envelope=({tenantId},{sessionId},{manifestId}) " + + $"manifest=({manifest.TenantId},{manifest.SessionId},{manifest.ManifestId})"); + } + + // (2) Download progress + ETag. If already completed, this is an idempotent re-pickup. + var (progress, etag) = await _blob.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, cancellationToken); + + // PR4c F6: enforce the snapshot↔progress SHA binding (plan §3). The producer wrote + // the manifest's SHA into progress.SnapshotSha256 at upload time; if they don't match + // here, the artifacts have been tampered with or the producer crashed in a bad way. + if (!string.IsNullOrEmpty(progress.SnapshotSha256) + && !string.Equals(progress.SnapshotSha256, snapshotSha, StringComparison.OrdinalIgnoreCase)) + { + throw new System.IO.InvalidDataException( + $"Snapshot/progress SHA binding mismatch: progress.SnapshotSha256={progress.SnapshotSha256} " + + $"manifest SHA={snapshotSha} — corruption signal. tenant={tenantId} session={sessionId} manifestId={manifestId}"); + } + + if (progress.CompletedAt != null) + { + _logger.LogInformation( + "SessionDeletionHandler skipping already-completed cascade: tenant={TenantId} session={SessionId} manifestId={ManifestId} completedAt={CompletedAt}", + tenantId, sessionId, manifestId, progress.CompletedAt.Value); + return; + } + + // (3) Inspect Sessions row and dispatch on state. Plan §16 R13 — Queued → CAS Running; + // Running w/ matching ManifestId → resume; anything else → poison. + await AcquireOrResumeRunningStateAsync(tenantId, sessionId, manifestId, cancellationToken); + + // (4) Per-step execution loop. Iterate by Order; skip those already complete; defer FINAL. + foreach (var step in manifest.Steps.OrderBy(s => s.Order)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (progress.CompletedSteps.Contains(step.Order)) continue; + if (step.Class == DeletionStepClass.Final) continue; // tombstone runs after verification + + GuardStepShape(step); + + var stepStart = sw.ElapsedMilliseconds; + DeletionBatchResult result = DeletionBatchResult.Empty; + int aggregateProcessed = 0; + + try + { + switch (step.Class) + { + case DeletionStepClass.PkBySession: + case DeletionStepClass.PkRkExact: + case DeletionStepClass.PropTenantPk: + case DeletionStepClass.DiscriminatorPkRkSuffix: + case DeletionStepClass.DiscriminatorPkRkExact: + case DeletionStepClass.DiscriminatorPkProp: + result = await ExecuteTableStepAsync(step, cancellationToken); + break; + + case DeletionStepClass.Aggregate: + // PR4c F1: per-key idempotent decrement. The helper takes + returns + // (progress, etag) because each key's success is persisted with + // ETag-CAS before the next key is attempted. + (aggregateProcessed, progress, etag) = await ExecuteAggregateStepWithIdempotencyAsync( + tenantId, sessionId, manifestId, step, progress, etag, cancellationToken); + break; + + default: + throw new InvalidOperationException( + $"Manifest step Order={step.Order} has unsupported class '{step.Class}'."); + } + } + catch (Exception ex) when (!(ex is OperationCanceledException)) + { + await AuditStepFailedAsync(tenantId, sessionId, manifestId, step, ex, cancellationToken); + throw; + } + + // Mark complete with bounded-retry CAS. On 412, re-read progress; if a concurrent + // worker already added this step.Order to CompletedSteps, treat as success. + (progress, etag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, etag, + mutate: p => p.CompletedSteps.Add(step.Order), + isAlreadyApplied: p => p.CompletedSteps.Contains(step.Order), + cancellationToken); + + await _maintenanceRepo.LogAuditEntryAsync( + tenantId, + action: "deletion_step_completed", + entityType: "Session", + entityId: sessionId, + performedBy: "system", + details: BuildStepCompletedDetails(manifestId, step, result, aggregateProcessed, sw.ElapsedMilliseconds - stepStart)) + .ConfigureAwait(false); + } + + // (5) Live verification (§1 P4). Outcome dictates whether tombstone runs at all. + if (!progress.VerificationDone) + { + var verification = await _verifier.VerifyAsync(manifest, cancellationToken); + if (!verification.IsClean) + { + await _maintenanceRepo.LogAuditEntryAsync( + tenantId, + action: "deletion_verification_failed", + entityType: "Session", + entityId: sessionId, + performedBy: "system", + details: BuildVerificationFailedDetails(manifestId, verification.Residuals)) + .ConfigureAwait(false); + + throw new InvalidOperationException( + $"Cascade verification found {verification.Residuals.Count} residual row(s) for " + + $"tenant={tenantId} session={sessionId} manifestId={manifestId}; refusing to tombstone."); + } + + (progress, etag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, etag, + mutate: p => p.VerificationDone = true, + isAlreadyApplied: p => p.VerificationDone, + cancellationToken); + } + + // (6) FINAL tombstone. Plan §5 PR4 + §16 R2: SessionsIndex first, then Sessions. + var tombstone = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Final) + ?? throw new InvalidOperationException($"Manifest has no FINAL step (tenant={tenantId} session={sessionId} manifestId={manifestId})."); + + if (!progress.CompletedSteps.Contains(tombstone.Order)) + { + // PR4c F2: persist TombstoneStarted=true BEFORE the first tombstone-row delete. + // Closes the "tombstone gap": if the worker dies after deleting the Sessions row + // but before writing CompletedAt, restore can still dispatch into full-restore + // by reading this flag (otherwise sessions=null + completedAt=null looks like + // corruption and restore rejects). + if (!progress.TombstoneStarted) + { + (progress, etag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, etag, + mutate: p => p.TombstoneStarted = true, + isAlreadyApplied: p => p.TombstoneStarted, + cancellationToken); + } + + await ExecuteTombstoneAsync(tombstone, cancellationToken); + } + + (progress, etag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, etag, + mutate: p => + { + p.CompletedSteps.Add(tombstone.Order); + p.CompletedAt = DateTime.UtcNow; + }, + isAlreadyApplied: p => p.CompletedAt != null, + cancellationToken); + + sw.Stop(); + + await _maintenanceRepo.LogAuditEntryAsync( + tenantId, + action: "deletion_completed", + entityType: "Session", + entityId: sessionId, + performedBy: "system", + details: BuildCompletedDetails(manifestId, manifest, sw.ElapsedMilliseconds)) + .ConfigureAwait(false); + + await _signalR.NotifySessionDeletedAsync(tenantId, sessionId); + + _logger.LogInformation( + "SessionDeletionHandler completed: tenant={TenantId} session={SessionId} manifestId={ManifestId} durationMs={DurationMs}", + tenantId, sessionId, manifestId, sw.ElapsedMilliseconds); + } + + // ============================================================ State-machine entry ==== + + private async Task AcquireOrResumeRunningStateAsync(string tenantId, string sessionId, string manifestId, CancellationToken ct) + { + // Read the Sessions row to determine current state. The reader interface is implemented + // by TableStorageService (PR1) and 404-returns-null is the documented contract. + var sessionRow = await ((ISessionDeletionInventoryReader)_storage).GetSessionRowAsync(tenantId, sessionId, ct); + if (sessionRow == null) + { + throw new InvalidOperationException( + $"Sessions row missing at worker pickup — corruption or stale envelope. " + + $"tenant={tenantId} session={sessionId} manifestId={manifestId}"); + } + + var pendingManifestId = sessionRow.GetString("PendingDeletionManifestId"); + if (!string.Equals(pendingManifestId, manifestId, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Stale envelope: Sessions.PendingDeletionManifestId={pendingManifestId} does not match envelope.ManifestId={manifestId} " + + $"(tenant={tenantId} session={sessionId})."); + } + + var currentState = sessionRow.GetString("DeletionState") ?? string.Empty; + if (string.IsNullOrEmpty(currentState)) currentState = SessionDeletionState.None; + + switch (currentState) + { + case SessionDeletionState.Queued: + var cas = await _storage.CasSetSessionDeletionStateAsync( + tenantId, sessionId, + fromState: SessionDeletionState.Queued, + toState: SessionDeletionState.Running, + newManifestId: null, + ct); + if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.Updated) + { + return; + } + if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.WrongState + && cas.CurrentState == SessionDeletionState.Running + && string.Equals(cas.CurrentManifestId, manifestId, StringComparison.Ordinal)) + { + // Concurrent worker already CAS'd to Running — resume directly. + return; + } + throw new InvalidOperationException( + $"Failed to CAS Queued→Running: tenant={tenantId} session={sessionId} manifestId={manifestId} " + + $"outcome={cas.Outcome} currentState={cas.CurrentState} currentManifestId={cas.CurrentManifestId}"); + + case SessionDeletionState.Running: + // Idempotent re-pickup — no CAS write (Running → Running is not a valid + // transition; skip the CAS guard entirely, the manifestId match above is the + // sufficient correctness check). + return; + + case SessionDeletionState.Poisoned: + throw new InvalidOperationException( + $"Sessions row already Poisoned for matching ManifestId — operator must run restore-from-poisoned. " + + $"tenant={tenantId} session={sessionId} manifestId={manifestId}"); + + default: + throw new InvalidOperationException( + $"Unexpected DeletionState '{currentState}' on Sessions row with matching ManifestId. " + + $"tenant={tenantId} session={sessionId} manifestId={manifestId}"); + } + } + + // ============================================================ Step execution ==== + + private static void GuardStepShape(DeletionStep step) + { + // Per §5 PR4: any DISCRIMINATOR_* or PROP_TENANT_PK step with non-zero RowCount must + // carry the matching Rows[] entries — empty rows[] + non-zero count is a corruption + // signal. PK_BY_SESSION can be empty (no rows present at preflight) so we don't gate + // it. PK_RK_EXACT also allows empty rows (target was absent at preflight, e.g. + // VulnerabilityReports for a session with no scan). + switch (step.Class) + { + case DeletionStepClass.PropTenantPk: + case DeletionStepClass.DiscriminatorPkRkSuffix: + case DeletionStepClass.DiscriminatorPkRkExact: + case DeletionStepClass.DiscriminatorPkProp: + if (step.RowCount > 0 && step.Rows.Count == 0) + { + throw new InvalidOperationException( + $"Corruption: step Order={step.Order} Class={step.Class} reports RowCount={step.RowCount} " + + "but Rows[] is empty."); + } + break; + } + } + + private async Task ExecuteTableStepAsync(DeletionStep step, CancellationToken ct) + { + if (string.IsNullOrEmpty(step.Table)) + { + throw new InvalidOperationException( + $"Manifest step Order={step.Order} Class={step.Class} has no Table — cannot execute."); + } + if (step.Rows.Count == 0) + { + return DeletionBatchResult.Empty; // nothing to delete (empty PK_BY_SESSION, absent PK_RK_EXACT, etc.) + } + + var keys = step.Rows.Select(r => (r.Pk, r.Rk)).ToList(); + return await _storage.DeleteByExactKeysInBatchesAsync(step.Table!, keys, ct); + } + + /// + /// PR4c F1: per-key idempotent AGGREGATE-step executor. Persists the composite key into + /// with ETag-CAS before + /// issuing the underlying decrement; on retry, keys already in the set are skipped. + /// + /// Ordering rationale: persist-first/decrement-second bounds the worst-case drift to + /// +1 per crash (rare key-was-marked-but-decrement-didn't-run case). The opposite + /// ordering would unbound the drift to -N per crash (every key re-decremented on + /// retry — the original Codex F1 finding). The clamp-≥-0 invariant in the underlying + /// helper provides the symmetric protection in the rare miss direction. + /// + /// + private async Task<(int Processed, DeletionProgress Progress, string Etag)> ExecuteAggregateStepWithIdempotencyAsync( + string tenantId, string sessionId, string manifestId, + DeletionStep step, DeletionProgress progress, string etag, CancellationToken ct) + { + // The only AGGREGATE step today is SoftwareInventoryDecrement (§17.5). + if (!string.Equals(step.Step, DeletionStepNames.SoftwareInventoryDecrement, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Manifest AGGREGATE step Order={step.Order} has unsupported synthetic step name '{step.Step}'."); + } + if (step.Decrements == null || step.Decrements.Count == 0) + { + return (0, progress, etag); + } + + // Lazy-initialize the applied-keys set on first AGGREGATE-step entry. Persist so + // PR1-PR4 progress blobs (which had no such field) gain it on first use. + if (progress.AggregateDecrementsApplied == null) + { + (progress, etag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, etag, + mutate: p => p.AggregateDecrementsApplied ??= new HashSet(StringComparer.Ordinal), + isAlreadyApplied: p => p.AggregateDecrementsApplied != null, + ct); + } + + var processed = 0; + foreach (var key in step.Decrements) + { + ct.ThrowIfCancellationRequested(); + var composite = BuildAggregateCompositeKey(key); + if (progress.AggregateDecrementsApplied!.Contains(composite)) + { + continue; // already applied on a prior attempt + } + + // F1 ordering: persist FIRST (mark intent), decrement SECOND (apply effect). + // Capture composite in a local so the closure doesn't surprise on retry. + var compositeForClosure = composite; + (progress, etag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, etag, + mutate: p => p.AggregateDecrementsApplied!.Add(compositeForClosure), + isAlreadyApplied: p => p.AggregateDecrementsApplied!.Contains(compositeForClosure), + ct); + + // Decrement helper is bounded-retry + clamp-≥-0 + 404-idempotent (PR2 §17.4). + await _storage.DecrementSoftwareInventoryEntryAsync(tenantId, key.Vendor, key.Name, key.Version, ct); + processed++; + } + return (processed, progress, etag); + } + + /// + /// Composite-key format for + /// and . Matches the colon- + /// separated shape that BuildSoftwareInventoryRowKey in + /// TableStorageService.Inventory.cs uses, so the same key identifies a row across + /// forward (decrement) and reverse (re-increment) operations. + /// + internal static string BuildAggregateCompositeKey(DeletionDecrementKey key) + => $"{key.Vendor ?? string.Empty}:{key.Name ?? string.Empty}:{key.Version ?? string.Empty}"; + + private async Task ExecuteTombstoneAsync(DeletionStep tombstone, CancellationToken ct) + { + if (tombstone.Rows.Count == 0) + { + // The Sessions row would not exist at this point in normal flow, so an empty + // FINAL step is acceptable on a re-run of an already-tombstoned cascade. The + // FINAL step is marked complete after this method returns; subsequent picks + // hit the `CompletedAt != null` early-return. + return; + } + + // The builder (DeletionManifestBuilder.AddTombstoneStep) emits SessionsIndex first + // (if present) then Sessions. Determine table by row.Rk shape: anything matching the + // session id is the Sessions row; everything else is SessionsIndex. This works even + // if the manifest carries only one of the two rows (corruption-recovery scenario). + foreach (var row in tombstone.Rows) + { + ct.ThrowIfCancellationRequested(); + var table = IsSessionsRow(row) ? Constants.TableNames.Sessions : Constants.TableNames.SessionsIndex; + await _storage.DeleteByExactKeysInBatchesAsync(table, new[] { (row.Pk, row.Rk) }, ct); + } + } + + private static bool IsSessionsRow(DeletionRowDump row) + { + // Sessions row: PK = {tenantId}, RK = {sessionId}. + // SessionsIndex row: PK = {tenantId}, RK = {indexRowKey} (typically inverted-timestamp + sessionId). + // The discriminator is whether the RowKey *equals* a sessionId-shaped string. We use a + // simple heuristic: if the RowKey contains a "_" the row is the SessionsIndex (its + // RK is always composed); otherwise it's the Sessions row. This matches the builder's + // current schema (Sessions RK is the bare session GUID, IndexRowKey is "inverted_…_GUID"). + return row.Rk != null && !row.Rk.Contains('_'); + } + + // ============================================================ Progress CAS retry ==== + + /// + /// Bounded-retry helper for ETag-CAS progress-blob writes. On 412 Precondition Failed + /// it re-reads the progress blob; if returns true + /// against the freshly-read progress, treats the write as already-done by a concurrent + /// worker and returns the fresh (progress, etag). Otherwise re-applies the mutation + /// against the fresh progress and retries. Caps at + /// attempts or wall-clock, whichever first. + /// + private async Task<(DeletionProgress, string)> UpdateProgressWithRetryAsync( + string tenantId, string sessionId, string manifestId, + DeletionProgress progress, string etag, + Action mutate, + Func isAlreadyApplied, + CancellationToken ct) + { + mutate(progress); + + var deadline = DateTime.UtcNow + ProgressCasMaxWallClock; + for (var attempt = 0; attempt < ProgressCasMaxAttempts && DateTime.UtcNow < deadline; attempt++) + { + ct.ThrowIfCancellationRequested(); + try + { + var newEtag = await _blob.UpdateDeletionProgressAsync(tenantId, sessionId, manifestId, progress, etag, ct); + return (progress, newEtag); + } + catch (RequestFailedException ex) when (ex.Status == 412) + { + var fresh = await _blob.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct); + if (isAlreadyApplied(fresh.Progress)) + { + _logger.LogInformation( + "SessionDeletionHandler progress-CAS conflict resolved: concurrent worker already applied mutation. " + + "tenant={TenantId} session={SessionId} manifestId={ManifestId} attempt={Attempt}", + tenantId, sessionId, manifestId, attempt + 1); + return (fresh.Progress, fresh.ETag); + } + progress = fresh.Progress; + etag = fresh.ETag; + mutate(progress); + } + } + + throw new InvalidOperationException( + $"Progress-blob ETag-CAS exhausted after {ProgressCasMaxAttempts} attempts or " + + $"{ProgressCasMaxWallClock.TotalSeconds:0}s wall-clock. " + + $"tenant={tenantId} session={sessionId} manifestId={manifestId}"); + } + + // ============================================================ Audit details builders ==== + + private static Dictionary BuildStepCompletedDetails( + string manifestId, DeletionStep step, DeletionBatchResult tableResult, int aggregateProcessed, long stepDurationMs) + { + var details = new Dictionary(StringComparer.Ordinal) + { + ["manifestId"] = manifestId, + ["stepOrder"] = step.Order.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["class"] = step.Class, + ["stepDurationMs"] = stepDurationMs.ToString(System.Globalization.CultureInfo.InvariantCulture), + }; + if (!string.IsNullOrEmpty(step.Table)) + { + details["table"] = step.Table!; + } + if (!string.IsNullOrEmpty(step.Step)) + { + details["syntheticStep"] = step.Step!; + } + if (step.Class == DeletionStepClass.Aggregate) + { + details["decrementsProcessed"] = aggregateProcessed.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + else + { + details["attempted"] = tableResult.Attempted.ToString(System.Globalization.CultureInfo.InvariantCulture); + details["deletedNow"] = tableResult.DeletedNow.ToString(System.Globalization.CultureInfo.InvariantCulture); + details["alreadyMissing"] = tableResult.AlreadyMissing.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + return details; + } + + private static Dictionary BuildVerificationFailedDetails( + string manifestId, IReadOnlyList residuals) + { + var sample = residuals + .Take(AuditResidualSampleSize) + .Select(r => new { table = r.Table, pk = r.Pk, rk = r.Rk }); + return new Dictionary(StringComparer.Ordinal) + { + ["manifestId"] = manifestId, + ["residualCount"] = residuals.Count.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["residualKeys"] = JsonSerializer.Serialize(sample), + }; + } + + private static Dictionary BuildCompletedDetails( + string manifestId, DeletionManifest manifest, long durationMs) + { + var totalsByTable = manifest.PreflightCounts; + return new Dictionary(StringComparer.Ordinal) + { + ["manifestId"] = manifestId, + ["durationMs"] = durationMs.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["totalsByTable"] = JsonSerializer.Serialize(totalsByTable), + ["stepsCount"] = manifest.Steps.Count.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["schemaHash"] = manifest.SchemaHash, + }; + } + + private Task AuditStepFailedAsync( + string tenantId, string sessionId, string manifestId, DeletionStep step, Exception ex, CancellationToken _) + { + var details = new Dictionary(StringComparer.Ordinal) + { + ["manifestId"] = manifestId, + ["stepOrder"] = step.Order.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["class"] = step.Class, + ["exceptionType"] = ex.GetType().FullName ?? "Unknown", + // Trim error message to a reasonable size for the audit row. + ["error"] = ex.Message.Length > 1024 ? ex.Message.Substring(0, 1024) : ex.Message, + }; + if (!string.IsNullOrEmpty(step.Table)) details["table"] = step.Table!; + + return _maintenanceRepo.LogAuditEntryAsync( + tenantId, + action: "deletion_step_failed", + entityType: "Session", + entityId: sessionId, + performedBy: "system", + details: details); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionWorker.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionWorker.cs new file mode 100644 index 00000000..a5d075ee --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionWorker.cs @@ -0,0 +1,502 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Azure.Identity; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Functions.Services.Deletion +{ + /// + /// Background poll-loop for the session-deletion queue (plan §5 PR4). Modeled on + /// + /// with four explicit deviations: + /// + /// = 1 (not 32) — a single cascade can be tens of MB + /// and minutes of wall-time; one-at-a-time bounds memory and poison blast radius. + /// Kill-switch on entry: every receive checks + /// AdminConfiguration.SessionDeletionKillSwitch; when active the worker idles + /// without dequeuing (envelope stays on the queue, no visibility timeout pressure). + /// Heartbeat: while runs, a sidecar task + /// calls UpdateMessageAsync every to extend the + /// message visibility by . Prevents queue re-delivery + /// from spawning a parallel worker on the same manifest while a step is in flight. + /// Poison audit: max-dequeue exhaustion writes a deletion_poisoned + /// audit entry alongside the poison-queue move so the operator can correlate the + /// poisoned cascade with its manifest. The Sessions row's DeletionState is NOT + /// auto-cleared — operator action via restore-from-poisoned (§13, PR4b) is required. + /// + /// + /// The worker is NOT registered in Program.cs in PR4 — that wires up in PR5 + /// alongside the flag-gated DeleteSessionFunction changes. Without the producer + /// emitting envelopes (gated by TenantConfiguration.EnableCascadeDeleteV2, default + /// false), having the worker idle on an empty queue is harmless; we hold off so the worker's + /// poll-loop doesn't appear in production telemetry until the cutover. + /// + /// + public sealed class SessionDeletionWorker : BackgroundService + { + /// + /// Explicit deviation from sibling workers — one cascade per receive bounds the worker's + /// memory footprint (a 35MB snapshot blob × 32 messages = 1.1GB) and limits poison-queue + /// blast radius on a corruption signal. Plan §5 PR4. + /// + internal const int BatchSize = 1; + + /// Initial message-visibility window — extended in-flight by the heartbeat task. + internal static readonly TimeSpan VisibilityTimeout = TimeSpan.FromMinutes(5); + + /// How often the heartbeat task extends the in-flight message's visibility. + internal static readonly TimeSpan DefaultHeartbeatInterval = TimeSpan.FromSeconds(60); + + /// How much the heartbeat task adds to visibility on each tick. + internal static readonly TimeSpan HeartbeatExtendBy = TimeSpan.FromMinutes(5); + + /// Idle sleep between empty receives or kill-switch-on cycles. + internal static readonly TimeSpan DefaultPollInterval = TimeSpan.FromSeconds(10); + + /// Instance-level overrides — production uses the defaults above, tests inject shorter values via the internal ctor. + private readonly TimeSpan _heartbeatInterval; + private readonly TimeSpan _pollInterval; + + /// Cool-down after an unhandled poll-loop exception. + internal static readonly TimeSpan ErrorBackoff = TimeSpan.FromSeconds(30); + + /// Match the QueueTrigger platform default + sibling workers: 5 attempts → poison. + internal const int MaxDequeueCount = 5; + + private const string PoisonQueueSuffix = "-poison"; + + private readonly QueueClient _mainQueue; + private readonly QueueClient _poisonQueue; + private readonly SessionDeletionHandler _handler; + private readonly TableStorageService _storage; + private readonly AdminConfigurationService _adminConfig; + private readonly IMaintenanceRepository _maintenanceRepo; + private readonly ILogger _logger; + + public SessionDeletionWorker( + IConfiguration configuration, + SessionDeletionHandler handler, + TableStorageService storage, + AdminConfigurationService adminConfig, + IMaintenanceRepository maintenanceRepo, + ILogger logger) + { + _handler = handler ?? throw new ArgumentNullException(nameof(handler)); + _storage = storage ?? throw new ArgumentNullException(nameof(storage)); + _adminConfig = adminConfig ?? throw new ArgumentNullException(nameof(adminConfig)); + _maintenanceRepo = maintenanceRepo ?? throw new ArgumentNullException(nameof(maintenanceRepo)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _heartbeatInterval = DefaultHeartbeatInterval; + _pollInterval = DefaultPollInterval; + + var options = new QueueClientOptions + { + MessageEncoding = QueueMessageEncoding.Base64, + }; + + var storageAccountName = configuration["AzureStorageAccountName"]; + var connectionString = configuration["AzureTableStorageConnectionString"]; + + if (!string.IsNullOrEmpty(storageAccountName)) + { + var mainUri = new Uri( + $"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.SessionDeletion}"); + var poisonUri = new Uri( + $"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.SessionDeletion}{PoisonQueueSuffix}"); + var credential = new DefaultAzureCredential(); + _mainQueue = new QueueClient(mainUri, credential, options); + _poisonQueue = new QueueClient(poisonUri, credential, options); + _logger.LogInformation( + "SessionDeletionWorker initialized with Managed Identity (account: {Account})", + storageAccountName); + } + else if (!string.IsNullOrEmpty(connectionString)) + { + _mainQueue = new QueueClient( + connectionString, Constants.QueueNames.SessionDeletion, options); + _poisonQueue = new QueueClient( + connectionString, Constants.QueueNames.SessionDeletion + PoisonQueueSuffix, options); + _logger.LogInformation("SessionDeletionWorker initialized with connection string"); + } + else + { + throw new InvalidOperationException( + "Queue Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureTableStorageConnectionString'."); + } + } + + /// + /// Test seam: construct directly with mock instances so the + /// poll loop can be exercised against an in-memory queue without spinning up Azurite. + /// Mirrors 's internal test ctor. + /// + internal SessionDeletionWorker( + QueueClient mainQueue, + QueueClient poisonQueue, + SessionDeletionHandler handler, + TableStorageService storage, + AdminConfigurationService adminConfig, + IMaintenanceRepository maintenanceRepo, + ILogger logger, + TimeSpan? heartbeatInterval = null, + TimeSpan? pollInterval = null) + { + _mainQueue = mainQueue ?? throw new ArgumentNullException(nameof(mainQueue)); + _poisonQueue = poisonQueue ?? throw new ArgumentNullException(nameof(poisonQueue)); + _handler = handler ?? throw new ArgumentNullException(nameof(handler)); + _storage = storage ?? throw new ArgumentNullException(nameof(storage)); + _adminConfig = adminConfig ?? throw new ArgumentNullException(nameof(adminConfig)); + _maintenanceRepo = maintenanceRepo ?? throw new ArgumentNullException(nameof(maintenanceRepo)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _heartbeatInterval = heartbeatInterval ?? DefaultHeartbeatInterval; + _pollInterval = pollInterval ?? DefaultPollInterval; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await TryCreateQueueAsync(_mainQueue, "main", stoppingToken).ConfigureAwait(false); + await TryCreateQueueAsync(_poisonQueue, "poison", stoppingToken).ConfigureAwait(false); + + _logger.LogInformation("SessionDeletionWorker: poll loop started (queue {Name})", _mainQueue.Name); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + // Kill-switch entry guard (plan §1 P8 / §9). When active, the worker does not + // dequeue — the message stays visible on the main queue until the operator + // flips the switch back. Envelope is preserved without losing dequeue budget. + var admin = await _adminConfig.GetConfigurationAsync().ConfigureAwait(false); + if (admin.SessionDeletionKillSwitch) + { + _logger.LogDebug("SessionDeletionWorker: kill-switch active; idling for {Interval}", _pollInterval); + await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false); + continue; + } + + var batch = await _mainQueue + .ReceiveMessagesAsync(BatchSize, VisibilityTimeout, stoppingToken) + .ConfigureAwait(false); + + if (batch?.Value is null || batch.Value.Length == 0) + { + await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false); + continue; + } + + foreach (var msg in batch.Value) + { + if (stoppingToken.IsCancellationRequested) break; + await ProcessOneAsync(msg, stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, + "SessionDeletionWorker: poll-loop error — backing off {Backoff}", + ErrorBackoff); + try { await Task.Delay(ErrorBackoff, stoppingToken).ConfigureAwait(false); } + catch (OperationCanceledException) { break; } + } + } + + _logger.LogInformation("SessionDeletionWorker: poll loop stopped"); + } + + private async Task ProcessOneAsync(QueueMessage msg, CancellationToken ct) + { + if (msg.DequeueCount > MaxDequeueCount) + { + await MoveToPoisonAsync(msg, ct).ConfigureAwait(false); + return; + } + + SessionDeletionEnvelope? envelope; + try + { + envelope = JsonConvert.DeserializeObject(msg.Body.ToString()); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, + "SessionDeletionWorker: malformed envelope JSON — dropping (msg {Id})", + msg.MessageId); + await SafeDeleteAsync(msg, ct).ConfigureAwait(false); + return; + } + + if (envelope is null + || string.IsNullOrEmpty(envelope.TenantId) + || string.IsNullOrEmpty(envelope.SessionId) + || string.IsNullOrEmpty(envelope.ManifestId)) + { + _logger.LogWarning( + "SessionDeletionWorker: envelope missing required fields — dropping (msg {Id})", + msg.MessageId); + await SafeDeleteAsync(msg, ct).ConfigureAwait(false); + return; + } + + // Heartbeat: extend visibility every _heartbeatInterval while the handler runs. We + // capture the latest PopReceipt locally because each UpdateMessageAsync returns a + // fresh receipt; the SafeDelete at the end uses whichever receipt was issued last. + using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + var heartbeatState = new HeartbeatState(msg.MessageId, msg.PopReceipt); + var heartbeatTask = Task.Run(() => HeartbeatLoopAsync(heartbeatState, heartbeatCts.Token), heartbeatCts.Token); + + try + { + await _handler.HandleAsync(envelope, ct).ConfigureAwait(false); + + // Handler succeeded — stop heartbeat and delete using the latest pop-receipt. + heartbeatCts.Cancel(); + try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { } + + await SafeDeleteAsync(msg.MessageId, heartbeatState.PopReceipt, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + heartbeatCts.Cancel(); + throw; + } + catch (Exception ex) + { + heartbeatCts.Cancel(); + try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { } + + _logger.LogWarning(ex, + "SessionDeletionWorker: handler failed (tenant={Tenant} session={Session} manifestId={ManifestId} dequeue={N}) — visibility-timeout retry", + envelope.TenantId, envelope.SessionId, envelope.ManifestId, msg.DequeueCount); + } + } + + private async Task HeartbeatLoopAsync(HeartbeatState state, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try { await Task.Delay(_heartbeatInterval, ct).ConfigureAwait(false); } + catch (OperationCanceledException) { return; } + + try + { + var updated = await _mainQueue.UpdateMessageAsync( + state.MessageId, state.PopReceipt, + visibilityTimeout: HeartbeatExtendBy, + cancellationToken: ct).ConfigureAwait(false); + state.PopReceipt = updated.Value.PopReceipt; + _logger.LogDebug( + "SessionDeletionWorker: heartbeat extended visibility for message {Id} by {Extend}", + state.MessageId, HeartbeatExtendBy); + } + catch (OperationCanceledException) { return; } + catch (Exception ex) + { + // Heartbeat failure (e.g. PopReceipt expired because the visibility window + // was missed) is logged and the loop exits. The handler keeps running; on + // its eventual return, SafeDelete will hit a 404 and the message will + // reappear after its current visibility expires — but the handler's own + // ETag-CAS on the progress blob makes a parallel pickup idempotent. + _logger.LogWarning(ex, + "SessionDeletionWorker: heartbeat UpdateMessageAsync failed for message {Id} — heartbeat exiting", + state.MessageId); + return; + } + } + } + + private async Task MoveToPoisonAsync(QueueMessage msg, CancellationToken ct) + { + // Best-effort: attempt to parse envelope so we can audit the manifest id alongside + // the poison move. If the body is malformed we still move it to the poison queue + // (the audit just lacks the manifestId field). + SessionDeletionEnvelope? envelope = null; + try { envelope = JsonConvert.DeserializeObject(msg.Body.ToString()); } + catch (JsonException) { /* swallow — non-parseable envelopes still get poisoned */ } + + // PR4b E1: transition Sessions.DeletionState → Poisoned BEFORE the poison-queue send. + // Without this, the row stays at Running (or Queued if poisoned before pickup) and the + // restore endpoint cannot dispatch into partial-restore mode. Best-effort: if CAS fails + // (concurrent writer, row gone, state mismatch) we still continue with the poison-queue + // move so the original poison flow completes and an audit trail exists. + if (envelope != null && !string.IsNullOrEmpty(envelope.TenantId) && !string.IsNullOrEmpty(envelope.SessionId)) + { + await TryTransitionToPoisonedStateAsync(envelope.TenantId, envelope.SessionId, envelope.ManifestId, ct).ConfigureAwait(false); + } + + try + { + await _poisonQueue.SendMessageAsync(msg.Body.ToString(), ct).ConfigureAwait(false); + await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false); + _logger.LogWarning( + "SessionDeletionWorker: moved message {Id} to poison queue after {N} failed attempts", + msg.MessageId, msg.DequeueCount - 1); + } + catch (Exception ex) + { + _logger.LogError(ex, + "SessionDeletionWorker: poison move failed for message {Id} (will retry)", + msg.MessageId); + return; + } + + if (envelope != null && !string.IsNullOrEmpty(envelope.TenantId) && !string.IsNullOrEmpty(envelope.SessionId)) + { + try + { + await _maintenanceRepo.LogAuditEntryAsync( + envelope.TenantId, + action: "deletion_poisoned", + entityType: "Session", + entityId: envelope.SessionId, + performedBy: "system", + details: new Dictionary(StringComparer.Ordinal) + { + ["manifestId"] = envelope.ManifestId ?? string.Empty, + ["reason"] = envelope.Reason ?? string.Empty, + ["messageId"] = msg.MessageId, + ["dequeueCount"] = (msg.DequeueCount - 1).ToString(System.Globalization.CultureInfo.InvariantCulture), + }).ConfigureAwait(false); + } + catch (Exception auditEx) + { + _logger.LogError(auditEx, + "SessionDeletionWorker: deletion_poisoned audit log write failed for tenant={Tenant} session={Session}", + envelope.TenantId, envelope.SessionId); + } + } + } + + private async Task SafeDeleteAsync(QueueMessage msg, CancellationToken ct) + => await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false); + + private async Task SafeDeleteAsync(string messageId, string popReceipt, CancellationToken ct) + { + try + { + await _mainQueue.DeleteMessageAsync(messageId, popReceipt, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "SessionDeletionWorker: delete failed for message {Id} (will reappear after visibility-timeout)", + messageId); + } + } + + private async Task TryCreateQueueAsync(QueueClient queue, string label, CancellationToken ct) + { + try + { + await queue.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "SessionDeletionWorker: CreateIfNotExists failed for {Label} queue — will continue, send/receive will retry", + label); + } + } + + /// + /// PR4b E1 + PR4c F5: CAS Sessions.DeletionState to Poisoned with manifestId pre-check. + /// Tries Running → Poisoned first (the common case — Worker successfully CAS'd + /// Queued→Running on first pickup and the handler then failed N times). Falls back to + /// Queued → Poisoned for the rare case where the cascade poisoned before the + /// Worker ever successfully transitioned to Running (handler always threw at the + /// state-acquire step). Logs warning on persistent CAS failure but does NOT throw — the + /// poison-queue move + audit must still complete so the operator has an observable trail + /// even if the row state is somehow inconsistent. + /// + /// PR4c F5: before issuing any CAS, read the Sessions row's + /// PendingDeletionManifestId and compare to the envelope's manifestId. If + /// they don't match, the envelope is stale (from a prior cascade attempt) — do NOT + /// touch the state of a possibly-fresh active cascade. The poison-queue + audit still + /// fire in the caller so the stale message is removed from the main queue. + /// + /// + private async Task TryTransitionToPoisonedStateAsync(string tenantId, string sessionId, string? manifestId, CancellationToken ct) + { + // PR4c F5: pre-check that the envelope's manifestId matches the row's + // PendingDeletionManifestId. A stale envelope from a prior cascade must not flip a + // fresh active cascade to Poisoned. + var row = await _storage.GetSessionRowAsync(tenantId, sessionId, ct).ConfigureAwait(false); + if (row == null) + { + _logger.LogInformation( + "SessionDeletionWorker: Sessions row gone for tenant={Tenant} session={Session} (cascade already tombstoned by another worker?) — skipping state transition", + tenantId, sessionId); + return; + } + + var currentPending = row.GetString("PendingDeletionManifestId"); + if (!string.Equals(currentPending, manifestId, StringComparison.Ordinal)) + { + _logger.LogWarning( + "SessionDeletionWorker: stale poison envelope for tenant={Tenant} session={Session} — envelope.ManifestId={Envelope} but row.PendingDeletionManifestId={Pending}; skipping state transition (active cascade is a different manifest)", + tenantId, sessionId, manifestId, currentPending); + return; + } + + // First attempt: Running → Poisoned (the expected case after a successful pickup). + var cas = await _storage.CasSetSessionDeletionStateAsync( + tenantId, sessionId, + fromState: SessionDeletionState.Running, + toState: SessionDeletionState.Poisoned, + newManifestId: null, + ct).ConfigureAwait(false); + if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.Updated) + { + _logger.LogInformation( + "SessionDeletionWorker: CAS Running→Poisoned succeeded for tenant={Tenant} session={Session} manifestId={ManifestId}", + tenantId, sessionId, manifestId); + return; + } + + // Fallback: Queued → Poisoned (poisoned before pickup ever succeeded). + if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.WrongState + && cas.CurrentState == SessionDeletionState.Queued) + { + cas = await _storage.CasSetSessionDeletionStateAsync( + tenantId, sessionId, + fromState: SessionDeletionState.Queued, + toState: SessionDeletionState.Poisoned, + newManifestId: null, + ct).ConfigureAwait(false); + if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.Updated) + { + _logger.LogInformation( + "SessionDeletionWorker: CAS Queued→Poisoned succeeded for tenant={Tenant} session={Session} manifestId={ManifestId}", + tenantId, sessionId, manifestId); + return; + } + } + + // Persistent CAS failure — log and continue so the poison-queue move + audit still fire. + _logger.LogWarning( + "SessionDeletionWorker: could not transition Sessions.DeletionState to Poisoned for tenant={Tenant} session={Session} manifestId={ManifestId} outcome={Outcome} currentState={State}; restore endpoint may reject with 409 \"active_cascade\" until operator manually intervenes.", + tenantId, sessionId, manifestId, cas.Outcome, cas.CurrentState); + } + + private sealed class HeartbeatState + { + public string MessageId { get; } + public string PopReceipt { get; set; } + public HeartbeatState(string messageId, string popReceipt) { MessageId = messageId; PopReceipt = popReceipt; } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRestoreService.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRestoreService.cs new file mode 100644 index 00000000..bd46cf0e --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRestoreService.cs @@ -0,0 +1,586 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Deletion +{ + /// + /// Outcome of a call. The HTTP function + /// maps these to status codes (Restored → 200 OK, every Reject* → 409 Conflict, etc.). + /// + public enum SessionRestoreOutcome + { + /// Restore succeeded; row counts populated. + Restored, + /// Dry run succeeded; populated, no writes. + DryRunOk, + /// Sessions row present + DeletionState=None — nothing to restore. + RejectAlreadyAtOriginalState, + /// Sessions row present + DeletionState ∈ {Preparing, Queued, Running} — cascade still in flight. + RejectActiveCascade, + /// Sessions row exists but PendingDeletionManifestId does not match the envelope. + RejectManifestIdMismatch, + /// Sessions row absent but progress.CompletedAt is null — corruption signal. + RejectCorruptState, + /// Manifest blob missing or 404 from storage. + RejectManifestNotFound, + /// Snapshot SHA-256 mismatch — manifest is corrupted. + RejectManifestCorruption, + /// CAS Poisoned → None failed at end of partial-restore (concurrent writer raced). + RejectCasConflictOnClear, + } + + /// + /// Restore-result payload. The HTTP function serializes this directly into the response body. + /// + public sealed class SessionRestoreResult + { + public SessionRestoreOutcome Outcome { get; set; } + public string? Mode { get; set; } // "full" | "partial" | "dryRun" + public string? Message { get; set; } // operator-readable reason on reject + public string? CurrentState { get; set; } // for reject diagnostics + public string? PendingManifestId { get; set; } // for reject diagnostics + public Dictionary RowsRestoredByTable { get; set; } = new Dictionary(StringComparer.Ordinal); + public Dictionary RowsSkippedByTable { get; set; } = new Dictionary(StringComparer.Ordinal); + public Dictionary WouldRestoreByTable { get; set; } = new Dictionary(StringComparer.Ordinal); + public int InventoryReIncrements { get; set; } + public long DurationMs { get; set; } + } + + /// + /// Live restore executor (PR4b, plan §13). Symmetric inverse of : + /// downloads the cascade manifest, dispatches to full or partial-poisoned-recovery + /// mode based on the (Sessions row state, progress.CompletedAt) tuple, re-inserts rows in + /// reverse cascade order, re-increments SoftwareInventory counters from the manifest's + /// AGGREGATE step, and (partial mode only) CAS-clears Sessions.DeletionState: Poisoned → None. + /// + /// Every restore attempt — even one that rejects post-download — emits a + /// deletion_manifest_downloaded audit entry before any other read (§13.1). Restore + /// successes emit deletion_restored. + /// + /// + public class SessionRestoreService + { + private readonly TableStorageService _storage; + private readonly BlobStorageService _blob; + private readonly IMaintenanceRepository _maintenanceRepo; + private readonly ILogger _logger; + + public SessionRestoreService( + TableStorageService storage, + BlobStorageService blob, + IMaintenanceRepository maintenanceRepo, + ILogger logger) + { + _storage = storage; + _blob = blob; + _maintenanceRepo = maintenanceRepo; + _logger = logger; + } + + public virtual async Task RestoreAsync( + string tenantId, string sessionId, string manifestId, + bool dryRun, string actor, + CancellationToken ct = default) + { + if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId)); + if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("sessionId is required", nameof(sessionId)); + if (string.IsNullOrEmpty(manifestId)) throw new ArgumentException("manifestId is required", nameof(manifestId)); + if (string.IsNullOrEmpty(actor)) actor = "system"; + + var sw = Stopwatch.StartNew(); + var result = new SessionRestoreResult(); + + // (1) §13.1 — audit BEFORE any read. + await AuditManifestDownloadedAsync(tenantId, sessionId, manifestId, actor, ct).ConfigureAwait(false); + + // (2) Download manifest (SHA-verified) + progress. PR4c F6: use the with-Sha variant + // and verify the snapshot↔progress binding once both blobs are loaded (plan §3). + DeletionManifest manifest; + string snapshotSha; + DeletionProgress progress; + string progressEtag; + try + { + (manifest, snapshotSha) = await _blob.DownloadDeletionManifestWithShaAsync(tenantId, sessionId, manifestId, ct).ConfigureAwait(false); + (progress, progressEtag) = await _blob.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct).ConfigureAwait(false); + + // PR4c F6: enforce the snapshot↔progress SHA binding. Producer wrote the manifest's + // SHA into progress.SnapshotSha256 at upload time; if they diverge, the artifacts + // were tampered with or the producer crashed in a bad way. + if (!string.IsNullOrEmpty(progress.SnapshotSha256) + && !string.Equals(progress.SnapshotSha256, snapshotSha, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidDataException( + $"Snapshot/progress SHA binding mismatch: progress.SnapshotSha256={progress.SnapshotSha256} " + + $"manifest SHA={snapshotSha} — corruption signal."); + } + } + catch (Azure.RequestFailedException ex) when (ex.Status == 404) + { + result.Outcome = SessionRestoreOutcome.RejectManifestNotFound; + result.Message = $"Manifest blob not found for tenant={tenantId} session={sessionId} manifestId={manifestId}. " + + "It may have been GC'd after the 33-day retention window — recovery is no longer possible."; + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + catch (InvalidDataException ex) + { + _logger.LogWarning(ex, + "Manifest corruption detected during restore for tenant={Tenant} session={Session} manifestId={ManifestId}", + tenantId, sessionId, manifestId); + result.Outcome = SessionRestoreOutcome.RejectManifestCorruption; + result.Message = $"Manifest corruption: {ex.Message}"; + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + + // (3) Read Sessions row, dispatch by state (§13.4 table). + var sessionRow = await _storage.GetSessionRowAsync(tenantId, sessionId, ct).ConfigureAwait(false); + var state = sessionRow?.GetString("DeletionState") ?? SessionDeletionState.None; + if (string.IsNullOrEmpty(state)) state = SessionDeletionState.None; + var pendingManifestId = sessionRow?.GetString("PendingDeletionManifestId"); + var completedAt = progress.CompletedAt; + + // Dispatch table (PR4c F2 extends this — TombstoneStarted closes the tombstone gap): + // sessionRow=null + completedAt set → Full restore (clean case) + // sessionRow=null + completedAt null + TombstoneStarted → Full restore (gap case, NEW) + // sessionRow=null + completedAt null + !TombstoneStarted → 409 corrupt_state + // exists + state=None → 409 already_at_original_state + // exists + state=Poisoned + matching → Partial restore + // exists + state=Poisoned + mismatch → 409 manifestid_mismatch + // exists + state ∈ {Preparing,Queued,Running} → 409 active_cascade + string mode; + if (sessionRow == null) + { + if (completedAt == null && !progress.TombstoneStarted) + { + result.Outcome = SessionRestoreOutcome.RejectCorruptState; + result.Message = "Cascade did not complete (no CompletedAt, TombstoneStarted=false) but Sessions row is missing. " + + "Operator must investigate before retrying — re-inserting may overwrite genuine new data."; + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + mode = "full"; + } + else if (string.Equals(state, SessionDeletionState.None, StringComparison.Ordinal)) + { + result.Outcome = SessionRestoreOutcome.RejectAlreadyAtOriginalState; + result.Message = "Session is already in DeletionState=None — nothing to restore. If the manifest is from a prior delete that was already restored, this is expected."; + result.CurrentState = state; + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + else if (string.Equals(state, SessionDeletionState.Poisoned, StringComparison.Ordinal)) + { + if (!string.Equals(pendingManifestId, manifestId, StringComparison.Ordinal)) + { + result.Outcome = SessionRestoreOutcome.RejectManifestIdMismatch; + result.Message = $"Sessions.PendingDeletionManifestId='{pendingManifestId}' does not match requested manifestId='{manifestId}'. " + + "Restore must target the manifest that owns the current lock."; + result.CurrentState = state; + result.PendingManifestId = pendingManifestId; + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + mode = "partial"; + } + else + { + // Preparing / Queued / Running + result.Outcome = SessionRestoreOutcome.RejectActiveCascade; + result.Message = $"Sessions.DeletionState='{state}' — cascade is still in flight. " + + "Flip AdminConfiguration.SessionDeletionKillSwitch to abort first, wait for the worker to finish or poison, then retry restore."; + result.CurrentState = state; + result.PendingManifestId = pendingManifestId; + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + + // (4) Dry-run path: simulate by counting what each step would restore, no writes. + if (dryRun) + { + result.Outcome = SessionRestoreOutcome.DryRunOk; + result.Mode = "dryRun"; + foreach (var step in manifest.Steps) + { + if (step.Class == DeletionStepClass.Aggregate || step.Class == DeletionStepClass.Final) continue; + if (string.IsNullOrEmpty(step.Table) || step.Rows.Count == 0) continue; + result.WouldRestoreByTable[step.Table!] = step.Rows.Count; + } + // Tombstone rows: split by Sessions vs SessionsIndex. + var tombstone = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Final); + if (tombstone != null) + { + foreach (var row in tombstone.Rows) + { + var t = IsSessionsRow(row) ? Constants.TableNames.Sessions : Constants.TableNames.SessionsIndex; + if (!result.WouldRestoreByTable.TryGetValue(t, out var c)) c = 0; + result.WouldRestoreByTable[t] = c + 1; + } + } + var aggregate = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Aggregate); + if (aggregate?.Decrements != null) result.InventoryReIncrements = aggregate.Decrements.Count; + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + + // (5) Execute the dispatched mode. PR4c F4: both modes thread progress + etag so the + // per-key re-increment idempotency can persist after each successful inventory write. + if (mode == "full") + { + (progress, progressEtag) = await RunFullRestoreAsync( + tenantId, sessionId, manifestId, manifest, progress, progressEtag, result, ct).ConfigureAwait(false); + } + else + { + (progress, progressEtag) = await RunPartialRestoreAsync( + tenantId, sessionId, manifestId, manifest, progress, progressEtag, result, ct).ConfigureAwait(false); + if (result.Outcome == SessionRestoreOutcome.RejectCasConflictOnClear) + { + result.DurationMs = sw.ElapsedMilliseconds; + return result; + } + } + + result.Outcome = SessionRestoreOutcome.Restored; + result.Mode = mode; + result.DurationMs = sw.ElapsedMilliseconds; + + // (6) Audit completion. + await AuditRestoredAsync(tenantId, sessionId, manifestId, mode, actor, result, ct).ConfigureAwait(false); + + _logger.LogInformation( + "SessionRestoreService completed: tenant={Tenant} session={Session} manifestId={ManifestId} mode={Mode} durationMs={Duration}", + tenantId, sessionId, manifestId, mode, result.DurationMs); + + return result; + } + + // ============================================================ Full restore ==== + + /// + /// PR4c F3a: full restore inserts Sessions row LAST, not first. Order: + /// + /// All non-FINAL non-AGGREGATE steps (foreign tables) in reverse order. + /// AGGREGATE step (inventory re-increment, with PR4c F4 per-key idempotency). + /// FINAL step's SessionsIndex row. + /// FINAL step's Sessions row (TRULY LAST — its presence signals + /// "session is back" + re-establishes the writer-lock semantics). + /// + /// Rationale: if any insert fails mid-flight, sessions=null on retry → dispatch + /// re-enters full-restore (no RejectAlreadyAtOriginalState trap from Codex F3). + /// + private async Task<(DeletionProgress, string)> RunFullRestoreAsync( + string tenantId, string sessionId, string manifestId, + DeletionManifest manifest, DeletionProgress progress, string progressEtag, + SessionRestoreResult result, CancellationToken ct) + { + // Pass 1: every non-FINAL step in REVERSE cascade order (17 → 16 → … → 1). + // AGGREGATE re-increment uses per-key idempotency (PR4c F4). + foreach (var step in manifest.Steps + .Where(s => s.Class != DeletionStepClass.Final) + .OrderByDescending(s => s.Order)) + { + ct.ThrowIfCancellationRequested(); + + switch (step.Class) + { + case DeletionStepClass.Aggregate: + (progress, progressEtag) = await RunInventoryReIncrementWithIdempotencyAsync( + tenantId, sessionId, manifestId, step, progress, progressEtag, result, ct).ConfigureAwait(false); + break; + + case DeletionStepClass.PkBySession: + case DeletionStepClass.PkRkExact: + case DeletionStepClass.PropTenantPk: + case DeletionStepClass.DiscriminatorPkRkSuffix: + case DeletionStepClass.DiscriminatorPkRkExact: + case DeletionStepClass.DiscriminatorPkProp: + { + if (string.IsNullOrEmpty(step.Table) || step.Rows.Count == 0) break; + var res = await _storage.RestoreRowsByExactKeysInBatchesAsync( + step.Table!, step.Rows, RestoreMode.Full, ct).ConfigureAwait(false); + AddCount(result.RowsRestoredByTable, step.Table!, res.Restored); + AddCount(result.RowsSkippedByTable, step.Table!, res.Skipped); + break; + } + + default: + throw new InvalidOperationException($"Unsupported step.Class='{step.Class}' in restore (Order={step.Order})."); + } + } + + // Pass 2: FINAL step in BUILDER ORDER (SessionsIndex first, Sessions LAST). + // Builder emits [SessionsIndex, Sessions] (DeletionManifestBuilder.AddTombstoneStep). + var tombstone = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Final); + if (tombstone != null) + { + foreach (var row in tombstone.Rows) + { + ct.ThrowIfCancellationRequested(); + var table = IsSessionsRow(row) ? Constants.TableNames.Sessions : Constants.TableNames.SessionsIndex; + var res = await _storage.RestoreRowsByExactKeysInBatchesAsync( + table, new[] { row }, RestoreMode.Full, ct).ConfigureAwait(false); + AddCount(result.RowsRestoredByTable, table, res.Restored); + AddCount(result.RowsSkippedByTable, table, res.Skipped); + } + } + + return (progress, progressEtag); + } + + // ============================================================ Partial restore (poisoned-recovery) ==== + + private async Task<(DeletionProgress, string)> RunPartialRestoreAsync( + string tenantId, string sessionId, string manifestId, + DeletionManifest manifest, DeletionProgress progress, string progressEtag, + SessionRestoreResult result, CancellationToken ct) + { + // Determine whether the AGGREGATE decrement step had already run before the cascade + // poisoned. If yes → re-increment to undo it. If no → leave counters alone (avoid + // double +1). + var aggregate = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Aggregate); + var decrementAlreadyRan = aggregate != null && progress.CompletedSteps.Contains(aggregate.Order); + + // Pass 1: every non-FINAL step in REVERSE cascade order. + foreach (var step in manifest.Steps + .Where(s => s.Class != DeletionStepClass.Final) + .OrderByDescending(s => s.Order)) + { + ct.ThrowIfCancellationRequested(); + + switch (step.Class) + { + case DeletionStepClass.Aggregate: + if (!decrementAlreadyRan) break; + (progress, progressEtag) = await RunInventoryReIncrementWithIdempotencyAsync( + tenantId, sessionId, manifestId, step, progress, progressEtag, result, ct).ConfigureAwait(false); + break; + + case DeletionStepClass.PkBySession: + case DeletionStepClass.PkRkExact: + case DeletionStepClass.PropTenantPk: + case DeletionStepClass.DiscriminatorPkRkSuffix: + case DeletionStepClass.DiscriminatorPkRkExact: + case DeletionStepClass.DiscriminatorPkProp: + { + if (string.IsNullOrEmpty(step.Table) || step.Rows.Count == 0) break; + var res = await _storage.RestoreRowsByExactKeysInBatchesAsync( + step.Table!, step.Rows, RestoreMode.Partial, ct).ConfigureAwait(false); + AddCount(result.RowsRestoredByTable, step.Table!, res.Restored); + AddCount(result.RowsSkippedByTable, step.Table!, res.Skipped); + break; + } + + default: + throw new InvalidOperationException($"Unsupported step.Class='{step.Class}' in restore (Order={step.Order})."); + } + } + + // Pass 2: FINAL step in BUILDER ORDER (SessionsIndex first, Sessions LAST). + // In partial-restore mode, Sessions + SessionsIndex usually still exist (cascade + // poisoned before tombstone) → 409-skip each. The Sessions LAST ordering doesn't + // matter in the typical partial case (both rows already there), but stays consistent + // with the full-restore order for retry-after-partial-fail safety. + var tombstone = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Final); + if (tombstone != null) + { + foreach (var row in tombstone.Rows) + { + ct.ThrowIfCancellationRequested(); + var table = IsSessionsRow(row) ? Constants.TableNames.Sessions : Constants.TableNames.SessionsIndex; + var res = await _storage.RestoreRowsByExactKeysInBatchesAsync( + table, new[] { row }, RestoreMode.Partial, ct).ConfigureAwait(false); + AddCount(result.RowsRestoredByTable, table, res.Restored); + AddCount(result.RowsSkippedByTable, table, res.Skipped); + } + } + + // Final atomic step: CAS Poisoned → None. On failure: row recovery succeeded but + // the lock is stuck — return RejectCasConflictOnClear so the operator can re-run + // restore (idempotent via AddEntity 409-ignore + per-key inventory idempotency). + var cas = await _storage.CasSetSessionDeletionStateAsync( + tenantId, sessionId, + fromState: SessionDeletionState.Poisoned, + toState: SessionDeletionState.None, + newManifestId: null, + ct).ConfigureAwait(false); + if (cas.Outcome != TableStorageService.SessionDeletionStateCasOutcome.Updated) + { + _logger.LogWarning( + "SessionRestoreService: CAS Poisoned→None failed for tenant={Tenant} session={Session} outcome={Outcome} currentState={State}; restore is idempotent — retry.", + tenantId, sessionId, cas.Outcome, cas.CurrentState); + result.Outcome = SessionRestoreOutcome.RejectCasConflictOnClear; + result.Message = $"Row recovery succeeded but DeletionState lock could not be cleared (outcome={cas.Outcome}, currentState={cas.CurrentState}). Re-run restore — it's idempotent."; + result.CurrentState = cas.CurrentState; + } + + return (progress, progressEtag); + } + + /// + /// PR4c F4: per-key idempotent inventory re-increment. Used by BOTH full and partial + /// restore (Codex F4 was scoped to partial, but the same drift risk exists in full + /// restore on retry after a mid-flight failure — symmetric fix). Persist-first / increment- + /// second ordering bounds drift to +1 per crash in the rare gap window, vs unbounded + /// over-increment per attempt if we did increment-first. + /// + private async Task<(DeletionProgress, string)> RunInventoryReIncrementWithIdempotencyAsync( + string tenantId, string sessionId, string manifestId, + DeletionStep step, DeletionProgress progress, string progressEtag, + SessionRestoreResult result, CancellationToken ct) + { + if (step.Decrements == null || step.Decrements.Count == 0) + { + return (progress, progressEtag); + } + + // Lazy-initialize the applied-keys set on first re-increment entry. + if (progress.RestoreReIncrementsApplied == null) + { + (progress, progressEtag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, progressEtag, + mutate: p => p.RestoreReIncrementsApplied ??= new HashSet(StringComparer.Ordinal), + isAlreadyApplied: p => p.RestoreReIncrementsApplied != null, + ct).ConfigureAwait(false); + } + + foreach (var key in step.Decrements) + { + ct.ThrowIfCancellationRequested(); + var composite = BuildAggregateCompositeKey(key); + if (progress.RestoreReIncrementsApplied!.Contains(composite)) + { + continue; // already applied on a prior attempt + } + + // F4 ordering: persist FIRST (mark intent), increment SECOND (apply effect). + var compositeForClosure = composite; + (progress, progressEtag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, progressEtag, + mutate: p => p.RestoreReIncrementsApplied!.Add(compositeForClosure), + isAlreadyApplied: p => p.RestoreReIncrementsApplied!.Contains(compositeForClosure), + ct).ConfigureAwait(false); + + await _storage.RestoreSoftwareInventoryEntryByKeyAsync(tenantId, key, ct).ConfigureAwait(false); + result.InventoryReIncrements++; + } + + return (progress, progressEtag); + } + + /// + /// Composite-key format for + /// and . Matches + /// SessionDeletionHandler.BuildAggregateCompositeKey so forward (decrement) and + /// reverse (re-increment) operations identify the same row. + /// + internal static string BuildAggregateCompositeKey(DeletionDecrementKey key) + => $"{key.Vendor ?? string.Empty}:{key.Name ?? string.Empty}:{key.Version ?? string.Empty}"; + + /// + /// Bounded-retry helper for ETag-CAS progress-blob writes during restore. Mirrors the + /// pattern in : on 412 re-read progress, check whether + /// the mutation is already applied (concurrent winner case), else retry with fresh ETag. + /// Cap: attempts or + /// wall-clock, whichever first. + /// + private async Task<(DeletionProgress, string)> UpdateProgressWithRetryAsync( + string tenantId, string sessionId, string manifestId, + DeletionProgress progress, string etag, + Action mutate, + Func isAlreadyApplied, + CancellationToken ct) + { + mutate(progress); + + var deadline = DateTime.UtcNow + ProgressCasMaxWallClock; + for (var attempt = 0; attempt < ProgressCasMaxAttempts && DateTime.UtcNow < deadline; attempt++) + { + ct.ThrowIfCancellationRequested(); + try + { + var newEtag = await _blob.UpdateDeletionProgressAsync(tenantId, sessionId, manifestId, progress, etag, ct).ConfigureAwait(false); + return (progress, newEtag); + } + catch (Azure.RequestFailedException ex) when (ex.Status == 412) + { + var fresh = await _blob.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct).ConfigureAwait(false); + if (isAlreadyApplied(fresh.Progress)) + { + return (fresh.Progress, fresh.ETag); + } + progress = fresh.Progress; + etag = fresh.ETag; + mutate(progress); + } + } + + throw new InvalidOperationException( + $"Progress-blob ETag-CAS exhausted after {ProgressCasMaxAttempts} attempts or " + + $"{ProgressCasMaxWallClock.TotalSeconds:0}s wall-clock. " + + $"tenant={tenantId} session={sessionId} manifestId={manifestId}"); + } + + // §12-Q10: 10 attempts OR 60s wall-clock, whichever first → throw. + internal const int ProgressCasMaxAttempts = 10; + internal static readonly TimeSpan ProgressCasMaxWallClock = TimeSpan.FromSeconds(60); + + // ============================================================ Helpers ==== + + private static bool IsSessionsRow(DeletionRowDump row) + => row.Rk != null && !row.Rk.Contains('_'); + + private static void AddCount(Dictionary map, string key, int delta) + { + if (delta == 0) return; + map[key] = map.TryGetValue(key, out var c) ? c + delta : delta; + } + + private async Task AuditManifestDownloadedAsync(string tenantId, string sessionId, string manifestId, string actor, CancellationToken ct) + { + await _maintenanceRepo.LogAuditEntryAsync( + tenantId, + action: "deletion_manifest_downloaded", + entityType: "Session", + entityId: sessionId, + performedBy: actor, + details: new Dictionary(StringComparer.Ordinal) + { + ["manifestId"] = manifestId, + ["reason"] = "restore_attempt", + }).ConfigureAwait(false); + } + + private async Task AuditRestoredAsync( + string tenantId, string sessionId, string manifestId, string mode, string actor, + SessionRestoreResult result, CancellationToken ct) + { + await _maintenanceRepo.LogAuditEntryAsync( + tenantId, + action: "deletion_restored", + entityType: "Session", + entityId: sessionId, + performedBy: actor, + details: new Dictionary(StringComparer.Ordinal) + { + ["manifestId"] = manifestId, + ["mode"] = mode, + ["rowsRestoredByTable"] = JsonSerializer.Serialize(result.RowsRestoredByTable), + ["rowsSkippedByTable"] = JsonSerializer.Serialize(result.RowsSkippedByTable), + ["inventoryReIncrements"] = result.InventoryReIncrements.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["durationMs"] = result.DurationMs.ToString(System.Globalization.CultureInfo.InvariantCulture), + }).ConfigureAwait(false); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/ISignalRNotificationService.cs b/src/Backend/AutopilotMonitor.Functions/Services/ISignalRNotificationService.cs index 8050daba..62a019b9 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/ISignalRNotificationService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/ISignalRNotificationService.cs @@ -13,6 +13,15 @@ namespace AutopilotMonitor.Functions.Services Task NotifyRuleResultsAvailableAsync(string tenantId, string sessionId, int resultCount); Task NotifyVulnerabilityReportAvailableAsync(string tenantId, string sessionId, string overallRisk); + /// + /// Emitted by the cascade-deletion worker after a session's tombstone step completes + /// (plan §5 PR4). The web UI subscribes to session-{tenantId}-{sessionId} and + /// removes the row + closes any open detail pane on receipt. Idempotent — a re-pickup of + /// an already-tombstoned session is a no-op at the worker level, so receivers may see + /// the event at most once per cascade. + /// + Task NotifySessionDeletedAsync(string tenantId, string sessionId); + Task SendTenantNotificationAsync(string tenantId, NotificationAudience audience, object dto); Task SendTenantNotificationDismissedAsync(string tenantId, string notificationId); Task SendTenantNotificationDismissedAllAsync(string tenantId); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/SignalRNotificationService.cs b/src/Backend/AutopilotMonitor.Functions/Services/SignalRNotificationService.cs index 0846c256..a5f3e127 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/SignalRNotificationService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/SignalRNotificationService.cs @@ -95,6 +95,37 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// Notify the session group that the cascade-deletion worker has tombstoned the session + /// (plan §5 PR4). Receivers (web UI) remove the row and close any open detail pane. + /// Failures are logged and swallowed — the SignalR push is observability, not correctness; + /// the audit log entry deletion_completed remains the canonical record. + /// + public async Task NotifySessionDeletedAsync(string tenantId, string sessionId) + { + try + { + var hub = await GetHubContextAsync(); + var groupName = $"session-{tenantId}-{sessionId}"; + + await hub.Clients.Group(groupName).SendCoreAsync("sessionDeleted", new object[] + { + new + { + sessionId, + tenantId, + timestamp = DateTime.UtcNow.ToString("o") + } + }); + + _logger.LogDebug("Sent sessionDeleted to group {Group}", groupName); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to send sessionDeleted notification for session {SessionId}", sessionId); + } + } + /// /// Push a newly created tenant notification to the appropriate audience-tier group. /// Member-tier notifications go to the -notify-member group (joined by all tenant members); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Deletion.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Deletion.cs index 8d30767d..e30c717e 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Deletion.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Deletion.cs @@ -41,7 +41,7 @@ namespace AutopilotMonitor.Functions.Services /// public partial class TableStorageService : ISessionDeletionInventoryReader { - public async Task GetSessionRowAsync(string tenantId, string sessionId, CancellationToken cancellationToken = default) + public virtual async Task GetSessionRowAsync(string tenantId, string sessionId, CancellationToken cancellationToken = default) { var tableClient = _tableServiceClient.GetTableClient(Shared.Constants.TableNames.Sessions); try @@ -108,7 +108,7 @@ namespace AutopilotMonitor.Functions.Services /// Returns for accurate progress accounting on /// re-runs of partially-completed cascades. /// - public async Task DeleteByExactKeysInBatchesAsync( + public virtual async Task DeleteByExactKeysInBatchesAsync( string tableName, IReadOnlyList<(string Pk, string Rk)> keys, CancellationToken cancellationToken = default) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Inventory.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Inventory.cs index 3d1e0504..cc0969b4 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Inventory.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Inventory.cs @@ -300,7 +300,7 @@ namespace AutopilotMonitor.Functions.Services /// zero. 404 = idempotent no-op. Bounded retry per / /// ; throws on exhaustion. /// - public async Task DecrementSoftwareInventoryEntryAsync( + public virtual async Task DecrementSoftwareInventoryEntryAsync( string tenantId, string normalizedVendor, string normalizedName, string normalizedVersion, CancellationToken ct = default) { @@ -362,6 +362,91 @@ namespace AutopilotMonitor.Functions.Services + $"{InventoryCasMaxWallClock.TotalSeconds:0}s wall-clock. tenant={tenantId} key={rowKey}"); } + /// + /// Re-increments a SoftwareInventory counter from a + /// (vendor, name, version only — no DisplayName/Publisher). Symmetric inverse of + /// ; used by the cascade-delete restore + /// path (PR4b plan §13.6) where the manifest's side-row only carries the decrement keys. + /// + /// If the inventory row exists → ETag-CAS SessionCount += 1, preserving DisplayName, + /// Publisher, and every other column untouched. If the row is missing (count had dropped + /// to zero and the cascade deleted it) → AddEntity with vendor/name/version + count=1 + + /// FirstSeenAt + LastSeenAt + empty DisplayName/Publisher. The fidelity loss on display + /// metadata is accepted per plan §13.6 ("re-creates the inventory row with SessionCount=1"); + /// the next live inventory-correlation will fill in DisplayName + Publisher. + /// + /// Bounded retry per / ; + /// throws on exhaustion per memory feedback_storage_helpers_fail_soft. + /// + public virtual async Task RestoreSoftwareInventoryEntryByKeyAsync( + string tenantId, DeletionDecrementKey key, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + if (key == null) throw new ArgumentNullException(nameof(key)); + var rowKey = BuildSoftwareInventoryRowKey(key.Vendor, key.Name, key.Version); + if (string.IsNullOrWhiteSpace(rowKey) || rowKey == "::") + { + throw new ArgumentException( + "DeletionDecrementKey has empty vendor/name/version — cannot restore a SoftwareInventory entry with no identity.", + nameof(key)); + } + + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.SoftwareInventory); + var deadline = DateTime.UtcNow + InventoryCasMaxWallClock; + + for (var attempt = 0; attempt < InventoryCasMaxAttempts && DateTime.UtcNow < deadline; attempt++) + { + ct.ThrowIfCancellationRequested(); + + TableEntity existing; + try + { + var response = await tableClient.GetEntityAsync(tenantId, rowKey, cancellationToken: ct); + existing = response.Value; + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + // Row was deleted by cascade (count reached zero) — re-create with minimal data. + var now = DateTime.UtcNow; + var fresh = new TableEntity(tenantId, rowKey) + { + ["NormalizedVendor"] = key.Vendor ?? string.Empty, + ["NormalizedName"] = key.Name ?? string.Empty, + ["NormalizedVersion"] = key.Version ?? string.Empty, + ["SessionCount"] = 1, + ["FirstSeenAt"] = now.ToString("o"), + ["LastSeenAt"] = now.ToString("o"), + }; + try + { + await tableClient.AddEntityAsync(fresh, ct); + return; + } + catch (RequestFailedException rfe) when (rfe.Status == 409) + { + // Race: another writer created the row between our Get and our Add — retry. + continue; + } + } + + existing["SessionCount"] = (existing.GetInt32("SessionCount") ?? 0) + 1; + existing["LastSeenAt"] = DateTime.UtcNow.ToString("o"); + try + { + await tableClient.UpdateEntityAsync(existing, existing.ETag, TableUpdateMode.Replace, ct); + return; + } + catch (RequestFailedException ex) when (ex.Status == 412) + { + continue; + } + } + + throw new InvalidOperationException( + $"SoftwareInventory restore ETag-CAS exhausted after {InventoryCasMaxAttempts} attempts or " + + $"{InventoryCasMaxWallClock.TotalSeconds:0}s wall-clock. tenant={tenantId} key={rowKey}"); + } + // ============================================================ Helpers ==== /// diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Restore.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Restore.cs new file mode 100644 index 00000000..c7723cdf --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Restore.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models.Deletion; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Restore mode for . + /// + /// PR4c F3b: both values now have identical 409-ignore semantics. The enum is kept for + /// API readability — callers still document intent ("I'm doing a full restore" vs "I'm doing + /// a partial-poisoned-recovery") — but the implementation no longer throws on row-already-exists. + /// This was relaxed because the original "Full mode throws on 409 as corruption signal" intent + /// blocked the common case: a previous restore attempt failed mid-flight and left some rows + /// behind, and the operator's retry would 409-throw on those leftover rows. With 409-ignore, + /// retries are idempotent; still surfaces the count + /// for operator inspection. The plan §13.6 "ghost row corruption signal" check shifts to a + /// future post-restore verification pass (out of PR4c scope). + /// + /// + public enum RestoreMode + { + /// Restore from a completed cascade (Sessions row absent + progress.CompletedAt set, + /// or PR4c F2 tombstone-gap case: Sessions row absent + TombstoneStarted=true). + Full, + /// Restore from a poisoned cascade (Sessions row present + DeletionState=Poisoned). + Partial, + } + + /// + /// Result of a call. + /// counts rows newly inserted; counts rows that + /// were already present (Partial mode only — Full mode throws on conflict). + /// + public class RestoreBatchResult + { + public int Attempted { get; } + public int Restored { get; } + public int Skipped { get; } + + public RestoreBatchResult(int attempted, int restored, int skipped) + { + Attempted = attempted; + Restored = restored; + Skipped = skipped; + } + + public static readonly RestoreBatchResult Empty = new RestoreBatchResult(0, 0, 0); + } + + /// + /// Cascade-deletion restore primitives (PR4b). Symmetric inverse of the PR2 delete helpers: + /// where DeleteByExactKeysInBatchesAsync consumes (Pk, Rk) tuples, this partial + /// consumes the full records (with EDM-tagged Props) and + /// re-Inserts them as typed rows. + /// + /// EDM round-trip fidelity (plan §13 "manifest IS the backup") is preserved by + /// : each tag + /// dispatches into the matching JsonElement.Get* + .NET-type conversion so the SDK + /// re-serializes with the original column type on the wire. Without the tag a DateTime + /// coming out of JSON would be inserted as Edm.String and silently change the column shape. + /// + /// + public partial class TableStorageService + { + /// + /// Mirror of : groups + /// by PartitionKey (batch transactions are partition-scoped), chunks each group at the + /// batch-action limit, and submits a single UpsertReplace/Add transaction + /// per chunk. On batch-level 409 (any row exists) the behaviour depends on + /// : + /// + /// — falls back to per-row AddEntityAsync; + /// any individual 409 throws as a corruption + /// signal (the cascade had marked these rows for deletion and they should not exist). + /// — falls back to per-row AddEntityAsync + /// with 409-ignore, counting skipped rows in the returned + /// . + /// + /// All other storage errors propagate unchanged per memory feedback_storage_helpers_fail_soft. + /// + public virtual async Task RestoreRowsByExactKeysInBatchesAsync( + string tableName, + IReadOnlyList rows, + RestoreMode mode, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName is required", nameof(tableName)); + if (rows == null) throw new ArgumentNullException(nameof(rows)); + if (rows.Count == 0) return RestoreBatchResult.Empty; + + var tableClient = _tableServiceClient.GetTableClient(tableName); + var restored = 0; + var skipped = 0; + + foreach (var group in rows.GroupBy(r => r.Pk, StringComparer.Ordinal)) + { + var rowsInGroup = group.ToList(); + for (var i = 0; i < rowsInGroup.Count; i += RestoreBatchActionLimit) + { + var chunk = rowsInGroup.Skip(i).Take(RestoreBatchActionLimit).ToList(); + var actions = chunk + .Select(dump => new TableTransactionAction( + TableTransactionActionType.Add, + ConvertDumpToEntity(dump, tableName))) + .ToList(); + + try + { + await tableClient.SubmitTransactionAsync(actions, cancellationToken); + restored += chunk.Count; + } + catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 400) + { + // Azure rolls back the entire transaction when any Add hits an existing + // row. Fall back to per-row AddEntity so we can distinguish "row already + // there" from genuine errors. Some Azure deployments map duplicate-row + // conflict to 400 with code EntityAlreadyExists; both surface through + // this fallback path. + // PR4c F3b: both Full and Partial modes count 409 as Skipped (idempotent + // retries after partial-failed restore). The `mode` parameter is kept for + // API readability but no longer changes behaviour here. + _ = mode; // intentional: relaxation documented above + foreach (var dump in chunk) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + await tableClient.AddEntityAsync(ConvertDumpToEntity(dump, tableName), cancellationToken); + restored++; + } + catch (RequestFailedException rfe) when (rfe.Status == 409) + { + skipped++; + } + } + } + } + } + + return new RestoreBatchResult(rows.Count, restored, skipped); + } + + /// + /// Symmetric inverse of : + /// parses the EDM-tagged back into a typed .NET object + /// suitable for assignment into a 's dictionary. The SDK then + /// re-serializes with the original column type on the wire. + /// + /// Null handling: EdmType=String + ValueKind=Null returns null + /// (mirrors the encode path's null fallback). + /// + /// + public static object? ConvertFromPropValue(DeletionPropValue prop) + { + if (prop == null) throw new ArgumentNullException(nameof(prop)); + + // Null is encoded as EdmType=String + JsonNull (no first-class null in Azure Tables). + if (prop.Value.ValueKind == JsonValueKind.Null) + { + return null; + } + + switch (prop.EdmType) + { + case DeletionPropEdmType.String: + return prop.Value.GetString(); + case DeletionPropEdmType.Boolean: + return prop.Value.GetBoolean(); + case DeletionPropEdmType.Int32: + return prop.Value.GetInt32(); + case DeletionPropEdmType.Int64: + return prop.Value.GetInt64(); + case DeletionPropEdmType.Double: + return prop.Value.GetDouble(); + case DeletionPropEdmType.DateTime: + { + var raw = prop.Value.GetString() + ?? throw new InvalidDataException("DateTime EdmType has null Value."); + // RoundtripKind preserves UTC Kind set by ConvertToPropValue. + return DateTime.Parse(raw, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + } + case DeletionPropEdmType.Guid: + { + var raw = prop.Value.GetString() + ?? throw new InvalidDataException("Guid EdmType has null Value."); + return Guid.Parse(raw); + } + case DeletionPropEdmType.Binary: + { + var raw = prop.Value.GetString() + ?? throw new InvalidDataException("Binary EdmType has null Value."); + return Convert.FromBase64String(raw); + } + default: + throw new InvalidDataException( + $"Unsupported EdmType '{prop.EdmType}' in DeletionPropValue. " + + "Manifest schema may be from a newer version than this restore endpoint supports."); + } + } + + /// + /// Builds a from a ready for + /// AddEntityAsync. For Sessions-table rows the DeletionState + + /// PendingDeletionManifestId columns are forcibly overridden to + /// None + null — the snapshot captured the row AFTER the producer's + /// CAS-Preparing, so a verbatim re-insert would leave the restored session permanently + /// locked. This override is the critical correctness invariant of the restore path. + /// + public static TableEntity ConvertDumpToEntity(DeletionRowDump dump, string tableName) + { + if (dump == null) throw new ArgumentNullException(nameof(dump)); + if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName is required", nameof(tableName)); + + var entity = new TableEntity(dump.Pk, dump.Rk); + foreach (var kvp in dump.Props) + { + entity[kvp.Key] = ConvertFromPropValue(kvp.Value); + } + + // Sessions-row override: snapshot was captured after CAS-None→Preparing so the row's + // DeletionState column carries "Preparing" + the current ManifestId. Restore must + // reset both so writers can lock the row again post-restore. + if (string.Equals(tableName, Constants.TableNames.Sessions, StringComparison.Ordinal)) + { + entity["DeletionState"] = SessionDeletionState.None; + entity["PendingDeletionManifestId"] = null; + } + + return entity; + } + + // Azure Tables batch transactions cap at 100 actions per submission, all sharing a + // PartitionKey. Mirrors the const in the deletion partial. + private const int RestoreBatchActionLimit = 100; + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionProgress.cs b/src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionProgress.cs index 891701ed..87dc4428 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionProgress.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionProgress.cs @@ -22,5 +22,42 @@ namespace AutopilotMonitor.Shared.Models.Deletion /// UTC timestamp once the FINAL tombstone step has completed; null while in flight. public DateTime? CompletedAt { get; set; } + + // ============================================================ PR4c additions ==== + // Three additive fields close the per-key idempotency + tombstone-gap correctness holes + // discovered by Codex review of PR4 + PR4b. All three default to safe values + // (null / false) so PR1–PR4 progress blobs without these fields deserialize cleanly. + + /// + /// PR4c F1: Composite keys ({Vendor}:{Name}:{Version}) of SoftwareInventory + /// decrements already applied by the cascade worker's AGGREGATE step. Per-key + /// persistence so a crash mid-decrement-loop doesn't double-decrement on retry. + /// Worker writes one entry after each successful DecrementSoftwareInventoryEntryAsync + /// and persists the progress blob with ETag-CAS, then moves to the next key. + /// Null on PR1-PR4 progress blobs (worker initializes on first AGGREGATE-step entry). + /// + public HashSet? AggregateDecrementsApplied { get; set; } + + /// + /// PR4c F4: Composite keys of SoftwareInventory re-increments already applied by the + /// partial-restore service. Per-key persistence so a crash between the re-increment + /// loop and the final Poisoned → None CAS doesn't double-increment counters on + /// retry. Service writes one entry after each successful + /// RestoreSoftwareInventoryEntryByKeyAsync and persists with ETag-CAS. + /// Null on PR1-PR4 progress blobs (service initializes on first re-increment). + /// + public HashSet? RestoreReIncrementsApplied { get; set; } + + /// + /// PR4c F2: Set to true by the cascade worker before issuing the first + /// FINAL-step row delete. Closes the "tombstone gap": if the worker dies between + /// deleting the Sessions row and writing , restore can still + /// dispatch into full-restore mode by reading this flag (otherwise + /// sessions=null + completedAt=null looks like corruption and restore rejects). + /// Default false for back-compat — PR1-PR4 progress blobs without this field + /// deserialize cleanly to false, preserving the existing "corrupt-state" reject + /// behaviour for genuine bugs (Sessions row removed outside the cascade). + /// + public bool TombstoneStarted { get; set; } } }