diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/BlobStorageServiceDeletionProgressTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/BlobStorageServiceDeletionProgressTests.cs
new file mode 100644
index 00000000..27b41578
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/BlobStorageServiceDeletionProgressTests.cs
@@ -0,0 +1,174 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure;
+using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
+using AutopilotMonitor.Functions.Services;
+using AutopilotMonitor.Shared.Models.Deletion;
+using Microsoft.Extensions.Logging.Abstractions;
+using Xunit;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+///
+/// Tests for the PR4 cascade-deletion progress-blob Download + CAS-Update helpers on
+/// . Same fake-subclass pattern as
+/// BlobStorageServiceDeletionManifestTests: override the blob-IO seams, capture bytes
+/// + options, replay on download. No Azurite.
+///
+public class BlobStorageServiceDeletionProgressTests
+{
+ private const string TenantId = "11111111-1111-1111-1111-111111111111";
+ private const string SessionId = "22222222-2222-2222-2222-222222222222";
+ private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
+ private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
+
+ [Fact]
+ public async Task Download_round_trips_progress_with_etag()
+ {
+ var sut = new FakeBlobStorageService();
+ // Seed the fake with an initial progress blob written via the existing helper.
+ var initialEtag = await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256);
+ Assert.False(string.IsNullOrEmpty(initialEtag));
+
+ var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
+
+ Assert.Equal(Sha256, progress.SnapshotSha256);
+ Assert.Empty(progress.CompletedSteps);
+ Assert.False(progress.VerificationDone);
+ Assert.Null(progress.CompletedAt);
+ Assert.False(string.IsNullOrEmpty(etag));
+ // ETag from a fresh download must match what UploadInitial returned.
+ Assert.Equal(initialEtag, etag);
+ }
+
+ [Fact]
+ public async Task Update_with_matching_etag_writes_new_state_and_returns_fresh_etag()
+ {
+ var sut = new FakeBlobStorageService();
+ var initialEtag = await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256);
+ var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
+
+ progress.CompletedSteps.Add(1);
+ progress.CompletedSteps.Add(2);
+ var newEtag = await sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, etag);
+
+ Assert.False(string.IsNullOrEmpty(newEtag));
+ Assert.NotEqual(initialEtag, newEtag);
+
+ var (refetched, _) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
+ Assert.Equal(new HashSet { 1, 2 }, refetched.CompletedSteps);
+ }
+
+ [Fact]
+ public async Task Update_with_mismatched_etag_throws_412_RequestFailedException()
+ {
+ var sut = new FakeBlobStorageService();
+ await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256);
+ var (progress, _) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
+
+ progress.CompletedSteps.Add(1);
+
+ var ex = await Assert.ThrowsAsync(
+ () => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, ifMatchEtag: "\"0xSTALE_ETAG\""));
+ Assert.Equal(412, ex.Status);
+ }
+
+ [Fact]
+ public async Task Update_writes_IfMatch_condition_on_the_upload_options()
+ {
+ var sut = new FakeBlobStorageService();
+ await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256);
+ var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
+
+ await sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, etag);
+
+ Assert.NotNull(sut.LastWrittenProgressOptions);
+ Assert.NotNull(sut.LastWrittenProgressOptions!.Conditions);
+ // PR4 §12-Q10 contract: every update is an ETag-CAS via IfMatch (not IfNoneMatch).
+ Assert.NotNull(sut.LastWrittenProgressOptions.Conditions!.IfMatch);
+ Assert.Equal(new ETag(etag), sut.LastWrittenProgressOptions.Conditions.IfMatch);
+ // Content type stays application/json — no gzip on the small progress blob.
+ Assert.Equal("application/json", sut.LastWrittenProgressOptions.HttpHeaders!.ContentType);
+ }
+
+ [Fact]
+ public async Task Update_throws_on_missing_required_fields()
+ {
+ var sut = new FakeBlobStorageService();
+
+ await Assert.ThrowsAsync(
+ () => sut.UpdateDeletionProgressAsync("", SessionId, ManifestId, new DeletionProgress(), "etag"));
+ await Assert.ThrowsAsync(
+ () => sut.UpdateDeletionProgressAsync(TenantId, "", ManifestId, new DeletionProgress(), "etag"));
+ await Assert.ThrowsAsync(
+ () => sut.UpdateDeletionProgressAsync(TenantId, SessionId, "", new DeletionProgress(), "etag"));
+ await Assert.ThrowsAsync(
+ () => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, null!, "etag"));
+ await Assert.ThrowsAsync(
+ () => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, new DeletionProgress(), ""));
+ }
+
+ [Fact]
+ public async Task Download_throws_on_missing_required_fields()
+ {
+ var sut = new FakeBlobStorageService();
+
+ await Assert.ThrowsAsync(
+ () => sut.DownloadDeletionProgressAsync("", SessionId, ManifestId));
+ await Assert.ThrowsAsync(
+ () => sut.DownloadDeletionProgressAsync(TenantId, "", ManifestId));
+ await Assert.ThrowsAsync(
+ () => sut.DownloadDeletionProgressAsync(TenantId, SessionId, ""));
+ }
+
+ // ---------------------------------------------------------------- Test fixture ----
+
+ ///
+ /// Captures the progress-blob bytes + options written by Upload/Update and replays them on
+ /// Download. Each write returns a fresh ETag; an Update with a stale ETag throws 412 so the
+ /// Handler's bounded-retry loop can be exercised against this fake.
+ ///
+ private sealed class FakeBlobStorageService : BlobStorageService
+ {
+ private byte[]? _lastBytes;
+ private BlobUploadOptions? _lastOptions;
+ private ETag _currentEtag;
+ private int _etagCounter;
+
+ public BlobUploadOptions? LastWrittenProgressOptions => _lastOptions;
+
+ public FakeBlobStorageService()
+ : base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger.Instance, usesManagedIdentity: false)
+ {
+ }
+
+ protected internal override Task WriteDeletionProgressBlobAsync(
+ string blobName, byte[] payload, BlobUploadOptions options, CancellationToken cancellationToken)
+ {
+ if (options.Conditions?.IfMatch != null
+ && _currentEtag != default
+ && options.Conditions.IfMatch != _currentEtag)
+ {
+ throw new RequestFailedException(412, "ConditionNotMet", "ConditionNotMet", innerException: null);
+ }
+ _lastBytes = payload;
+ _lastOptions = options;
+ _currentEtag = new ETag($"\"0xFAKE_ETAG_{++_etagCounter}\"");
+ return Task.FromResult(_currentEtag);
+ }
+
+ protected internal override Task<(byte[] Payload, ETag ETag)> ReadDeletionProgressBlobAsync(
+ string blobName, CancellationToken cancellationToken)
+ {
+ if (_lastBytes == null)
+ {
+ throw new InvalidOperationException(
+ "Test must call UploadInitialDeletionProgressAsync before DownloadDeletionProgressAsync.");
+ }
+ return Task.FromResult((_lastBytes, _currentEtag));
+ }
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/CascadeVerificationServiceTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/CascadeVerificationServiceTests.cs
new file mode 100644
index 00000000..5a5fd0bd
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/CascadeVerificationServiceTests.cs
@@ -0,0 +1,399 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure.Data.Tables;
+using AutopilotMonitor.Functions.Services.Deletion;
+using AutopilotMonitor.Shared.Models.Deletion;
+using Microsoft.Extensions.Logging.Abstractions;
+using Xunit;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+///
+/// Per-class behaviour tests for . Verifies that each
+/// step class drives the right inventory-reader call shape (filter for scan-based classes,
+/// direct GetEntity for manifest-key-only classes), that AGGREGATE + FINAL are skipped, and
+/// that the residual-sample cap is honoured.
+///
+public class CascadeVerificationServiceTests
+{
+ private const string TenantId = "11111111-1111-1111-1111-111111111111";
+ private const string SessionId = "22222222-2222-2222-2222-222222222222";
+
+ [Fact]
+ public async Task Verify_returns_clean_when_all_tables_empty()
+ {
+ var reader = new FakeReader();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = BuildManifestWithAllClasses();
+
+ var result = await sut.VerifyAsync(manifest);
+
+ Assert.True(result.IsClean);
+ Assert.Empty(result.Residuals);
+ }
+
+ [Fact]
+ public async Task PkBySession_uses_partition_filter()
+ {
+ var reader = new FakeReader();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
+ },
+ };
+
+ await sut.VerifyAsync(manifest);
+
+ var call = Assert.Single(reader.QueryCalls);
+ Assert.Equal("Events", call.Table);
+ Assert.Equal($"PartitionKey eq '{TenantId}_{SessionId}'", call.Filter);
+ }
+
+ [Fact]
+ public async Task PropTenantPk_uses_partition_and_sessionid_filter()
+ {
+ var reader = new FakeReader();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep { Order = 3, Table = "AppInstallSummaries", Class = DeletionStepClass.PropTenantPk, RowCount = 0 },
+ },
+ };
+
+ await sut.VerifyAsync(manifest);
+
+ var call = Assert.Single(reader.QueryCalls);
+ Assert.Equal($"PartitionKey eq '{TenantId}' and SessionId eq '{SessionId}'", call.Filter);
+ }
+
+ [Fact]
+ public async Task PkRkExact_uses_direct_GetEntity_per_manifest_row()
+ {
+ var reader = new FakeReader();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep
+ {
+ Order = 5, Table = "DeviceSnapshot", Class = DeletionStepClass.PkRkExact, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } },
+ },
+ },
+ };
+
+ await sut.VerifyAsync(manifest);
+
+ Assert.Empty(reader.QueryCalls);
+ var get = Assert.Single(reader.GetCalls);
+ Assert.Equal("DeviceSnapshot", get.Table);
+ Assert.Equal(TenantId, get.Pk);
+ Assert.Equal(SessionId, get.Rk);
+ }
+
+ [Fact]
+ public async Task DiscriminatorPkRkExact_uses_manifest_key_GetEntity_not_live_scan()
+ {
+ // Plan §12-Q8: CveIndex is the deliberate exception. Manifest-key only — verifier MUST NOT
+ // do a tenant-prefix scan on this class.
+ var reader = new FakeReader();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep
+ {
+ Order = 10, Table = "CveIndex", Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 2,
+ Rows = new List
+ {
+ new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId },
+ new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0002", Rk = SessionId },
+ },
+ },
+ },
+ };
+
+ await sut.VerifyAsync(manifest);
+
+ Assert.Empty(reader.QueryCalls); // no tenant-prefix scan
+ Assert.Equal(2, reader.GetCalls.Count);
+ Assert.All(reader.GetCalls, c => Assert.Equal("CveIndex", c.Table));
+ }
+
+ [Fact]
+ public async Task DiscriminatorPkRkSuffix_uses_tenant_prefix_scan_with_client_side_suffix_filter()
+ {
+ var reader = new FakeReader();
+ // Seed with a row matching the suffix and one that doesn't — client-side filter must drop the non-match.
+ reader.QueryResults["EventTypeIndex"] = new List
+ {
+ BuildEntity($"{TenantId}_install_failed", $"6299999999999999_{SessionId}"), // matches
+ BuildEntity($"{TenantId}_install_failed", "6299999999999999_other-session"), // does not match
+ };
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep { Order = 9, Table = "EventTypeIndex", Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 0 },
+ },
+ };
+
+ var result = await sut.VerifyAsync(manifest);
+
+ var call = Assert.Single(reader.QueryCalls);
+ Assert.Contains($"PartitionKey ge '{TenantId}_'", call.Filter);
+ Assert.Contains($"PartitionKey lt '{TenantId}_~'", call.Filter);
+ // Suffix-matching row is the only residual reported.
+ Assert.False(result.IsClean);
+ Assert.Single(result.Residuals);
+ Assert.EndsWith($"_{SessionId}", result.Residuals[0].Rk);
+ }
+
+ [Fact]
+ public async Task DiscriminatorPkProp_uses_tenant_prefix_and_sessionid_filter()
+ {
+ var reader = new FakeReader();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep { Order = 11, Table = "SessionsByTerminal", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0 },
+ },
+ };
+
+ await sut.VerifyAsync(manifest);
+
+ var call = Assert.Single(reader.QueryCalls);
+ Assert.Contains($"PartitionKey ge '{TenantId}_'", call.Filter);
+ Assert.Contains($"PartitionKey lt '{TenantId}_~'", call.Filter);
+ Assert.Contains($"SessionId eq '{SessionId}'", call.Filter);
+ }
+
+ [Fact]
+ public async Task Aggregate_step_is_skipped_without_verification_call()
+ {
+ var reader = new FakeReader();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep
+ {
+ Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement,
+ Class = DeletionStepClass.Aggregate, RowCount = 3,
+ Decrements = new List
+ {
+ new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" },
+ },
+ },
+ },
+ };
+
+ var result = await sut.VerifyAsync(manifest);
+
+ Assert.True(result.IsClean);
+ Assert.Empty(reader.QueryCalls);
+ Assert.Empty(reader.GetCalls);
+ }
+
+ [Fact]
+ public async Task Final_step_is_skipped_without_verification_call()
+ {
+ var reader = new FakeReader();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep
+ {
+ Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2,
+ Rows = new List
+ {
+ new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
+ new DeletionRowDump { Pk = TenantId, Rk = SessionId },
+ },
+ },
+ },
+ };
+
+ var result = await sut.VerifyAsync(manifest);
+
+ Assert.True(result.IsClean);
+ Assert.Empty(reader.QueryCalls);
+ Assert.Empty(reader.GetCalls);
+ }
+
+ [Fact]
+ public async Task Single_ghost_row_in_PkBySession_returns_not_clean_with_residual()
+ {
+ var reader = new FakeReader();
+ reader.QueryResults["Events"] = new List
+ {
+ BuildEntity($"{TenantId}_{SessionId}", "ghost-event-rk"),
+ };
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
+ },
+ };
+
+ var result = await sut.VerifyAsync(manifest);
+
+ Assert.False(result.IsClean);
+ var residual = Assert.Single(result.Residuals);
+ Assert.Equal("Events", residual.Table);
+ Assert.Equal($"{TenantId}_{SessionId}", residual.Pk);
+ Assert.Equal("ghost-event-rk", residual.Rk);
+ }
+
+ [Fact]
+ public async Task Residual_sample_is_capped_at_MaxResidualSampleSize()
+ {
+ var reader = new FakeReader();
+ reader.QueryResults["Events"] = Enumerable.Range(0, 200)
+ .Select(i => BuildEntity($"{TenantId}_{SessionId}", $"rk-{i:D3}"))
+ .ToList();
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
+ },
+ };
+
+ var result = await sut.VerifyAsync(manifest);
+
+ Assert.False(result.IsClean);
+ Assert.Equal(CascadeVerificationService.MaxResidualSampleSize, result.Residuals.Count);
+ }
+
+ [Fact]
+ public async Task First_table_failure_short_circuits_remaining_steps()
+ {
+ var reader = new FakeReader();
+ reader.QueryResults["Events"] = new List
+ {
+ BuildEntity($"{TenantId}_{SessionId}", "ghost-rk"),
+ };
+ // RuleResults would have ghosts too, but the verifier must NOT enumerate it after the
+ // first-table failure — the short-circuit keeps verification cost bounded.
+ reader.QueryResults["RuleResults"] = new List
+ {
+ BuildEntity($"{TenantId}_{SessionId}", "should-not-be-seen"),
+ };
+ var sut = new CascadeVerificationService(reader, NullLogger.Instance);
+ var manifest = new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
+ new DeletionStep { Order = 2, Table = "RuleResults", Class = DeletionStepClass.PkBySession, RowCount = 0 },
+ },
+ };
+
+ var result = await sut.VerifyAsync(manifest);
+
+ Assert.False(result.IsClean);
+ // Only Events was queried; RuleResults short-circuited.
+ Assert.Single(reader.QueryCalls);
+ Assert.Equal("Events", reader.QueryCalls[0].Table);
+ }
+
+ // ---------------------------------------------------------------- Helpers ----
+
+ private static TableEntity BuildEntity(string pk, string rk) => new TableEntity(pk, rk);
+
+ private static DeletionManifest BuildManifestWithAllClasses() => new DeletionManifest
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ Steps = new List
+ {
+ new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
+ new DeletionStep { Order = 3, Table = "AppInstallSummaries", Class = DeletionStepClass.PropTenantPk, RowCount = 0 },
+ new DeletionStep { Order = 5, Table = "DeviceSnapshot", Class = DeletionStepClass.PkRkExact, RowCount = 0 },
+ new DeletionStep { Order = 9, Table = "EventTypeIndex", Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 0 },
+ new DeletionStep { Order = 10, Table = "CveIndex", Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 0 },
+ new DeletionStep { Order = 11, Table = "SessionsByTerminal", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0 },
+ new DeletionStep { Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement, Class = DeletionStepClass.Aggregate, RowCount = 0 },
+ new DeletionStep { Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 0 },
+ },
+ };
+
+ private sealed class FakeReader : ISessionDeletionInventoryReader
+ {
+ public List<(string Table, string Filter)> QueryCalls { get; } = new List<(string, string)>();
+ public List<(string Table, string Pk, string Rk)> GetCalls { get; } = new List<(string, string, string)>();
+ public Dictionary> QueryResults { get; } = new Dictionary>(StringComparer.Ordinal);
+ public Dictionary<(string Table, string Pk, string Rk), TableEntity> GetResults { get; }
+ = new Dictionary<(string, string, string), TableEntity>();
+
+ public Task GetSessionRowAsync(string tenantId, string sessionId, CancellationToken ct = default)
+ => Task.FromResult(null);
+
+ public Task GetSessionsIndexRowAsync(string tenantId, string indexRowKey, CancellationToken ct = default)
+ => Task.FromResult(null);
+
+ public async IAsyncEnumerable QueryAsync(
+ string tableName, string filter, [EnumeratorCancellation] CancellationToken ct = default)
+ {
+ QueryCalls.Add((tableName, filter));
+ if (QueryResults.TryGetValue(tableName, out var rows))
+ {
+ foreach (var row in rows)
+ {
+ await Task.Yield();
+ yield return row;
+ }
+ }
+ }
+
+ public Task GetEntityOrNullAsync(string tableName, string partitionKey, string rowKey, CancellationToken ct = default)
+ {
+ GetCalls.Add((tableName, partitionKey, rowKey));
+ GetResults.TryGetValue((tableName, partitionKey, rowKey), out var entity);
+ return Task.FromResult(entity);
+ }
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/FakeSignalRNotificationService.cs b/src/Backend/AutopilotMonitor.Functions.Tests/FakeSignalRNotificationService.cs
index bc3be033..fc833a7f 100644
--- a/src/Backend/AutopilotMonitor.Functions.Tests/FakeSignalRNotificationService.cs
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/FakeSignalRNotificationService.cs
@@ -20,6 +20,7 @@ public class FakeSignalRNotificationService : ISignalRNotificationService
public int GlobalDismissAllCount { get; private set; }
public List<(string TenantId, string SessionId, int Count)> RuleResultsCalls { get; } = new();
public List<(string TenantId, string SessionId, string Risk)> VulnReportCalls { get; } = new();
+ public List<(string TenantId, string SessionId)> SessionDeletedCalls { get; } = new();
public Task NotifyRuleResultsAvailableAsync(string tenantId, string sessionId, int resultCount)
{
@@ -33,6 +34,12 @@ public class FakeSignalRNotificationService : ISignalRNotificationService
return Task.CompletedTask;
}
+ public Task NotifySessionDeletedAsync(string tenantId, string sessionId)
+ {
+ SessionDeletedCalls.Add((tenantId, sessionId));
+ return Task.CompletedTask;
+ }
+
public Task SendTenantNotificationAsync(string tenantId, NotificationAudience audience, object dto)
{
TenantSends.Add((tenantId, audience, dto));
diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RestoreSessionFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RestoreSessionFunctionTests.cs
new file mode 100644
index 00000000..412ba0c7
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/RestoreSessionFunctionTests.cs
@@ -0,0 +1,55 @@
+using System.Linq;
+using System.Net;
+using AutopilotMonitor.Functions.Functions.Sessions;
+using AutopilotMonitor.Functions.Security;
+using AutopilotMonitor.Functions.Services.Deletion;
+using Xunit;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+///
+/// Thin HTTP-layer tests for (PR4b). The bulk of behaviour
+/// lives in (covered by SessionRestoreServiceTests);
+/// this file only verifies the layer-specific pieces: status mapping, route registration in
+/// the policy catalog (GA-only enforcement), and body shape.
+///
+public class RestoreSessionFunctionTests
+{
+ [Theory]
+ [InlineData(SessionRestoreOutcome.Restored, HttpStatusCode.OK)]
+ [InlineData(SessionRestoreOutcome.DryRunOk, HttpStatusCode.OK)]
+ [InlineData(SessionRestoreOutcome.RejectManifestNotFound, HttpStatusCode.NotFound)]
+ [InlineData(SessionRestoreOutcome.RejectAlreadyAtOriginalState, HttpStatusCode.Conflict)]
+ [InlineData(SessionRestoreOutcome.RejectActiveCascade, HttpStatusCode.Conflict)]
+ [InlineData(SessionRestoreOutcome.RejectManifestIdMismatch, HttpStatusCode.Conflict)]
+ [InlineData(SessionRestoreOutcome.RejectCorruptState, HttpStatusCode.Conflict)]
+ [InlineData(SessionRestoreOutcome.RejectManifestCorruption, HttpStatusCode.Conflict)]
+ [InlineData(SessionRestoreOutcome.RejectCasConflictOnClear, HttpStatusCode.Conflict)]
+ public void MapOutcomeToStatus_returns_expected_status(SessionRestoreOutcome outcome, HttpStatusCode expected)
+ {
+ Assert.Equal(expected, RestoreSessionFunction.MapOutcomeToStatus(outcome));
+ }
+
+ [Fact]
+ public void Restore_route_is_registered_in_policy_catalog_as_GlobalAdminOnly()
+ {
+ // Per memory feedback_route_policy_catalog — every HTTP route MUST be registered in
+ // EndpointAccessPolicyCatalog. An unregistered route fail-closes to 403. This test
+ // guards the restore endpoint's GA-only enforcement.
+ var entry = EndpointAccessPolicyCatalog.FindPolicy("POST", "admin/sessions/some-id/restore");
+
+ Assert.NotNull(entry);
+ Assert.Equal(EndpointPolicy.GlobalAdminOnly, entry!.Policy);
+ }
+
+ [Fact]
+ public void Restore_route_is_NOT_registered_for_non_POST_methods()
+ {
+ // GET to the restore route should not match the POST registration.
+ var getEntry = EndpointAccessPolicyCatalog.FindPolicy("GET", "admin/sessions/some-id/restore");
+ Assert.Null(getEntry);
+
+ var putEntry = EndpointAccessPolicyCatalog.FindPolicy("PUT", "admin/sessions/some-id/restore");
+ Assert.Null(putEntry);
+ }
+}
diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs
new file mode 100644
index 00000000..5e5bfc50
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs
@@ -0,0 +1,667 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure;
+using Azure.Data.Tables;
+using AutopilotMonitor.Functions.Services;
+using AutopilotMonitor.Functions.Services.Deletion;
+using AutopilotMonitor.Shared;
+using AutopilotMonitor.Shared.DataAccess;
+using AutopilotMonitor.Shared.Models.Deletion;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Xunit;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+///
+/// End-to-end Handler facts for plan §5 PR4. Moq-based, no Azurite. Every TableStorageService /
+/// BlobStorageService / CascadeVerificationService dependency is mocked through the existing
+/// virtual seams; the Handler exercises the full §3 schema flow without touching real storage.
+///
+public class SessionDeletionHandlerTests
+{
+ private const string TenantId = "11111111-1111-1111-1111-111111111111";
+ private const string SessionId = "22222222-2222-2222-2222-222222222222";
+ private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
+ private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
+
+ // ============================================================ §5 PR4 plan tests ====
+
+ [Fact]
+ public async Task Cascade_deletes_full_session_round_trip()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ var manifest = harness.SetFullSessionManifest();
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ // All 6 table-step deletes invoked with the manifest's keys.
+ foreach (var step in manifest.Steps.Where(s => s.Class != DeletionStepClass.Aggregate && s.Class != DeletionStepClass.Final && s.Rows.Count > 0))
+ {
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ step.Table!, It.Is>(keys => keys.Count == step.Rows.Count), It.IsAny()),
+ Times.AtLeastOnce, $"step {step.Order} ({step.Table}) was not deleted");
+ }
+ // AGGREGATE decrement: 3 keys → 3 calls.
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()),
+ Times.Exactly(3));
+ // Tombstone: 2 deletes (Index + Sessions).
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.SessionsIndex, It.IsAny>(), It.IsAny()),
+ Times.AtLeastOnce);
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.Sessions, It.IsAny>(), It.IsAny()),
+ Times.AtLeastOnce);
+
+ Assert.Single(harness.SignalR.SessionDeletedCalls);
+ Assert.Equal((TenantId, SessionId), harness.SignalR.SessionDeletedCalls[0]);
+
+ // Final audit asserts both step-completed entries AND deletion_completed.
+ Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_completed");
+ Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_step_completed");
+ }
+
+ [Fact]
+ public async Task Cascade_includes_CveIndex_step()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.CveIndex,
+ It.Is>(keys =>
+ keys.Count == 1 && keys[0].Pk == $"{TenantId}_CVE-2024-0001" && keys[0].Rk == SessionId),
+ It.IsAny()),
+ Times.Once);
+ }
+
+ [Fact]
+ public async Task Cascade_is_idempotent_when_re_run_after_completion()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ // Progress shows CompletedAt already set — handler must early-return.
+ harness.ProgressFake.CompletedAt = DateTime.UtcNow;
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ It.IsAny(), It.IsAny>(), It.IsAny()),
+ Times.Never);
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()),
+ Times.Never);
+ Assert.Empty(harness.SignalR.SessionDeletedCalls);
+ Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_completed");
+ }
+
+ [Fact]
+ public async Task Cascade_resumes_after_partial_failure_by_skipping_completed_steps()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ // Steps 1–8 already complete on prior pickup; cascade resumes from step 9 onward.
+ harness.ProgressFake.CompletedSteps.UnionWith(new[] { 1, 2, 3, 4, 5, 6, 7, 8 });
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ // Steps 1–8 must NOT be re-issued.
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.Events, It.IsAny>(), It.IsAny()),
+ Times.Never);
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.RuleResults, It.IsAny>(), It.IsAny()),
+ Times.Never);
+ // Step 9 onward DOES get issued.
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.EventTypeIndex, It.IsAny>(), It.IsAny()),
+ Times.AtLeastOnce);
+ }
+
+ [Fact]
+ public async Task Cascade_corrupted_snapshot_hash_mismatch_throws_so_worker_poisons()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ // BlobStorageService throws InvalidDataException on SHA mismatch (existing helper contract).
+ // PR4c F6: Handler uses the with-Sha variant; mock it to throw on the same condition.
+ harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
+ TenantId, SessionId, ManifestId, It.IsAny()))
+ .ThrowsAsync(new System.IO.InvalidDataException("snapshot SHA mismatch"));
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+
+ // No deletes, no signalR, no completion audit — the handler must throw before touching anything.
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ It.IsAny(), It.IsAny>(), It.IsAny()),
+ Times.Never);
+ Assert.Empty(harness.SignalR.SessionDeletedCalls);
+ }
+
+ [Fact]
+ public async Task Cascade_throws_on_DISCRIMINATOR_step_with_zero_rows_but_nonzero_count()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ var manifest = harness.SetSingleStepManifest(new DeletionStep
+ {
+ Order = 11, Table = "SessionsByTerminal",
+ Class = DeletionStepClass.DiscriminatorPkProp,
+ RowCount = 5,
+ Rows = new List(), // empty despite RowCount = 5
+ });
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ "SessionsByTerminal", It.IsAny>(), It.IsAny()),
+ Times.Never);
+ }
+
+ [Fact]
+ public async Task Cascade_does_not_tombstone_on_live_verification_failure()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ // Verifier reports residuals → handler aborts before tombstone.
+ harness.Verifier.Setup(v => v.VerifyAsync(It.IsAny(), It.IsAny()))
+ .ReturnsAsync(new CascadeVerificationResult(
+ isClean: false,
+ residuals: new List { new CascadeResidualKey("Events", "ghost-pk", "ghost-rk") }));
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+
+ // The two FINAL-step deletes (SessionsIndex + Sessions) MUST NOT have fired.
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.SessionsIndex, It.IsAny>(), It.IsAny()),
+ Times.Never);
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.Sessions, It.IsAny>(), It.IsAny()),
+ Times.Never);
+ Assert.Empty(harness.SignalR.SessionDeletedCalls);
+ Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_verification_failed");
+ }
+
+ [Fact]
+ public async Task Cascade_decrements_software_inventory_via_contributions_row()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ // The 3 keys in the AGGREGATE step → 3 decrement calls.
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, "Microsoft", "Office", "16.0", It.IsAny()), Times.Once);
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, "Adobe", "Acrobat", "23.1", It.IsAny()), Times.Once);
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, "Mozilla", "Firefox", "120.0", It.IsAny()), Times.Once);
+ }
+
+ [Fact]
+ public async Task Cascade_skips_software_inventory_when_no_contributions_row()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ // Manifest WITHOUT the side-row step 16 + 17 (pre-side-row session).
+ harness.SetMinimalManifestNoInventory();
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()),
+ Times.Never);
+ }
+
+ [Fact]
+ public async Task Cascade_aggregate_step_runs_before_side_row_delete()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+
+ var callOrder = new List();
+ harness.Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()))
+ .Callback(() => callOrder.Add("decrement"))
+ .Returns(Task.CompletedTask);
+ harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.SessionInventoryContributions, It.IsAny>(), It.IsAny()))
+ .Callback(() => callOrder.Add("delete-side-row"))
+ .ReturnsAsync(new DeletionBatchResult(1, 1, 0));
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ // All decrement calls must happen before the side-row delete.
+ var firstSideRow = callOrder.IndexOf("delete-side-row");
+ Assert.True(firstSideRow >= 0, "side-row delete was never issued");
+ Assert.True(callOrder.Take(firstSideRow).All(c => c == "decrement"),
+ "decrements must complete before the SessionInventoryContributions side-row is deleted (§17.5)");
+ }
+
+ [Fact]
+ public async Task Cascade_does_not_touch_diagnostics_blob_even_when_recorded_in_manifest()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ var manifest = harness.SetFullSessionManifest();
+ manifest.DiagnosticsBlobName = "diagnostics-uploads/contoso/session-abc/diag.zip";
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ // The Blob mock surfaces ONLY deletion-manifest container operations; no other blob calls.
+ // Assertion: no Verify on diagnostics blob methods (we never set them up, so nothing fires).
+ // Stronger: assert no Blob mock method other than the manifest helpers was ever invoked.
+ // We do this by verifying the only Setup'd methods received calls — anything else would
+ // have thrown via Moq's strict-mode... but we're using loose mode. Instead: assert no
+ // Storage call mentions the diag blob name.
+ harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
+ It.Is(table => table.Contains("diagnostics", StringComparison.OrdinalIgnoreCase)),
+ It.IsAny>(), It.IsAny()),
+ Times.Never);
+ }
+
+ [Fact]
+ public async Task Cascade_emits_signalr_notification_on_completion()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ var call = Assert.Single(harness.SignalR.SessionDeletedCalls);
+ Assert.Equal(TenantId, call.TenantId);
+ Assert.Equal(SessionId, call.SessionId);
+ }
+
+ [Fact]
+ public async Task Cascade_deletes_sessionsIndex_before_sessions_row()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+
+ var deleteOrder = new List();
+ harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.SessionsIndex, It.IsAny>(), It.IsAny()))
+ .Callback(() => deleteOrder.Add(Constants.TableNames.SessionsIndex))
+ .ReturnsAsync(new DeletionBatchResult(1, 1, 0));
+ harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.Sessions, It.IsAny>(), It.IsAny()))
+ .Callback(() => deleteOrder.Add(Constants.TableNames.Sessions))
+ .ReturnsAsync(new DeletionBatchResult(1, 1, 0));
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ var indexIdx = deleteOrder.IndexOf(Constants.TableNames.SessionsIndex);
+ var sessIdx = deleteOrder.IndexOf(Constants.TableNames.Sessions);
+ Assert.True(indexIdx >= 0, "SessionsIndex was not deleted");
+ Assert.True(sessIdx >= 0, "Sessions was not deleted");
+ Assert.True(indexIdx < sessIdx, "SessionsIndex must be deleted before Sessions");
+ }
+
+ // ============================================================ State-machine + resume tests ====
+
+ [Fact]
+ public async Task Handler_throws_when_state_is_None_with_matching_manifestId()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ harness.SessionRow["DeletionState"] = SessionDeletionState.None;
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+ }
+
+ [Fact]
+ public async Task Handler_throws_when_PendingDeletionManifestId_does_not_match_envelope()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ harness.SessionRow["PendingDeletionManifestId"] = "DIFFERENT-MANIFEST-ID";
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+ }
+
+ [Fact]
+ public async Task Handler_throws_when_sessions_row_missing_at_pickup()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ // Override Sessions read to return null.
+ harness.Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny()))
+ .ReturnsAsync((TableEntity?)null);
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+ }
+
+ [Fact]
+ public async Task Handler_resumes_when_DeletionState_already_Running_with_matching_manifestId()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ harness.SessionRow["DeletionState"] = SessionDeletionState.Running;
+ // Track CAS calls — Running → Running is NOT a valid transition; handler must NOT CAS.
+ harness.Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
+ It.IsAny(), It.IsAny(),
+ SessionDeletionState.Queued, SessionDeletionState.Running,
+ It.IsAny(), It.IsAny()))
+ .ThrowsAsync(new InvalidOperationException("CAS Queued→Running should not be issued on resume"));
+
+ // Should not throw — handler proceeds with cascade.
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ Assert.Single(harness.SignalR.SessionDeletedCalls);
+ }
+
+ [Fact]
+ public async Task Handler_throws_when_state_is_Poisoned()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ harness.SessionRow["DeletionState"] = SessionDeletionState.Poisoned;
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+ }
+
+ [Fact]
+ public async Task Handler_audits_deletion_step_failed_when_table_delete_throws()
+ {
+ var harness = new Harness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ // First delete throws — Handler must audit and rethrow.
+ harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
+ Constants.TableNames.Events, It.IsAny>(), It.IsAny()))
+ .ThrowsAsync(new RequestFailedException(503, "ServiceUnavailable"));
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+
+ Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_step_failed");
+ Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_completed");
+ Assert.Empty(harness.SignalR.SessionDeletedCalls);
+ }
+
+ // ============================================================ Harness ====
+
+ private sealed class Harness
+ {
+ public Mock Storage { get; }
+ public Mock Blob { get; }
+ public Mock Verifier { get; }
+ public Mock Maintenance { get; }
+ public FakeSignalRNotificationService SignalR { get; }
+ public SessionDeletionHandler Sut { get; }
+
+ public SessionDeletionEnvelope Envelope { get; }
+ public TableEntity SessionRow { get; }
+ public DeletionProgress ProgressFake { get; }
+
+ public List AuditCalls { get; } = new List();
+
+ public Harness()
+ {
+ Storage = new Mock(
+ Mock.Of(),
+ NullLogger.Instance);
+
+ Blob = new Mock(
+ new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
+ NullLogger.Instance,
+ false);
+
+ Verifier = new Mock(
+ Mock.Of(),
+ NullLogger.Instance);
+
+ Maintenance = new Mock();
+ Maintenance.Setup(m => m.LogAuditEntryAsync(
+ It.IsAny(), It.IsAny(), It.IsAny(),
+ It.IsAny(), It.IsAny(), It.IsAny?>()))
+ .Returns?>(
+ (tenantId, action, entityType, entityId, performedBy, details) =>
+ {
+ AuditCalls.Add(new AuditEntry(tenantId, action, entityType, entityId, performedBy, details));
+ return Task.FromResult(true);
+ });
+
+ SignalR = new FakeSignalRNotificationService();
+
+ Envelope = new SessionDeletionEnvelope
+ {
+ TenantId = TenantId,
+ SessionId = SessionId,
+ ManifestId = ManifestId,
+ Reason = "admin_delete",
+ EnqueuedAt = DateTime.UtcNow,
+ };
+
+ SessionRow = new TableEntity(TenantId, SessionId)
+ {
+ ["DeletionState"] = SessionDeletionState.Queued,
+ ["PendingDeletionManifestId"] = ManifestId,
+ };
+
+ ProgressFake = new DeletionProgress
+ {
+ SnapshotSha256 = Sha256,
+ CompletedSteps = new HashSet(),
+ VerificationDone = false,
+ CompletedAt = null,
+ };
+
+ Sut = new SessionDeletionHandler(
+ Storage.Object, Blob.Object, Verifier.Object,
+ Maintenance.Object, SignalR,
+ NullLogger.Instance);
+ }
+
+ public void SetHappyPath()
+ {
+ // Sessions row read (state-machine).
+ Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny()))
+ .ReturnsAsync(() => SessionRow);
+
+ // CAS Queued → Running succeeds.
+ Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
+ TenantId, SessionId,
+ SessionDeletionState.Queued, SessionDeletionState.Running,
+ It.IsAny(), It.IsAny()))
+ .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
+ {
+ Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
+ CurrentState = SessionDeletionState.Running,
+ CurrentManifestId = ManifestId,
+ });
+
+ // Per-table deletes succeed with token counts.
+ Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
+ It.IsAny(), It.IsAny>(), It.IsAny()))
+ .ReturnsAsync((string _, IReadOnlyList<(string, string)> keys, CancellationToken _) =>
+ new DeletionBatchResult(keys.Count, keys.Count, 0));
+
+ Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync(
+ It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()))
+ .Returns(Task.CompletedTask);
+
+ // Blob: progress download returns the live ProgressFake snapshot + a stable ETag.
+ Blob.Setup(b => b.DownloadDeletionProgressAsync(
+ TenantId, SessionId, ManifestId, It.IsAny()))
+ .ReturnsAsync(() => (CloneProgress(ProgressFake), "\"0xFAKE_ETAG\""));
+
+ // Update mutates the ProgressFake in place so each subsequent download sees the latest state.
+ Blob.Setup(b => b.UpdateDeletionProgressAsync(
+ TenantId, SessionId, ManifestId, It.IsAny(), It.IsAny(), It.IsAny()))
+ .ReturnsAsync((string _, string _, string _, DeletionProgress p, string _, CancellationToken _) =>
+ {
+ ProgressFake.CompletedSteps = new HashSet(p.CompletedSteps);
+ ProgressFake.VerificationDone = p.VerificationDone;
+ ProgressFake.CompletedAt = p.CompletedAt;
+ // PR4c F1 + F2: propagate the new per-key + tombstone fields too so the harness
+ // mirrors live behaviour for tests that inspect them post-Handle.
+ ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null
+ ? null : new HashSet(p.AggregateDecrementsApplied, StringComparer.Ordinal);
+ ProgressFake.RestoreReIncrementsApplied = p.RestoreReIncrementsApplied == null
+ ? null : new HashSet(p.RestoreReIncrementsApplied, StringComparer.Ordinal);
+ ProgressFake.TombstoneStarted = p.TombstoneStarted;
+ return "\"0xFAKE_ETAG_2\"";
+ });
+
+ // Verifier reports clean by default.
+ Verifier.Setup(v => v.VerifyAsync(It.IsAny(), It.IsAny()))
+ .ReturnsAsync(new CascadeVerificationResult(true, new List()));
+ }
+
+ // PR4c F6: Handler uses DownloadDeletionManifestWithShaAsync; return a stable sha hex
+ // string that matches the harness's progress.SnapshotSha256 so the binding check passes.
+ private const string FakeManifestSha = "1111111111111111111111111111111111111111111111111111111111111111";
+
+ public DeletionManifest SetFullSessionManifest()
+ {
+ var manifest = BuildFullManifest();
+ Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
+ TenantId, SessionId, ManifestId, It.IsAny()))
+ .ReturnsAsync((manifest, FakeManifestSha));
+ return manifest;
+ }
+
+ public DeletionManifest SetMinimalManifestNoInventory()
+ {
+ // Same as full but omit the AGGREGATE step 16 and side-row step 17.
+ var manifest = BuildFullManifest();
+ manifest.Steps = manifest.Steps
+ .Where(s => s.Class != DeletionStepClass.Aggregate
+ && !(s.Class == DeletionStepClass.PkRkExact && s.Table == Constants.TableNames.SessionInventoryContributions))
+ .ToList();
+ Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
+ TenantId, SessionId, ManifestId, It.IsAny()))
+ .ReturnsAsync((manifest, FakeManifestSha));
+ return manifest;
+ }
+
+ public DeletionManifest SetSingleStepManifest(DeletionStep step)
+ {
+ var manifest = new DeletionManifest
+ {
+ ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId,
+ CreatedAt = DateTime.UtcNow,
+ Steps = new List
+ {
+ step,
+ new DeletionStep
+ {
+ Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2,
+ Rows = new List
+ {
+ new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
+ new DeletionRowDump { Pk = TenantId, Rk = SessionId },
+ },
+ },
+ },
+ };
+ Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
+ TenantId, SessionId, ManifestId, It.IsAny()))
+ .ReturnsAsync((manifest, FakeManifestSha));
+ return manifest;
+ }
+
+ private static DeletionManifest BuildFullManifest() => new DeletionManifest
+ {
+ ManifestId = ManifestId,
+ TenantId = TenantId,
+ SessionId = SessionId,
+ CreatedAt = new DateTime(2026, 5, 12, 10, 0, 0, DateTimeKind.Utc),
+ CreatedBy = new DeletionActor { Type = "admin", Actor = "alice@example.com" },
+ Reason = "admin_delete",
+ RetentionContext = new DeletionRetentionContext { TenantRetentionDays = 90 },
+ SchemaHash = "sha256:test",
+ Steps = new List
+ {
+ new DeletionStep { Order = 1, Table = Constants.TableNames.Events, Class = DeletionStepClass.PkBySession, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "evt-1" } } },
+ new DeletionStep { Order = 2, Table = Constants.TableNames.RuleResults, Class = DeletionStepClass.PkBySession, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "rule-1" } } },
+ new DeletionStep { Order = 3, Table = Constants.TableNames.AppInstallSummaries, Class = DeletionStepClass.PropTenantPk, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = "app-1" } } },
+ new DeletionStep { Order = 4, Table = Constants.TableNames.VulnerabilityReports, Class = DeletionStepClass.PkRkExact, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "report" } } },
+ new DeletionStep { Order = 5, Table = Constants.TableNames.DeviceSnapshot, Class = DeletionStepClass.PkRkExact, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } },
+ new DeletionStep { Order = 6, Table = Constants.TableNames.EventSessionIndex, Class = DeletionStepClass.PkRkExact, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } },
+ new DeletionStep { Order = 7, Table = Constants.TableNames.Signals, Class = DeletionStepClass.PkBySession, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "sig-1" } } },
+ new DeletionStep { Order = 8, Table = Constants.TableNames.DecisionTransitions, Class = DeletionStepClass.PkBySession, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "trans-1" } } },
+ new DeletionStep { Order = 9, Table = Constants.TableNames.EventTypeIndex, Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_install_failed", Rk = $"6299999999999999_{SessionId}" } } },
+ new DeletionStep { Order = 10, Table = Constants.TableNames.CveIndex, Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId } } },
+ new DeletionStep { Order = 11, Table = Constants.TableNames.SessionsByTerminal, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_failed", Rk = "rk-1" } } },
+ new DeletionStep { Order = 12, Table = Constants.TableNames.SessionsByStage, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_AccountSetup", Rk = "rk-2" } } },
+ new DeletionStep { Order = 13, Table = Constants.TableNames.DeadEndsByReason, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0,
+ Rows = new List() },
+ new DeletionStep { Order = 14, Table = Constants.TableNames.ClassifierVerdictsByIdLevel, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_clf_warn", Rk = "rk-3" } } },
+ new DeletionStep { Order = 15, Table = Constants.TableNames.SignalsByKind, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_hello", Rk = "rk-4" } } },
+ new DeletionStep
+ {
+ Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement,
+ Class = DeletionStepClass.Aggregate, RowCount = 3,
+ Decrements = new List
+ {
+ new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" },
+ new DeletionDecrementKey { Vendor = "Adobe", Name = "Acrobat", Version = "23.1" },
+ new DeletionDecrementKey { Vendor = "Mozilla", Name = "Firefox", Version = "120.0" },
+ },
+ },
+ new DeletionStep { Order = 17, Table = Constants.TableNames.SessionInventoryContributions, Class = DeletionStepClass.PkRkExact, RowCount = 1,
+ Rows = new List { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } },
+ new DeletionStep
+ {
+ Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2,
+ Rows = new List
+ {
+ new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
+ new DeletionRowDump { Pk = TenantId, Rk = SessionId },
+ },
+ },
+ },
+ PreflightCounts = new Dictionary(),
+ };
+
+ private static DeletionProgress CloneProgress(DeletionProgress source) => new DeletionProgress
+ {
+ SnapshotSha256 = source.SnapshotSha256,
+ CompletedSteps = new HashSet(source.CompletedSteps),
+ VerificationDone = source.VerificationDone,
+ CompletedAt = source.CompletedAt,
+ };
+ }
+
+ private sealed record AuditEntry(
+ string TenantId, string Action, string EntityType, string EntityId, string PerformedBy,
+ Dictionary? Details);
+}
diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs
new file mode 100644
index 00000000..a32a54c4
--- /dev/null
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs
@@ -0,0 +1,736 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure;
+using Azure.Data.Tables;
+using Azure.Storage.Blobs;
+using Azure.Storage.Queues;
+using Azure.Storage.Queues.Models;
+using AutopilotMonitor.Functions.Services;
+using AutopilotMonitor.Functions.Services.Deletion;
+using AutopilotMonitor.Shared;
+using AutopilotMonitor.Shared.DataAccess;
+using AutopilotMonitor.Shared.Models;
+using AutopilotMonitor.Shared.Models.Deletion;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Newtonsoft.Json;
+using Xunit;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+///
+/// Codex review fixes for PR4 + PR4b — six findings, all High/Medium severity. Tests here are
+/// organized one section per finding (F1 → F6). Where a finding is best exercised via the
+/// existing test harnesses (Handler / RestoreService / Worker), the new fact extends that file;
+/// where the assertion is layer-pure (e.g. BlobStorage SHA round-trip), it lives here as a
+/// focused unit test.
+///
+public class SessionDeletionPr4cCodexFixesTests
+{
+ private const string TenantId = "11111111-1111-1111-1111-111111111111";
+ private const string SessionId = "22222222-2222-2222-2222-222222222222";
+ private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
+ private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
+
+ // ============================================================ F1: AGGREGATE per-key idempotency ====
+
+ [Fact]
+ public async Task F1_Cascade_aggregate_skips_already_applied_decrement_keys_on_retry()
+ {
+ var harness = new HandlerHarness();
+ harness.SetHappyPath();
+ var manifest = harness.SetFullSessionManifest();
+ // Simulate a prior partial run: pretend keys for "Microsoft:Office:16.0" and
+ // "Adobe:Acrobat:23.1" were already decremented, but the AGGREGATE step never marked
+ // complete. On retry the Handler must skip those two and only decrement Firefox.
+ harness.ProgressFake.AggregateDecrementsApplied = new HashSet(StringComparer.Ordinal)
+ {
+ "Microsoft:Office:16.0",
+ "Adobe:Acrobat:23.1",
+ };
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, "Microsoft", "Office", "16.0", It.IsAny()), Times.Never);
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, "Adobe", "Acrobat", "23.1", It.IsAny()), Times.Never);
+ harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, "Mozilla", "Firefox", "120.0", It.IsAny()), Times.Once);
+ }
+
+ [Fact]
+ public async Task F1_Cascade_aggregate_persists_per_key_progress_before_each_decrement()
+ {
+ // Ordering invariant: persist-first / decrement-second bounds drift to +1 per crash.
+ var harness = new HandlerHarness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+
+ var callOrder = new List();
+ harness.Blob.Setup(b => b.UpdateDeletionProgressAsync(
+ TenantId, SessionId, ManifestId,
+ It.IsAny(), It.IsAny(), It.IsAny()))
+ .Returns((_, _, _, p, _, _) =>
+ {
+ // Capture progress state mutations that include AggregateDecrementsApplied changes
+ // so we can correlate them with the decrement order.
+ var snapshot = p.AggregateDecrementsApplied == null
+ ? "init"
+ : string.Join(",", p.AggregateDecrementsApplied);
+ callOrder.Add($"persist:{snapshot}");
+ harness.ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null
+ ? null : new HashSet(p.AggregateDecrementsApplied, StringComparer.Ordinal);
+ harness.ProgressFake.CompletedSteps = new HashSet(p.CompletedSteps);
+ harness.ProgressFake.VerificationDone = p.VerificationDone;
+ harness.ProgressFake.CompletedAt = p.CompletedAt;
+ harness.ProgressFake.TombstoneStarted = p.TombstoneStarted;
+ return Task.FromResult("\"0xFAKE_ETAG\"");
+ });
+ harness.Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync(
+ TenantId, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()))
+ .Callback((string _, string vendor, string name, string ver, CancellationToken _) =>
+ callOrder.Add($"decrement:{vendor}:{name}:{ver}"))
+ .Returns(Task.CompletedTask);
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ // Each "decrement:X" must be preceded by a "persist:…X…" entry. We verify the pattern by
+ // checking that for every decrement, the most-recent prior persist contains the composite.
+ for (var i = 0; i < callOrder.Count; i++)
+ {
+ if (!callOrder[i].StartsWith("decrement:", StringComparison.Ordinal)) continue;
+ var composite = callOrder[i].Substring("decrement:".Length);
+ var priorPersists = callOrder.Take(i).Where(c => c.StartsWith("persist:", StringComparison.Ordinal)).ToList();
+ Assert.NotEmpty(priorPersists);
+ Assert.Contains(priorPersists, p => p.Contains(composite, StringComparison.Ordinal));
+ }
+ }
+
+ // ============================================================ F2: TombstoneStarted gap closure ====
+
+ [Fact]
+ public async Task F2_Cascade_persists_TombstoneStarted_before_first_tombstone_delete()
+ {
+ var harness = new HandlerHarness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+
+ var callOrder = new List();
+ harness.Blob.Setup(b => b.UpdateDeletionProgressAsync(
+ TenantId, SessionId, ManifestId,
+ It.IsAny(), It.IsAny(), It.IsAny()))
+ .Returns((_, _, _, p, _, _) =>
+ {
+ if (p.TombstoneStarted && !harness.ProgressFake.TombstoneStarted)
+ callOrder.Add("persist:tombstone-started");
+ harness.ProgressFake.CompletedSteps = new HashSet(p.CompletedSteps);
+ harness.ProgressFake.VerificationDone = p.VerificationDone;
+ harness.ProgressFake.CompletedAt = p.CompletedAt;
+ harness.ProgressFake.TombstoneStarted = p.TombstoneStarted;
+ harness.ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null
+ ? null : new HashSet(p.AggregateDecrementsApplied, StringComparer.Ordinal);
+ return Task.FromResult("\"0xFAKE_ETAG\"");
+ });
+ harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
+ It.Is(t => t == Constants.TableNames.SessionsIndex || t == Constants.TableNames.Sessions),
+ It.IsAny>(), It.IsAny()))
+ .Callback((string t, IReadOnlyList<(string, string)> _, CancellationToken _) =>
+ callOrder.Add($"delete:{t}"))
+ .ReturnsAsync(new DeletionBatchResult(1, 1, 0));
+
+ await harness.Sut.HandleAsync(harness.Envelope);
+
+ var tombstonePersistIdx = callOrder.IndexOf("persist:tombstone-started");
+ var firstTombstoneDeleteIdx = callOrder.FindIndex(c => c.StartsWith("delete:Sessions", StringComparison.Ordinal));
+ Assert.True(tombstonePersistIdx >= 0, "TombstoneStarted persist was not observed");
+ Assert.True(firstTombstoneDeleteIdx >= 0, "Tombstone delete was not observed");
+ Assert.True(tombstonePersistIdx < firstTombstoneDeleteIdx,
+ "TombstoneStarted=true must be persisted BEFORE any tombstone-row delete (Codex F2 gap closure).");
+ }
+
+ [Fact]
+ public async Task F2_Restore_full_dispatches_on_TombstoneStarted_gap_when_sessions_null_and_completedAt_null()
+ {
+ // The exact gap scenario: cascade wrote TombstoneStarted=true, deleted Sessions row, but
+ // crashed before writing CompletedAt. Restore must dispatch as Full (not corrupt-state).
+ var harness = new RestoreHarness();
+ harness.SetPoisonedCascade(); // gives us a manifest
+ harness.SessionRow = null; // tombstone removed it
+ harness.Progress.CompletedAt = null;
+ harness.Progress.TombstoneStarted = true; // PR4c F2: gap signal
+
+ var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
+
+ Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome);
+ Assert.Equal("full", result.Mode);
+ }
+
+ // ============================================================ F3: Full restore retry safety ====
+
+ [Fact]
+ public async Task F3_Restore_full_inserts_sessions_row_last_not_first()
+ {
+ var harness = new RestoreHarness();
+ harness.SetCompletedCascade();
+
+ var insertOrder = new List();
+ harness.Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync(
+ It.IsAny(), It.IsAny>(),
+ It.IsAny(), It.IsAny()))
+ .Callback((string table, IReadOnlyList _, RestoreMode _, CancellationToken _) =>
+ insertOrder.Add(table))
+ .ReturnsAsync((string _, IReadOnlyList rows, RestoreMode _, CancellationToken _) =>
+ new RestoreBatchResult(rows.Count, rows.Count, 0));
+
+ await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
+
+ // Sessions row must be the very LAST insert.
+ var lastInsert = insertOrder.LastOrDefault();
+ Assert.Equal(Constants.TableNames.Sessions, lastInsert);
+ // SessionsIndex must come BEFORE Sessions.
+ var sessionsIdx = insertOrder.LastIndexOf(Constants.TableNames.Sessions);
+ var sessionsIndexIdx = insertOrder.LastIndexOf(Constants.TableNames.SessionsIndex);
+ Assert.True(sessionsIndexIdx < sessionsIdx,
+ "SessionsIndex must be restored before Sessions (PR4c F3a — Sessions LAST).");
+ }
+
+ [Fact]
+ public async Task F3_Restore_full_409_now_counts_as_skipped_no_throw()
+ {
+ // Simulates retry-after-partial-failure: some rows already inserted by the previous
+ // attempt. PR4c F3b: Full mode now 409-ignores like Partial; the result reports them
+ // in RowsSkippedByTable, no exception.
+ var harness = new RestoreHarness();
+ harness.SetCompletedCascade();
+ harness.Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync(
+ Constants.TableNames.Events, It.IsAny>(),
+ RestoreMode.Full, It.IsAny()))
+ .ReturnsAsync(new RestoreBatchResult(attempted: 1, restored: 0, skipped: 1));
+
+ var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
+
+ Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome);
+ Assert.Equal(1, result.RowsSkippedByTable[Constants.TableNames.Events]);
+ }
+
+ // ============================================================ F4: Partial restore per-key idempotency ====
+
+ [Fact]
+ public async Task F4_Restore_partial_skips_already_applied_re_increment_keys_on_retry()
+ {
+ var harness = new RestoreHarness();
+ harness.SetPoisonedCascade();
+ // Simulate a prior partial restore that already re-incremented Office + Acrobat but
+ // crashed before the CAS Poisoned → None (the exact Codex F4 scenario).
+ var aggregateOrder = harness.Manifest.Steps.First(s => s.Class == DeletionStepClass.Aggregate).Order;
+ harness.Progress.CompletedSteps.Add(aggregateOrder);
+ harness.Progress.RestoreReIncrementsApplied = new HashSet(StringComparer.Ordinal)
+ {
+ "Microsoft:Office:16.0",
+ "Adobe:Acrobat:23.1",
+ };
+
+ await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
+
+ harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
+ TenantId, It.Is(k => k.Vendor == "Microsoft" && k.Name == "Office"),
+ It.IsAny()), Times.Never);
+ harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
+ TenantId, It.Is(k => k.Vendor == "Adobe" && k.Name == "Acrobat"),
+ It.IsAny()), Times.Never);
+ harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
+ TenantId, It.Is(k => k.Vendor == "Mozilla" && k.Name == "Firefox"),
+ It.IsAny()), Times.Once);
+ }
+
+ // ============================================================ F5: Stale poison envelope ====
+
+ [Fact]
+ public async Task F5_Worker_skips_poison_state_transition_when_envelope_manifestId_does_not_match_current_pending()
+ {
+ var harness = new WorkerHarness();
+ var staleEnvelope = new SessionDeletionEnvelope
+ {
+ TenantId = TenantId, SessionId = SessionId,
+ ManifestId = "STALE-OLD-MANIFEST-ID", // ≠ row.PendingDeletionManifestId
+ Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow,
+ };
+ harness.EnqueueMessage(JsonConvert.SerializeObject(staleEnvelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1);
+ // Default GetSessionRowAsync setup returns PendingDeletionManifestId = ManifestId
+ // (a fresh active cascade); the envelope's STALE-OLD-MANIFEST-ID must NOT touch state.
+
+ await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
+
+ // CAS must NOT have been issued — fresh cascade preserved.
+ harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync(
+ TenantId, SessionId,
+ It.IsAny(), SessionDeletionState.Poisoned,
+ It.IsAny(), It.IsAny()),
+ Times.Never);
+ // Poison-queue + audit still fire so the stale message is off the main queue.
+ harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny(), It.IsAny()),
+ Times.AtLeastOnce);
+ }
+
+ // ============================================================ F6: SHA binding ====
+
+ [Fact]
+ public async Task F6_DownloadDeletionManifestWithShaAsync_returns_hex_sha_matching_blob_metadata()
+ {
+ // Layer-pure: build a manifest, upload through the fake, download via with-Sha variant,
+ // verify the returned SHA equals the metadata SHA.
+ var fake = new FakeBlobService();
+ var manifest = new DeletionManifest
+ {
+ ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId,
+ Reason = "admin_delete", SchemaHash = "sha256:test",
+ };
+
+ var pointer = await fake.UploadDeletionManifestAsync(manifest);
+ var (downloaded, sha) = await fake.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId);
+
+ Assert.Equal(manifest.ManifestId, downloaded.ManifestId);
+ Assert.Equal(pointer.SnapshotSha256, sha);
+ Assert.Matches("^[0-9a-f]{64}$", sha);
+ }
+
+ [Fact]
+ public async Task F6_Cascade_throws_when_progress_snapshotSha256_does_not_match_manifest_sha()
+ {
+ var harness = new HandlerHarness();
+ harness.SetHappyPath();
+ harness.SetFullSessionManifest();
+ // Force a mismatch: progress carries one SHA, manifest download returns a different one.
+ harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
+ TenantId, SessionId, ManifestId, It.IsAny()))
+ .ReturnsAsync((new DeletionManifest
+ {
+ ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId,
+ Steps = new List(),
+ }, "deadbeef" + new string('0', 56)));
+ harness.ProgressFake.SnapshotSha256 = Sha256; // different value
+
+ await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope));
+ }
+
+ [Fact]
+ public async Task F6_Restore_rejects_when_progress_snapshotSha256_does_not_match_manifest_sha()
+ {
+ var harness = new RestoreHarness();
+ harness.SetCompletedCascade();
+ // Force the SHA binding to mismatch.
+ harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
+ TenantId, SessionId, ManifestId, It.IsAny()))
+ .ReturnsAsync((harness.Manifest, "deadbeef" + new string('0', 56)));
+
+ var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
+
+ Assert.Equal(SessionRestoreOutcome.RejectManifestCorruption, result.Outcome);
+ Assert.Contains("SHA binding", result.Message ?? string.Empty);
+ }
+
+ // ============================================================ DeletionProgress back-compat ====
+
+ [Fact]
+ public void DeletionProgress_defaults_keep_back_compat_for_PR1_PR4_progress_blobs()
+ {
+ // New fields all default to safe values so PR1-PR4 progress blobs (no new fields)
+ // deserialize cleanly without surprises.
+ var progress = new DeletionProgress();
+
+ Assert.Null(progress.AggregateDecrementsApplied);
+ Assert.Null(progress.RestoreReIncrementsApplied);
+ Assert.False(progress.TombstoneStarted);
+ }
+
+ // ============================================================ Test harnesses ====
+ // These mirror the per-test-file harnesses in SessionDeletionHandlerTests.cs and
+ // SessionRestoreServiceTests.cs, kept private so PR4c facts don't depend on the
+ // other files' internals.
+
+ private const string FakeManifestSha = Sha256;
+
+ private sealed class HandlerHarness
+ {
+ public Mock