mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat(backend): cascade-delete worker + restore + codex-fixes (PR4/7)
PR4 ships the cascade-delete executor for the safe-cascade-delete plan: the
SessionDeletionWorker (BackgroundService queue poll-loop) plus the
SessionDeletionHandler (per-step execution, live verification, tombstone).
Worker is NOT registered in Program.cs — that lights up in PR5.
Three logical phases land together as one commit per Oliver's call:
PR4 — Worker + Handler + CascadeVerificationService
- SessionDeletionWorker: BatchSize=1, kill-switch entry guard, heartbeat task
extending visibility every 60s during long handler runs, poison after
max-dequeue
- SessionDeletionHandler: download manifest+progress, CAS Queued→Running
(with Running→Running resume), per-step dispatch by Class, AGGREGATE
decrement, live verification, ordered tombstone (SessionsIndex first,
Sessions last)
- CascadeVerificationService: per-Class live residual scans (CveIndex is
manifest-key-only per §12-Q8; AGGREGATE+FINAL skipped)
- NotifySessionDeletedAsync added to ISignalRNotificationService
- BlobStorageService: DownloadDeletionProgressAsync + ETag-CAS
UpdateDeletionProgressAsync
PR4b — Restore endpoint
- POST /api/admin/sessions/{sessionId}/restore (GA-only via Policy Catalog)
- SessionRestoreService: dispatch table per §13.4, full + partial-poisoned-
recovery modes
- TableStorageService.Restore.cs: RestoreRowsByExactKeysInBatchesAsync +
ConvertFromPropValue + ConvertDumpToEntity (Sessions-row override forces
DeletionState=None on restore so the row isn't permanently locked)
- RestoreSoftwareInventoryEntryByKeyAsync helper
- Worker MoveToPoisonAsync now CAS-transitions Running→Poisoned (E1 fix)
so restore can dispatch into partial-restore mode
PR4c — Codex review fixes (6 findings)
- F1: AGGREGATE per-key idempotency via DeletionProgress.AggregateDecrementsApplied
(persist-first/decrement-second bounds drift to +1 per crash vs unbounded -N)
- F2: TombstoneStarted flag closes the gap where sessions=null+completedAt=null
blocked restore after a tombstone crash
- F3a: Sessions row inserted LAST in full restore (was first → trapped retry
with RejectAlreadyAtOriginalState)
- F3b: RestoreMode.Full now 409-ignores (was throw-on-409); converges with
Partial mode, makes retry-after-partial-failure idempotent
- F4: Partial-restore per-key re-increment idempotency via
DeletionProgress.RestoreReIncrementsApplied
- F5: Poison transition pre-checks PendingDeletionManifestId so stale
envelopes can't flip a fresh active cascade to Poisoned
- F6: DownloadDeletionManifestWithShaAsync exposes snapshot SHA; Handler +
Restore both verify against progress.SnapshotSha256 (snapshot↔progress
binding per plan §3)
DeletionProgress schema gains 3 additive optional fields (AggregateDecrementsApplied,
RestoreReIncrementsApplied, TombstoneStarted) with safe defaults so PR1-PR4
progress blobs deserialize cleanly.
Tests: 1457/1457 backend xUnit facts green (1391 pre-PR4 + 66 new across PR4,
PR4b, PR4c). Full solution build green. Program.cs has zero
SessionDeletionWorker references (PR5 territory stays untouched).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+174
@@ -0,0 +1,174 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for the PR4 cascade-deletion progress-blob Download + CAS-Update helpers on
|
||||
/// <see cref="BlobStorageService"/>. Same fake-subclass pattern as
|
||||
/// <c>BlobStorageServiceDeletionManifestTests</c>: override the blob-IO seams, capture bytes
|
||||
/// + options, replay on download. No Azurite.
|
||||
/// </summary>
|
||||
public class BlobStorageServiceDeletionProgressTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
|
||||
private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
|
||||
|
||||
[Fact]
|
||||
public async Task Download_round_trips_progress_with_etag()
|
||||
{
|
||||
var sut = new FakeBlobStorageService();
|
||||
// Seed the fake with an initial progress blob written via the existing helper.
|
||||
var initialEtag = await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256);
|
||||
Assert.False(string.IsNullOrEmpty(initialEtag));
|
||||
|
||||
var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
|
||||
|
||||
Assert.Equal(Sha256, progress.SnapshotSha256);
|
||||
Assert.Empty(progress.CompletedSteps);
|
||||
Assert.False(progress.VerificationDone);
|
||||
Assert.Null(progress.CompletedAt);
|
||||
Assert.False(string.IsNullOrEmpty(etag));
|
||||
// ETag from a fresh download must match what UploadInitial returned.
|
||||
Assert.Equal(initialEtag, etag);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Update_with_matching_etag_writes_new_state_and_returns_fresh_etag()
|
||||
{
|
||||
var sut = new FakeBlobStorageService();
|
||||
var initialEtag = await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256);
|
||||
var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
|
||||
|
||||
progress.CompletedSteps.Add(1);
|
||||
progress.CompletedSteps.Add(2);
|
||||
var newEtag = await sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, etag);
|
||||
|
||||
Assert.False(string.IsNullOrEmpty(newEtag));
|
||||
Assert.NotEqual(initialEtag, newEtag);
|
||||
|
||||
var (refetched, _) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
|
||||
Assert.Equal(new HashSet<int> { 1, 2 }, refetched.CompletedSteps);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Update_with_mismatched_etag_throws_412_RequestFailedException()
|
||||
{
|
||||
var sut = new FakeBlobStorageService();
|
||||
await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256);
|
||||
var (progress, _) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
|
||||
|
||||
progress.CompletedSteps.Add(1);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RequestFailedException>(
|
||||
() => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, ifMatchEtag: "\"0xSTALE_ETAG\""));
|
||||
Assert.Equal(412, ex.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Update_writes_IfMatch_condition_on_the_upload_options()
|
||||
{
|
||||
var sut = new FakeBlobStorageService();
|
||||
await sut.UploadInitialDeletionProgressAsync(TenantId, SessionId, ManifestId, Sha256);
|
||||
var (progress, etag) = await sut.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId);
|
||||
|
||||
await sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, progress, etag);
|
||||
|
||||
Assert.NotNull(sut.LastWrittenProgressOptions);
|
||||
Assert.NotNull(sut.LastWrittenProgressOptions!.Conditions);
|
||||
// PR4 §12-Q10 contract: every update is an ETag-CAS via IfMatch (not IfNoneMatch).
|
||||
Assert.NotNull(sut.LastWrittenProgressOptions.Conditions!.IfMatch);
|
||||
Assert.Equal(new ETag(etag), sut.LastWrittenProgressOptions.Conditions.IfMatch);
|
||||
// Content type stays application/json — no gzip on the small progress blob.
|
||||
Assert.Equal("application/json", sut.LastWrittenProgressOptions.HttpHeaders!.ContentType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Update_throws_on_missing_required_fields()
|
||||
{
|
||||
var sut = new FakeBlobStorageService();
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => sut.UpdateDeletionProgressAsync("", SessionId, ManifestId, new DeletionProgress(), "etag"));
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => sut.UpdateDeletionProgressAsync(TenantId, "", ManifestId, new DeletionProgress(), "etag"));
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => sut.UpdateDeletionProgressAsync(TenantId, SessionId, "", new DeletionProgress(), "etag"));
|
||||
await Assert.ThrowsAsync<ArgumentNullException>(
|
||||
() => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, null!, "etag"));
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => sut.UpdateDeletionProgressAsync(TenantId, SessionId, ManifestId, new DeletionProgress(), ""));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Download_throws_on_missing_required_fields()
|
||||
{
|
||||
var sut = new FakeBlobStorageService();
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => sut.DownloadDeletionProgressAsync("", SessionId, ManifestId));
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => sut.DownloadDeletionProgressAsync(TenantId, "", ManifestId));
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => sut.DownloadDeletionProgressAsync(TenantId, SessionId, ""));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Test fixture ----
|
||||
|
||||
/// <summary>
|
||||
/// Captures the progress-blob bytes + options written by Upload/Update and replays them on
|
||||
/// Download. Each write returns a fresh ETag; an Update with a stale ETag throws 412 so the
|
||||
/// Handler's bounded-retry loop can be exercised against this fake.
|
||||
/// </summary>
|
||||
private sealed class FakeBlobStorageService : BlobStorageService
|
||||
{
|
||||
private byte[]? _lastBytes;
|
||||
private BlobUploadOptions? _lastOptions;
|
||||
private ETag _currentEtag;
|
||||
private int _etagCounter;
|
||||
|
||||
public BlobUploadOptions? LastWrittenProgressOptions => _lastOptions;
|
||||
|
||||
public FakeBlobStorageService()
|
||||
: base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false)
|
||||
{
|
||||
}
|
||||
|
||||
protected internal override Task<ETag> WriteDeletionProgressBlobAsync(
|
||||
string blobName, byte[] payload, BlobUploadOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
if (options.Conditions?.IfMatch != null
|
||||
&& _currentEtag != default
|
||||
&& options.Conditions.IfMatch != _currentEtag)
|
||||
{
|
||||
throw new RequestFailedException(412, "ConditionNotMet", "ConditionNotMet", innerException: null);
|
||||
}
|
||||
_lastBytes = payload;
|
||||
_lastOptions = options;
|
||||
_currentEtag = new ETag($"\"0xFAKE_ETAG_{++_etagCounter}\"");
|
||||
return Task.FromResult(_currentEtag);
|
||||
}
|
||||
|
||||
protected internal override Task<(byte[] Payload, ETag ETag)> ReadDeletionProgressBlobAsync(
|
||||
string blobName, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_lastBytes == null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Test must call UploadInitialDeletionProgressAsync before DownloadDeletionProgressAsync.");
|
||||
}
|
||||
return Task.FromResult((_lastBytes, _currentEtag));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Per-class behaviour tests for <see cref="CascadeVerificationService"/>. Verifies that each
|
||||
/// step class drives the right inventory-reader call shape (filter for scan-based classes,
|
||||
/// direct GetEntity for manifest-key-only classes), that AGGREGATE + FINAL are skipped, and
|
||||
/// that the residual-sample cap is honoured.
|
||||
/// </summary>
|
||||
public class CascadeVerificationServiceTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
|
||||
[Fact]
|
||||
public async Task Verify_returns_clean_when_all_tables_empty()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = BuildManifestWithAllClasses();
|
||||
|
||||
var result = await sut.VerifyAsync(manifest);
|
||||
|
||||
Assert.True(result.IsClean);
|
||||
Assert.Empty(result.Residuals);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PkBySession_uses_partition_filter()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
|
||||
},
|
||||
};
|
||||
|
||||
await sut.VerifyAsync(manifest);
|
||||
|
||||
var call = Assert.Single(reader.QueryCalls);
|
||||
Assert.Equal("Events", call.Table);
|
||||
Assert.Equal($"PartitionKey eq '{TenantId}_{SessionId}'", call.Filter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PropTenantPk_uses_partition_and_sessionid_filter()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 3, Table = "AppInstallSummaries", Class = DeletionStepClass.PropTenantPk, RowCount = 0 },
|
||||
},
|
||||
};
|
||||
|
||||
await sut.VerifyAsync(manifest);
|
||||
|
||||
var call = Assert.Single(reader.QueryCalls);
|
||||
Assert.Equal($"PartitionKey eq '{TenantId}' and SessionId eq '{SessionId}'", call.Filter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PkRkExact_uses_direct_GetEntity_per_manifest_row()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 5, Table = "DeviceSnapshot", Class = DeletionStepClass.PkRkExact, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = TenantId, Rk = SessionId } },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await sut.VerifyAsync(manifest);
|
||||
|
||||
Assert.Empty(reader.QueryCalls);
|
||||
var get = Assert.Single(reader.GetCalls);
|
||||
Assert.Equal("DeviceSnapshot", get.Table);
|
||||
Assert.Equal(TenantId, get.Pk);
|
||||
Assert.Equal(SessionId, get.Rk);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscriminatorPkRkExact_uses_manifest_key_GetEntity_not_live_scan()
|
||||
{
|
||||
// Plan §12-Q8: CveIndex is the deliberate exception. Manifest-key only — verifier MUST NOT
|
||||
// do a tenant-prefix scan on this class.
|
||||
var reader = new FakeReader();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 10, Table = "CveIndex", Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 2,
|
||||
Rows = new List<DeletionRowDump>
|
||||
{
|
||||
new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId },
|
||||
new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0002", Rk = SessionId },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await sut.VerifyAsync(manifest);
|
||||
|
||||
Assert.Empty(reader.QueryCalls); // no tenant-prefix scan
|
||||
Assert.Equal(2, reader.GetCalls.Count);
|
||||
Assert.All(reader.GetCalls, c => Assert.Equal("CveIndex", c.Table));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscriminatorPkRkSuffix_uses_tenant_prefix_scan_with_client_side_suffix_filter()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
// Seed with a row matching the suffix and one that doesn't — client-side filter must drop the non-match.
|
||||
reader.QueryResults["EventTypeIndex"] = new List<TableEntity>
|
||||
{
|
||||
BuildEntity($"{TenantId}_install_failed", $"6299999999999999_{SessionId}"), // matches
|
||||
BuildEntity($"{TenantId}_install_failed", "6299999999999999_other-session"), // does not match
|
||||
};
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 9, Table = "EventTypeIndex", Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 0 },
|
||||
},
|
||||
};
|
||||
|
||||
var result = await sut.VerifyAsync(manifest);
|
||||
|
||||
var call = Assert.Single(reader.QueryCalls);
|
||||
Assert.Contains($"PartitionKey ge '{TenantId}_'", call.Filter);
|
||||
Assert.Contains($"PartitionKey lt '{TenantId}_~'", call.Filter);
|
||||
// Suffix-matching row is the only residual reported.
|
||||
Assert.False(result.IsClean);
|
||||
Assert.Single(result.Residuals);
|
||||
Assert.EndsWith($"_{SessionId}", result.Residuals[0].Rk);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscriminatorPkProp_uses_tenant_prefix_and_sessionid_filter()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 11, Table = "SessionsByTerminal", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0 },
|
||||
},
|
||||
};
|
||||
|
||||
await sut.VerifyAsync(manifest);
|
||||
|
||||
var call = Assert.Single(reader.QueryCalls);
|
||||
Assert.Contains($"PartitionKey ge '{TenantId}_'", call.Filter);
|
||||
Assert.Contains($"PartitionKey lt '{TenantId}_~'", call.Filter);
|
||||
Assert.Contains($"SessionId eq '{SessionId}'", call.Filter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Aggregate_step_is_skipped_without_verification_call()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement,
|
||||
Class = DeletionStepClass.Aggregate, RowCount = 3,
|
||||
Decrements = new List<DeletionDecrementKey>
|
||||
{
|
||||
new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var result = await sut.VerifyAsync(manifest);
|
||||
|
||||
Assert.True(result.IsClean);
|
||||
Assert.Empty(reader.QueryCalls);
|
||||
Assert.Empty(reader.GetCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Final_step_is_skipped_without_verification_call()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2,
|
||||
Rows = new List<DeletionRowDump>
|
||||
{
|
||||
new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
|
||||
new DeletionRowDump { Pk = TenantId, Rk = SessionId },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var result = await sut.VerifyAsync(manifest);
|
||||
|
||||
Assert.True(result.IsClean);
|
||||
Assert.Empty(reader.QueryCalls);
|
||||
Assert.Empty(reader.GetCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Single_ghost_row_in_PkBySession_returns_not_clean_with_residual()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
reader.QueryResults["Events"] = new List<TableEntity>
|
||||
{
|
||||
BuildEntity($"{TenantId}_{SessionId}", "ghost-event-rk"),
|
||||
};
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
|
||||
},
|
||||
};
|
||||
|
||||
var result = await sut.VerifyAsync(manifest);
|
||||
|
||||
Assert.False(result.IsClean);
|
||||
var residual = Assert.Single(result.Residuals);
|
||||
Assert.Equal("Events", residual.Table);
|
||||
Assert.Equal($"{TenantId}_{SessionId}", residual.Pk);
|
||||
Assert.Equal("ghost-event-rk", residual.Rk);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Residual_sample_is_capped_at_MaxResidualSampleSize()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
reader.QueryResults["Events"] = Enumerable.Range(0, 200)
|
||||
.Select(i => BuildEntity($"{TenantId}_{SessionId}", $"rk-{i:D3}"))
|
||||
.ToList();
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
|
||||
},
|
||||
};
|
||||
|
||||
var result = await sut.VerifyAsync(manifest);
|
||||
|
||||
Assert.False(result.IsClean);
|
||||
Assert.Equal(CascadeVerificationService.MaxResidualSampleSize, result.Residuals.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task First_table_failure_short_circuits_remaining_steps()
|
||||
{
|
||||
var reader = new FakeReader();
|
||||
reader.QueryResults["Events"] = new List<TableEntity>
|
||||
{
|
||||
BuildEntity($"{TenantId}_{SessionId}", "ghost-rk"),
|
||||
};
|
||||
// RuleResults would have ghosts too, but the verifier must NOT enumerate it after the
|
||||
// first-table failure — the short-circuit keeps verification cost bounded.
|
||||
reader.QueryResults["RuleResults"] = new List<TableEntity>
|
||||
{
|
||||
BuildEntity($"{TenantId}_{SessionId}", "should-not-be-seen"),
|
||||
};
|
||||
var sut = new CascadeVerificationService(reader, NullLogger<CascadeVerificationService>.Instance);
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
|
||||
new DeletionStep { Order = 2, Table = "RuleResults", Class = DeletionStepClass.PkBySession, RowCount = 0 },
|
||||
},
|
||||
};
|
||||
|
||||
var result = await sut.VerifyAsync(manifest);
|
||||
|
||||
Assert.False(result.IsClean);
|
||||
// Only Events was queried; RuleResults short-circuited.
|
||||
Assert.Single(reader.QueryCalls);
|
||||
Assert.Equal("Events", reader.QueryCalls[0].Table);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Helpers ----
|
||||
|
||||
private static TableEntity BuildEntity(string pk, string rk) => new TableEntity(pk, rk);
|
||||
|
||||
private static DeletionManifest BuildManifestWithAllClasses() => new DeletionManifest
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 1, Table = "Events", Class = DeletionStepClass.PkBySession, RowCount = 0 },
|
||||
new DeletionStep { Order = 3, Table = "AppInstallSummaries", Class = DeletionStepClass.PropTenantPk, RowCount = 0 },
|
||||
new DeletionStep { Order = 5, Table = "DeviceSnapshot", Class = DeletionStepClass.PkRkExact, RowCount = 0 },
|
||||
new DeletionStep { Order = 9, Table = "EventTypeIndex", Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 0 },
|
||||
new DeletionStep { Order = 10, Table = "CveIndex", Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 0 },
|
||||
new DeletionStep { Order = 11, Table = "SessionsByTerminal", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0 },
|
||||
new DeletionStep { Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement, Class = DeletionStepClass.Aggregate, RowCount = 0 },
|
||||
new DeletionStep { Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 0 },
|
||||
},
|
||||
};
|
||||
|
||||
private sealed class FakeReader : ISessionDeletionInventoryReader
|
||||
{
|
||||
public List<(string Table, string Filter)> QueryCalls { get; } = new List<(string, string)>();
|
||||
public List<(string Table, string Pk, string Rk)> GetCalls { get; } = new List<(string, string, string)>();
|
||||
public Dictionary<string, List<TableEntity>> QueryResults { get; } = new Dictionary<string, List<TableEntity>>(StringComparer.Ordinal);
|
||||
public Dictionary<(string Table, string Pk, string Rk), TableEntity> GetResults { get; }
|
||||
= new Dictionary<(string, string, string), TableEntity>();
|
||||
|
||||
public Task<TableEntity?> GetSessionRowAsync(string tenantId, string sessionId, CancellationToken ct = default)
|
||||
=> Task.FromResult<TableEntity?>(null);
|
||||
|
||||
public Task<TableEntity?> GetSessionsIndexRowAsync(string tenantId, string indexRowKey, CancellationToken ct = default)
|
||||
=> Task.FromResult<TableEntity?>(null);
|
||||
|
||||
public async IAsyncEnumerable<TableEntity> QueryAsync(
|
||||
string tableName, string filter, [EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
QueryCalls.Add((tableName, filter));
|
||||
if (QueryResults.TryGetValue(tableName, out var rows))
|
||||
{
|
||||
foreach (var row in rows)
|
||||
{
|
||||
await Task.Yield();
|
||||
yield return row;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Task<TableEntity?> GetEntityOrNullAsync(string tableName, string partitionKey, string rowKey, CancellationToken ct = default)
|
||||
{
|
||||
GetCalls.Add((tableName, partitionKey, rowKey));
|
||||
GetResults.TryGetValue((tableName, partitionKey, rowKey), out var entity);
|
||||
return Task.FromResult<TableEntity?>(entity);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ public class FakeSignalRNotificationService : ISignalRNotificationService
|
||||
public int GlobalDismissAllCount { get; private set; }
|
||||
public List<(string TenantId, string SessionId, int Count)> RuleResultsCalls { get; } = new();
|
||||
public List<(string TenantId, string SessionId, string Risk)> VulnReportCalls { get; } = new();
|
||||
public List<(string TenantId, string SessionId)> SessionDeletedCalls { get; } = new();
|
||||
|
||||
public Task NotifyRuleResultsAvailableAsync(string tenantId, string sessionId, int resultCount)
|
||||
{
|
||||
@@ -33,6 +34,12 @@ public class FakeSignalRNotificationService : ISignalRNotificationService
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task NotifySessionDeletedAsync(string tenantId, string sessionId)
|
||||
{
|
||||
SessionDeletedCalls.Add((tenantId, sessionId));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task SendTenantNotificationAsync(string tenantId, NotificationAudience audience, object dto)
|
||||
{
|
||||
TenantSends.Add((tenantId, audience, dto));
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Functions.Sessions;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Thin HTTP-layer tests for <see cref="RestoreSessionFunction"/> (PR4b). The bulk of behaviour
|
||||
/// lives in <see cref="SessionRestoreService"/> (covered by <c>SessionRestoreServiceTests</c>);
|
||||
/// this file only verifies the layer-specific pieces: status mapping, route registration in
|
||||
/// the policy catalog (GA-only enforcement), and body shape.
|
||||
/// </summary>
|
||||
public class RestoreSessionFunctionTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(SessionRestoreOutcome.Restored, HttpStatusCode.OK)]
|
||||
[InlineData(SessionRestoreOutcome.DryRunOk, HttpStatusCode.OK)]
|
||||
[InlineData(SessionRestoreOutcome.RejectManifestNotFound, HttpStatusCode.NotFound)]
|
||||
[InlineData(SessionRestoreOutcome.RejectAlreadyAtOriginalState, HttpStatusCode.Conflict)]
|
||||
[InlineData(SessionRestoreOutcome.RejectActiveCascade, HttpStatusCode.Conflict)]
|
||||
[InlineData(SessionRestoreOutcome.RejectManifestIdMismatch, HttpStatusCode.Conflict)]
|
||||
[InlineData(SessionRestoreOutcome.RejectCorruptState, HttpStatusCode.Conflict)]
|
||||
[InlineData(SessionRestoreOutcome.RejectManifestCorruption, HttpStatusCode.Conflict)]
|
||||
[InlineData(SessionRestoreOutcome.RejectCasConflictOnClear, HttpStatusCode.Conflict)]
|
||||
public void MapOutcomeToStatus_returns_expected_status(SessionRestoreOutcome outcome, HttpStatusCode expected)
|
||||
{
|
||||
Assert.Equal(expected, RestoreSessionFunction.MapOutcomeToStatus(outcome));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Restore_route_is_registered_in_policy_catalog_as_GlobalAdminOnly()
|
||||
{
|
||||
// Per memory feedback_route_policy_catalog — every HTTP route MUST be registered in
|
||||
// EndpointAccessPolicyCatalog. An unregistered route fail-closes to 403. This test
|
||||
// guards the restore endpoint's GA-only enforcement.
|
||||
var entry = EndpointAccessPolicyCatalog.FindPolicy("POST", "admin/sessions/some-id/restore");
|
||||
|
||||
Assert.NotNull(entry);
|
||||
Assert.Equal(EndpointPolicy.GlobalAdminOnly, entry!.Policy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Restore_route_is_NOT_registered_for_non_POST_methods()
|
||||
{
|
||||
// GET to the restore route should not match the POST registration.
|
||||
var getEntry = EndpointAccessPolicyCatalog.FindPolicy("GET", "admin/sessions/some-id/restore");
|
||||
Assert.Null(getEntry);
|
||||
|
||||
var putEntry = EndpointAccessPolicyCatalog.FindPolicy("PUT", "admin/sessions/some-id/restore");
|
||||
Assert.Null(putEntry);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,667 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end Handler facts for plan §5 PR4. Moq-based, no Azurite. Every TableStorageService /
|
||||
/// BlobStorageService / CascadeVerificationService dependency is mocked through the existing
|
||||
/// virtual seams; the Handler exercises the full §3 schema flow without touching real storage.
|
||||
/// </summary>
|
||||
public class SessionDeletionHandlerTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
|
||||
private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
|
||||
|
||||
// ============================================================ §5 PR4 plan tests ====
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_deletes_full_session_round_trip()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
var manifest = harness.SetFullSessionManifest();
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
// All 6 table-step deletes invoked with the manifest's keys.
|
||||
foreach (var step in manifest.Steps.Where(s => s.Class != DeletionStepClass.Aggregate && s.Class != DeletionStepClass.Final && s.Rows.Count > 0))
|
||||
{
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
step.Table!, It.Is<IReadOnlyList<(string, string)>>(keys => keys.Count == step.Rows.Count), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce, $"step {step.Order} ({step.Table}) was not deleted");
|
||||
}
|
||||
// AGGREGATE decrement: 3 keys → 3 calls.
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Exactly(3));
|
||||
// Tombstone: 2 deletes (Index + Sessions).
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.SessionsIndex, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.Sessions, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
|
||||
Assert.Single(harness.SignalR.SessionDeletedCalls);
|
||||
Assert.Equal((TenantId, SessionId), harness.SignalR.SessionDeletedCalls[0]);
|
||||
|
||||
// Final audit asserts both step-completed entries AND deletion_completed.
|
||||
Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_completed");
|
||||
Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_step_completed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_includes_CveIndex_step()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.CveIndex,
|
||||
It.Is<IReadOnlyList<(string Pk, string Rk)>>(keys =>
|
||||
keys.Count == 1 && keys[0].Pk == $"{TenantId}_CVE-2024-0001" && keys[0].Rk == SessionId),
|
||||
It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_is_idempotent_when_re_run_after_completion()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
// Progress shows CompletedAt already set — handler must early-return.
|
||||
harness.ProgressFake.CompletedAt = DateTime.UtcNow;
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
Assert.Empty(harness.SignalR.SessionDeletedCalls);
|
||||
Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_completed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_resumes_after_partial_failure_by_skipping_completed_steps()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
// Steps 1–8 already complete on prior pickup; cascade resumes from step 9 onward.
|
||||
harness.ProgressFake.CompletedSteps.UnionWith(new[] { 1, 2, 3, 4, 5, 6, 7, 8 });
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
// Steps 1–8 must NOT be re-issued.
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.Events, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.RuleResults, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
// Step 9 onward DOES get issued.
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.EventTypeIndex, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_corrupted_snapshot_hash_mismatch_throws_so_worker_poisons()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
// BlobStorageService throws InvalidDataException on SHA mismatch (existing helper contract).
|
||||
// PR4c F6: Handler uses the with-Sha variant; mock it to throw on the same condition.
|
||||
harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new System.IO.InvalidDataException("snapshot SHA mismatch"));
|
||||
|
||||
await Assert.ThrowsAsync<System.IO.InvalidDataException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
|
||||
// No deletes, no signalR, no completion audit — the handler must throw before touching anything.
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
Assert.Empty(harness.SignalR.SessionDeletedCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_throws_on_DISCRIMINATOR_step_with_zero_rows_but_nonzero_count()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
var manifest = harness.SetSingleStepManifest(new DeletionStep
|
||||
{
|
||||
Order = 11, Table = "SessionsByTerminal",
|
||||
Class = DeletionStepClass.DiscriminatorPkProp,
|
||||
RowCount = 5,
|
||||
Rows = new List<DeletionRowDump>(), // empty despite RowCount = 5
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
"SessionsByTerminal", It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_does_not_tombstone_on_live_verification_failure()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
// Verifier reports residuals → handler aborts before tombstone.
|
||||
harness.Verifier.Setup(v => v.VerifyAsync(It.IsAny<DeletionManifest>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new CascadeVerificationResult(
|
||||
isClean: false,
|
||||
residuals: new List<CascadeResidualKey> { new CascadeResidualKey("Events", "ghost-pk", "ghost-rk") }));
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
|
||||
// The two FINAL-step deletes (SessionsIndex + Sessions) MUST NOT have fired.
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.SessionsIndex, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.Sessions, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
Assert.Empty(harness.SignalR.SessionDeletedCalls);
|
||||
Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_verification_failed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_decrements_software_inventory_via_contributions_row()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
// The 3 keys in the AGGREGATE step → 3 decrement calls.
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, "Microsoft", "Office", "16.0", It.IsAny<CancellationToken>()), Times.Once);
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, "Adobe", "Acrobat", "23.1", It.IsAny<CancellationToken>()), Times.Once);
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, "Mozilla", "Firefox", "120.0", It.IsAny<CancellationToken>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_skips_software_inventory_when_no_contributions_row()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
// Manifest WITHOUT the side-row step 16 + 17 (pre-side-row session).
|
||||
harness.SetMinimalManifestNoInventory();
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_aggregate_step_runs_before_side_row_delete()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
|
||||
var callOrder = new List<string>();
|
||||
harness.Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => callOrder.Add("decrement"))
|
||||
.Returns(Task.CompletedTask);
|
||||
harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.SessionInventoryContributions, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => callOrder.Add("delete-side-row"))
|
||||
.ReturnsAsync(new DeletionBatchResult(1, 1, 0));
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
// All decrement calls must happen before the side-row delete.
|
||||
var firstSideRow = callOrder.IndexOf("delete-side-row");
|
||||
Assert.True(firstSideRow >= 0, "side-row delete was never issued");
|
||||
Assert.True(callOrder.Take(firstSideRow).All(c => c == "decrement"),
|
||||
"decrements must complete before the SessionInventoryContributions side-row is deleted (§17.5)");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_does_not_touch_diagnostics_blob_even_when_recorded_in_manifest()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
var manifest = harness.SetFullSessionManifest();
|
||||
manifest.DiagnosticsBlobName = "diagnostics-uploads/contoso/session-abc/diag.zip";
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
// The Blob mock surfaces ONLY deletion-manifest container operations; no other blob calls.
|
||||
// Assertion: no Verify on diagnostics blob methods (we never set them up, so nothing fires).
|
||||
// Stronger: assert no Blob mock method other than the manifest helpers was ever invoked.
|
||||
// We do this by verifying the only Setup'd methods received calls — anything else would
|
||||
// have thrown via Moq's strict-mode... but we're using loose mode. Instead: assert no
|
||||
// Storage call mentions the diag blob name.
|
||||
harness.Storage.Verify(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
It.Is<string>(table => table.Contains("diagnostics", StringComparison.OrdinalIgnoreCase)),
|
||||
It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_emits_signalr_notification_on_completion()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
var call = Assert.Single(harness.SignalR.SessionDeletedCalls);
|
||||
Assert.Equal(TenantId, call.TenantId);
|
||||
Assert.Equal(SessionId, call.SessionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cascade_deletes_sessionsIndex_before_sessions_row()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
|
||||
var deleteOrder = new List<string>();
|
||||
harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.SessionsIndex, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => deleteOrder.Add(Constants.TableNames.SessionsIndex))
|
||||
.ReturnsAsync(new DeletionBatchResult(1, 1, 0));
|
||||
harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.Sessions, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => deleteOrder.Add(Constants.TableNames.Sessions))
|
||||
.ReturnsAsync(new DeletionBatchResult(1, 1, 0));
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
var indexIdx = deleteOrder.IndexOf(Constants.TableNames.SessionsIndex);
|
||||
var sessIdx = deleteOrder.IndexOf(Constants.TableNames.Sessions);
|
||||
Assert.True(indexIdx >= 0, "SessionsIndex was not deleted");
|
||||
Assert.True(sessIdx >= 0, "Sessions was not deleted");
|
||||
Assert.True(indexIdx < sessIdx, "SessionsIndex must be deleted before Sessions");
|
||||
}
|
||||
|
||||
// ============================================================ State-machine + resume tests ====
|
||||
|
||||
[Fact]
|
||||
public async Task Handler_throws_when_state_is_None_with_matching_manifestId()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
harness.SessionRow["DeletionState"] = SessionDeletionState.None;
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handler_throws_when_PendingDeletionManifestId_does_not_match_envelope()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
harness.SessionRow["PendingDeletionManifestId"] = "DIFFERENT-MANIFEST-ID";
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handler_throws_when_sessions_row_missing_at_pickup()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
// Override Sessions read to return null.
|
||||
harness.Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((TableEntity?)null);
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handler_resumes_when_DeletionState_already_Running_with_matching_manifestId()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
harness.SessionRow["DeletionState"] = SessionDeletionState.Running;
|
||||
// Track CAS calls — Running → Running is NOT a valid transition; handler must NOT CAS.
|
||||
harness.Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
SessionDeletionState.Queued, SessionDeletionState.Running,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new InvalidOperationException("CAS Queued→Running should not be issued on resume"));
|
||||
|
||||
// Should not throw — handler proceeds with cascade.
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
Assert.Single(harness.SignalR.SessionDeletedCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handler_throws_when_state_is_Poisoned()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
harness.SessionRow["DeletionState"] = SessionDeletionState.Poisoned;
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handler_audits_deletion_step_failed_when_table_delete_throws()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
// First delete throws — Handler must audit and rethrow.
|
||||
harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.Events, It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new RequestFailedException(503, "ServiceUnavailable"));
|
||||
|
||||
await Assert.ThrowsAsync<RequestFailedException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
|
||||
Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_step_failed");
|
||||
Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_completed");
|
||||
Assert.Empty(harness.SignalR.SessionDeletedCalls);
|
||||
}
|
||||
|
||||
// ============================================================ Harness ====
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public Mock<TableStorageService> Storage { get; }
|
||||
public Mock<BlobStorageService> Blob { get; }
|
||||
public Mock<CascadeVerificationService> Verifier { get; }
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; }
|
||||
public FakeSignalRNotificationService SignalR { get; }
|
||||
public SessionDeletionHandler Sut { get; }
|
||||
|
||||
public SessionDeletionEnvelope Envelope { get; }
|
||||
public TableEntity SessionRow { get; }
|
||||
public DeletionProgress ProgressFake { get; }
|
||||
|
||||
public List<AuditEntry> AuditCalls { get; } = new List<AuditEntry>();
|
||||
|
||||
public Harness()
|
||||
{
|
||||
Storage = new Mock<TableStorageService>(
|
||||
Mock.Of<TableServiceClient>(),
|
||||
NullLogger<TableStorageService>.Instance);
|
||||
|
||||
Blob = new Mock<BlobStorageService>(
|
||||
new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance,
|
||||
false);
|
||||
|
||||
Verifier = new Mock<CascadeVerificationService>(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(),
|
||||
NullLogger<CascadeVerificationService>.Instance);
|
||||
|
||||
Maintenance = new Mock<IMaintenanceRepository>();
|
||||
Maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.Returns<string, string, string, string, string, Dictionary<string, string>?>(
|
||||
(tenantId, action, entityType, entityId, performedBy, details) =>
|
||||
{
|
||||
AuditCalls.Add(new AuditEntry(tenantId, action, entityType, entityId, performedBy, details));
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
|
||||
SignalR = new FakeSignalRNotificationService();
|
||||
|
||||
Envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
ManifestId = ManifestId,
|
||||
Reason = "admin_delete",
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
SessionRow = new TableEntity(TenantId, SessionId)
|
||||
{
|
||||
["DeletionState"] = SessionDeletionState.Queued,
|
||||
["PendingDeletionManifestId"] = ManifestId,
|
||||
};
|
||||
|
||||
ProgressFake = new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = Sha256,
|
||||
CompletedSteps = new HashSet<int>(),
|
||||
VerificationDone = false,
|
||||
CompletedAt = null,
|
||||
};
|
||||
|
||||
Sut = new SessionDeletionHandler(
|
||||
Storage.Object, Blob.Object, Verifier.Object,
|
||||
Maintenance.Object, SignalR,
|
||||
NullLogger<SessionDeletionHandler>.Instance);
|
||||
}
|
||||
|
||||
public void SetHappyPath()
|
||||
{
|
||||
// Sessions row read (state-machine).
|
||||
Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => SessionRow);
|
||||
|
||||
// CAS Queued → Running succeeds.
|
||||
Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Queued, SessionDeletionState.Running,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.Running,
|
||||
CurrentManifestId = ManifestId,
|
||||
});
|
||||
|
||||
// Per-table deletes succeed with token counts.
|
||||
Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, IReadOnlyList<(string, string)> keys, CancellationToken _) =>
|
||||
new DeletionBatchResult(keys.Count, keys.Count, 0));
|
||||
|
||||
Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
// Blob: progress download returns the live ProgressFake snapshot + a stable ETag.
|
||||
Blob.Setup(b => b.DownloadDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => (CloneProgress(ProgressFake), "\"0xFAKE_ETAG\""));
|
||||
|
||||
// Update mutates the ProgressFake in place so each subsequent download sees the latest state.
|
||||
Blob.Setup(b => b.UpdateDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<DeletionProgress>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string _, string _, DeletionProgress p, string _, CancellationToken _) =>
|
||||
{
|
||||
ProgressFake.CompletedSteps = new HashSet<int>(p.CompletedSteps);
|
||||
ProgressFake.VerificationDone = p.VerificationDone;
|
||||
ProgressFake.CompletedAt = p.CompletedAt;
|
||||
// PR4c F1 + F2: propagate the new per-key + tombstone fields too so the harness
|
||||
// mirrors live behaviour for tests that inspect them post-Handle.
|
||||
ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null
|
||||
? null : new HashSet<string>(p.AggregateDecrementsApplied, StringComparer.Ordinal);
|
||||
ProgressFake.RestoreReIncrementsApplied = p.RestoreReIncrementsApplied == null
|
||||
? null : new HashSet<string>(p.RestoreReIncrementsApplied, StringComparer.Ordinal);
|
||||
ProgressFake.TombstoneStarted = p.TombstoneStarted;
|
||||
return "\"0xFAKE_ETAG_2\"";
|
||||
});
|
||||
|
||||
// Verifier reports clean by default.
|
||||
Verifier.Setup(v => v.VerifyAsync(It.IsAny<DeletionManifest>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new CascadeVerificationResult(true, new List<CascadeResidualKey>()));
|
||||
}
|
||||
|
||||
// PR4c F6: Handler uses DownloadDeletionManifestWithShaAsync; return a stable sha hex
|
||||
// string that matches the harness's progress.SnapshotSha256 so the binding check passes.
|
||||
private const string FakeManifestSha = "1111111111111111111111111111111111111111111111111111111111111111";
|
||||
|
||||
public DeletionManifest SetFullSessionManifest()
|
||||
{
|
||||
var manifest = BuildFullManifest();
|
||||
Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((manifest, FakeManifestSha));
|
||||
return manifest;
|
||||
}
|
||||
|
||||
public DeletionManifest SetMinimalManifestNoInventory()
|
||||
{
|
||||
// Same as full but omit the AGGREGATE step 16 and side-row step 17.
|
||||
var manifest = BuildFullManifest();
|
||||
manifest.Steps = manifest.Steps
|
||||
.Where(s => s.Class != DeletionStepClass.Aggregate
|
||||
&& !(s.Class == DeletionStepClass.PkRkExact && s.Table == Constants.TableNames.SessionInventoryContributions))
|
||||
.ToList();
|
||||
Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((manifest, FakeManifestSha));
|
||||
return manifest;
|
||||
}
|
||||
|
||||
public DeletionManifest SetSingleStepManifest(DeletionStep step)
|
||||
{
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
step,
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2,
|
||||
Rows = new List<DeletionRowDump>
|
||||
{
|
||||
new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
|
||||
new DeletionRowDump { Pk = TenantId, Rk = SessionId },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((manifest, FakeManifestSha));
|
||||
return manifest;
|
||||
}
|
||||
|
||||
private static DeletionManifest BuildFullManifest() => new DeletionManifest
|
||||
{
|
||||
ManifestId = ManifestId,
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
CreatedAt = new DateTime(2026, 5, 12, 10, 0, 0, DateTimeKind.Utc),
|
||||
CreatedBy = new DeletionActor { Type = "admin", Actor = "alice@example.com" },
|
||||
Reason = "admin_delete",
|
||||
RetentionContext = new DeletionRetentionContext { TenantRetentionDays = 90 },
|
||||
SchemaHash = "sha256:test",
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 1, Table = Constants.TableNames.Events, Class = DeletionStepClass.PkBySession, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "evt-1" } } },
|
||||
new DeletionStep { Order = 2, Table = Constants.TableNames.RuleResults, Class = DeletionStepClass.PkBySession, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "rule-1" } } },
|
||||
new DeletionStep { Order = 3, Table = Constants.TableNames.AppInstallSummaries, Class = DeletionStepClass.PropTenantPk, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = TenantId, Rk = "app-1" } } },
|
||||
new DeletionStep { Order = 4, Table = Constants.TableNames.VulnerabilityReports, Class = DeletionStepClass.PkRkExact, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "report" } } },
|
||||
new DeletionStep { Order = 5, Table = Constants.TableNames.DeviceSnapshot, Class = DeletionStepClass.PkRkExact, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } },
|
||||
new DeletionStep { Order = 6, Table = Constants.TableNames.EventSessionIndex, Class = DeletionStepClass.PkRkExact, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } },
|
||||
new DeletionStep { Order = 7, Table = Constants.TableNames.Signals, Class = DeletionStepClass.PkBySession, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "sig-1" } } },
|
||||
new DeletionStep { Order = 8, Table = Constants.TableNames.DecisionTransitions, Class = DeletionStepClass.PkBySession, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "trans-1" } } },
|
||||
new DeletionStep { Order = 9, Table = Constants.TableNames.EventTypeIndex, Class = DeletionStepClass.DiscriminatorPkRkSuffix, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_install_failed", Rk = $"6299999999999999_{SessionId}" } } },
|
||||
new DeletionStep { Order = 10, Table = Constants.TableNames.CveIndex, Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId } } },
|
||||
new DeletionStep { Order = 11, Table = Constants.TableNames.SessionsByTerminal, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_failed", Rk = "rk-1" } } },
|
||||
new DeletionStep { Order = 12, Table = Constants.TableNames.SessionsByStage, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_AccountSetup", Rk = "rk-2" } } },
|
||||
new DeletionStep { Order = 13, Table = Constants.TableNames.DeadEndsByReason, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0,
|
||||
Rows = new List<DeletionRowDump>() },
|
||||
new DeletionStep { Order = 14, Table = Constants.TableNames.ClassifierVerdictsByIdLevel, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_clf_warn", Rk = "rk-3" } } },
|
||||
new DeletionStep { Order = 15, Table = Constants.TableNames.SignalsByKind, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_hello", Rk = "rk-4" } } },
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement,
|
||||
Class = DeletionStepClass.Aggregate, RowCount = 3,
|
||||
Decrements = new List<DeletionDecrementKey>
|
||||
{
|
||||
new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" },
|
||||
new DeletionDecrementKey { Vendor = "Adobe", Name = "Acrobat", Version = "23.1" },
|
||||
new DeletionDecrementKey { Vendor = "Mozilla", Name = "Firefox", Version = "120.0" },
|
||||
},
|
||||
},
|
||||
new DeletionStep { Order = 17, Table = Constants.TableNames.SessionInventoryContributions, Class = DeletionStepClass.PkRkExact, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } },
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2,
|
||||
Rows = new List<DeletionRowDump>
|
||||
{
|
||||
new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
|
||||
new DeletionRowDump { Pk = TenantId, Rk = SessionId },
|
||||
},
|
||||
},
|
||||
},
|
||||
PreflightCounts = new Dictionary<string, int>(),
|
||||
};
|
||||
|
||||
private static DeletionProgress CloneProgress(DeletionProgress source) => new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = source.SnapshotSha256,
|
||||
CompletedSteps = new HashSet<int>(source.CompletedSteps),
|
||||
VerificationDone = source.VerificationDone,
|
||||
CompletedAt = source.CompletedAt,
|
||||
};
|
||||
}
|
||||
|
||||
private sealed record AuditEntry(
|
||||
string TenantId, string Action, string EntityType, string EntityId, string PerformedBy,
|
||||
Dictionary<string, string>? Details);
|
||||
}
|
||||
@@ -0,0 +1,736 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Queues;
|
||||
using Azure.Storage.Queues.Models;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Newtonsoft.Json;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Codex review fixes for PR4 + PR4b — six findings, all High/Medium severity. Tests here are
|
||||
/// organized one section per finding (F1 → F6). Where a finding is best exercised via the
|
||||
/// existing test harnesses (Handler / RestoreService / Worker), the new fact extends that file;
|
||||
/// where the assertion is layer-pure (e.g. BlobStorage SHA round-trip), it lives here as a
|
||||
/// focused unit test.
|
||||
/// </summary>
|
||||
public class SessionDeletionPr4cCodexFixesTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
|
||||
private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
|
||||
|
||||
// ============================================================ F1: AGGREGATE per-key idempotency ====
|
||||
|
||||
[Fact]
|
||||
public async Task F1_Cascade_aggregate_skips_already_applied_decrement_keys_on_retry()
|
||||
{
|
||||
var harness = new HandlerHarness();
|
||||
harness.SetHappyPath();
|
||||
var manifest = harness.SetFullSessionManifest();
|
||||
// Simulate a prior partial run: pretend keys for "Microsoft:Office:16.0" and
|
||||
// "Adobe:Acrobat:23.1" were already decremented, but the AGGREGATE step never marked
|
||||
// complete. On retry the Handler must skip those two and only decrement Firefox.
|
||||
harness.ProgressFake.AggregateDecrementsApplied = new HashSet<string>(StringComparer.Ordinal)
|
||||
{
|
||||
"Microsoft:Office:16.0",
|
||||
"Adobe:Acrobat:23.1",
|
||||
};
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, "Microsoft", "Office", "16.0", It.IsAny<CancellationToken>()), Times.Never);
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, "Adobe", "Acrobat", "23.1", It.IsAny<CancellationToken>()), Times.Never);
|
||||
harness.Storage.Verify(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, "Mozilla", "Firefox", "120.0", It.IsAny<CancellationToken>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task F1_Cascade_aggregate_persists_per_key_progress_before_each_decrement()
|
||||
{
|
||||
// Ordering invariant: persist-first / decrement-second bounds drift to +1 per crash.
|
||||
var harness = new HandlerHarness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
|
||||
var callOrder = new List<string>();
|
||||
harness.Blob.Setup(b => b.UpdateDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId,
|
||||
It.IsAny<DeletionProgress>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, string, DeletionProgress, string, CancellationToken>((_, _, _, p, _, _) =>
|
||||
{
|
||||
// Capture progress state mutations that include AggregateDecrementsApplied changes
|
||||
// so we can correlate them with the decrement order.
|
||||
var snapshot = p.AggregateDecrementsApplied == null
|
||||
? "init"
|
||||
: string.Join(",", p.AggregateDecrementsApplied);
|
||||
callOrder.Add($"persist:{snapshot}");
|
||||
harness.ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null
|
||||
? null : new HashSet<string>(p.AggregateDecrementsApplied, StringComparer.Ordinal);
|
||||
harness.ProgressFake.CompletedSteps = new HashSet<int>(p.CompletedSteps);
|
||||
harness.ProgressFake.VerificationDone = p.VerificationDone;
|
||||
harness.ProgressFake.CompletedAt = p.CompletedAt;
|
||||
harness.ProgressFake.TombstoneStarted = p.TombstoneStarted;
|
||||
return Task.FromResult("\"0xFAKE_ETAG\"");
|
||||
});
|
||||
harness.Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback((string _, string vendor, string name, string ver, CancellationToken _) =>
|
||||
callOrder.Add($"decrement:{vendor}:{name}:{ver}"))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
// Each "decrement:X" must be preceded by a "persist:…X…" entry. We verify the pattern by
|
||||
// checking that for every decrement, the most-recent prior persist contains the composite.
|
||||
for (var i = 0; i < callOrder.Count; i++)
|
||||
{
|
||||
if (!callOrder[i].StartsWith("decrement:", StringComparison.Ordinal)) continue;
|
||||
var composite = callOrder[i].Substring("decrement:".Length);
|
||||
var priorPersists = callOrder.Take(i).Where(c => c.StartsWith("persist:", StringComparison.Ordinal)).ToList();
|
||||
Assert.NotEmpty(priorPersists);
|
||||
Assert.Contains(priorPersists, p => p.Contains(composite, StringComparison.Ordinal));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================ F2: TombstoneStarted gap closure ====
|
||||
|
||||
[Fact]
|
||||
public async Task F2_Cascade_persists_TombstoneStarted_before_first_tombstone_delete()
|
||||
{
|
||||
var harness = new HandlerHarness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
|
||||
var callOrder = new List<string>();
|
||||
harness.Blob.Setup(b => b.UpdateDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId,
|
||||
It.IsAny<DeletionProgress>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, string, DeletionProgress, string, CancellationToken>((_, _, _, p, _, _) =>
|
||||
{
|
||||
if (p.TombstoneStarted && !harness.ProgressFake.TombstoneStarted)
|
||||
callOrder.Add("persist:tombstone-started");
|
||||
harness.ProgressFake.CompletedSteps = new HashSet<int>(p.CompletedSteps);
|
||||
harness.ProgressFake.VerificationDone = p.VerificationDone;
|
||||
harness.ProgressFake.CompletedAt = p.CompletedAt;
|
||||
harness.ProgressFake.TombstoneStarted = p.TombstoneStarted;
|
||||
harness.ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null
|
||||
? null : new HashSet<string>(p.AggregateDecrementsApplied, StringComparer.Ordinal);
|
||||
return Task.FromResult("\"0xFAKE_ETAG\"");
|
||||
});
|
||||
harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
It.Is<string>(t => t == Constants.TableNames.SessionsIndex || t == Constants.TableNames.Sessions),
|
||||
It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
|
||||
.Callback((string t, IReadOnlyList<(string, string)> _, CancellationToken _) =>
|
||||
callOrder.Add($"delete:{t}"))
|
||||
.ReturnsAsync(new DeletionBatchResult(1, 1, 0));
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
var tombstonePersistIdx = callOrder.IndexOf("persist:tombstone-started");
|
||||
var firstTombstoneDeleteIdx = callOrder.FindIndex(c => c.StartsWith("delete:Sessions", StringComparison.Ordinal));
|
||||
Assert.True(tombstonePersistIdx >= 0, "TombstoneStarted persist was not observed");
|
||||
Assert.True(firstTombstoneDeleteIdx >= 0, "Tombstone delete was not observed");
|
||||
Assert.True(tombstonePersistIdx < firstTombstoneDeleteIdx,
|
||||
"TombstoneStarted=true must be persisted BEFORE any tombstone-row delete (Codex F2 gap closure).");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task F2_Restore_full_dispatches_on_TombstoneStarted_gap_when_sessions_null_and_completedAt_null()
|
||||
{
|
||||
// The exact gap scenario: cascade wrote TombstoneStarted=true, deleted Sessions row, but
|
||||
// crashed before writing CompletedAt. Restore must dispatch as Full (not corrupt-state).
|
||||
var harness = new RestoreHarness();
|
||||
harness.SetPoisonedCascade(); // gives us a manifest
|
||||
harness.SessionRow = null; // tombstone removed it
|
||||
harness.Progress.CompletedAt = null;
|
||||
harness.Progress.TombstoneStarted = true; // PR4c F2: gap signal
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome);
|
||||
Assert.Equal("full", result.Mode);
|
||||
}
|
||||
|
||||
// ============================================================ F3: Full restore retry safety ====
|
||||
|
||||
[Fact]
|
||||
public async Task F3_Restore_full_inserts_sessions_row_last_not_first()
|
||||
{
|
||||
var harness = new RestoreHarness();
|
||||
harness.SetCompletedCascade();
|
||||
|
||||
var insertOrder = new List<string>();
|
||||
harness.Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<DeletionRowDump>>(),
|
||||
It.IsAny<RestoreMode>(), It.IsAny<CancellationToken>()))
|
||||
.Callback((string table, IReadOnlyList<DeletionRowDump> _, RestoreMode _, CancellationToken _) =>
|
||||
insertOrder.Add(table))
|
||||
.ReturnsAsync((string _, IReadOnlyList<DeletionRowDump> rows, RestoreMode _, CancellationToken _) =>
|
||||
new RestoreBatchResult(rows.Count, rows.Count, 0));
|
||||
|
||||
await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
// Sessions row must be the very LAST insert.
|
||||
var lastInsert = insertOrder.LastOrDefault();
|
||||
Assert.Equal(Constants.TableNames.Sessions, lastInsert);
|
||||
// SessionsIndex must come BEFORE Sessions.
|
||||
var sessionsIdx = insertOrder.LastIndexOf(Constants.TableNames.Sessions);
|
||||
var sessionsIndexIdx = insertOrder.LastIndexOf(Constants.TableNames.SessionsIndex);
|
||||
Assert.True(sessionsIndexIdx < sessionsIdx,
|
||||
"SessionsIndex must be restored before Sessions (PR4c F3a — Sessions LAST).");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task F3_Restore_full_409_now_counts_as_skipped_no_throw()
|
||||
{
|
||||
// Simulates retry-after-partial-failure: some rows already inserted by the previous
|
||||
// attempt. PR4c F3b: Full mode now 409-ignores like Partial; the result reports them
|
||||
// in RowsSkippedByTable, no exception.
|
||||
var harness = new RestoreHarness();
|
||||
harness.SetCompletedCascade();
|
||||
harness.Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.Events, It.IsAny<IReadOnlyList<DeletionRowDump>>(),
|
||||
RestoreMode.Full, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new RestoreBatchResult(attempted: 1, restored: 0, skipped: 1));
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome);
|
||||
Assert.Equal(1, result.RowsSkippedByTable[Constants.TableNames.Events]);
|
||||
}
|
||||
|
||||
// ============================================================ F4: Partial restore per-key idempotency ====
|
||||
|
||||
[Fact]
|
||||
public async Task F4_Restore_partial_skips_already_applied_re_increment_keys_on_retry()
|
||||
{
|
||||
var harness = new RestoreHarness();
|
||||
harness.SetPoisonedCascade();
|
||||
// Simulate a prior partial restore that already re-incremented Office + Acrobat but
|
||||
// crashed before the CAS Poisoned → None (the exact Codex F4 scenario).
|
||||
var aggregateOrder = harness.Manifest.Steps.First(s => s.Class == DeletionStepClass.Aggregate).Order;
|
||||
harness.Progress.CompletedSteps.Add(aggregateOrder);
|
||||
harness.Progress.RestoreReIncrementsApplied = new HashSet<string>(StringComparer.Ordinal)
|
||||
{
|
||||
"Microsoft:Office:16.0",
|
||||
"Adobe:Acrobat:23.1",
|
||||
};
|
||||
|
||||
await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
TenantId, It.Is<DeletionDecrementKey>(k => k.Vendor == "Microsoft" && k.Name == "Office"),
|
||||
It.IsAny<CancellationToken>()), Times.Never);
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
TenantId, It.Is<DeletionDecrementKey>(k => k.Vendor == "Adobe" && k.Name == "Acrobat"),
|
||||
It.IsAny<CancellationToken>()), Times.Never);
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
TenantId, It.Is<DeletionDecrementKey>(k => k.Vendor == "Mozilla" && k.Name == "Firefox"),
|
||||
It.IsAny<CancellationToken>()), Times.Once);
|
||||
}
|
||||
|
||||
// ============================================================ F5: Stale poison envelope ====
|
||||
|
||||
[Fact]
|
||||
public async Task F5_Worker_skips_poison_state_transition_when_envelope_manifestId_does_not_match_current_pending()
|
||||
{
|
||||
var harness = new WorkerHarness();
|
||||
var staleEnvelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId,
|
||||
ManifestId = "STALE-OLD-MANIFEST-ID", // ≠ row.PendingDeletionManifestId
|
||||
Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(staleEnvelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1);
|
||||
// Default GetSessionRowAsync setup returns PendingDeletionManifestId = ManifestId
|
||||
// (a fresh active cascade); the envelope's STALE-OLD-MANIFEST-ID must NOT touch state.
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
// CAS must NOT have been issued — fresh cascade preserved.
|
||||
harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
It.IsAny<string>(), SessionDeletionState.Poisoned,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
// Poison-queue + audit still fire so the stale message is off the main queue.
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
}
|
||||
|
||||
// ============================================================ F6: SHA binding ====
|
||||
|
||||
[Fact]
|
||||
public async Task F6_DownloadDeletionManifestWithShaAsync_returns_hex_sha_matching_blob_metadata()
|
||||
{
|
||||
// Layer-pure: build a manifest, upload through the fake, download via with-Sha variant,
|
||||
// verify the returned SHA equals the metadata SHA.
|
||||
var fake = new FakeBlobService();
|
||||
var manifest = new DeletionManifest
|
||||
{
|
||||
ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId,
|
||||
Reason = "admin_delete", SchemaHash = "sha256:test",
|
||||
};
|
||||
|
||||
var pointer = await fake.UploadDeletionManifestAsync(manifest);
|
||||
var (downloaded, sha) = await fake.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId);
|
||||
|
||||
Assert.Equal(manifest.ManifestId, downloaded.ManifestId);
|
||||
Assert.Equal(pointer.SnapshotSha256, sha);
|
||||
Assert.Matches("^[0-9a-f]{64}$", sha);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task F6_Cascade_throws_when_progress_snapshotSha256_does_not_match_manifest_sha()
|
||||
{
|
||||
var harness = new HandlerHarness();
|
||||
harness.SetHappyPath();
|
||||
harness.SetFullSessionManifest();
|
||||
// Force a mismatch: progress carries one SHA, manifest download returns a different one.
|
||||
harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((new DeletionManifest
|
||||
{
|
||||
ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId,
|
||||
Steps = new List<DeletionStep>(),
|
||||
}, "deadbeef" + new string('0', 56)));
|
||||
harness.ProgressFake.SnapshotSha256 = Sha256; // different value
|
||||
|
||||
await Assert.ThrowsAsync<InvalidDataException>(() => harness.Sut.HandleAsync(harness.Envelope));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task F6_Restore_rejects_when_progress_snapshotSha256_does_not_match_manifest_sha()
|
||||
{
|
||||
var harness = new RestoreHarness();
|
||||
harness.SetCompletedCascade();
|
||||
// Force the SHA binding to mismatch.
|
||||
harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((harness.Manifest, "deadbeef" + new string('0', 56)));
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.RejectManifestCorruption, result.Outcome);
|
||||
Assert.Contains("SHA binding", result.Message ?? string.Empty);
|
||||
}
|
||||
|
||||
// ============================================================ DeletionProgress back-compat ====
|
||||
|
||||
[Fact]
|
||||
public void DeletionProgress_defaults_keep_back_compat_for_PR1_PR4_progress_blobs()
|
||||
{
|
||||
// New fields all default to safe values so PR1-PR4 progress blobs (no new fields)
|
||||
// deserialize cleanly without surprises.
|
||||
var progress = new DeletionProgress();
|
||||
|
||||
Assert.Null(progress.AggregateDecrementsApplied);
|
||||
Assert.Null(progress.RestoreReIncrementsApplied);
|
||||
Assert.False(progress.TombstoneStarted);
|
||||
}
|
||||
|
||||
// ============================================================ Test harnesses ====
|
||||
// These mirror the per-test-file harnesses in SessionDeletionHandlerTests.cs and
|
||||
// SessionRestoreServiceTests.cs, kept private so PR4c facts don't depend on the
|
||||
// other files' internals.
|
||||
|
||||
private const string FakeManifestSha = Sha256;
|
||||
|
||||
private sealed class HandlerHarness
|
||||
{
|
||||
public Mock<TableStorageService> Storage { get; }
|
||||
public Mock<BlobStorageService> Blob { get; }
|
||||
public Mock<CascadeVerificationService> Verifier { get; }
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; }
|
||||
public FakeSignalRNotificationService SignalR { get; }
|
||||
public SessionDeletionHandler Sut { get; }
|
||||
|
||||
public SessionDeletionEnvelope Envelope { get; }
|
||||
public TableEntity SessionRow { get; set; }
|
||||
public DeletionProgress ProgressFake { get; }
|
||||
|
||||
public List<AuditEntry> AuditCalls { get; } = new List<AuditEntry>();
|
||||
|
||||
public HandlerHarness()
|
||||
{
|
||||
Storage = new Mock<TableStorageService>(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance);
|
||||
Blob = new Mock<BlobStorageService>(
|
||||
new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, false);
|
||||
Verifier = new Mock<CascadeVerificationService>(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(),
|
||||
NullLogger<CascadeVerificationService>.Instance);
|
||||
Maintenance = new Mock<IMaintenanceRepository>();
|
||||
Maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.Returns<string, string, string, string, string, Dictionary<string, string>?>(
|
||||
(t, a, e, eid, p, d) => { AuditCalls.Add(new AuditEntry(t, a, e, eid, p, d)); return Task.FromResult(true); });
|
||||
|
||||
SignalR = new FakeSignalRNotificationService();
|
||||
Envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId,
|
||||
Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
SessionRow = new TableEntity(TenantId, SessionId)
|
||||
{
|
||||
["DeletionState"] = SessionDeletionState.Queued,
|
||||
["PendingDeletionManifestId"] = ManifestId,
|
||||
};
|
||||
ProgressFake = new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = Sha256,
|
||||
CompletedSteps = new HashSet<int>(),
|
||||
VerificationDone = false,
|
||||
CompletedAt = null,
|
||||
};
|
||||
|
||||
Sut = new SessionDeletionHandler(
|
||||
Storage.Object, Blob.Object, Verifier.Object,
|
||||
Maintenance.Object, SignalR,
|
||||
NullLogger<SessionDeletionHandler>.Instance);
|
||||
}
|
||||
|
||||
public void SetHappyPath()
|
||||
{
|
||||
Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => SessionRow);
|
||||
Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Queued, SessionDeletionState.Running,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.Running, CurrentManifestId = ManifestId,
|
||||
});
|
||||
Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, IReadOnlyList<(string, string)> keys, CancellationToken _) =>
|
||||
new DeletionBatchResult(keys.Count, keys.Count, 0));
|
||||
Storage.Setup(s => s.DecrementSoftwareInventoryEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
Blob.Setup(b => b.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => (CloneProgress(ProgressFake), "\"0xFAKE_ETAG\""));
|
||||
Blob.Setup(b => b.UpdateDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<DeletionProgress>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string _, string _, DeletionProgress p, string _, CancellationToken _) =>
|
||||
{
|
||||
ProgressFake.CompletedSteps = new HashSet<int>(p.CompletedSteps);
|
||||
ProgressFake.VerificationDone = p.VerificationDone;
|
||||
ProgressFake.CompletedAt = p.CompletedAt;
|
||||
ProgressFake.TombstoneStarted = p.TombstoneStarted;
|
||||
ProgressFake.AggregateDecrementsApplied = p.AggregateDecrementsApplied == null
|
||||
? null : new HashSet<string>(p.AggregateDecrementsApplied, StringComparer.Ordinal);
|
||||
return "\"0xFAKE_ETAG_2\"";
|
||||
});
|
||||
Verifier.Setup(v => v.VerifyAsync(It.IsAny<DeletionManifest>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new CascadeVerificationResult(true, new List<CascadeResidualKey>()));
|
||||
}
|
||||
|
||||
public DeletionManifest SetFullSessionManifest()
|
||||
{
|
||||
var manifest = BuildFullManifest();
|
||||
Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((manifest, FakeManifestSha));
|
||||
return manifest;
|
||||
}
|
||||
|
||||
private static DeletionProgress CloneProgress(DeletionProgress s) => new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = s.SnapshotSha256,
|
||||
CompletedSteps = new HashSet<int>(s.CompletedSteps),
|
||||
VerificationDone = s.VerificationDone,
|
||||
CompletedAt = s.CompletedAt,
|
||||
TombstoneStarted = s.TombstoneStarted,
|
||||
AggregateDecrementsApplied = s.AggregateDecrementsApplied == null
|
||||
? null : new HashSet<string>(s.AggregateDecrementsApplied, StringComparer.Ordinal),
|
||||
RestoreReIncrementsApplied = s.RestoreReIncrementsApplied == null
|
||||
? null : new HashSet<string>(s.RestoreReIncrementsApplied, StringComparer.Ordinal),
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class RestoreHarness
|
||||
{
|
||||
public Mock<TableStorageService> Storage { get; }
|
||||
public Mock<BlobStorageService> Blob { get; }
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; }
|
||||
public SessionRestoreService Sut { get; }
|
||||
public TableEntity? SessionRow { get; set; }
|
||||
public DeletionProgress Progress { get; set; } = new DeletionProgress();
|
||||
public DeletionManifest Manifest { get; set; } = new DeletionManifest();
|
||||
|
||||
public RestoreHarness()
|
||||
{
|
||||
Storage = new Mock<TableStorageService>(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance);
|
||||
Blob = new Mock<BlobStorageService>(
|
||||
new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, false);
|
||||
Maintenance = new Mock<IMaintenanceRepository>();
|
||||
Maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.ReturnsAsync(true);
|
||||
|
||||
Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => SessionRow);
|
||||
Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => (Manifest, Sha256));
|
||||
Blob.Setup(b => b.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => (Progress, "\"0xETAG\""));
|
||||
Blob.Setup(b => b.UpdateDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId,
|
||||
It.IsAny<DeletionProgress>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync("\"0xETAG-NEW\"");
|
||||
Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<DeletionRowDump>>(),
|
||||
It.IsAny<RestoreMode>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, IReadOnlyList<DeletionRowDump> rows, RestoreMode _, CancellationToken _) =>
|
||||
new RestoreBatchResult(rows.Count, rows.Count, 0));
|
||||
Storage.Setup(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
It.IsAny<string>(), It.IsAny<DeletionDecrementKey>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId, SessionDeletionState.Poisoned, SessionDeletionState.None,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.None,
|
||||
});
|
||||
|
||||
Sut = new SessionRestoreService(Storage.Object, Blob.Object, Maintenance.Object,
|
||||
NullLogger<SessionRestoreService>.Instance);
|
||||
}
|
||||
|
||||
public void SetCompletedCascade()
|
||||
{
|
||||
SessionRow = null;
|
||||
Progress = new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = Sha256,
|
||||
CompletedSteps = new HashSet<int>(Enumerable.Range(1, 18)),
|
||||
VerificationDone = true,
|
||||
CompletedAt = DateTime.UtcNow,
|
||||
TombstoneStarted = true,
|
||||
};
|
||||
Manifest = BuildFullManifest();
|
||||
}
|
||||
|
||||
public void SetPoisonedCascade()
|
||||
{
|
||||
SessionRow = new TableEntity(TenantId, SessionId)
|
||||
{
|
||||
["DeletionState"] = SessionDeletionState.Poisoned,
|
||||
["PendingDeletionManifestId"] = ManifestId,
|
||||
};
|
||||
Progress = new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = Sha256,
|
||||
CompletedSteps = new HashSet<int> { 1, 2, 3 },
|
||||
VerificationDone = false,
|
||||
CompletedAt = null,
|
||||
};
|
||||
Manifest = BuildFullManifest();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class WorkerHarness
|
||||
{
|
||||
public Mock<QueueClient> MainQueue { get; }
|
||||
public Mock<QueueClient> PoisonQueue { get; }
|
||||
public Mock<TableStorageService> StorageMock { get; }
|
||||
public Mock<AdminConfigurationService> AdminConfig { get; }
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; }
|
||||
public SessionDeletionWorker Sut { get; }
|
||||
private readonly Queue<QueueMessage> _pendingMessages = new Queue<QueueMessage>();
|
||||
|
||||
public WorkerHarness()
|
||||
{
|
||||
MainQueue = new Mock<QueueClient>();
|
||||
PoisonQueue = new Mock<QueueClient>();
|
||||
MainQueue.Setup(q => q.CreateIfNotExistsAsync(
|
||||
It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((Response?)null);
|
||||
PoisonQueue.Setup(q => q.CreateIfNotExistsAsync(
|
||||
It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((Response?)null);
|
||||
|
||||
MainQueue.Setup(q => q.ReceiveMessagesAsync(
|
||||
It.IsAny<int>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<int, TimeSpan?, CancellationToken>((max, _, _) =>
|
||||
{
|
||||
var batch = new List<QueueMessage>();
|
||||
while (batch.Count < max && _pendingMessages.Count > 0) batch.Add(_pendingMessages.Dequeue());
|
||||
return Task.FromResult(Response.FromValue(batch.ToArray(), new Mock<Response>().Object));
|
||||
});
|
||||
MainQueue.Setup(q => q.DeleteMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new Mock<Response>().Object);
|
||||
PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, CancellationToken>((_, _) =>
|
||||
{
|
||||
var receipt = QueuesModelFactory.SendReceipt("p", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(7), "pr", DateTimeOffset.UtcNow);
|
||||
return Task.FromResult(Response.FromValue(receipt, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
StorageMock = new Mock<TableStorageService>(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance);
|
||||
// PR4c F5: default GetSessionRowAsync returns a row whose PendingDeletionManifestId
|
||||
// matches the constant ManifestId, simulating a FRESH ACTIVE cascade. The stale-
|
||||
// envelope test then sends an envelope with a DIFFERENT manifestId.
|
||||
StorageMock.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => new TableEntity(TenantId, SessionId)
|
||||
{
|
||||
["DeletionState"] = SessionDeletionState.Running,
|
||||
["PendingDeletionManifestId"] = ManifestId, // fresh cascade
|
||||
});
|
||||
StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.Poisoned,
|
||||
});
|
||||
|
||||
var blobMock = new Mock<BlobStorageService>(
|
||||
new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, false);
|
||||
var verifierMock = new Mock<CascadeVerificationService>(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(),
|
||||
NullLogger<CascadeVerificationService>.Instance);
|
||||
var handlerMock = new Mock<SessionDeletionHandler>(
|
||||
StorageMock.Object, blobMock.Object, verifierMock.Object,
|
||||
Mock.Of<IMaintenanceRepository>(),
|
||||
new FakeSignalRNotificationService(),
|
||||
NullLogger<SessionDeletionHandler>.Instance);
|
||||
handlerMock.Setup(h => h.HandleAsync(It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
AdminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(),
|
||||
NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions()));
|
||||
AdminConfig.Setup(a => a.GetConfigurationAsync())
|
||||
.ReturnsAsync(new AdminConfiguration { SessionDeletionKillSwitch = false });
|
||||
|
||||
Maintenance = new Mock<IMaintenanceRepository>();
|
||||
Maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.ReturnsAsync(true);
|
||||
|
||||
Sut = new SessionDeletionWorker(
|
||||
MainQueue.Object, PoisonQueue.Object,
|
||||
handlerMock.Object, StorageMock.Object,
|
||||
AdminConfig.Object, Maintenance.Object,
|
||||
NullLogger<SessionDeletionWorker>.Instance,
|
||||
heartbeatInterval: TimeSpan.FromMilliseconds(200),
|
||||
pollInterval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
|
||||
public void EnqueueMessage(string body, int dequeueCount)
|
||||
{
|
||||
var msg = QueuesModelFactory.QueueMessage(
|
||||
messageId: "msg-" + Guid.NewGuid().ToString("N"),
|
||||
popReceipt: "pop-" + Guid.NewGuid().ToString("N"),
|
||||
body: new BinaryData(body),
|
||||
dequeueCount: dequeueCount);
|
||||
_pendingMessages.Enqueue(msg);
|
||||
}
|
||||
|
||||
public async Task RunForAsync(TimeSpan duration)
|
||||
{
|
||||
using var cts = new CancellationTokenSource(duration);
|
||||
try { await Sut.StartAsync(cts.Token); } catch (OperationCanceledException) { }
|
||||
try { await Task.Delay(duration, cts.Token); } catch (OperationCanceledException) { }
|
||||
try { await Sut.StopAsync(CancellationToken.None); } catch (OperationCanceledException) { }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fake that captures the manifest upload and replays it on download, mirroring the
|
||||
/// <c>BlobStorageServiceDeletionManifestTests</c> pattern.
|
||||
/// </summary>
|
||||
private sealed class FakeBlobService : BlobStorageService
|
||||
{
|
||||
private byte[]? _bytes;
|
||||
private System.Collections.Generic.IDictionary<string, string>? _metadata;
|
||||
|
||||
public FakeBlobService()
|
||||
: base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger<BlobStorageService>.Instance, false) { }
|
||||
|
||||
protected internal override Task WriteDeletionManifestBlobAsync(
|
||||
string blobName, byte[] gzipped, Azure.Storage.Blobs.Models.BlobUploadOptions options, CancellationToken ct)
|
||||
{
|
||||
_bytes = gzipped;
|
||||
_metadata = options.Metadata;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
protected internal override Task<(byte[] Gzipped, System.Collections.Generic.IDictionary<string, string>? Metadata)>
|
||||
ReadDeletionManifestBlobAsync(string blobName, CancellationToken ct)
|
||||
=> Task.FromResult((_bytes!, _metadata));
|
||||
}
|
||||
|
||||
// ============================================================ Shared helpers ====
|
||||
|
||||
private static DeletionManifest BuildFullManifest() => new DeletionManifest
|
||||
{
|
||||
ManifestId = ManifestId, TenantId = TenantId, SessionId = SessionId,
|
||||
Reason = "admin_delete",
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 1, Table = Constants.TableNames.Events, Class = DeletionStepClass.PkBySession, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "evt-1" } } },
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 16, Step = DeletionStepNames.SoftwareInventoryDecrement,
|
||||
Class = DeletionStepClass.Aggregate, RowCount = 3,
|
||||
Decrements = new List<DeletionDecrementKey>
|
||||
{
|
||||
new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" },
|
||||
new DeletionDecrementKey { Vendor = "Adobe", Name = "Acrobat", Version = "23.1" },
|
||||
new DeletionDecrementKey { Vendor = "Mozilla", Name = "Firefox", Version = "120.0" },
|
||||
},
|
||||
},
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 18, Step = DeletionStepNames.Tombstone, Class = DeletionStepClass.Final, RowCount = 2,
|
||||
Rows = new List<DeletionRowDump>
|
||||
{
|
||||
new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
|
||||
new DeletionRowDump { Pk = TenantId, Rk = SessionId },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
private sealed record AuditEntry(
|
||||
string TenantId, string Action, string EntityType, string EntityId, string PerformedBy,
|
||||
Dictionary<string, string>? Details);
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Storage.Queues;
|
||||
using Azure.Storage.Queues.Models;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Newtonsoft.Json;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Worker-level tests for plan §5 PR4. Wraps the BackgroundService poll-loop in a
|
||||
/// CancellationTokenSource and exercises the four explicit behaviours:
|
||||
/// kill-switch entry guard, poison after max-dequeue (+ audit), malformed-envelope drop,
|
||||
/// heartbeat extends visibility on long handler runs.
|
||||
/// </summary>
|
||||
public class SessionDeletionWorkerTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_does_not_receive_messages_while_kill_switch_active()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetKillSwitch(true);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
harness.MainQueue.Verify(q => q.ReceiveMessagesAsync(
|
||||
It.IsAny<int>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
harness.HandlerMock.Verify(h => h.HandleAsync(
|
||||
It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_moves_message_to_poison_after_max_dequeue_and_audits()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId,
|
||||
Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
// Poison queue received the body and main queue deleted the original.
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
harness.MainQueue.Verify(q => q.DeleteMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
// Audit deletion_poisoned written with the parsed envelope's manifestId.
|
||||
Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_poisoned"
|
||||
&& a.Details != null && a.Details["manifestId"] == ManifestId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_transitions_state_to_Poisoned_before_sending_to_poison_queue()
|
||||
{
|
||||
// PR4b E1 fix: MoveToPoisonAsync must CAS Sessions.DeletionState: Running → Poisoned
|
||||
// BEFORE sending to the poison queue. Without this transition, the restore endpoint
|
||||
// (PR4b) cannot dispatch into partial-restore mode (it keys off DeletionState=Poisoned).
|
||||
var harness = new Harness();
|
||||
var envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId,
|
||||
Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1);
|
||||
|
||||
// Track call order: the CAS must come BEFORE the poison-queue send.
|
||||
var callOrder = new List<string>();
|
||||
harness.StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Running, SessionDeletionState.Poisoned,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => callOrder.Add("cas-running-to-poisoned"))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.Poisoned,
|
||||
});
|
||||
harness.PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback(() => callOrder.Add("poison-queue-send"))
|
||||
.Returns<string, CancellationToken>((body, _) =>
|
||||
{
|
||||
var receipt = QueuesModelFactory.SendReceipt("poison-msg", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(7), "poison-pop", DateTimeOffset.UtcNow);
|
||||
return Task.FromResult(Response.FromValue(receipt, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
var casIdx = callOrder.IndexOf("cas-running-to-poisoned");
|
||||
var poisonIdx = callOrder.IndexOf("poison-queue-send");
|
||||
Assert.True(casIdx >= 0, "CAS Running→Poisoned was not issued");
|
||||
Assert.True(poisonIdx >= 0, "Poison-queue send was not issued");
|
||||
Assert.True(casIdx < poisonIdx,
|
||||
"CAS Running→Poisoned must precede the poison-queue send so the restore endpoint can dispatch.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_falls_back_to_Queued_to_Poisoned_when_Running_CAS_misses()
|
||||
{
|
||||
// Rare case: cascade poisoned before the worker ever successfully transitioned Queued→Running
|
||||
// (handler always threw at the state-acquire step). In that case Running→Poisoned hits
|
||||
// WrongState and the fallback Queued→Poisoned must succeed.
|
||||
var harness = new Harness();
|
||||
var envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId,
|
||||
Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1);
|
||||
|
||||
// First attempt: Running→Poisoned WrongState (current=Queued).
|
||||
harness.StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Running, SessionDeletionState.Poisoned,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.WrongState,
|
||||
CurrentState = SessionDeletionState.Queued,
|
||||
});
|
||||
// Fallback attempt: Queued→Poisoned Updated.
|
||||
harness.StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Queued, SessionDeletionState.Poisoned,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.Poisoned,
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
// Both CAS attempts fired, and the poison-queue send still completed.
|
||||
harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Running, SessionDeletionState.Poisoned,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Queued, SessionDeletionState.Poisoned,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_still_poisons_message_when_state_CAS_persistently_fails()
|
||||
{
|
||||
// Best-effort contract: if both CAS attempts fail (concurrent writer chaos), the
|
||||
// poison-queue move + audit must STILL complete so the operator has an observable
|
||||
// trail. Restore endpoint may reject with 409 "active_cascade" until the operator
|
||||
// manually intervenes — but the cascade message itself is off the main queue.
|
||||
var harness = new Harness();
|
||||
var envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId,
|
||||
Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1);
|
||||
|
||||
harness.StorageMock.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
It.IsAny<string>(), SessionDeletionState.Poisoned,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.WrongState,
|
||||
CurrentState = "Unknown",
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_poisoned");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_drops_malformed_envelope_without_invoking_handler()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.EnqueueMessage("{ \"this is not valid JSON for an envelope: garbage", dequeueCount: 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
harness.HandlerMock.Verify(h => h.HandleAsync(
|
||||
It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
// Malformed messages are removed from the main queue (otherwise they'd loop forever).
|
||||
harness.MainQueue.Verify(q => q.DeleteMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
// Poison queue should NOT receive a malformed body — those are dropped, not poisoned.
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_heartbeat_extends_visibility_while_handler_runs()
|
||||
{
|
||||
var harness = new Harness(heartbeatInterval: TimeSpan.FromMilliseconds(50));
|
||||
var envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId,
|
||||
Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: 1);
|
||||
|
||||
// Handler hangs for 350ms so the heartbeat task ticks at least 5×.
|
||||
harness.HandlerMock.Setup(h => h.HandleAsync(
|
||||
It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(async (SessionDeletionEnvelope _, CancellationToken ct) =>
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(350), ct);
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(800));
|
||||
|
||||
// Heartbeat called at least 3× while the handler was busy (350ms / 50ms ≈ 7, minus a few
|
||||
// due to scheduling jitter). One call is enough to prove the heartbeat task is wired.
|
||||
harness.MainQueue.Verify(q => q.UpdateMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeast(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Worker_drops_envelope_missing_required_fields_without_invoking_handler()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
// ManifestId deliberately empty — required-fields guard must catch this.
|
||||
TenantId = TenantId, SessionId = SessionId, ManifestId = string.Empty,
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
harness.HandlerMock.Verify(h => h.HandleAsync(
|
||||
It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
harness.MainQueue.Verify(q => q.DeleteMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
}
|
||||
|
||||
// ============================================================ Harness ====
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public Mock<QueueClient> MainQueue { get; }
|
||||
public Mock<QueueClient> PoisonQueue { get; }
|
||||
public Mock<SessionDeletionHandler> HandlerMock { get; }
|
||||
public Mock<AdminConfigurationService> AdminConfig { get; }
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; }
|
||||
public List<AuditEntry> AuditCalls { get; } = new List<AuditEntry>();
|
||||
public SessionDeletionWorker Sut { get; }
|
||||
|
||||
private readonly Queue<QueueMessage> _pendingMessages = new Queue<QueueMessage>();
|
||||
|
||||
public Harness(TimeSpan? heartbeatInterval = null)
|
||||
{
|
||||
MainQueue = new Mock<QueueClient>();
|
||||
PoisonQueue = new Mock<QueueClient>();
|
||||
|
||||
MainQueue.Setup(q => q.CreateIfNotExistsAsync(
|
||||
It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((Response?)null);
|
||||
PoisonQueue.Setup(q => q.CreateIfNotExistsAsync(
|
||||
It.IsAny<IDictionary<string, string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((Response?)null);
|
||||
|
||||
MainQueue.Setup(q => q.ReceiveMessagesAsync(
|
||||
It.IsAny<int>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<int, TimeSpan?, CancellationToken>((maxMessages, _, _) =>
|
||||
{
|
||||
var batch = new List<QueueMessage>();
|
||||
while (batch.Count < maxMessages && _pendingMessages.Count > 0)
|
||||
{
|
||||
batch.Add(_pendingMessages.Dequeue());
|
||||
}
|
||||
var response = QueuesModelFactory.QueueMessage(
|
||||
messageId: "msg-batch", popReceipt: "pop-batch",
|
||||
body: new BinaryData(string.Empty), dequeueCount: 0);
|
||||
return Task.FromResult(Response.FromValue(batch.ToArray(), new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
MainQueue.Setup(q => q.DeleteMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new Mock<Response>().Object);
|
||||
|
||||
MainQueue.Setup(q => q.UpdateMessageAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, string, TimeSpan, CancellationToken>((id, pr, body, vis, ct) =>
|
||||
{
|
||||
var updated = QueuesModelFactory.UpdateReceipt(
|
||||
"pop-extended-" + Guid.NewGuid().ToString("N"),
|
||||
DateTimeOffset.UtcNow.Add(vis));
|
||||
return Task.FromResult(Response.FromValue(updated, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, CancellationToken>((body, _) =>
|
||||
{
|
||||
var receipt = QueuesModelFactory.SendReceipt("poison-msg", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddMinutes(7), "poison-pop", DateTimeOffset.UtcNow);
|
||||
return Task.FromResult(Response.FromValue(receipt, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
// Handler — virtual HandleAsync; default behaviour is no-op (handler succeeded).
|
||||
// Moq needs real ctor args for the proxy; the worker only calls HandleAsync so the
|
||||
// inner deps stay untouched.
|
||||
var storageMock = new Mock<TableStorageService>(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance);
|
||||
// PR4c F5: Worker now reads the Sessions row to verify PendingDeletionManifestId
|
||||
// matches the envelope's manifestId before issuing the poison CAS. Default mock
|
||||
// returns a row whose pending matches the test's ManifestId so the existing PR4b
|
||||
// tests (which exercise the post-pre-check CAS paths) keep working.
|
||||
storageMock.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => new TableEntity(TenantId, SessionId)
|
||||
{
|
||||
["DeletionState"] = SessionDeletionState.Running,
|
||||
["PendingDeletionManifestId"] = ManifestId,
|
||||
});
|
||||
// Default: poison-state CAS succeeds. Individual tests override to verify the E1 fix.
|
||||
storageMock.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.Poisoned,
|
||||
});
|
||||
var blobMock = new Mock<BlobStorageService>(
|
||||
new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, false);
|
||||
var verifierMock = new Mock<CascadeVerificationService>(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(),
|
||||
NullLogger<CascadeVerificationService>.Instance);
|
||||
HandlerMock = new Mock<SessionDeletionHandler>(
|
||||
storageMock.Object, blobMock.Object, verifierMock.Object,
|
||||
Mock.Of<IMaintenanceRepository>(),
|
||||
new FakeSignalRNotificationService(),
|
||||
NullLogger<SessionDeletionHandler>.Instance);
|
||||
HandlerMock.Setup(h => h.HandleAsync(It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
AdminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(),
|
||||
NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions()));
|
||||
AdminConfig.Setup(a => a.GetConfigurationAsync())
|
||||
.ReturnsAsync(new AdminConfiguration { SessionDeletionKillSwitch = false });
|
||||
|
||||
Maintenance = new Mock<IMaintenanceRepository>();
|
||||
Maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.Returns<string, string, string, string, string, Dictionary<string, string>?>(
|
||||
(tenantId, action, entityType, entityId, performedBy, details) =>
|
||||
{
|
||||
AuditCalls.Add(new AuditEntry(tenantId, action, entityType, entityId, performedBy, details));
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
|
||||
Sut = new SessionDeletionWorker(
|
||||
MainQueue.Object, PoisonQueue.Object,
|
||||
HandlerMock.Object, storageMock.Object,
|
||||
AdminConfig.Object, Maintenance.Object,
|
||||
NullLogger<SessionDeletionWorker>.Instance,
|
||||
heartbeatInterval: heartbeatInterval ?? TimeSpan.FromMilliseconds(200),
|
||||
pollInterval: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
StorageMock = storageMock;
|
||||
}
|
||||
|
||||
public Mock<TableStorageService> StorageMock { get; private set; } = null!;
|
||||
|
||||
public void SetKillSwitch(bool active)
|
||||
{
|
||||
AdminConfig.Setup(a => a.GetConfigurationAsync())
|
||||
.ReturnsAsync(new AdminConfiguration { SessionDeletionKillSwitch = active });
|
||||
}
|
||||
|
||||
public void EnqueueMessage(string body, int dequeueCount)
|
||||
{
|
||||
var msg = QueuesModelFactory.QueueMessage(
|
||||
messageId: "msg-" + Guid.NewGuid().ToString("N"),
|
||||
popReceipt: "pop-" + Guid.NewGuid().ToString("N"),
|
||||
body: new BinaryData(body),
|
||||
dequeueCount: dequeueCount);
|
||||
_pendingMessages.Enqueue(msg);
|
||||
}
|
||||
|
||||
public async Task RunForAsync(TimeSpan duration)
|
||||
{
|
||||
using var cts = new CancellationTokenSource(duration);
|
||||
try { await Sut.StartAsync(cts.Token); }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
try { await Task.Delay(duration, cts.Token); }
|
||||
catch (OperationCanceledException) { /* expected on timeout */ }
|
||||
try { await Sut.StopAsync(CancellationToken.None); }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record AuditEntry(
|
||||
string TenantId, string Action, string EntityType, string EntityId, string PerformedBy,
|
||||
Dictionary<string, string>? Details);
|
||||
}
|
||||
@@ -0,0 +1,509 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end Service facts for plan §13 (PR4b). All 14 plan-listed tests + dispatch-table
|
||||
/// edge cases. Moq-based, no Azurite — matches PR1–PR4 convention.
|
||||
/// </summary>
|
||||
public class SessionRestoreServiceTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
|
||||
private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
|
||||
|
||||
// ============================================================ Full restore (8 facts) ====
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_full_round_trips_session_after_completed_cascade()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade(); // sessionRow=null + progress.CompletedAt set
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome);
|
||||
Assert.Equal("full", result.Mode);
|
||||
// Sessions row inserted (manifest.FINAL.rows[1] in builder order).
|
||||
harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.Sessions, It.IsAny<IReadOnlyList<DeletionRowDump>>(), RestoreMode.Full, It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
// SessionsIndex inserted.
|
||||
harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
Constants.TableNames.SessionsIndex, It.IsAny<IReadOnlyList<DeletionRowDump>>(), RestoreMode.Full, It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
// Inventory re-increments fired.
|
||||
Assert.Equal(3, result.InventoryReIncrements);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_full_re_increments_software_inventory_via_contributions()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade();
|
||||
|
||||
await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
TenantId,
|
||||
It.Is<DeletionDecrementKey>(k => k.Vendor == "Microsoft" && k.Name == "Office" && k.Version == "16.0"),
|
||||
It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
TenantId,
|
||||
It.Is<DeletionDecrementKey>(k => k.Vendor == "Adobe" && k.Name == "Acrobat" && k.Version == "23.1"),
|
||||
It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_full_skips_software_inventory_when_no_contributions_row_in_snapshot()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade(includeInventorySteps: false); // pre-side-row session
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome);
|
||||
Assert.Equal(0, result.InventoryReIncrements);
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
It.IsAny<string>(), It.IsAny<DeletionDecrementKey>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_full_rejects_when_session_already_exists_with_state_None()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade();
|
||||
harness.SessionRow = MakeSessionRow(SessionDeletionState.None, pendingManifestId: null);
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.RejectAlreadyAtOriginalState, result.Outcome);
|
||||
harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<DeletionRowDump>>(), It.IsAny<RestoreMode>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_full_rejects_on_snapshot_hash_mismatch()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade();
|
||||
harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new InvalidDataException("Snapshot SHA-256 mismatch"));
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.RejectManifestCorruption, result.Outcome);
|
||||
Assert.Empty(result.RowsRestoredByTable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_full_audits_manifest_download_before_any_read()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade();
|
||||
// Force a post-download reject path so we can verify the audit fired BEFORE the reject.
|
||||
harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new InvalidDataException("corruption"));
|
||||
|
||||
await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_manifest_downloaded"
|
||||
&& a.Details != null && a.Details["reason"] == "restore_attempt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_full_does_not_touch_SessionReports_or_session_reports_container()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade();
|
||||
|
||||
await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
// The harness's RestoreRowsByExactKeysInBatchesAsync mock records the tableName. We
|
||||
// assert it was never invoked for SessionReports.
|
||||
harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
"SessionReports", It.IsAny<IReadOnlyList<DeletionRowDump>>(), It.IsAny<RestoreMode>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_rejects_manifest_not_found_with_404_outcome()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade();
|
||||
harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new RequestFailedException(404, "BlobNotFound"));
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.RejectManifestNotFound, result.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_rejects_corrupt_state_when_sessions_row_missing_but_no_completedAt()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetPoisonedCascade();
|
||||
harness.SessionRow = null; // Sessions row gone but progress NOT completed → corruption
|
||||
harness.Progress.CompletedAt = null;
|
||||
// PR4c F2: TombstoneStarted=false (default) — distinguishes "Sessions row removed by
|
||||
// something outside the cascade" (true corruption) from the new gap-recovery case where
|
||||
// TombstoneStarted=true allows full-restore to proceed.
|
||||
harness.Progress.TombstoneStarted = false;
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.RejectCorruptState, result.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_dryRun_writes_no_data_but_reports_what_would_restore()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade();
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: true, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.DryRunOk, result.Outcome);
|
||||
Assert.Equal("dryRun", result.Mode);
|
||||
Assert.NotEmpty(result.WouldRestoreByTable);
|
||||
// No live writes.
|
||||
harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<DeletionRowDump>>(), It.IsAny<RestoreMode>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
It.IsAny<string>(), It.IsAny<DeletionDecrementKey>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
// dryRun does NOT emit deletion_restored (only the download audit).
|
||||
Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_restored");
|
||||
}
|
||||
|
||||
// ============================================================ Partial restore (6 facts) ====
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_partial_recovers_session_poisoned_mid_cascade()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetPoisonedCascade();
|
||||
harness.Progress.CompletedSteps.UnionWith(new[] { 1, 2, 3, 4, 5, 6, 7 });
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.Restored, result.Outcome);
|
||||
Assert.Equal("partial", result.Mode);
|
||||
// Partial mode used for at least one restore call.
|
||||
harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<DeletionRowDump>>(), RestoreMode.Partial, It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
// CAS Poisoned→None fired at end.
|
||||
harness.Storage.Verify(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Poisoned, SessionDeletionState.None,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_partial_skips_inventory_re_increment_when_decrement_never_ran()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetPoisonedCascade();
|
||||
// CompletedSteps does NOT include the AGGREGATE step's order (16).
|
||||
harness.Progress.CompletedSteps.Clear();
|
||||
harness.Progress.CompletedSteps.Add(1); // poisoned right after step 1
|
||||
|
||||
await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
It.IsAny<string>(), It.IsAny<DeletionDecrementKey>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_partial_re_increments_inventory_when_decrement_already_ran()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetPoisonedCascade();
|
||||
// CompletedSteps includes the AGGREGATE step's order (16).
|
||||
harness.Progress.CompletedSteps.UnionWith(new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 });
|
||||
|
||||
await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
harness.Storage.Verify(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
TenantId, It.IsAny<DeletionDecrementKey>(), It.IsAny<CancellationToken>()),
|
||||
Times.Exactly(3));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(SessionDeletionState.Running)]
|
||||
[InlineData(SessionDeletionState.Preparing)]
|
||||
[InlineData(SessionDeletionState.Queued)]
|
||||
public async Task Restore_partial_rejects_when_state_is_not_Poisoned(string state)
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetPoisonedCascade();
|
||||
harness.SessionRow = MakeSessionRow(state, pendingManifestId: ManifestId);
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.RejectActiveCascade, result.Outcome);
|
||||
Assert.Equal(state, result.CurrentState);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_partial_rejects_manifestid_mismatch()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetPoisonedCascade();
|
||||
harness.SessionRow = MakeSessionRow(SessionDeletionState.Poisoned, pendingManifestId: "DIFFERENT-MANIFEST");
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.RejectManifestIdMismatch, result.Outcome);
|
||||
Assert.Equal("DIFFERENT-MANIFEST", result.PendingManifestId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_partial_returns_RejectCasConflictOnClear_when_final_CAS_fails()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetPoisonedCascade();
|
||||
harness.Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Poisoned, SessionDeletionState.None,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.EtagConflict,
|
||||
CurrentState = SessionDeletionState.Poisoned,
|
||||
});
|
||||
|
||||
var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
Assert.Equal(SessionRestoreOutcome.RejectCasConflictOnClear, result.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_audits_deletion_restored_on_success_with_full_details()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetCompletedCascade();
|
||||
|
||||
await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com");
|
||||
|
||||
var audit = Assert.Single(harness.AuditCalls, a => a.Action == "deletion_restored");
|
||||
Assert.NotNull(audit.Details);
|
||||
Assert.Equal(ManifestId, audit.Details!["manifestId"]);
|
||||
Assert.Equal("full", audit.Details["mode"]);
|
||||
Assert.True(audit.Details.ContainsKey("rowsRestoredByTable"));
|
||||
Assert.True(audit.Details.ContainsKey("durationMs"));
|
||||
}
|
||||
|
||||
// ============================================================ Harness ====
|
||||
|
||||
private static TableEntity MakeSessionRow(string state, string? pendingManifestId)
|
||||
{
|
||||
var entity = new TableEntity(TenantId, SessionId)
|
||||
{
|
||||
["DeletionState"] = state,
|
||||
};
|
||||
if (pendingManifestId != null)
|
||||
{
|
||||
entity["PendingDeletionManifestId"] = pendingManifestId;
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public Mock<TableStorageService> Storage { get; }
|
||||
public Mock<BlobStorageService> Blob { get; }
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; }
|
||||
public SessionRestoreService Sut { get; }
|
||||
public List<AuditEntry> AuditCalls { get; } = new List<AuditEntry>();
|
||||
|
||||
// Per-test state — modify via Set*() methods.
|
||||
public TableEntity? SessionRow { get; set; }
|
||||
public DeletionProgress Progress { get; set; } = new DeletionProgress();
|
||||
public DeletionManifest Manifest { get; set; } = new DeletionManifest();
|
||||
|
||||
public Harness()
|
||||
{
|
||||
Storage = new Mock<TableStorageService>(
|
||||
Mock.Of<TableServiceClient>(),
|
||||
NullLogger<TableStorageService>.Instance);
|
||||
|
||||
Blob = new Mock<BlobStorageService>(
|
||||
new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, false);
|
||||
|
||||
Maintenance = new Mock<IMaintenanceRepository>();
|
||||
Maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.Returns<string, string, string, string, string, Dictionary<string, string>?>(
|
||||
(tenantId, action, entityType, entityId, performedBy, details) =>
|
||||
{
|
||||
AuditCalls.Add(new AuditEntry(tenantId, action, entityType, entityId, performedBy, details));
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
|
||||
// Defaults for the live-storage interactions.
|
||||
Storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => SessionRow);
|
||||
|
||||
// PR4c F6: RestoreService uses DownloadDeletionManifestWithShaAsync; the harness's
|
||||
// Progress.SnapshotSha256 (set to the same Sha256 constant) matches the fake SHA so
|
||||
// the snapshot↔progress binding check passes on the happy path.
|
||||
Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => (Manifest, Sha256));
|
||||
Blob.Setup(b => b.DownloadDeletionProgressAsync(TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => (Progress, "\"0xETAG\""));
|
||||
|
||||
// PR4c F4: RestoreService now persists per-key re-increment progress via ETag-CAS.
|
||||
// The harness default lets UpdateDeletionProgressAsync succeed unconditionally so
|
||||
// tests focus on behaviour, not on simulating the bounded-retry loop.
|
||||
Blob.Setup(b => b.UpdateDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId,
|
||||
It.IsAny<DeletionProgress>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync("\"0xETAG-NEW\"");
|
||||
|
||||
Storage.Setup(s => s.RestoreRowsByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<DeletionRowDump>>(),
|
||||
It.IsAny<RestoreMode>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, IReadOnlyList<DeletionRowDump> rows, RestoreMode _, CancellationToken _) =>
|
||||
new RestoreBatchResult(rows.Count, rows.Count, 0));
|
||||
|
||||
Storage.Setup(s => s.RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
It.IsAny<string>(), It.IsAny<DeletionDecrementKey>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Poisoned, SessionDeletionState.None,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.None,
|
||||
});
|
||||
|
||||
Sut = new SessionRestoreService(
|
||||
Storage.Object, Blob.Object, Maintenance.Object,
|
||||
NullLogger<SessionRestoreService>.Instance);
|
||||
}
|
||||
|
||||
public void SetCompletedCascade(bool includeInventorySteps = true)
|
||||
{
|
||||
SessionRow = null;
|
||||
Progress = new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = Sha256,
|
||||
CompletedSteps = new HashSet<int>(Enumerable.Range(1, 18)),
|
||||
VerificationDone = true,
|
||||
CompletedAt = DateTime.UtcNow,
|
||||
};
|
||||
Manifest = BuildFullManifest(includeInventorySteps);
|
||||
}
|
||||
|
||||
public void SetPoisonedCascade(bool includeInventorySteps = true)
|
||||
{
|
||||
SessionRow = MakeSessionRow(SessionDeletionState.Poisoned, pendingManifestId: ManifestId);
|
||||
Progress = new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = Sha256,
|
||||
CompletedSteps = new HashSet<int> { 1, 2, 3 },
|
||||
VerificationDone = false,
|
||||
CompletedAt = null,
|
||||
};
|
||||
Manifest = BuildFullManifest(includeInventorySteps);
|
||||
}
|
||||
}
|
||||
|
||||
private static DeletionManifest BuildFullManifest(bool includeInventorySteps)
|
||||
{
|
||||
var steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep { Order = 1, Table = Constants.TableNames.Events, Class = DeletionStepClass.PkBySession, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "evt-1" } } },
|
||||
new DeletionStep { Order = 2, Table = Constants.TableNames.RuleResults, Class = DeletionStepClass.PkBySession, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_{SessionId}", Rk = "rule-1" } } },
|
||||
new DeletionStep { Order = 5, Table = Constants.TableNames.DeviceSnapshot, Class = DeletionStepClass.PkRkExact, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = TenantId, Rk = SessionId } } },
|
||||
new DeletionStep { Order = 10, Table = Constants.TableNames.CveIndex, Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId } } },
|
||||
};
|
||||
|
||||
if (includeInventorySteps)
|
||||
{
|
||||
steps.Add(new DeletionStep
|
||||
{
|
||||
Order = 16,
|
||||
Step = DeletionStepNames.SoftwareInventoryDecrement,
|
||||
Class = DeletionStepClass.Aggregate,
|
||||
RowCount = 3,
|
||||
Decrements = new List<DeletionDecrementKey>
|
||||
{
|
||||
new DeletionDecrementKey { Vendor = "Microsoft", Name = "Office", Version = "16.0" },
|
||||
new DeletionDecrementKey { Vendor = "Adobe", Name = "Acrobat", Version = "23.1" },
|
||||
new DeletionDecrementKey { Vendor = "Mozilla", Name = "Firefox", Version = "120.0" },
|
||||
},
|
||||
});
|
||||
steps.Add(new DeletionStep
|
||||
{
|
||||
Order = 17,
|
||||
Table = Constants.TableNames.SessionInventoryContributions,
|
||||
Class = DeletionStepClass.PkRkExact,
|
||||
RowCount = 1,
|
||||
Rows = new List<DeletionRowDump> { new DeletionRowDump { Pk = TenantId, Rk = SessionId } },
|
||||
});
|
||||
}
|
||||
|
||||
steps.Add(new DeletionStep
|
||||
{
|
||||
Order = 18,
|
||||
Step = DeletionStepNames.Tombstone,
|
||||
Class = DeletionStepClass.Final,
|
||||
RowCount = 2,
|
||||
Rows = new List<DeletionRowDump>
|
||||
{
|
||||
new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" }, // SessionsIndex
|
||||
new DeletionRowDump { Pk = TenantId, Rk = SessionId }, // Sessions
|
||||
},
|
||||
});
|
||||
|
||||
return new DeletionManifest
|
||||
{
|
||||
ManifestId = ManifestId,
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
Steps = steps,
|
||||
PreflightCounts = new Dictionary<string, int>(),
|
||||
};
|
||||
}
|
||||
|
||||
private sealed record AuditEntry(
|
||||
string TenantId, string Action, string EntityType, string EntityId, string PerformedBy,
|
||||
Dictionary<string, string>? Details);
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Decode-direction tests for <see cref="TableStorageService.ConvertFromPropValue"/> +
|
||||
/// <see cref="TableStorageService.ConvertDumpToEntity"/> (PR4b). Pairs with the existing
|
||||
/// <c>DeletionManifestSerializationTests</c> which covers the encode direction. Together they
|
||||
/// establish the byte-faithful EDM round-trip the manifest-as-backup guarantee depends on
|
||||
/// (plan §13 / §3 "types preserved").
|
||||
/// </summary>
|
||||
public class TableStoragePropValueConversionTests
|
||||
{
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_round_trips_String()
|
||||
{
|
||||
var prop = MakeProp(DeletionPropEdmType.String, "\"hello world\"");
|
||||
Assert.Equal("hello world", TableStorageService.ConvertFromPropValue(prop));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_round_trips_Boolean_true_and_false()
|
||||
{
|
||||
Assert.Equal(true, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Boolean, "true")));
|
||||
Assert.Equal(false, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Boolean, "false")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_round_trips_Int32()
|
||||
{
|
||||
Assert.Equal(42, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int32, "42")));
|
||||
Assert.Equal(-1, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int32, "-1")));
|
||||
Assert.Equal(int.MaxValue, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int32, int.MaxValue.ToString(CultureInfo.InvariantCulture))));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_round_trips_Int64()
|
||||
{
|
||||
Assert.Equal(123456789012L, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int64, "123456789012")));
|
||||
Assert.Equal(long.MaxValue, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Int64, long.MaxValue.ToString(CultureInfo.InvariantCulture))));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_round_trips_Double()
|
||||
{
|
||||
Assert.Equal(3.14, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Double, "3.14")));
|
||||
Assert.Equal(0.0, TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Double, "0")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_round_trips_DateTime_as_UTC()
|
||||
{
|
||||
// Encode side stamps with "o" round-trip format and forces UTC kind.
|
||||
var json = JsonSerializer.Serialize(new DateTime(2026, 5, 12, 10, 0, 0, DateTimeKind.Utc).ToString("o", CultureInfo.InvariantCulture));
|
||||
var result = TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.DateTime, json));
|
||||
var dt = Assert.IsType<DateTime>(result);
|
||||
Assert.Equal(new DateTime(2026, 5, 12, 10, 0, 0, DateTimeKind.Utc), dt.ToUniversalTime());
|
||||
Assert.Equal(DateTimeKind.Utc, dt.Kind);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_round_trips_Guid_in_D_format()
|
||||
{
|
||||
var guid = new Guid("33333333-3333-3333-3333-333333333333");
|
||||
var json = JsonSerializer.Serialize(guid.ToString("D"));
|
||||
var result = TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Guid, json));
|
||||
Assert.Equal(guid, Assert.IsType<Guid>(result));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_round_trips_Binary_as_Base64()
|
||||
{
|
||||
var bytes = new byte[] { 0x01, 0x02, 0x03, 0xff };
|
||||
var json = JsonSerializer.Serialize(Convert.ToBase64String(bytes));
|
||||
var result = TableStorageService.ConvertFromPropValue(MakeProp(DeletionPropEdmType.Binary, json));
|
||||
var decoded = Assert.IsType<byte[]>(result);
|
||||
Assert.Equal(bytes, decoded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_returns_null_for_JsonNull_value()
|
||||
{
|
||||
// Encode path stamps EdmType=String + JsonElement(null) for .NET null values.
|
||||
var prop = MakeProp(DeletionPropEdmType.String, "null");
|
||||
Assert.Null(TableStorageService.ConvertFromPropValue(prop));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertFromPropValue_throws_on_unknown_EdmType()
|
||||
{
|
||||
var prop = new DeletionPropValue
|
||||
{
|
||||
EdmType = "ThisEdmTypeDoesNotExist",
|
||||
Value = ParseJson("\"x\""),
|
||||
};
|
||||
Assert.Throws<System.IO.InvalidDataException>(() => TableStorageService.ConvertFromPropValue(prop));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertDumpToEntity_overrides_DeletionState_to_None_for_Sessions_row()
|
||||
{
|
||||
// Snapshot was captured after CAS-Preparing — without the override the restored row
|
||||
// would carry DeletionState=Preparing + the old manifestId, locking the session forever.
|
||||
var dump = new DeletionRowDump
|
||||
{
|
||||
Pk = "tenant-1",
|
||||
Rk = "session-1",
|
||||
Etag = "\"0xETAG\"",
|
||||
Props = new Dictionary<string, DeletionPropValue>(StringComparer.Ordinal)
|
||||
{
|
||||
["DeletionState"] = MakeProp(DeletionPropEdmType.String, "\"Preparing\""),
|
||||
["PendingDeletionManifestId"] = MakeProp(DeletionPropEdmType.String, "\"some-old-manifest\""),
|
||||
["CurrentPhase"] = MakeProp(DeletionPropEdmType.String, "\"AccountSetup\""),
|
||||
},
|
||||
};
|
||||
|
||||
var entity = TableStorageService.ConvertDumpToEntity(dump, Constants.TableNames.Sessions);
|
||||
|
||||
Assert.Equal(SessionDeletionState.None, entity.GetString("DeletionState"));
|
||||
Assert.Null(entity["PendingDeletionManifestId"]);
|
||||
// Other props preserved.
|
||||
Assert.Equal("AccountSetup", entity.GetString("CurrentPhase"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertDumpToEntity_does_NOT_override_DeletionState_for_other_tables()
|
||||
{
|
||||
// SessionsIndex (or any other table) shouldn't have DeletionState forced.
|
||||
var dump = new DeletionRowDump
|
||||
{
|
||||
Pk = "tenant-1",
|
||||
Rk = "rk-1",
|
||||
Props = new Dictionary<string, DeletionPropValue>(StringComparer.Ordinal)
|
||||
{
|
||||
["SomeColumn"] = MakeProp(DeletionPropEdmType.String, "\"keep-me\""),
|
||||
},
|
||||
};
|
||||
|
||||
var entity = TableStorageService.ConvertDumpToEntity(dump, Constants.TableNames.SessionsIndex);
|
||||
|
||||
Assert.Equal("keep-me", entity.GetString("SomeColumn"));
|
||||
Assert.False(entity.ContainsKey("DeletionState")); // Not added by override
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertDumpToEntity_preserves_pk_and_rk()
|
||||
{
|
||||
var dump = new DeletionRowDump
|
||||
{
|
||||
Pk = "tenant-foo",
|
||||
Rk = "session-bar",
|
||||
Props = new Dictionary<string, DeletionPropValue>(),
|
||||
};
|
||||
var entity = TableStorageService.ConvertDumpToEntity(dump, "AnyTable");
|
||||
Assert.Equal("tenant-foo", entity.PartitionKey);
|
||||
Assert.Equal("session-bar", entity.RowKey);
|
||||
}
|
||||
|
||||
private static DeletionPropValue MakeProp(string edmType, string jsonValue) => new DeletionPropValue
|
||||
{
|
||||
EdmType = edmType,
|
||||
Value = ParseJson(jsonValue),
|
||||
};
|
||||
|
||||
private static JsonElement ParseJson(string json)
|
||||
{
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
return doc.RootElement.Clone();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// SDK-mocked behaviour tests for <see cref="TableStorageService.RestoreRowsByExactKeysInBatchesAsync"/>
|
||||
/// (PR4b). Mirror of <see cref="TableStorageDeletionBatchTests"/>: PK-grouped batches, 100-action
|
||||
/// chunks, per-row fallback when batch surfaces 409. Verifies the Full vs Partial 409 semantics
|
||||
/// asymmetry that distinguishes "row already there is a corruption signal" (Full) from "row
|
||||
/// already there is expected" (Partial).
|
||||
/// </summary>
|
||||
public class TableStorageRestoreBatchTests
|
||||
{
|
||||
private const string TableName = Constants.TableNames.Events;
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_returns_empty_result_for_empty_row_list()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var result = await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(TableName, new List<DeletionRowDump>(), RestoreMode.Full);
|
||||
|
||||
Assert.Equal(0, result.Attempted);
|
||||
Assert.Equal(0, result.Restored);
|
||||
Assert.Equal(0, result.Skipped);
|
||||
Assert.Empty(harness.SubmittedBatches);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_groups_rows_by_partition_so_one_batch_per_pk()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var rows = new[]
|
||||
{
|
||||
MakeDump("pkA", "rk1"), MakeDump("pkA", "rk2"),
|
||||
MakeDump("pkB", "rk3"), MakeDump("pkB", "rk4"), MakeDump("pkB", "rk5"),
|
||||
MakeDump("pkA", "rk6"),
|
||||
}.ToList();
|
||||
|
||||
var result = await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(TableName, rows, RestoreMode.Full);
|
||||
|
||||
Assert.Equal(6, result.Attempted);
|
||||
Assert.Equal(6, result.Restored);
|
||||
Assert.Equal(0, result.Skipped);
|
||||
// Two batches: pkA (3 rows) + pkB (3 rows).
|
||||
Assert.Equal(2, harness.SubmittedBatches.Count);
|
||||
Assert.All(harness.SubmittedBatches, batch =>
|
||||
{
|
||||
var pks = batch.Select(a => a.Entity.PartitionKey).Distinct().ToList();
|
||||
Assert.Single(pks); // batch is partition-scoped
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_uses_Add_action_so_409_surfaces_for_existing_rows()
|
||||
{
|
||||
var harness = new Harness();
|
||||
await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(
|
||||
TableName, new[] { MakeDump("pkA", "rk_0") }, RestoreMode.Full);
|
||||
|
||||
var batch = harness.SubmittedBatches.Single();
|
||||
Assert.Equal(TableTransactionActionType.Add, batch.Single().ActionType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_Full_mode_now_counts_409_as_skipped_like_Partial()
|
||||
{
|
||||
// PR4c F3b: Full mode used to throw InvalidDataException on per-row 409 (as a "corruption
|
||||
// signal"). That intent was hypothetical (Session GUID reuse is unlikely) and blocked the
|
||||
// operationally-real case of retrying a partial-failed restore. Both modes now share
|
||||
// 409-ignore semantics; RowsSkippedByTable surfaces the count for operator inspection.
|
||||
var harness = new Harness
|
||||
{
|
||||
BatchBehavior = _ => throw new RequestFailedException(409, "EntityAlreadyExists"),
|
||||
PerRowAddBehavior = _ => throw new RequestFailedException(409, "EntityAlreadyExists"),
|
||||
};
|
||||
|
||||
var result = await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(
|
||||
TableName, new[] { MakeDump("pkA", "rk_x") }, RestoreMode.Full);
|
||||
|
||||
Assert.Equal(1, result.Attempted);
|
||||
Assert.Equal(0, result.Restored);
|
||||
Assert.Equal(1, result.Skipped);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_Partial_mode_counts_409_as_skipped_not_thrown()
|
||||
{
|
||||
var harness = new Harness
|
||||
{
|
||||
BatchBehavior = _ => throw new RequestFailedException(409, "EntityAlreadyExists"),
|
||||
PerRowAddBehavior = rk => rk == "rk_already_there"
|
||||
? throw new RequestFailedException(409, "EntityAlreadyExists")
|
||||
: (Response)new Mock<Response>().Object,
|
||||
};
|
||||
|
||||
var result = await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(
|
||||
TableName,
|
||||
new[] { MakeDump("pkA", "rk_already_there"), MakeDump("pkA", "rk_new") },
|
||||
RestoreMode.Partial);
|
||||
|
||||
Assert.Equal(2, result.Attempted);
|
||||
Assert.Equal(1, result.Restored);
|
||||
Assert.Equal(1, result.Skipped);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_propagates_non_409_non_400_storage_errors()
|
||||
{
|
||||
var harness = new Harness
|
||||
{
|
||||
BatchBehavior = _ => throw new RequestFailedException(503, "ServiceUnavailable"),
|
||||
};
|
||||
|
||||
await Assert.ThrowsAsync<RequestFailedException>(() =>
|
||||
harness.Sut.RestoreRowsByExactKeysInBatchesAsync(
|
||||
TableName, new[] { MakeDump("pkA", "rk_0") }, RestoreMode.Full));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_throws_on_null_args()
|
||||
{
|
||||
var harness = new Harness();
|
||||
await Assert.ThrowsAsync<ArgumentException>(() =>
|
||||
harness.Sut.RestoreRowsByExactKeysInBatchesAsync("", new List<DeletionRowDump>(), RestoreMode.Full));
|
||||
await Assert.ThrowsAsync<ArgumentNullException>(() =>
|
||||
harness.Sut.RestoreRowsByExactKeysInBatchesAsync(TableName, null!, RestoreMode.Full));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Restore_preserves_dumps_props_via_ConvertDumpToEntity()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var dump = new DeletionRowDump
|
||||
{
|
||||
Pk = "pkA",
|
||||
Rk = "rk_0",
|
||||
Props = new Dictionary<string, DeletionPropValue>(StringComparer.Ordinal)
|
||||
{
|
||||
["MyStringCol"] = MakeProp(DeletionPropEdmType.String, "\"abc\""),
|
||||
["MyIntCol"] = MakeProp(DeletionPropEdmType.Int32, "42"),
|
||||
},
|
||||
};
|
||||
|
||||
await harness.Sut.RestoreRowsByExactKeysInBatchesAsync(TableName, new[] { dump }, RestoreMode.Full);
|
||||
|
||||
var inserted = Assert.IsType<TableEntity>(harness.SubmittedBatches.Single().Single().Entity);
|
||||
Assert.Equal("abc", inserted.GetString("MyStringCol"));
|
||||
Assert.Equal(42, inserted.GetInt32("MyIntCol"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Helpers ----
|
||||
|
||||
private static DeletionRowDump MakeDump(string pk, string rk) => new DeletionRowDump
|
||||
{
|
||||
Pk = pk,
|
||||
Rk = rk,
|
||||
Props = new Dictionary<string, DeletionPropValue>(),
|
||||
};
|
||||
|
||||
private static DeletionPropValue MakeProp(string edmType, string jsonValue)
|
||||
{
|
||||
using var doc = JsonDocument.Parse(jsonValue);
|
||||
return new DeletionPropValue { EdmType = edmType, Value = doc.RootElement.Clone() };
|
||||
}
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public List<List<TableTransactionAction>> SubmittedBatches { get; } = new List<List<TableTransactionAction>>();
|
||||
public List<(string Pk, string Rk)> PerRowAdds { get; } = new List<(string, string)>();
|
||||
|
||||
public Func<List<TableTransactionAction>, Response<IReadOnlyList<Response>>>? BatchBehavior { get; set; }
|
||||
public Func<string, Response>? PerRowAddBehavior { get; set; }
|
||||
|
||||
public TableStorageService Sut { get; }
|
||||
|
||||
public Harness()
|
||||
{
|
||||
var mockTableClient = new Mock<TableClient>();
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.SubmitTransactionAsync(It.IsAny<IEnumerable<TableTransactionAction>>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<IEnumerable<TableTransactionAction>, CancellationToken>((actions, _) =>
|
||||
{
|
||||
var snapshot = actions.ToList();
|
||||
SubmittedBatches.Add(snapshot);
|
||||
if (BatchBehavior != null) return Task.FromResult(BatchBehavior(snapshot));
|
||||
var inner = (IReadOnlyList<Response>)snapshot.Select(_ => new Mock<Response>().Object).ToList();
|
||||
return Task.FromResult(Response.FromValue(inner, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.AddEntityAsync(It.IsAny<ITableEntity>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<ITableEntity, CancellationToken>((entity, _) =>
|
||||
{
|
||||
PerRowAdds.Add((entity.PartitionKey, entity.RowKey));
|
||||
if (PerRowAddBehavior != null) return Task.FromResult(PerRowAddBehavior(entity.RowKey));
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
var mockServiceClient = new Mock<TableServiceClient>();
|
||||
mockServiceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(mockTableClient.Object);
|
||||
|
||||
Sut = new TableStorageService(mockServiceClient.Object, NullLogger<TableStorageService>.Instance);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
{
|
||||
/// <summary>
|
||||
/// <c>POST /api/admin/sessions/{sessionId}/restore</c> — GA-only cascade-delete restore
|
||||
/// endpoint (plan §13, PR4b). Body: <c>{ "manifestId": "...", "dryRun": false }</c>.
|
||||
/// Dispatches into <see cref="SessionRestoreService"/> which auto-selects full vs
|
||||
/// partial-poisoned-recovery mode based on the (Sessions row state, progress.CompletedAt)
|
||||
/// tuple. Status mapping:
|
||||
/// <list type="bullet">
|
||||
/// <item>200 OK — restore (or dry-run) succeeded, body carries the row counts.</item>
|
||||
/// <item>400 Bad Request — missing/invalid body, missing manifestId.</item>
|
||||
/// <item>404 Not Found — manifest blob missing (GC'd past 33-day window) or session unknown.</item>
|
||||
/// <item>409 Conflict — every <c>Reject*</c> outcome (state mismatch, manifest corruption, already at None, manifestId mismatch, CAS conflict on clear).</item>
|
||||
/// </list>
|
||||
/// Route registration in <c>EndpointAccessPolicyCatalog</c> with
|
||||
/// <c>EndpointPolicy.GlobalAdminOnly</c> is what makes this GA-only; unregistered routes
|
||||
/// fail-closed → 403 (per memory <c>feedback_route_policy_catalog</c>).
|
||||
/// </summary>
|
||||
public class RestoreSessionFunction
|
||||
{
|
||||
private readonly ILogger<RestoreSessionFunction> _logger;
|
||||
private readonly SessionRestoreService _restoreService;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
|
||||
public RestoreSessionFunction(
|
||||
ILogger<RestoreSessionFunction> logger,
|
||||
SessionRestoreService restoreService,
|
||||
ISessionRepository sessionRepo)
|
||||
{
|
||||
_logger = logger;
|
||||
_restoreService = restoreService;
|
||||
_sessionRepo = sessionRepo;
|
||||
}
|
||||
|
||||
[Function("RestoreSession")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "admin/sessions/{sessionId}/restore")] HttpRequestData req,
|
||||
string sessionId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sessionId))
|
||||
{
|
||||
return await WriteJsonAsync(req, HttpStatusCode.BadRequest, new { success = false, message = "sessionId is required" });
|
||||
}
|
||||
|
||||
// Parse body.
|
||||
RestoreRequestBody? body;
|
||||
try
|
||||
{
|
||||
body = await JsonSerializer.DeserializeAsync<RestoreRequestBody>(
|
||||
req.Body, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "RestoreSession: malformed body for session {SessionId}", sessionId);
|
||||
return await WriteJsonAsync(req, HttpStatusCode.BadRequest,
|
||||
new { success = false, message = "Request body is not valid JSON. Expected: { \"manifestId\": \"...\", \"dryRun\": false }" });
|
||||
}
|
||||
if (body == null || string.IsNullOrWhiteSpace(body.ManifestId))
|
||||
{
|
||||
return await WriteJsonAsync(req, HttpStatusCode.BadRequest,
|
||||
new { success = false, message = "manifestId is required in the request body." });
|
||||
}
|
||||
|
||||
// Tenant resolution: prefer JWT TargetTenantId, fall back to SessionsIndex lookup
|
||||
// (Global Admin can target any session by id alone). Mirrors the preview endpoint.
|
||||
var requestCtx = req.GetRequestContext();
|
||||
var actorEmail = TenantHelper.GetUserIdentifier(req);
|
||||
var tenantId = requestCtx.TargetTenantId;
|
||||
if (string.IsNullOrEmpty(tenantId) || string.Equals(tenantId, "global", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
tenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId);
|
||||
}
|
||||
if (string.IsNullOrEmpty(tenantId))
|
||||
{
|
||||
// Sessions row is gone — but a full restore IS legal in that case. We need a way
|
||||
// to determine the tenantId without the SessionsIndex (which is also gone).
|
||||
// The manifest blob path encodes tenantId, but the operator only gives us
|
||||
// sessionId + manifestId. Require the operator to scope the call by providing
|
||||
// the tenantId in the request body (or via the X-Target-Tenant header).
|
||||
if (string.IsNullOrWhiteSpace(body.TenantId))
|
||||
{
|
||||
return await WriteJsonAsync(req, HttpStatusCode.NotFound, new
|
||||
{
|
||||
success = false,
|
||||
message = $"Session {sessionId} not found in any active SessionsIndex entry. " +
|
||||
"If the cascade completed (Sessions row gone), pass the tenantId explicitly in the request body."
|
||||
});
|
||||
}
|
||||
tenantId = body.TenantId;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"RestoreSession: tenant={TenantId} session={SessionId} manifestId={ManifestId} dryRun={DryRun} actor={Actor}",
|
||||
tenantId, sessionId, body.ManifestId, body.DryRun, actorEmail);
|
||||
|
||||
SessionRestoreResult result;
|
||||
try
|
||||
{
|
||||
result = await _restoreService.RestoreAsync(
|
||||
tenantId, sessionId, body.ManifestId,
|
||||
body.DryRun, actorEmail);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"RestoreSession: unhandled exception for tenant={TenantId} session={SessionId} manifestId={ManifestId}",
|
||||
tenantId, sessionId, body.ManifestId);
|
||||
return await WriteJsonAsync(req, HttpStatusCode.InternalServerError,
|
||||
new { success = false, message = "Internal error during restore — see audit log + telemetry for details.", exceptionType = ex.GetType().Name });
|
||||
}
|
||||
|
||||
return await WriteResultAsync(req, result);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure status-mapping function for the restore outcome → HTTP status code translation.
|
||||
/// Extracted as static + public so unit tests can verify it without spinning up the full
|
||||
/// HttpRequestData mocking machinery.
|
||||
/// </summary>
|
||||
public static HttpStatusCode MapOutcomeToStatus(SessionRestoreOutcome outcome) => outcome switch
|
||||
{
|
||||
SessionRestoreOutcome.Restored => HttpStatusCode.OK,
|
||||
SessionRestoreOutcome.DryRunOk => HttpStatusCode.OK,
|
||||
SessionRestoreOutcome.RejectManifestNotFound => HttpStatusCode.NotFound,
|
||||
_ => HttpStatusCode.Conflict,
|
||||
};
|
||||
|
||||
private static async Task<HttpResponseData> WriteResultAsync(HttpRequestData req, SessionRestoreResult result)
|
||||
{
|
||||
var status = MapOutcomeToStatus(result.Outcome);
|
||||
|
||||
var body = new
|
||||
{
|
||||
success = status == HttpStatusCode.OK,
|
||||
outcome = result.Outcome.ToString(),
|
||||
mode = result.Mode,
|
||||
message = result.Message,
|
||||
currentState = result.CurrentState,
|
||||
pendingManifestId = result.PendingManifestId,
|
||||
rowsRestoredByTable = result.RowsRestoredByTable,
|
||||
rowsSkippedByTable = result.RowsSkippedByTable,
|
||||
wouldRestoreByTable = result.WouldRestoreByTable,
|
||||
inventoryReIncrements = result.InventoryReIncrements,
|
||||
durationMs = result.DurationMs,
|
||||
};
|
||||
|
||||
return await WriteJsonAsync(req, status, body);
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseData> WriteJsonAsync(HttpRequestData req, HttpStatusCode status, object body)
|
||||
{
|
||||
var response = req.CreateResponse(status);
|
||||
await response.WriteAsJsonAsync(body);
|
||||
return response;
|
||||
}
|
||||
|
||||
private sealed class RestoreRequestBody
|
||||
{
|
||||
public string ManifestId { get; set; } = string.Empty;
|
||||
public bool DryRun { get; set; }
|
||||
/// <summary>
|
||||
/// Optional: when the Sessions row is already gone (full-restore case after a completed
|
||||
/// cascade), there's no SessionsIndex entry to look up the tenant from. Operators
|
||||
/// should provide tenantId explicitly in this scenario.
|
||||
/// </summary>
|
||||
public string? TenantId { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -136,6 +136,10 @@ builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.Delet
|
||||
// the producer into DeleteSessionFunction + the dedicated SessionDeletionMaintenanceFunction.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionDeletionGuard>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionDeletionProducer>();
|
||||
// PR4b: cascade-delete restore endpoint (full + partial-poisoned-recovery). Consumed by
|
||||
// RestoreSessionFunction; GA-only via EndpointAccessPolicyCatalog. No hosted-service registration
|
||||
// needed — the function is HTTP-triggered.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionRestoreService>();
|
||||
builder.Services.AddHostedService<TableInitializerService>(); // Initialize all tables at startup
|
||||
|
||||
// Data Access Layer — repository interfaces backed by Table Storage.
|
||||
|
||||
@@ -214,6 +214,7 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("GET", "sessions/{sessionId}/decision-graph", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "sessions/{sessionId}/reducer-verification", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "admin/sessions/{sessionId}/delete/preview", EndpointPolicy.GlobalAdminOnly),
|
||||
new("POST", "admin/sessions/{sessionId}/restore", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "global/raw/sessions", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "global/raw/events", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "global/raw/events/search", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
@@ -191,7 +191,22 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// restore semantics rely on this); rethrows the underlying <see cref="RequestFailedException"/>
|
||||
/// on 404 or other storage error.
|
||||
/// </summary>
|
||||
public async Task<DeletionManifest> DownloadDeletionManifestAsync(
|
||||
public virtual async Task<DeletionManifest> DownloadDeletionManifestAsync(
|
||||
string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (manifest, _) = await DownloadDeletionManifestWithShaAsync(tenantId, sessionId, manifestId, cancellationToken).ConfigureAwait(false);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PR4c F6: variant of <see cref="DownloadDeletionManifestAsync"/> that also returns the
|
||||
/// verified snapshot SHA-256 (hex string, lowercase) so the caller can enforce the
|
||||
/// snapshot↔progress binding (plan §3): the caller compares this against
|
||||
/// <see cref="DeletionProgress.SnapshotSha256"/> and refuses to proceed on mismatch.
|
||||
/// Existing behaviour is preserved — the helper still throws
|
||||
/// <see cref="InvalidDataException"/> on blob-metadata vs payload SHA mismatch.
|
||||
/// </summary>
|
||||
public virtual async Task<(DeletionManifest Manifest, string Sha256Hex)> DownloadDeletionManifestWithShaAsync(
|
||||
string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId));
|
||||
@@ -238,7 +253,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
throw new InvalidDataException($"Deletion manifest blob {blobName} deserialized to null.");
|
||||
}
|
||||
return manifest;
|
||||
return (manifest, actualSha256);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -327,6 +342,79 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return response.Value.ETag;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Downloads the cascade-deletion progress blob and returns the parsed
|
||||
/// <see cref="DeletionProgress"/> alongside the current ETag. The ETag must be passed
|
||||
/// back to <see cref="UpdateDeletionProgressAsync"/> for the ETag-CAS that prevents two
|
||||
/// parallel worker invocations (e.g. queue re-delivery before visibility expires) from
|
||||
/// clobbering each other's progress writes (plan §3 + §12-Q10). Fails loud on 404 — the
|
||||
/// progress blob is uploaded by the producer before the queue message is sent, so its
|
||||
/// absence at worker pickup time is a corruption signal.
|
||||
/// </summary>
|
||||
public virtual async Task<(DeletionProgress Progress, string ETag)> DownloadDeletionProgressAsync(
|
||||
string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId));
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("sessionId is required", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(manifestId)) throw new ArgumentException("manifestId is required", nameof(manifestId));
|
||||
|
||||
var blobName = BuildProgressBlobName(tenantId, sessionId, manifestId);
|
||||
var (payload, etag) = await ReadDeletionProgressBlobAsync(blobName, cancellationToken);
|
||||
|
||||
var progress = JsonSerializer.Deserialize<DeletionProgress>(payload, DeletionManifestJson.SerializerOptions);
|
||||
if (progress == null)
|
||||
{
|
||||
throw new InvalidDataException($"Deletion progress blob {blobName} deserialized to null.");
|
||||
}
|
||||
return (progress, etag.ToString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test seam: reads the progress blob bytes + current ETag. Production override calls
|
||||
/// Azure Blob Storage; tests subclass to return canned bytes without spinning up the SDK.
|
||||
/// </summary>
|
||||
protected internal virtual async Task<(byte[] Payload, ETag ETag)> ReadDeletionProgressBlobAsync(
|
||||
string blobName, CancellationToken cancellationToken)
|
||||
{
|
||||
var containerClient = _blobServiceClient.GetBlobContainerClient(DeletionManifestsContainer);
|
||||
var blobClient = containerClient.GetBlobClient(blobName);
|
||||
using var downloaded = new MemoryStream();
|
||||
var response = await blobClient.DownloadToAsync(downloaded, cancellationToken);
|
||||
var properties = await blobClient.GetPropertiesAsync(cancellationToken: cancellationToken);
|
||||
return (downloaded.ToArray(), properties.Value.ETag);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the progress blob with <c>If-Match=ifMatchEtag</c> — single CAS attempt. On
|
||||
/// ETag mismatch (412 Precondition Failed) the underlying
|
||||
/// <see cref="RequestFailedException"/> propagates unchanged so the caller can implement
|
||||
/// the §12-Q10 bounded-retry loop (re-download progress, check whether the target step
|
||||
/// is already in <see cref="DeletionProgress.CompletedSteps"/> — concurrent-winner case
|
||||
/// — or retry the write). Returns the freshly-assigned ETag on success.
|
||||
/// </summary>
|
||||
public virtual async Task<string> UpdateDeletionProgressAsync(
|
||||
string tenantId, string sessionId, string manifestId,
|
||||
DeletionProgress progress, string ifMatchEtag,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId));
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("sessionId is required", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(manifestId)) throw new ArgumentException("manifestId is required", nameof(manifestId));
|
||||
if (progress == null) throw new ArgumentNullException(nameof(progress));
|
||||
if (string.IsNullOrEmpty(ifMatchEtag)) throw new ArgumentException("ifMatchEtag is required", nameof(ifMatchEtag));
|
||||
|
||||
var json = JsonSerializer.SerializeToUtf8Bytes(progress, DeletionManifestJson.SerializerOptions);
|
||||
var blobName = BuildProgressBlobName(tenantId, sessionId, manifestId);
|
||||
var options = new BlobUploadOptions
|
||||
{
|
||||
HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" },
|
||||
Conditions = new BlobRequestConditions { IfMatch = new ETag(ifMatchEtag) },
|
||||
};
|
||||
|
||||
var etag = await WriteDeletionProgressBlobAsync(blobName, json, options, cancellationToken);
|
||||
return etag.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lightweight existence check for a cascade-deletion snapshot blob. Used by the
|
||||
/// producer's resume-from-Preparing path to distinguish "the prior producer crashed
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>
|
||||
/// Live verification pass between the table-step phase and the FINAL tombstone phase of the
|
||||
/// cascade worker (plan §1 P4 + §3 verification table). For every cascade-targeted table the
|
||||
/// service issues a LIVE query against current data — not a manifest-key existence check — so
|
||||
/// it catches "ghost rows" written by paths that bypassed the writer-block guard within the
|
||||
/// T2-T4 race window (§16 R13). A single hit on any table aborts the cascade: caller audits
|
||||
/// <c>deletion_verification_failed</c> with a capped residual-key sample and throws so the
|
||||
/// worker poisons after max-dequeue; operator then runs §13 restore-from-poisoned.
|
||||
/// <para>
|
||||
/// CveIndex is the deliberate exception (§12-Q8): tenant-wide live scan would be cost-prohibitive
|
||||
/// on a per-CVE-partition projection, so the verification falls back to per-manifest-key
|
||||
/// <c>GetEntity</c> checks. Safety is preserved upstream by the <c>SessionDeletionGuard</c>
|
||||
/// wired into every CveIndex writer (§5 PR3).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <see cref="DeletionStepClass.Aggregate"/> is excluded — SoftwareInventory counters are
|
||||
/// statistical (approximate by design; clamp ≥ 0) and a live residual check would always
|
||||
/// over-report. <see cref="DeletionStepClass.Final"/> is excluded — it IS the tombstone the
|
||||
/// verification gate sits in front of.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class CascadeVerificationService
|
||||
{
|
||||
private readonly ISessionDeletionInventoryReader _reader;
|
||||
private readonly ILogger<CascadeVerificationService> _logger;
|
||||
|
||||
// The verification residual sample passed to the audit log is bounded so a pathological
|
||||
// ghost-row scenario doesn't blow up the AuditLogs payload. Operator sees the first
|
||||
// sample; the precise total is recorded as a separate count field.
|
||||
internal const int MaxResidualSampleSize = 50;
|
||||
|
||||
public CascadeVerificationService(ISessionDeletionInventoryReader reader, ILogger<CascadeVerificationService> logger)
|
||||
{
|
||||
_reader = reader;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the verification pass over every non-FINAL, non-AGGREGATE step in the manifest.
|
||||
/// Returns the outcome plus the residual sample to attach to the audit log. The caller
|
||||
/// (Handler) decides whether to throw on a non-clean outcome.
|
||||
/// </summary>
|
||||
public virtual async Task<CascadeVerificationResult> VerifyAsync(DeletionManifest manifest, CancellationToken ct = default)
|
||||
{
|
||||
if (manifest == null) throw new ArgumentNullException(nameof(manifest));
|
||||
|
||||
var residuals = new List<CascadeResidualKey>();
|
||||
var safeTenantId = ODataSanitizer.EscapeValue(manifest.TenantId);
|
||||
var safeSessionId = ODataSanitizer.EscapeValue(manifest.SessionId);
|
||||
|
||||
foreach (var step in manifest.Steps)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
if (step.Class == DeletionStepClass.Aggregate || step.Class == DeletionStepClass.Final)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(step.Table))
|
||||
{
|
||||
// Synthetic non-AGGREGATE step is a corruption signal — every table-targeted
|
||||
// step must carry a Table name. Defensive: caller already validates the
|
||||
// manifest shape but the verifier must not silently skip.
|
||||
throw new InvalidOperationException(
|
||||
$"Manifest step Order={step.Order} Class={step.Class} has no Table — verification cannot proceed.");
|
||||
}
|
||||
|
||||
var found = await VerifyStepAsync(step, safeTenantId, safeSessionId, manifest.SessionId, ct);
|
||||
if (found.Count == 0) continue;
|
||||
|
||||
_logger.LogWarning(
|
||||
"Cascade verification found {Count} residual rows in table {Table} for tenant={TenantId} session={SessionId} manifestId={ManifestId}",
|
||||
found.Count, step.Table, manifest.TenantId, manifest.SessionId, manifest.ManifestId);
|
||||
|
||||
foreach (var key in found)
|
||||
{
|
||||
if (residuals.Count >= MaxResidualSampleSize) break;
|
||||
residuals.Add(key);
|
||||
}
|
||||
|
||||
// First-table-failure short-circuit: once we know the verification gate must reject
|
||||
// tombstone, the audit sample is informational; we do not need to enumerate every
|
||||
// remaining table. The Handler's poison + restore-from-poisoned path is the
|
||||
// recovery mechanism, not a per-table inventory.
|
||||
return new CascadeVerificationResult(isClean: false, residuals: residuals);
|
||||
}
|
||||
|
||||
return new CascadeVerificationResult(isClean: true, residuals: residuals);
|
||||
}
|
||||
|
||||
private async Task<List<CascadeResidualKey>> VerifyStepAsync(
|
||||
DeletionStep step, string safeTenantId, string safeSessionId, string sessionId, CancellationToken ct)
|
||||
{
|
||||
var found = new List<CascadeResidualKey>();
|
||||
var table = step.Table!;
|
||||
|
||||
switch (step.Class)
|
||||
{
|
||||
case DeletionStepClass.PkBySession:
|
||||
{
|
||||
var filter = $"PartitionKey eq '{safeTenantId}_{safeSessionId}'";
|
||||
await foreach (var entity in _reader.QueryAsync(table, filter, ct))
|
||||
{
|
||||
found.Add(new CascadeResidualKey(table, entity.PartitionKey ?? string.Empty, entity.RowKey ?? string.Empty));
|
||||
if (found.Count >= MaxResidualSampleSize) break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case DeletionStepClass.PropTenantPk:
|
||||
{
|
||||
var filter = $"PartitionKey eq '{safeTenantId}' and SessionId eq '{safeSessionId}'";
|
||||
await foreach (var entity in _reader.QueryAsync(table, filter, ct))
|
||||
{
|
||||
found.Add(new CascadeResidualKey(table, entity.PartitionKey ?? string.Empty, entity.RowKey ?? string.Empty));
|
||||
if (found.Count >= MaxResidualSampleSize) break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case DeletionStepClass.PkRkExact:
|
||||
{
|
||||
// The manifest stamped (PK, RK) at preflight; verification is the same lookup.
|
||||
foreach (var row in step.Rows)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var entity = await _reader.GetEntityOrNullAsync(table, row.Pk, row.Rk, ct);
|
||||
if (entity != null)
|
||||
{
|
||||
found.Add(new CascadeResidualKey(table, row.Pk, row.Rk));
|
||||
if (found.Count >= MaxResidualSampleSize) break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case DeletionStepClass.DiscriminatorPkRkSuffix:
|
||||
{
|
||||
var filter = $"PartitionKey ge '{safeTenantId}_' and PartitionKey lt '{safeTenantId}_~'";
|
||||
var suffix = $"_{sessionId}";
|
||||
await foreach (var entity in _reader.QueryAsync(table, filter, ct))
|
||||
{
|
||||
if (entity.RowKey != null && entity.RowKey.EndsWith(suffix, StringComparison.Ordinal))
|
||||
{
|
||||
found.Add(new CascadeResidualKey(table, entity.PartitionKey ?? string.Empty, entity.RowKey));
|
||||
if (found.Count >= MaxResidualSampleSize) break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case DeletionStepClass.DiscriminatorPkRkExact:
|
||||
{
|
||||
// §12-Q8: manifest-key only — direct GetEntity per (PK, RK).
|
||||
foreach (var row in step.Rows)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var entity = await _reader.GetEntityOrNullAsync(table, row.Pk, row.Rk, ct);
|
||||
if (entity != null)
|
||||
{
|
||||
found.Add(new CascadeResidualKey(table, row.Pk, row.Rk));
|
||||
if (found.Count >= MaxResidualSampleSize) break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case DeletionStepClass.DiscriminatorPkProp:
|
||||
{
|
||||
var filter = $"PartitionKey ge '{safeTenantId}_' and PartitionKey lt '{safeTenantId}_~' and SessionId eq '{safeSessionId}'";
|
||||
await foreach (var entity in _reader.QueryAsync(table, filter, ct))
|
||||
{
|
||||
found.Add(new CascadeResidualKey(table, entity.PartitionKey ?? string.Empty, entity.RowKey ?? string.Empty));
|
||||
if (found.Count >= MaxResidualSampleSize) break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
throw new InvalidOperationException(
|
||||
$"Manifest step Order={step.Order} has unsupported verification class '{step.Class}' (expected one of the table-targeted classes).");
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Outcome of a <see cref="CascadeVerificationService.VerifyAsync"/> call. When
|
||||
/// <see cref="IsClean"/> is false, <see cref="Residuals"/> carries the first up to
|
||||
/// <c>MaxResidualSampleSize</c> ghost rows for the audit log.
|
||||
/// </summary>
|
||||
public sealed class CascadeVerificationResult
|
||||
{
|
||||
public bool IsClean { get; }
|
||||
public IReadOnlyList<CascadeResidualKey> Residuals { get; }
|
||||
|
||||
public CascadeVerificationResult(bool isClean, IReadOnlyList<CascadeResidualKey> residuals)
|
||||
{
|
||||
IsClean = isClean;
|
||||
Residuals = residuals;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One residual key found during verification. Carried in the
|
||||
/// <c>deletion_verification_failed</c> audit entry's details Dictionary for operator triage.
|
||||
/// </summary>
|
||||
public sealed class CascadeResidualKey
|
||||
{
|
||||
public string Table { get; }
|
||||
public string Pk { get; }
|
||||
public string Rk { get; }
|
||||
|
||||
public CascadeResidualKey(string table, string pk, string rk)
|
||||
{
|
||||
Table = table;
|
||||
Pk = pk;
|
||||
Rk = rk;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,617 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-envelope cascade executor (plan §5 PR4). Consumed by
|
||||
/// <see cref="SessionDeletionWorker"/>; pure DI, no queue knowledge. Runs the §3 schema:
|
||||
/// <list type="number">
|
||||
/// <item>Download snapshot + progress; refuse on hash mismatch or already-completed.</item>
|
||||
/// <item>Inspect Sessions row state (Plan §16 R13): <c>Queued</c> → CAS to <c>Running</c>;
|
||||
/// <c>Running</c> with matching ManifestId → resume (no CAS); anything else with
|
||||
/// matching ManifestId → poison; mismatched ManifestId → poison.</item>
|
||||
/// <item>Iterate manifest steps in order, skipping those already in
|
||||
/// <see cref="DeletionProgress.CompletedSteps"/>. Dispatch by
|
||||
/// <see cref="DeletionStep.Class"/>; AGGREGATE runs the per-key decrement;
|
||||
/// FINAL is deferred until after live verification.</item>
|
||||
/// <item>Live verification pass (<see cref="CascadeVerificationService"/>). One residual
|
||||
/// row aborts the cascade — audit <c>deletion_verification_failed</c> and throw so
|
||||
/// the worker poisons after max-dequeue.</item>
|
||||
/// <item>Tombstone: delete <c>SessionsIndex</c> first (UI listings update immediately),
|
||||
/// then <c>Sessions</c> (row removal IS the completion signal — Plan §1 P7 + §16 R2).</item>
|
||||
/// <item>Audit <c>deletion_completed</c> + SignalR <c>sessionDeleted</c>.</item>
|
||||
/// </list>
|
||||
/// All progress-blob writes are ETag-CAS with a bounded retry loop per §12-Q10
|
||||
/// (10 attempts or 60s wall-clock); on a 412 the loop re-reads the progress blob and
|
||||
/// treats the mutation as already-applied if a concurrent worker already wrote it.
|
||||
/// </summary>
|
||||
public class SessionDeletionHandler
|
||||
{
|
||||
// §12-Q10: 10 attempts OR 60s wall-clock, whichever first → throw so the worker poisons.
|
||||
internal const int ProgressCasMaxAttempts = 10;
|
||||
internal static readonly TimeSpan ProgressCasMaxWallClock = TimeSpan.FromSeconds(60);
|
||||
|
||||
// Cap on the per-table audit-detail `residualKeys` sample — paired with
|
||||
// CascadeVerificationService.MaxResidualSampleSize. The Dictionary<string, string> entry
|
||||
// holds a small JSON array of "{table}/{pk}/{rk}" strings.
|
||||
internal const int AuditResidualSampleSize = 50;
|
||||
|
||||
private readonly TableStorageService _storage;
|
||||
private readonly BlobStorageService _blob;
|
||||
private readonly CascadeVerificationService _verifier;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly ISignalRNotificationService _signalR;
|
||||
private readonly ILogger<SessionDeletionHandler> _logger;
|
||||
|
||||
public SessionDeletionHandler(
|
||||
TableStorageService storage,
|
||||
BlobStorageService blob,
|
||||
CascadeVerificationService verifier,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
ISignalRNotificationService signalR,
|
||||
ILogger<SessionDeletionHandler> logger)
|
||||
{
|
||||
_storage = storage;
|
||||
_blob = blob;
|
||||
_verifier = verifier;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_signalR = signalR;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public virtual async Task HandleAsync(SessionDeletionEnvelope envelope, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
|
||||
if (string.IsNullOrEmpty(envelope.TenantId)) throw new ArgumentException("envelope.TenantId is required", nameof(envelope));
|
||||
if (string.IsNullOrEmpty(envelope.SessionId)) throw new ArgumentException("envelope.SessionId is required", nameof(envelope));
|
||||
if (string.IsNullOrEmpty(envelope.ManifestId)) throw new ArgumentException("envelope.ManifestId is required", nameof(envelope));
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var tenantId = envelope.TenantId;
|
||||
var sessionId = envelope.SessionId;
|
||||
var manifestId = envelope.ManifestId;
|
||||
|
||||
// (1) Download the snapshot. SHA-256 is verified inside DownloadDeletionManifestWithShaAsync;
|
||||
// any InvalidDataException propagates and the worker will poison after max-dequeue.
|
||||
// PR4c F6: we use the with-Sha variant so we can additionally verify the
|
||||
// snapshot↔progress binding once both blobs are loaded.
|
||||
var (manifest, snapshotSha) = await _blob.DownloadDeletionManifestWithShaAsync(tenantId, sessionId, manifestId, cancellationToken);
|
||||
if (!string.Equals(manifest.TenantId, tenantId, StringComparison.Ordinal)
|
||||
|| !string.Equals(manifest.SessionId, sessionId, StringComparison.Ordinal)
|
||||
|| !string.Equals(manifest.ManifestId, manifestId, StringComparison.Ordinal))
|
||||
{
|
||||
throw new System.IO.InvalidDataException(
|
||||
$"Manifest envelope mismatch: envelope=({tenantId},{sessionId},{manifestId}) " +
|
||||
$"manifest=({manifest.TenantId},{manifest.SessionId},{manifest.ManifestId})");
|
||||
}
|
||||
|
||||
// (2) Download progress + ETag. If already completed, this is an idempotent re-pickup.
|
||||
var (progress, etag) = await _blob.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, cancellationToken);
|
||||
|
||||
// PR4c F6: enforce the snapshot↔progress SHA binding (plan §3). The producer wrote
|
||||
// the manifest's SHA into progress.SnapshotSha256 at upload time; if they don't match
|
||||
// here, the artifacts have been tampered with or the producer crashed in a bad way.
|
||||
if (!string.IsNullOrEmpty(progress.SnapshotSha256)
|
||||
&& !string.Equals(progress.SnapshotSha256, snapshotSha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new System.IO.InvalidDataException(
|
||||
$"Snapshot/progress SHA binding mismatch: progress.SnapshotSha256={progress.SnapshotSha256} " +
|
||||
$"manifest SHA={snapshotSha} — corruption signal. tenant={tenantId} session={sessionId} manifestId={manifestId}");
|
||||
}
|
||||
|
||||
if (progress.CompletedAt != null)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionHandler skipping already-completed cascade: tenant={TenantId} session={SessionId} manifestId={ManifestId} completedAt={CompletedAt}",
|
||||
tenantId, sessionId, manifestId, progress.CompletedAt.Value);
|
||||
return;
|
||||
}
|
||||
|
||||
// (3) Inspect Sessions row and dispatch on state. Plan §16 R13 — Queued → CAS Running;
|
||||
// Running w/ matching ManifestId → resume; anything else → poison.
|
||||
await AcquireOrResumeRunningStateAsync(tenantId, sessionId, manifestId, cancellationToken);
|
||||
|
||||
// (4) Per-step execution loop. Iterate by Order; skip those already complete; defer FINAL.
|
||||
foreach (var step in manifest.Steps.OrderBy(s => s.Order))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (progress.CompletedSteps.Contains(step.Order)) continue;
|
||||
if (step.Class == DeletionStepClass.Final) continue; // tombstone runs after verification
|
||||
|
||||
GuardStepShape(step);
|
||||
|
||||
var stepStart = sw.ElapsedMilliseconds;
|
||||
DeletionBatchResult result = DeletionBatchResult.Empty;
|
||||
int aggregateProcessed = 0;
|
||||
|
||||
try
|
||||
{
|
||||
switch (step.Class)
|
||||
{
|
||||
case DeletionStepClass.PkBySession:
|
||||
case DeletionStepClass.PkRkExact:
|
||||
case DeletionStepClass.PropTenantPk:
|
||||
case DeletionStepClass.DiscriminatorPkRkSuffix:
|
||||
case DeletionStepClass.DiscriminatorPkRkExact:
|
||||
case DeletionStepClass.DiscriminatorPkProp:
|
||||
result = await ExecuteTableStepAsync(step, cancellationToken);
|
||||
break;
|
||||
|
||||
case DeletionStepClass.Aggregate:
|
||||
// PR4c F1: per-key idempotent decrement. The helper takes + returns
|
||||
// (progress, etag) because each key's success is persisted with
|
||||
// ETag-CAS before the next key is attempted.
|
||||
(aggregateProcessed, progress, etag) = await ExecuteAggregateStepWithIdempotencyAsync(
|
||||
tenantId, sessionId, manifestId, step, progress, etag, cancellationToken);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new InvalidOperationException(
|
||||
$"Manifest step Order={step.Order} has unsupported class '{step.Class}'.");
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (!(ex is OperationCanceledException))
|
||||
{
|
||||
await AuditStepFailedAsync(tenantId, sessionId, manifestId, step, ex, cancellationToken);
|
||||
throw;
|
||||
}
|
||||
|
||||
// Mark complete with bounded-retry CAS. On 412, re-read progress; if a concurrent
|
||||
// worker already added this step.Order to CompletedSteps, treat as success.
|
||||
(progress, etag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, etag,
|
||||
mutate: p => p.CompletedSteps.Add(step.Order),
|
||||
isAlreadyApplied: p => p.CompletedSteps.Contains(step.Order),
|
||||
cancellationToken);
|
||||
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
action: "deletion_step_completed",
|
||||
entityType: "Session",
|
||||
entityId: sessionId,
|
||||
performedBy: "system",
|
||||
details: BuildStepCompletedDetails(manifestId, step, result, aggregateProcessed, sw.ElapsedMilliseconds - stepStart))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// (5) Live verification (§1 P4). Outcome dictates whether tombstone runs at all.
|
||||
if (!progress.VerificationDone)
|
||||
{
|
||||
var verification = await _verifier.VerifyAsync(manifest, cancellationToken);
|
||||
if (!verification.IsClean)
|
||||
{
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
action: "deletion_verification_failed",
|
||||
entityType: "Session",
|
||||
entityId: sessionId,
|
||||
performedBy: "system",
|
||||
details: BuildVerificationFailedDetails(manifestId, verification.Residuals))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
throw new InvalidOperationException(
|
||||
$"Cascade verification found {verification.Residuals.Count} residual row(s) for " +
|
||||
$"tenant={tenantId} session={sessionId} manifestId={manifestId}; refusing to tombstone.");
|
||||
}
|
||||
|
||||
(progress, etag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, etag,
|
||||
mutate: p => p.VerificationDone = true,
|
||||
isAlreadyApplied: p => p.VerificationDone,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
// (6) FINAL tombstone. Plan §5 PR4 + §16 R2: SessionsIndex first, then Sessions.
|
||||
var tombstone = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Final)
|
||||
?? throw new InvalidOperationException($"Manifest has no FINAL step (tenant={tenantId} session={sessionId} manifestId={manifestId}).");
|
||||
|
||||
if (!progress.CompletedSteps.Contains(tombstone.Order))
|
||||
{
|
||||
// PR4c F2: persist TombstoneStarted=true BEFORE the first tombstone-row delete.
|
||||
// Closes the "tombstone gap": if the worker dies after deleting the Sessions row
|
||||
// but before writing CompletedAt, restore can still dispatch into full-restore
|
||||
// by reading this flag (otherwise sessions=null + completedAt=null looks like
|
||||
// corruption and restore rejects).
|
||||
if (!progress.TombstoneStarted)
|
||||
{
|
||||
(progress, etag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, etag,
|
||||
mutate: p => p.TombstoneStarted = true,
|
||||
isAlreadyApplied: p => p.TombstoneStarted,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
await ExecuteTombstoneAsync(tombstone, cancellationToken);
|
||||
}
|
||||
|
||||
(progress, etag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, etag,
|
||||
mutate: p =>
|
||||
{
|
||||
p.CompletedSteps.Add(tombstone.Order);
|
||||
p.CompletedAt = DateTime.UtcNow;
|
||||
},
|
||||
isAlreadyApplied: p => p.CompletedAt != null,
|
||||
cancellationToken);
|
||||
|
||||
sw.Stop();
|
||||
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
action: "deletion_completed",
|
||||
entityType: "Session",
|
||||
entityId: sessionId,
|
||||
performedBy: "system",
|
||||
details: BuildCompletedDetails(manifestId, manifest, sw.ElapsedMilliseconds))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await _signalR.NotifySessionDeletedAsync(tenantId, sessionId);
|
||||
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionHandler completed: tenant={TenantId} session={SessionId} manifestId={ManifestId} durationMs={DurationMs}",
|
||||
tenantId, sessionId, manifestId, sw.ElapsedMilliseconds);
|
||||
}
|
||||
|
||||
// ============================================================ State-machine entry ====
|
||||
|
||||
private async Task AcquireOrResumeRunningStateAsync(string tenantId, string sessionId, string manifestId, CancellationToken ct)
|
||||
{
|
||||
// Read the Sessions row to determine current state. The reader interface is implemented
|
||||
// by TableStorageService (PR1) and 404-returns-null is the documented contract.
|
||||
var sessionRow = await ((ISessionDeletionInventoryReader)_storage).GetSessionRowAsync(tenantId, sessionId, ct);
|
||||
if (sessionRow == null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Sessions row missing at worker pickup — corruption or stale envelope. " +
|
||||
$"tenant={tenantId} session={sessionId} manifestId={manifestId}");
|
||||
}
|
||||
|
||||
var pendingManifestId = sessionRow.GetString("PendingDeletionManifestId");
|
||||
if (!string.Equals(pendingManifestId, manifestId, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Stale envelope: Sessions.PendingDeletionManifestId={pendingManifestId} does not match envelope.ManifestId={manifestId} " +
|
||||
$"(tenant={tenantId} session={sessionId}).");
|
||||
}
|
||||
|
||||
var currentState = sessionRow.GetString("DeletionState") ?? string.Empty;
|
||||
if (string.IsNullOrEmpty(currentState)) currentState = SessionDeletionState.None;
|
||||
|
||||
switch (currentState)
|
||||
{
|
||||
case SessionDeletionState.Queued:
|
||||
var cas = await _storage.CasSetSessionDeletionStateAsync(
|
||||
tenantId, sessionId,
|
||||
fromState: SessionDeletionState.Queued,
|
||||
toState: SessionDeletionState.Running,
|
||||
newManifestId: null,
|
||||
ct);
|
||||
if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.Updated)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.WrongState
|
||||
&& cas.CurrentState == SessionDeletionState.Running
|
||||
&& string.Equals(cas.CurrentManifestId, manifestId, StringComparison.Ordinal))
|
||||
{
|
||||
// Concurrent worker already CAS'd to Running — resume directly.
|
||||
return;
|
||||
}
|
||||
throw new InvalidOperationException(
|
||||
$"Failed to CAS Queued→Running: tenant={tenantId} session={sessionId} manifestId={manifestId} " +
|
||||
$"outcome={cas.Outcome} currentState={cas.CurrentState} currentManifestId={cas.CurrentManifestId}");
|
||||
|
||||
case SessionDeletionState.Running:
|
||||
// Idempotent re-pickup — no CAS write (Running → Running is not a valid
|
||||
// transition; skip the CAS guard entirely, the manifestId match above is the
|
||||
// sufficient correctness check).
|
||||
return;
|
||||
|
||||
case SessionDeletionState.Poisoned:
|
||||
throw new InvalidOperationException(
|
||||
$"Sessions row already Poisoned for matching ManifestId — operator must run restore-from-poisoned. " +
|
||||
$"tenant={tenantId} session={sessionId} manifestId={manifestId}");
|
||||
|
||||
default:
|
||||
throw new InvalidOperationException(
|
||||
$"Unexpected DeletionState '{currentState}' on Sessions row with matching ManifestId. " +
|
||||
$"tenant={tenantId} session={sessionId} manifestId={manifestId}");
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================ Step execution ====
|
||||
|
||||
private static void GuardStepShape(DeletionStep step)
|
||||
{
|
||||
// Per §5 PR4: any DISCRIMINATOR_* or PROP_TENANT_PK step with non-zero RowCount must
|
||||
// carry the matching Rows[] entries — empty rows[] + non-zero count is a corruption
|
||||
// signal. PK_BY_SESSION can be empty (no rows present at preflight) so we don't gate
|
||||
// it. PK_RK_EXACT also allows empty rows (target was absent at preflight, e.g.
|
||||
// VulnerabilityReports for a session with no scan).
|
||||
switch (step.Class)
|
||||
{
|
||||
case DeletionStepClass.PropTenantPk:
|
||||
case DeletionStepClass.DiscriminatorPkRkSuffix:
|
||||
case DeletionStepClass.DiscriminatorPkRkExact:
|
||||
case DeletionStepClass.DiscriminatorPkProp:
|
||||
if (step.RowCount > 0 && step.Rows.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Corruption: step Order={step.Order} Class={step.Class} reports RowCount={step.RowCount} " +
|
||||
"but Rows[] is empty.");
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DeletionBatchResult> ExecuteTableStepAsync(DeletionStep step, CancellationToken ct)
|
||||
{
|
||||
if (string.IsNullOrEmpty(step.Table))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Manifest step Order={step.Order} Class={step.Class} has no Table — cannot execute.");
|
||||
}
|
||||
if (step.Rows.Count == 0)
|
||||
{
|
||||
return DeletionBatchResult.Empty; // nothing to delete (empty PK_BY_SESSION, absent PK_RK_EXACT, etc.)
|
||||
}
|
||||
|
||||
var keys = step.Rows.Select(r => (r.Pk, r.Rk)).ToList();
|
||||
return await _storage.DeleteByExactKeysInBatchesAsync(step.Table!, keys, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PR4c F1: per-key idempotent AGGREGATE-step executor. Persists the composite key into
|
||||
/// <see cref="DeletionProgress.AggregateDecrementsApplied"/> with ETag-CAS <b>before</b>
|
||||
/// issuing the underlying decrement; on retry, keys already in the set are skipped.
|
||||
/// <para>
|
||||
/// Ordering rationale: persist-first/decrement-second bounds the worst-case drift to
|
||||
/// <c>+1 per crash</c> (rare key-was-marked-but-decrement-didn't-run case). The opposite
|
||||
/// ordering would unbound the drift to <c>-N per crash</c> (every key re-decremented on
|
||||
/// retry — the original Codex F1 finding). The clamp-≥-0 invariant in the underlying
|
||||
/// helper provides the symmetric protection in the rare miss direction.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private async Task<(int Processed, DeletionProgress Progress, string Etag)> ExecuteAggregateStepWithIdempotencyAsync(
|
||||
string tenantId, string sessionId, string manifestId,
|
||||
DeletionStep step, DeletionProgress progress, string etag, CancellationToken ct)
|
||||
{
|
||||
// The only AGGREGATE step today is SoftwareInventoryDecrement (§17.5).
|
||||
if (!string.Equals(step.Step, DeletionStepNames.SoftwareInventoryDecrement, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Manifest AGGREGATE step Order={step.Order} has unsupported synthetic step name '{step.Step}'.");
|
||||
}
|
||||
if (step.Decrements == null || step.Decrements.Count == 0)
|
||||
{
|
||||
return (0, progress, etag);
|
||||
}
|
||||
|
||||
// Lazy-initialize the applied-keys set on first AGGREGATE-step entry. Persist so
|
||||
// PR1-PR4 progress blobs (which had no such field) gain it on first use.
|
||||
if (progress.AggregateDecrementsApplied == null)
|
||||
{
|
||||
(progress, etag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, etag,
|
||||
mutate: p => p.AggregateDecrementsApplied ??= new HashSet<string>(StringComparer.Ordinal),
|
||||
isAlreadyApplied: p => p.AggregateDecrementsApplied != null,
|
||||
ct);
|
||||
}
|
||||
|
||||
var processed = 0;
|
||||
foreach (var key in step.Decrements)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var composite = BuildAggregateCompositeKey(key);
|
||||
if (progress.AggregateDecrementsApplied!.Contains(composite))
|
||||
{
|
||||
continue; // already applied on a prior attempt
|
||||
}
|
||||
|
||||
// F1 ordering: persist FIRST (mark intent), decrement SECOND (apply effect).
|
||||
// Capture composite in a local so the closure doesn't surprise on retry.
|
||||
var compositeForClosure = composite;
|
||||
(progress, etag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, etag,
|
||||
mutate: p => p.AggregateDecrementsApplied!.Add(compositeForClosure),
|
||||
isAlreadyApplied: p => p.AggregateDecrementsApplied!.Contains(compositeForClosure),
|
||||
ct);
|
||||
|
||||
// Decrement helper is bounded-retry + clamp-≥-0 + 404-idempotent (PR2 §17.4).
|
||||
await _storage.DecrementSoftwareInventoryEntryAsync(tenantId, key.Vendor, key.Name, key.Version, ct);
|
||||
processed++;
|
||||
}
|
||||
return (processed, progress, etag);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Composite-key format for <see cref="DeletionProgress.AggregateDecrementsApplied"/>
|
||||
/// and <see cref="DeletionProgress.RestoreReIncrementsApplied"/>. Matches the colon-
|
||||
/// separated shape that <c>BuildSoftwareInventoryRowKey</c> in
|
||||
/// <c>TableStorageService.Inventory.cs</c> uses, so the same key identifies a row across
|
||||
/// forward (decrement) and reverse (re-increment) operations.
|
||||
/// </summary>
|
||||
internal static string BuildAggregateCompositeKey(DeletionDecrementKey key)
|
||||
=> $"{key.Vendor ?? string.Empty}:{key.Name ?? string.Empty}:{key.Version ?? string.Empty}";
|
||||
|
||||
private async Task ExecuteTombstoneAsync(DeletionStep tombstone, CancellationToken ct)
|
||||
{
|
||||
if (tombstone.Rows.Count == 0)
|
||||
{
|
||||
// The Sessions row would not exist at this point in normal flow, so an empty
|
||||
// FINAL step is acceptable on a re-run of an already-tombstoned cascade. The
|
||||
// FINAL step is marked complete after this method returns; subsequent picks
|
||||
// hit the `CompletedAt != null` early-return.
|
||||
return;
|
||||
}
|
||||
|
||||
// The builder (DeletionManifestBuilder.AddTombstoneStep) emits SessionsIndex first
|
||||
// (if present) then Sessions. Determine table by row.Rk shape: anything matching the
|
||||
// session id is the Sessions row; everything else is SessionsIndex. This works even
|
||||
// if the manifest carries only one of the two rows (corruption-recovery scenario).
|
||||
foreach (var row in tombstone.Rows)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var table = IsSessionsRow(row) ? Constants.TableNames.Sessions : Constants.TableNames.SessionsIndex;
|
||||
await _storage.DeleteByExactKeysInBatchesAsync(table, new[] { (row.Pk, row.Rk) }, ct);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsSessionsRow(DeletionRowDump row)
|
||||
{
|
||||
// Sessions row: PK = {tenantId}, RK = {sessionId}.
|
||||
// SessionsIndex row: PK = {tenantId}, RK = {indexRowKey} (typically inverted-timestamp + sessionId).
|
||||
// The discriminator is whether the RowKey *equals* a sessionId-shaped string. We use a
|
||||
// simple heuristic: if the RowKey contains a "_" the row is the SessionsIndex (its
|
||||
// RK is always composed); otherwise it's the Sessions row. This matches the builder's
|
||||
// current schema (Sessions RK is the bare session GUID, IndexRowKey is "inverted_…_GUID").
|
||||
return row.Rk != null && !row.Rk.Contains('_');
|
||||
}
|
||||
|
||||
// ============================================================ Progress CAS retry ====
|
||||
|
||||
/// <summary>
|
||||
/// Bounded-retry helper for ETag-CAS progress-blob writes. On 412 Precondition Failed
|
||||
/// it re-reads the progress blob; if <paramref name="isAlreadyApplied"/> returns true
|
||||
/// against the freshly-read progress, treats the write as already-done by a concurrent
|
||||
/// worker and returns the fresh (progress, etag). Otherwise re-applies the mutation
|
||||
/// against the fresh progress and retries. Caps at <see cref="ProgressCasMaxAttempts"/>
|
||||
/// attempts or <see cref="ProgressCasMaxWallClock"/> wall-clock, whichever first.
|
||||
/// </summary>
|
||||
private async Task<(DeletionProgress, string)> UpdateProgressWithRetryAsync(
|
||||
string tenantId, string sessionId, string manifestId,
|
||||
DeletionProgress progress, string etag,
|
||||
Action<DeletionProgress> mutate,
|
||||
Func<DeletionProgress, bool> isAlreadyApplied,
|
||||
CancellationToken ct)
|
||||
{
|
||||
mutate(progress);
|
||||
|
||||
var deadline = DateTime.UtcNow + ProgressCasMaxWallClock;
|
||||
for (var attempt = 0; attempt < ProgressCasMaxAttempts && DateTime.UtcNow < deadline; attempt++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
var newEtag = await _blob.UpdateDeletionProgressAsync(tenantId, sessionId, manifestId, progress, etag, ct);
|
||||
return (progress, newEtag);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412)
|
||||
{
|
||||
var fresh = await _blob.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct);
|
||||
if (isAlreadyApplied(fresh.Progress))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionHandler progress-CAS conflict resolved: concurrent worker already applied mutation. " +
|
||||
"tenant={TenantId} session={SessionId} manifestId={ManifestId} attempt={Attempt}",
|
||||
tenantId, sessionId, manifestId, attempt + 1);
|
||||
return (fresh.Progress, fresh.ETag);
|
||||
}
|
||||
progress = fresh.Progress;
|
||||
etag = fresh.ETag;
|
||||
mutate(progress);
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(
|
||||
$"Progress-blob ETag-CAS exhausted after {ProgressCasMaxAttempts} attempts or " +
|
||||
$"{ProgressCasMaxWallClock.TotalSeconds:0}s wall-clock. " +
|
||||
$"tenant={tenantId} session={sessionId} manifestId={manifestId}");
|
||||
}
|
||||
|
||||
// ============================================================ Audit details builders ====
|
||||
|
||||
private static Dictionary<string, string> BuildStepCompletedDetails(
|
||||
string manifestId, DeletionStep step, DeletionBatchResult tableResult, int aggregateProcessed, long stepDurationMs)
|
||||
{
|
||||
var details = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["manifestId"] = manifestId,
|
||||
["stepOrder"] = step.Order.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
["class"] = step.Class,
|
||||
["stepDurationMs"] = stepDurationMs.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
};
|
||||
if (!string.IsNullOrEmpty(step.Table))
|
||||
{
|
||||
details["table"] = step.Table!;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(step.Step))
|
||||
{
|
||||
details["syntheticStep"] = step.Step!;
|
||||
}
|
||||
if (step.Class == DeletionStepClass.Aggregate)
|
||||
{
|
||||
details["decrementsProcessed"] = aggregateProcessed.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
else
|
||||
{
|
||||
details["attempted"] = tableResult.Attempted.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
details["deletedNow"] = tableResult.DeletedNow.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
details["alreadyMissing"] = tableResult.AlreadyMissing.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
return details;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildVerificationFailedDetails(
|
||||
string manifestId, IReadOnlyList<CascadeResidualKey> residuals)
|
||||
{
|
||||
var sample = residuals
|
||||
.Take(AuditResidualSampleSize)
|
||||
.Select(r => new { table = r.Table, pk = r.Pk, rk = r.Rk });
|
||||
return new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["manifestId"] = manifestId,
|
||||
["residualCount"] = residuals.Count.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
["residualKeys"] = JsonSerializer.Serialize(sample),
|
||||
};
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> BuildCompletedDetails(
|
||||
string manifestId, DeletionManifest manifest, long durationMs)
|
||||
{
|
||||
var totalsByTable = manifest.PreflightCounts;
|
||||
return new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["manifestId"] = manifestId,
|
||||
["durationMs"] = durationMs.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
["totalsByTable"] = JsonSerializer.Serialize(totalsByTable),
|
||||
["stepsCount"] = manifest.Steps.Count.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
["schemaHash"] = manifest.SchemaHash,
|
||||
};
|
||||
}
|
||||
|
||||
private Task AuditStepFailedAsync(
|
||||
string tenantId, string sessionId, string manifestId, DeletionStep step, Exception ex, CancellationToken _)
|
||||
{
|
||||
var details = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["manifestId"] = manifestId,
|
||||
["stepOrder"] = step.Order.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
["class"] = step.Class,
|
||||
["exceptionType"] = ex.GetType().FullName ?? "Unknown",
|
||||
// Trim error message to a reasonable size for the audit row.
|
||||
["error"] = ex.Message.Length > 1024 ? ex.Message.Substring(0, 1024) : ex.Message,
|
||||
};
|
||||
if (!string.IsNullOrEmpty(step.Table)) details["table"] = step.Table!;
|
||||
|
||||
return _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
action: "deletion_step_failed",
|
||||
entityType: "Session",
|
||||
entityId: sessionId,
|
||||
performedBy: "system",
|
||||
details: details);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Identity;
|
||||
using Azure.Storage.Queues;
|
||||
using Azure.Storage.Queues.Models;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>
|
||||
/// Background poll-loop for the <c>session-deletion</c> queue (plan §5 PR4). Modeled on
|
||||
/// <see cref="AutopilotMonitor.Functions.Services.Analyze.AnalyzeOnEnrollmentEndQueueWorker"/>
|
||||
/// with four explicit deviations:
|
||||
/// <list type="bullet">
|
||||
/// <item><see cref="BatchSize"/> = <b>1</b> (not 32) — a single cascade can be tens of MB
|
||||
/// and minutes of wall-time; one-at-a-time bounds memory and poison blast radius.</item>
|
||||
/// <item><b>Kill-switch on entry</b>: every receive checks
|
||||
/// <c>AdminConfiguration.SessionDeletionKillSwitch</c>; when active the worker idles
|
||||
/// without dequeuing (envelope stays on the queue, no visibility timeout pressure).</item>
|
||||
/// <item><b>Heartbeat</b>: while <see cref="SessionDeletionHandler"/> runs, a sidecar task
|
||||
/// calls <c>UpdateMessageAsync</c> every <see cref="_heartbeatInterval"/> to extend the
|
||||
/// message visibility by <see cref="HeartbeatExtendBy"/>. Prevents queue re-delivery
|
||||
/// from spawning a parallel worker on the same manifest while a step is in flight.</item>
|
||||
/// <item><b>Poison audit</b>: max-dequeue exhaustion writes a <c>deletion_poisoned</c>
|
||||
/// audit entry alongside the poison-queue move so the operator can correlate the
|
||||
/// poisoned cascade with its manifest. The Sessions row's <c>DeletionState</c> is NOT
|
||||
/// auto-cleared — operator action via restore-from-poisoned (§13, PR4b) is required.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// <b>The worker is NOT registered in <c>Program.cs</c> in PR4</b> — that wires up in PR5
|
||||
/// alongside the flag-gated <c>DeleteSessionFunction</c> changes. Without the producer
|
||||
/// emitting envelopes (gated by <c>TenantConfiguration.EnableCascadeDeleteV2</c>, default
|
||||
/// false), having the worker idle on an empty queue is harmless; we hold off so the worker's
|
||||
/// poll-loop doesn't appear in production telemetry until the cutover.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class SessionDeletionWorker : BackgroundService
|
||||
{
|
||||
/// <summary>
|
||||
/// Explicit deviation from sibling workers — one cascade per receive bounds the worker's
|
||||
/// memory footprint (a 35MB snapshot blob × 32 messages = 1.1GB) and limits poison-queue
|
||||
/// blast radius on a corruption signal. Plan §5 PR4.
|
||||
/// </summary>
|
||||
internal const int BatchSize = 1;
|
||||
|
||||
/// <summary>Initial message-visibility window — extended in-flight by the heartbeat task.</summary>
|
||||
internal static readonly TimeSpan VisibilityTimeout = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>How often the heartbeat task extends the in-flight message's visibility.</summary>
|
||||
internal static readonly TimeSpan DefaultHeartbeatInterval = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>How much the heartbeat task adds to visibility on each tick.</summary>
|
||||
internal static readonly TimeSpan HeartbeatExtendBy = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>Idle sleep between empty receives or kill-switch-on cycles.</summary>
|
||||
internal static readonly TimeSpan DefaultPollInterval = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>Instance-level overrides — production uses the defaults above, tests inject shorter values via the internal ctor.</summary>
|
||||
private readonly TimeSpan _heartbeatInterval;
|
||||
private readonly TimeSpan _pollInterval;
|
||||
|
||||
/// <summary>Cool-down after an unhandled poll-loop exception.</summary>
|
||||
internal static readonly TimeSpan ErrorBackoff = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Match the QueueTrigger platform default + sibling workers: 5 attempts → poison.</summary>
|
||||
internal const int MaxDequeueCount = 5;
|
||||
|
||||
private const string PoisonQueueSuffix = "-poison";
|
||||
|
||||
private readonly QueueClient _mainQueue;
|
||||
private readonly QueueClient _poisonQueue;
|
||||
private readonly SessionDeletionHandler _handler;
|
||||
private readonly TableStorageService _storage;
|
||||
private readonly AdminConfigurationService _adminConfig;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly ILogger<SessionDeletionWorker> _logger;
|
||||
|
||||
public SessionDeletionWorker(
|
||||
IConfiguration configuration,
|
||||
SessionDeletionHandler handler,
|
||||
TableStorageService storage,
|
||||
AdminConfigurationService adminConfig,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
ILogger<SessionDeletionWorker> logger)
|
||||
{
|
||||
_handler = handler ?? throw new ArgumentNullException(nameof(handler));
|
||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||
_adminConfig = adminConfig ?? throw new ArgumentNullException(nameof(adminConfig));
|
||||
_maintenanceRepo = maintenanceRepo ?? throw new ArgumentNullException(nameof(maintenanceRepo));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_heartbeatInterval = DefaultHeartbeatInterval;
|
||||
_pollInterval = DefaultPollInterval;
|
||||
|
||||
var options = new QueueClientOptions
|
||||
{
|
||||
MessageEncoding = QueueMessageEncoding.Base64,
|
||||
};
|
||||
|
||||
var storageAccountName = configuration["AzureStorageAccountName"];
|
||||
var connectionString = configuration["AzureTableStorageConnectionString"];
|
||||
|
||||
if (!string.IsNullOrEmpty(storageAccountName))
|
||||
{
|
||||
var mainUri = new Uri(
|
||||
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.SessionDeletion}");
|
||||
var poisonUri = new Uri(
|
||||
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.SessionDeletion}{PoisonQueueSuffix}");
|
||||
var credential = new DefaultAzureCredential();
|
||||
_mainQueue = new QueueClient(mainUri, credential, options);
|
||||
_poisonQueue = new QueueClient(poisonUri, credential, options);
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionWorker initialized with Managed Identity (account: {Account})",
|
||||
storageAccountName);
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(connectionString))
|
||||
{
|
||||
_mainQueue = new QueueClient(
|
||||
connectionString, Constants.QueueNames.SessionDeletion, options);
|
||||
_poisonQueue = new QueueClient(
|
||||
connectionString, Constants.QueueNames.SessionDeletion + PoisonQueueSuffix, options);
|
||||
_logger.LogInformation("SessionDeletionWorker initialized with connection string");
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Queue Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureTableStorageConnectionString'.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test seam: construct directly with mock <see cref="QueueClient"/> instances so the
|
||||
/// poll loop can be exercised against an in-memory queue without spinning up Azurite.
|
||||
/// Mirrors <see cref="SessionDeletionProducer"/>'s internal test ctor.
|
||||
/// </summary>
|
||||
internal SessionDeletionWorker(
|
||||
QueueClient mainQueue,
|
||||
QueueClient poisonQueue,
|
||||
SessionDeletionHandler handler,
|
||||
TableStorageService storage,
|
||||
AdminConfigurationService adminConfig,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
ILogger<SessionDeletionWorker> logger,
|
||||
TimeSpan? heartbeatInterval = null,
|
||||
TimeSpan? pollInterval = null)
|
||||
{
|
||||
_mainQueue = mainQueue ?? throw new ArgumentNullException(nameof(mainQueue));
|
||||
_poisonQueue = poisonQueue ?? throw new ArgumentNullException(nameof(poisonQueue));
|
||||
_handler = handler ?? throw new ArgumentNullException(nameof(handler));
|
||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||
_adminConfig = adminConfig ?? throw new ArgumentNullException(nameof(adminConfig));
|
||||
_maintenanceRepo = maintenanceRepo ?? throw new ArgumentNullException(nameof(maintenanceRepo));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_heartbeatInterval = heartbeatInterval ?? DefaultHeartbeatInterval;
|
||||
_pollInterval = pollInterval ?? DefaultPollInterval;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await TryCreateQueueAsync(_mainQueue, "main", stoppingToken).ConfigureAwait(false);
|
||||
await TryCreateQueueAsync(_poisonQueue, "poison", stoppingToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("SessionDeletionWorker: poll loop started (queue {Name})", _mainQueue.Name);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Kill-switch entry guard (plan §1 P8 / §9). When active, the worker does not
|
||||
// dequeue — the message stays visible on the main queue until the operator
|
||||
// flips the switch back. Envelope is preserved without losing dequeue budget.
|
||||
var admin = await _adminConfig.GetConfigurationAsync().ConfigureAwait(false);
|
||||
if (admin.SessionDeletionKillSwitch)
|
||||
{
|
||||
_logger.LogDebug("SessionDeletionWorker: kill-switch active; idling for {Interval}", _pollInterval);
|
||||
await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
var batch = await _mainQueue
|
||||
.ReceiveMessagesAsync(BatchSize, VisibilityTimeout, stoppingToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (batch?.Value is null || batch.Value.Length == 0)
|
||||
{
|
||||
await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var msg in batch.Value)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) break;
|
||||
await ProcessOneAsync(msg, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"SessionDeletionWorker: poll-loop error — backing off {Backoff}",
|
||||
ErrorBackoff);
|
||||
try { await Task.Delay(ErrorBackoff, stoppingToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("SessionDeletionWorker: poll loop stopped");
|
||||
}
|
||||
|
||||
private async Task ProcessOneAsync(QueueMessage msg, CancellationToken ct)
|
||||
{
|
||||
if (msg.DequeueCount > MaxDequeueCount)
|
||||
{
|
||||
await MoveToPoisonAsync(msg, ct).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
SessionDeletionEnvelope? envelope;
|
||||
try
|
||||
{
|
||||
envelope = JsonConvert.DeserializeObject<SessionDeletionEnvelope>(msg.Body.ToString());
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"SessionDeletionWorker: malformed envelope JSON — dropping (msg {Id})",
|
||||
msg.MessageId);
|
||||
await SafeDeleteAsync(msg, ct).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (envelope is null
|
||||
|| string.IsNullOrEmpty(envelope.TenantId)
|
||||
|| string.IsNullOrEmpty(envelope.SessionId)
|
||||
|| string.IsNullOrEmpty(envelope.ManifestId))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SessionDeletionWorker: envelope missing required fields — dropping (msg {Id})",
|
||||
msg.MessageId);
|
||||
await SafeDeleteAsync(msg, ct).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Heartbeat: extend visibility every _heartbeatInterval while the handler runs. We
|
||||
// capture the latest PopReceipt locally because each UpdateMessageAsync returns a
|
||||
// fresh receipt; the SafeDelete at the end uses whichever receipt was issued last.
|
||||
using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var heartbeatState = new HeartbeatState(msg.MessageId, msg.PopReceipt);
|
||||
var heartbeatTask = Task.Run(() => HeartbeatLoopAsync(heartbeatState, heartbeatCts.Token), heartbeatCts.Token);
|
||||
|
||||
try
|
||||
{
|
||||
await _handler.HandleAsync(envelope, ct).ConfigureAwait(false);
|
||||
|
||||
// Handler succeeded — stop heartbeat and delete using the latest pop-receipt.
|
||||
heartbeatCts.Cancel();
|
||||
try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { }
|
||||
|
||||
await SafeDeleteAsync(msg.MessageId, heartbeatState.PopReceipt, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
heartbeatCts.Cancel();
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
heartbeatCts.Cancel();
|
||||
try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { }
|
||||
|
||||
_logger.LogWarning(ex,
|
||||
"SessionDeletionWorker: handler failed (tenant={Tenant} session={Session} manifestId={ManifestId} dequeue={N}) — visibility-timeout retry",
|
||||
envelope.TenantId, envelope.SessionId, envelope.ManifestId, msg.DequeueCount);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HeartbeatLoopAsync(HeartbeatState state, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await Task.Delay(_heartbeatInterval, ct).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
|
||||
try
|
||||
{
|
||||
var updated = await _mainQueue.UpdateMessageAsync(
|
||||
state.MessageId, state.PopReceipt,
|
||||
visibilityTimeout: HeartbeatExtendBy,
|
||||
cancellationToken: ct).ConfigureAwait(false);
|
||||
state.PopReceipt = updated.Value.PopReceipt;
|
||||
_logger.LogDebug(
|
||||
"SessionDeletionWorker: heartbeat extended visibility for message {Id} by {Extend}",
|
||||
state.MessageId, HeartbeatExtendBy);
|
||||
}
|
||||
catch (OperationCanceledException) { return; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Heartbeat failure (e.g. PopReceipt expired because the visibility window
|
||||
// was missed) is logged and the loop exits. The handler keeps running; on
|
||||
// its eventual return, SafeDelete will hit a 404 and the message will
|
||||
// reappear after its current visibility expires — but the handler's own
|
||||
// ETag-CAS on the progress blob makes a parallel pickup idempotent.
|
||||
_logger.LogWarning(ex,
|
||||
"SessionDeletionWorker: heartbeat UpdateMessageAsync failed for message {Id} — heartbeat exiting",
|
||||
state.MessageId);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MoveToPoisonAsync(QueueMessage msg, CancellationToken ct)
|
||||
{
|
||||
// Best-effort: attempt to parse envelope so we can audit the manifest id alongside
|
||||
// the poison move. If the body is malformed we still move it to the poison queue
|
||||
// (the audit just lacks the manifestId field).
|
||||
SessionDeletionEnvelope? envelope = null;
|
||||
try { envelope = JsonConvert.DeserializeObject<SessionDeletionEnvelope>(msg.Body.ToString()); }
|
||||
catch (JsonException) { /* swallow — non-parseable envelopes still get poisoned */ }
|
||||
|
||||
// PR4b E1: transition Sessions.DeletionState → Poisoned BEFORE the poison-queue send.
|
||||
// Without this, the row stays at Running (or Queued if poisoned before pickup) and the
|
||||
// restore endpoint cannot dispatch into partial-restore mode. Best-effort: if CAS fails
|
||||
// (concurrent writer, row gone, state mismatch) we still continue with the poison-queue
|
||||
// move so the original poison flow completes and an audit trail exists.
|
||||
if (envelope != null && !string.IsNullOrEmpty(envelope.TenantId) && !string.IsNullOrEmpty(envelope.SessionId))
|
||||
{
|
||||
await TryTransitionToPoisonedStateAsync(envelope.TenantId, envelope.SessionId, envelope.ManifestId, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _poisonQueue.SendMessageAsync(msg.Body.ToString(), ct).ConfigureAwait(false);
|
||||
await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false);
|
||||
_logger.LogWarning(
|
||||
"SessionDeletionWorker: moved message {Id} to poison queue after {N} failed attempts",
|
||||
msg.MessageId, msg.DequeueCount - 1);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"SessionDeletionWorker: poison move failed for message {Id} (will retry)",
|
||||
msg.MessageId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (envelope != null && !string.IsNullOrEmpty(envelope.TenantId) && !string.IsNullOrEmpty(envelope.SessionId))
|
||||
{
|
||||
try
|
||||
{
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
envelope.TenantId,
|
||||
action: "deletion_poisoned",
|
||||
entityType: "Session",
|
||||
entityId: envelope.SessionId,
|
||||
performedBy: "system",
|
||||
details: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["manifestId"] = envelope.ManifestId ?? string.Empty,
|
||||
["reason"] = envelope.Reason ?? string.Empty,
|
||||
["messageId"] = msg.MessageId,
|
||||
["dequeueCount"] = (msg.DequeueCount - 1).ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception auditEx)
|
||||
{
|
||||
_logger.LogError(auditEx,
|
||||
"SessionDeletionWorker: deletion_poisoned audit log write failed for tenant={Tenant} session={Session}",
|
||||
envelope.TenantId, envelope.SessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SafeDeleteAsync(QueueMessage msg, CancellationToken ct)
|
||||
=> await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false);
|
||||
|
||||
private async Task SafeDeleteAsync(string messageId, string popReceipt, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _mainQueue.DeleteMessageAsync(messageId, popReceipt, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"SessionDeletionWorker: delete failed for message {Id} (will reappear after visibility-timeout)",
|
||||
messageId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TryCreateQueueAsync(QueueClient queue, string label, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await queue.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"SessionDeletionWorker: CreateIfNotExists failed for {Label} queue — will continue, send/receive will retry",
|
||||
label);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PR4b E1 + PR4c F5: CAS Sessions.DeletionState to Poisoned with manifestId pre-check.
|
||||
/// Tries <c>Running → Poisoned</c> first (the common case — Worker successfully CAS'd
|
||||
/// Queued→Running on first pickup and the handler then failed N times). Falls back to
|
||||
/// <c>Queued → Poisoned</c> for the rare case where the cascade poisoned before the
|
||||
/// Worker ever successfully transitioned to Running (handler always threw at the
|
||||
/// state-acquire step). Logs warning on persistent CAS failure but does NOT throw — the
|
||||
/// poison-queue move + audit must still complete so the operator has an observable trail
|
||||
/// even if the row state is somehow inconsistent.
|
||||
/// <para>
|
||||
/// <b>PR4c F5</b>: before issuing any CAS, read the Sessions row's
|
||||
/// <c>PendingDeletionManifestId</c> and compare to the envelope's <c>manifestId</c>. If
|
||||
/// they don't match, the envelope is stale (from a prior cascade attempt) — do NOT
|
||||
/// touch the state of a possibly-fresh active cascade. The poison-queue + audit still
|
||||
/// fire in the caller so the stale message is removed from the main queue.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private async Task TryTransitionToPoisonedStateAsync(string tenantId, string sessionId, string? manifestId, CancellationToken ct)
|
||||
{
|
||||
// PR4c F5: pre-check that the envelope's manifestId matches the row's
|
||||
// PendingDeletionManifestId. A stale envelope from a prior cascade must not flip a
|
||||
// fresh active cascade to Poisoned.
|
||||
var row = await _storage.GetSessionRowAsync(tenantId, sessionId, ct).ConfigureAwait(false);
|
||||
if (row == null)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionWorker: Sessions row gone for tenant={Tenant} session={Session} (cascade already tombstoned by another worker?) — skipping state transition",
|
||||
tenantId, sessionId);
|
||||
return;
|
||||
}
|
||||
|
||||
var currentPending = row.GetString("PendingDeletionManifestId");
|
||||
if (!string.Equals(currentPending, manifestId, StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SessionDeletionWorker: stale poison envelope for tenant={Tenant} session={Session} — envelope.ManifestId={Envelope} but row.PendingDeletionManifestId={Pending}; skipping state transition (active cascade is a different manifest)",
|
||||
tenantId, sessionId, manifestId, currentPending);
|
||||
return;
|
||||
}
|
||||
|
||||
// First attempt: Running → Poisoned (the expected case after a successful pickup).
|
||||
var cas = await _storage.CasSetSessionDeletionStateAsync(
|
||||
tenantId, sessionId,
|
||||
fromState: SessionDeletionState.Running,
|
||||
toState: SessionDeletionState.Poisoned,
|
||||
newManifestId: null,
|
||||
ct).ConfigureAwait(false);
|
||||
if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.Updated)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionWorker: CAS Running→Poisoned succeeded for tenant={Tenant} session={Session} manifestId={ManifestId}",
|
||||
tenantId, sessionId, manifestId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback: Queued → Poisoned (poisoned before pickup ever succeeded).
|
||||
if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.WrongState
|
||||
&& cas.CurrentState == SessionDeletionState.Queued)
|
||||
{
|
||||
cas = await _storage.CasSetSessionDeletionStateAsync(
|
||||
tenantId, sessionId,
|
||||
fromState: SessionDeletionState.Queued,
|
||||
toState: SessionDeletionState.Poisoned,
|
||||
newManifestId: null,
|
||||
ct).ConfigureAwait(false);
|
||||
if (cas.Outcome == TableStorageService.SessionDeletionStateCasOutcome.Updated)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionWorker: CAS Queued→Poisoned succeeded for tenant={Tenant} session={Session} manifestId={ManifestId}",
|
||||
tenantId, sessionId, manifestId);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Persistent CAS failure — log and continue so the poison-queue move + audit still fire.
|
||||
_logger.LogWarning(
|
||||
"SessionDeletionWorker: could not transition Sessions.DeletionState to Poisoned for tenant={Tenant} session={Session} manifestId={ManifestId} outcome={Outcome} currentState={State}; restore endpoint may reject with 409 \"active_cascade\" until operator manually intervenes.",
|
||||
tenantId, sessionId, manifestId, cas.Outcome, cas.CurrentState);
|
||||
}
|
||||
|
||||
private sealed class HeartbeatState
|
||||
{
|
||||
public string MessageId { get; }
|
||||
public string PopReceipt { get; set; }
|
||||
public HeartbeatState(string messageId, string popReceipt) { MessageId = messageId; PopReceipt = popReceipt; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,586 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>
|
||||
/// Outcome of a <see cref="SessionRestoreService.RestoreAsync"/> call. The HTTP function
|
||||
/// maps these to status codes (Restored → 200 OK, every Reject* → 409 Conflict, etc.).
|
||||
/// </summary>
|
||||
public enum SessionRestoreOutcome
|
||||
{
|
||||
/// <summary>Restore succeeded; row counts populated.</summary>
|
||||
Restored,
|
||||
/// <summary>Dry run succeeded; <see cref="SessionRestoreResult.WouldRestoreByTable"/> populated, no writes.</summary>
|
||||
DryRunOk,
|
||||
/// <summary>Sessions row present + DeletionState=None — nothing to restore.</summary>
|
||||
RejectAlreadyAtOriginalState,
|
||||
/// <summary>Sessions row present + DeletionState ∈ {Preparing, Queued, Running} — cascade still in flight.</summary>
|
||||
RejectActiveCascade,
|
||||
/// <summary>Sessions row exists but PendingDeletionManifestId does not match the envelope.</summary>
|
||||
RejectManifestIdMismatch,
|
||||
/// <summary>Sessions row absent but progress.CompletedAt is null — corruption signal.</summary>
|
||||
RejectCorruptState,
|
||||
/// <summary>Manifest blob missing or 404 from storage.</summary>
|
||||
RejectManifestNotFound,
|
||||
/// <summary>Snapshot SHA-256 mismatch — manifest is corrupted.</summary>
|
||||
RejectManifestCorruption,
|
||||
/// <summary>CAS Poisoned → None failed at end of partial-restore (concurrent writer raced).</summary>
|
||||
RejectCasConflictOnClear,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Restore-result payload. The HTTP function serializes this directly into the response body.
|
||||
/// </summary>
|
||||
public sealed class SessionRestoreResult
|
||||
{
|
||||
public SessionRestoreOutcome Outcome { get; set; }
|
||||
public string? Mode { get; set; } // "full" | "partial" | "dryRun"
|
||||
public string? Message { get; set; } // operator-readable reason on reject
|
||||
public string? CurrentState { get; set; } // for reject diagnostics
|
||||
public string? PendingManifestId { get; set; } // for reject diagnostics
|
||||
public Dictionary<string, int> RowsRestoredByTable { get; set; } = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
public Dictionary<string, int> RowsSkippedByTable { get; set; } = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
public Dictionary<string, int> WouldRestoreByTable { get; set; } = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
public int InventoryReIncrements { get; set; }
|
||||
public long DurationMs { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Live restore executor (PR4b, plan §13). Symmetric inverse of <see cref="SessionDeletionHandler"/>:
|
||||
/// downloads the cascade manifest, dispatches to <b>full</b> or <b>partial-poisoned-recovery</b>
|
||||
/// mode based on the (Sessions row state, progress.CompletedAt) tuple, re-inserts rows in
|
||||
/// reverse cascade order, re-increments SoftwareInventory counters from the manifest's
|
||||
/// AGGREGATE step, and (partial mode only) CAS-clears <c>Sessions.DeletionState: Poisoned → None</c>.
|
||||
/// <para>
|
||||
/// Every restore attempt — even one that rejects post-download — emits a
|
||||
/// <c>deletion_manifest_downloaded</c> audit entry before any other read (§13.1). Restore
|
||||
/// successes emit <c>deletion_restored</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class SessionRestoreService
|
||||
{
|
||||
private readonly TableStorageService _storage;
|
||||
private readonly BlobStorageService _blob;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly ILogger<SessionRestoreService> _logger;
|
||||
|
||||
public SessionRestoreService(
|
||||
TableStorageService storage,
|
||||
BlobStorageService blob,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
ILogger<SessionRestoreService> logger)
|
||||
{
|
||||
_storage = storage;
|
||||
_blob = blob;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public virtual async Task<SessionRestoreResult> RestoreAsync(
|
||||
string tenantId, string sessionId, string manifestId,
|
||||
bool dryRun, string actor,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId));
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("sessionId is required", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(manifestId)) throw new ArgumentException("manifestId is required", nameof(manifestId));
|
||||
if (string.IsNullOrEmpty(actor)) actor = "system";
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = new SessionRestoreResult();
|
||||
|
||||
// (1) §13.1 — audit BEFORE any read.
|
||||
await AuditManifestDownloadedAsync(tenantId, sessionId, manifestId, actor, ct).ConfigureAwait(false);
|
||||
|
||||
// (2) Download manifest (SHA-verified) + progress. PR4c F6: use the with-Sha variant
|
||||
// and verify the snapshot↔progress binding once both blobs are loaded (plan §3).
|
||||
DeletionManifest manifest;
|
||||
string snapshotSha;
|
||||
DeletionProgress progress;
|
||||
string progressEtag;
|
||||
try
|
||||
{
|
||||
(manifest, snapshotSha) = await _blob.DownloadDeletionManifestWithShaAsync(tenantId, sessionId, manifestId, ct).ConfigureAwait(false);
|
||||
(progress, progressEtag) = await _blob.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct).ConfigureAwait(false);
|
||||
|
||||
// PR4c F6: enforce the snapshot↔progress SHA binding. Producer wrote the manifest's
|
||||
// SHA into progress.SnapshotSha256 at upload time; if they diverge, the artifacts
|
||||
// were tampered with or the producer crashed in a bad way.
|
||||
if (!string.IsNullOrEmpty(progress.SnapshotSha256)
|
||||
&& !string.Equals(progress.SnapshotSha256, snapshotSha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"Snapshot/progress SHA binding mismatch: progress.SnapshotSha256={progress.SnapshotSha256} " +
|
||||
$"manifest SHA={snapshotSha} — corruption signal.");
|
||||
}
|
||||
}
|
||||
catch (Azure.RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
result.Outcome = SessionRestoreOutcome.RejectManifestNotFound;
|
||||
result.Message = $"Manifest blob not found for tenant={tenantId} session={sessionId} manifestId={manifestId}. " +
|
||||
"It may have been GC'd after the 33-day retention window — recovery is no longer possible.";
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Manifest corruption detected during restore for tenant={Tenant} session={Session} manifestId={ManifestId}",
|
||||
tenantId, sessionId, manifestId);
|
||||
result.Outcome = SessionRestoreOutcome.RejectManifestCorruption;
|
||||
result.Message = $"Manifest corruption: {ex.Message}";
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
|
||||
// (3) Read Sessions row, dispatch by state (§13.4 table).
|
||||
var sessionRow = await _storage.GetSessionRowAsync(tenantId, sessionId, ct).ConfigureAwait(false);
|
||||
var state = sessionRow?.GetString("DeletionState") ?? SessionDeletionState.None;
|
||||
if (string.IsNullOrEmpty(state)) state = SessionDeletionState.None;
|
||||
var pendingManifestId = sessionRow?.GetString("PendingDeletionManifestId");
|
||||
var completedAt = progress.CompletedAt;
|
||||
|
||||
// Dispatch table (PR4c F2 extends this — TombstoneStarted closes the tombstone gap):
|
||||
// sessionRow=null + completedAt set → Full restore (clean case)
|
||||
// sessionRow=null + completedAt null + TombstoneStarted → Full restore (gap case, NEW)
|
||||
// sessionRow=null + completedAt null + !TombstoneStarted → 409 corrupt_state
|
||||
// exists + state=None → 409 already_at_original_state
|
||||
// exists + state=Poisoned + matching → Partial restore
|
||||
// exists + state=Poisoned + mismatch → 409 manifestid_mismatch
|
||||
// exists + state ∈ {Preparing,Queued,Running} → 409 active_cascade
|
||||
string mode;
|
||||
if (sessionRow == null)
|
||||
{
|
||||
if (completedAt == null && !progress.TombstoneStarted)
|
||||
{
|
||||
result.Outcome = SessionRestoreOutcome.RejectCorruptState;
|
||||
result.Message = "Cascade did not complete (no CompletedAt, TombstoneStarted=false) but Sessions row is missing. " +
|
||||
"Operator must investigate before retrying — re-inserting may overwrite genuine new data.";
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
mode = "full";
|
||||
}
|
||||
else if (string.Equals(state, SessionDeletionState.None, StringComparison.Ordinal))
|
||||
{
|
||||
result.Outcome = SessionRestoreOutcome.RejectAlreadyAtOriginalState;
|
||||
result.Message = "Session is already in DeletionState=None — nothing to restore. If the manifest is from a prior delete that was already restored, this is expected.";
|
||||
result.CurrentState = state;
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
else if (string.Equals(state, SessionDeletionState.Poisoned, StringComparison.Ordinal))
|
||||
{
|
||||
if (!string.Equals(pendingManifestId, manifestId, StringComparison.Ordinal))
|
||||
{
|
||||
result.Outcome = SessionRestoreOutcome.RejectManifestIdMismatch;
|
||||
result.Message = $"Sessions.PendingDeletionManifestId='{pendingManifestId}' does not match requested manifestId='{manifestId}'. " +
|
||||
"Restore must target the manifest that owns the current lock.";
|
||||
result.CurrentState = state;
|
||||
result.PendingManifestId = pendingManifestId;
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
mode = "partial";
|
||||
}
|
||||
else
|
||||
{
|
||||
// Preparing / Queued / Running
|
||||
result.Outcome = SessionRestoreOutcome.RejectActiveCascade;
|
||||
result.Message = $"Sessions.DeletionState='{state}' — cascade is still in flight. " +
|
||||
"Flip AdminConfiguration.SessionDeletionKillSwitch to abort first, wait for the worker to finish or poison, then retry restore.";
|
||||
result.CurrentState = state;
|
||||
result.PendingManifestId = pendingManifestId;
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
|
||||
// (4) Dry-run path: simulate by counting what each step would restore, no writes.
|
||||
if (dryRun)
|
||||
{
|
||||
result.Outcome = SessionRestoreOutcome.DryRunOk;
|
||||
result.Mode = "dryRun";
|
||||
foreach (var step in manifest.Steps)
|
||||
{
|
||||
if (step.Class == DeletionStepClass.Aggregate || step.Class == DeletionStepClass.Final) continue;
|
||||
if (string.IsNullOrEmpty(step.Table) || step.Rows.Count == 0) continue;
|
||||
result.WouldRestoreByTable[step.Table!] = step.Rows.Count;
|
||||
}
|
||||
// Tombstone rows: split by Sessions vs SessionsIndex.
|
||||
var tombstone = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Final);
|
||||
if (tombstone != null)
|
||||
{
|
||||
foreach (var row in tombstone.Rows)
|
||||
{
|
||||
var t = IsSessionsRow(row) ? Constants.TableNames.Sessions : Constants.TableNames.SessionsIndex;
|
||||
if (!result.WouldRestoreByTable.TryGetValue(t, out var c)) c = 0;
|
||||
result.WouldRestoreByTable[t] = c + 1;
|
||||
}
|
||||
}
|
||||
var aggregate = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Aggregate);
|
||||
if (aggregate?.Decrements != null) result.InventoryReIncrements = aggregate.Decrements.Count;
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
|
||||
// (5) Execute the dispatched mode. PR4c F4: both modes thread progress + etag so the
|
||||
// per-key re-increment idempotency can persist after each successful inventory write.
|
||||
if (mode == "full")
|
||||
{
|
||||
(progress, progressEtag) = await RunFullRestoreAsync(
|
||||
tenantId, sessionId, manifestId, manifest, progress, progressEtag, result, ct).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
(progress, progressEtag) = await RunPartialRestoreAsync(
|
||||
tenantId, sessionId, manifestId, manifest, progress, progressEtag, result, ct).ConfigureAwait(false);
|
||||
if (result.Outcome == SessionRestoreOutcome.RejectCasConflictOnClear)
|
||||
{
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
result.Outcome = SessionRestoreOutcome.Restored;
|
||||
result.Mode = mode;
|
||||
result.DurationMs = sw.ElapsedMilliseconds;
|
||||
|
||||
// (6) Audit completion.
|
||||
await AuditRestoredAsync(tenantId, sessionId, manifestId, mode, actor, result, ct).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"SessionRestoreService completed: tenant={Tenant} session={Session} manifestId={ManifestId} mode={Mode} durationMs={Duration}",
|
||||
tenantId, sessionId, manifestId, mode, result.DurationMs);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ============================================================ Full restore ====
|
||||
|
||||
/// <summary>
|
||||
/// PR4c F3a: full restore inserts <b>Sessions row LAST</b>, not first. Order:
|
||||
/// <list type="number">
|
||||
/// <item>All non-FINAL non-AGGREGATE steps (foreign tables) in reverse order.</item>
|
||||
/// <item>AGGREGATE step (inventory re-increment, with PR4c F4 per-key idempotency).</item>
|
||||
/// <item>FINAL step's <c>SessionsIndex</c> row.</item>
|
||||
/// <item>FINAL step's <c>Sessions</c> row (TRULY LAST — its presence signals
|
||||
/// "session is back" + re-establishes the writer-lock semantics).</item>
|
||||
/// </list>
|
||||
/// Rationale: if any insert fails mid-flight, <c>sessions=null</c> on retry → dispatch
|
||||
/// re-enters full-restore (no <c>RejectAlreadyAtOriginalState</c> trap from Codex F3).
|
||||
/// </summary>
|
||||
private async Task<(DeletionProgress, string)> RunFullRestoreAsync(
|
||||
string tenantId, string sessionId, string manifestId,
|
||||
DeletionManifest manifest, DeletionProgress progress, string progressEtag,
|
||||
SessionRestoreResult result, CancellationToken ct)
|
||||
{
|
||||
// Pass 1: every non-FINAL step in REVERSE cascade order (17 → 16 → … → 1).
|
||||
// AGGREGATE re-increment uses per-key idempotency (PR4c F4).
|
||||
foreach (var step in manifest.Steps
|
||||
.Where(s => s.Class != DeletionStepClass.Final)
|
||||
.OrderByDescending(s => s.Order))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
switch (step.Class)
|
||||
{
|
||||
case DeletionStepClass.Aggregate:
|
||||
(progress, progressEtag) = await RunInventoryReIncrementWithIdempotencyAsync(
|
||||
tenantId, sessionId, manifestId, step, progress, progressEtag, result, ct).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
case DeletionStepClass.PkBySession:
|
||||
case DeletionStepClass.PkRkExact:
|
||||
case DeletionStepClass.PropTenantPk:
|
||||
case DeletionStepClass.DiscriminatorPkRkSuffix:
|
||||
case DeletionStepClass.DiscriminatorPkRkExact:
|
||||
case DeletionStepClass.DiscriminatorPkProp:
|
||||
{
|
||||
if (string.IsNullOrEmpty(step.Table) || step.Rows.Count == 0) break;
|
||||
var res = await _storage.RestoreRowsByExactKeysInBatchesAsync(
|
||||
step.Table!, step.Rows, RestoreMode.Full, ct).ConfigureAwait(false);
|
||||
AddCount(result.RowsRestoredByTable, step.Table!, res.Restored);
|
||||
AddCount(result.RowsSkippedByTable, step.Table!, res.Skipped);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
throw new InvalidOperationException($"Unsupported step.Class='{step.Class}' in restore (Order={step.Order}).");
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2: FINAL step in BUILDER ORDER (SessionsIndex first, Sessions LAST).
|
||||
// Builder emits [SessionsIndex, Sessions] (DeletionManifestBuilder.AddTombstoneStep).
|
||||
var tombstone = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Final);
|
||||
if (tombstone != null)
|
||||
{
|
||||
foreach (var row in tombstone.Rows)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var table = IsSessionsRow(row) ? Constants.TableNames.Sessions : Constants.TableNames.SessionsIndex;
|
||||
var res = await _storage.RestoreRowsByExactKeysInBatchesAsync(
|
||||
table, new[] { row }, RestoreMode.Full, ct).ConfigureAwait(false);
|
||||
AddCount(result.RowsRestoredByTable, table, res.Restored);
|
||||
AddCount(result.RowsSkippedByTable, table, res.Skipped);
|
||||
}
|
||||
}
|
||||
|
||||
return (progress, progressEtag);
|
||||
}
|
||||
|
||||
// ============================================================ Partial restore (poisoned-recovery) ====
|
||||
|
||||
private async Task<(DeletionProgress, string)> RunPartialRestoreAsync(
|
||||
string tenantId, string sessionId, string manifestId,
|
||||
DeletionManifest manifest, DeletionProgress progress, string progressEtag,
|
||||
SessionRestoreResult result, CancellationToken ct)
|
||||
{
|
||||
// Determine whether the AGGREGATE decrement step had already run before the cascade
|
||||
// poisoned. If yes → re-increment to undo it. If no → leave counters alone (avoid
|
||||
// double +1).
|
||||
var aggregate = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Aggregate);
|
||||
var decrementAlreadyRan = aggregate != null && progress.CompletedSteps.Contains(aggregate.Order);
|
||||
|
||||
// Pass 1: every non-FINAL step in REVERSE cascade order.
|
||||
foreach (var step in manifest.Steps
|
||||
.Where(s => s.Class != DeletionStepClass.Final)
|
||||
.OrderByDescending(s => s.Order))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
switch (step.Class)
|
||||
{
|
||||
case DeletionStepClass.Aggregate:
|
||||
if (!decrementAlreadyRan) break;
|
||||
(progress, progressEtag) = await RunInventoryReIncrementWithIdempotencyAsync(
|
||||
tenantId, sessionId, manifestId, step, progress, progressEtag, result, ct).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
case DeletionStepClass.PkBySession:
|
||||
case DeletionStepClass.PkRkExact:
|
||||
case DeletionStepClass.PropTenantPk:
|
||||
case DeletionStepClass.DiscriminatorPkRkSuffix:
|
||||
case DeletionStepClass.DiscriminatorPkRkExact:
|
||||
case DeletionStepClass.DiscriminatorPkProp:
|
||||
{
|
||||
if (string.IsNullOrEmpty(step.Table) || step.Rows.Count == 0) break;
|
||||
var res = await _storage.RestoreRowsByExactKeysInBatchesAsync(
|
||||
step.Table!, step.Rows, RestoreMode.Partial, ct).ConfigureAwait(false);
|
||||
AddCount(result.RowsRestoredByTable, step.Table!, res.Restored);
|
||||
AddCount(result.RowsSkippedByTable, step.Table!, res.Skipped);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
throw new InvalidOperationException($"Unsupported step.Class='{step.Class}' in restore (Order={step.Order}).");
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2: FINAL step in BUILDER ORDER (SessionsIndex first, Sessions LAST).
|
||||
// In partial-restore mode, Sessions + SessionsIndex usually still exist (cascade
|
||||
// poisoned before tombstone) → 409-skip each. The Sessions LAST ordering doesn't
|
||||
// matter in the typical partial case (both rows already there), but stays consistent
|
||||
// with the full-restore order for retry-after-partial-fail safety.
|
||||
var tombstone = manifest.Steps.FirstOrDefault(s => s.Class == DeletionStepClass.Final);
|
||||
if (tombstone != null)
|
||||
{
|
||||
foreach (var row in tombstone.Rows)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var table = IsSessionsRow(row) ? Constants.TableNames.Sessions : Constants.TableNames.SessionsIndex;
|
||||
var res = await _storage.RestoreRowsByExactKeysInBatchesAsync(
|
||||
table, new[] { row }, RestoreMode.Partial, ct).ConfigureAwait(false);
|
||||
AddCount(result.RowsRestoredByTable, table, res.Restored);
|
||||
AddCount(result.RowsSkippedByTable, table, res.Skipped);
|
||||
}
|
||||
}
|
||||
|
||||
// Final atomic step: CAS Poisoned → None. On failure: row recovery succeeded but
|
||||
// the lock is stuck — return RejectCasConflictOnClear so the operator can re-run
|
||||
// restore (idempotent via AddEntity 409-ignore + per-key inventory idempotency).
|
||||
var cas = await _storage.CasSetSessionDeletionStateAsync(
|
||||
tenantId, sessionId,
|
||||
fromState: SessionDeletionState.Poisoned,
|
||||
toState: SessionDeletionState.None,
|
||||
newManifestId: null,
|
||||
ct).ConfigureAwait(false);
|
||||
if (cas.Outcome != TableStorageService.SessionDeletionStateCasOutcome.Updated)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SessionRestoreService: CAS Poisoned→None failed for tenant={Tenant} session={Session} outcome={Outcome} currentState={State}; restore is idempotent — retry.",
|
||||
tenantId, sessionId, cas.Outcome, cas.CurrentState);
|
||||
result.Outcome = SessionRestoreOutcome.RejectCasConflictOnClear;
|
||||
result.Message = $"Row recovery succeeded but DeletionState lock could not be cleared (outcome={cas.Outcome}, currentState={cas.CurrentState}). Re-run restore — it's idempotent.";
|
||||
result.CurrentState = cas.CurrentState;
|
||||
}
|
||||
|
||||
return (progress, progressEtag);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PR4c F4: per-key idempotent inventory re-increment. Used by BOTH full and partial
|
||||
/// restore (Codex F4 was scoped to partial, but the same drift risk exists in full
|
||||
/// restore on retry after a mid-flight failure — symmetric fix). Persist-first / increment-
|
||||
/// second ordering bounds drift to <c>+1 per crash</c> in the rare gap window, vs unbounded
|
||||
/// over-increment per attempt if we did increment-first.
|
||||
/// </summary>
|
||||
private async Task<(DeletionProgress, string)> RunInventoryReIncrementWithIdempotencyAsync(
|
||||
string tenantId, string sessionId, string manifestId,
|
||||
DeletionStep step, DeletionProgress progress, string progressEtag,
|
||||
SessionRestoreResult result, CancellationToken ct)
|
||||
{
|
||||
if (step.Decrements == null || step.Decrements.Count == 0)
|
||||
{
|
||||
return (progress, progressEtag);
|
||||
}
|
||||
|
||||
// Lazy-initialize the applied-keys set on first re-increment entry.
|
||||
if (progress.RestoreReIncrementsApplied == null)
|
||||
{
|
||||
(progress, progressEtag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, progressEtag,
|
||||
mutate: p => p.RestoreReIncrementsApplied ??= new HashSet<string>(StringComparer.Ordinal),
|
||||
isAlreadyApplied: p => p.RestoreReIncrementsApplied != null,
|
||||
ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
foreach (var key in step.Decrements)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var composite = BuildAggregateCompositeKey(key);
|
||||
if (progress.RestoreReIncrementsApplied!.Contains(composite))
|
||||
{
|
||||
continue; // already applied on a prior attempt
|
||||
}
|
||||
|
||||
// F4 ordering: persist FIRST (mark intent), increment SECOND (apply effect).
|
||||
var compositeForClosure = composite;
|
||||
(progress, progressEtag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, progressEtag,
|
||||
mutate: p => p.RestoreReIncrementsApplied!.Add(compositeForClosure),
|
||||
isAlreadyApplied: p => p.RestoreReIncrementsApplied!.Contains(compositeForClosure),
|
||||
ct).ConfigureAwait(false);
|
||||
|
||||
await _storage.RestoreSoftwareInventoryEntryByKeyAsync(tenantId, key, ct).ConfigureAwait(false);
|
||||
result.InventoryReIncrements++;
|
||||
}
|
||||
|
||||
return (progress, progressEtag);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Composite-key format for <see cref="DeletionProgress.AggregateDecrementsApplied"/>
|
||||
/// and <see cref="DeletionProgress.RestoreReIncrementsApplied"/>. Matches
|
||||
/// <c>SessionDeletionHandler.BuildAggregateCompositeKey</c> so forward (decrement) and
|
||||
/// reverse (re-increment) operations identify the same row.
|
||||
/// </summary>
|
||||
internal static string BuildAggregateCompositeKey(DeletionDecrementKey key)
|
||||
=> $"{key.Vendor ?? string.Empty}:{key.Name ?? string.Empty}:{key.Version ?? string.Empty}";
|
||||
|
||||
/// <summary>
|
||||
/// Bounded-retry helper for ETag-CAS progress-blob writes during restore. Mirrors the
|
||||
/// pattern in <see cref="SessionDeletionHandler"/>: on 412 re-read progress, check whether
|
||||
/// the mutation is already applied (concurrent winner case), else retry with fresh ETag.
|
||||
/// Cap: <see cref="ProgressCasMaxAttempts"/> attempts or
|
||||
/// <see cref="ProgressCasMaxWallClock"/> wall-clock, whichever first.
|
||||
/// </summary>
|
||||
private async Task<(DeletionProgress, string)> UpdateProgressWithRetryAsync(
|
||||
string tenantId, string sessionId, string manifestId,
|
||||
DeletionProgress progress, string etag,
|
||||
Action<DeletionProgress> mutate,
|
||||
Func<DeletionProgress, bool> isAlreadyApplied,
|
||||
CancellationToken ct)
|
||||
{
|
||||
mutate(progress);
|
||||
|
||||
var deadline = DateTime.UtcNow + ProgressCasMaxWallClock;
|
||||
for (var attempt = 0; attempt < ProgressCasMaxAttempts && DateTime.UtcNow < deadline; attempt++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
var newEtag = await _blob.UpdateDeletionProgressAsync(tenantId, sessionId, manifestId, progress, etag, ct).ConfigureAwait(false);
|
||||
return (progress, newEtag);
|
||||
}
|
||||
catch (Azure.RequestFailedException ex) when (ex.Status == 412)
|
||||
{
|
||||
var fresh = await _blob.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct).ConfigureAwait(false);
|
||||
if (isAlreadyApplied(fresh.Progress))
|
||||
{
|
||||
return (fresh.Progress, fresh.ETag);
|
||||
}
|
||||
progress = fresh.Progress;
|
||||
etag = fresh.ETag;
|
||||
mutate(progress);
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(
|
||||
$"Progress-blob ETag-CAS exhausted after {ProgressCasMaxAttempts} attempts or " +
|
||||
$"{ProgressCasMaxWallClock.TotalSeconds:0}s wall-clock. " +
|
||||
$"tenant={tenantId} session={sessionId} manifestId={manifestId}");
|
||||
}
|
||||
|
||||
// §12-Q10: 10 attempts OR 60s wall-clock, whichever first → throw.
|
||||
internal const int ProgressCasMaxAttempts = 10;
|
||||
internal static readonly TimeSpan ProgressCasMaxWallClock = TimeSpan.FromSeconds(60);
|
||||
|
||||
// ============================================================ Helpers ====
|
||||
|
||||
private static bool IsSessionsRow(DeletionRowDump row)
|
||||
=> row.Rk != null && !row.Rk.Contains('_');
|
||||
|
||||
private static void AddCount(Dictionary<string, int> map, string key, int delta)
|
||||
{
|
||||
if (delta == 0) return;
|
||||
map[key] = map.TryGetValue(key, out var c) ? c + delta : delta;
|
||||
}
|
||||
|
||||
private async Task AuditManifestDownloadedAsync(string tenantId, string sessionId, string manifestId, string actor, CancellationToken ct)
|
||||
{
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
action: "deletion_manifest_downloaded",
|
||||
entityType: "Session",
|
||||
entityId: sessionId,
|
||||
performedBy: actor,
|
||||
details: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["manifestId"] = manifestId,
|
||||
["reason"] = "restore_attempt",
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task AuditRestoredAsync(
|
||||
string tenantId, string sessionId, string manifestId, string mode, string actor,
|
||||
SessionRestoreResult result, CancellationToken ct)
|
||||
{
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
action: "deletion_restored",
|
||||
entityType: "Session",
|
||||
entityId: sessionId,
|
||||
performedBy: actor,
|
||||
details: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["manifestId"] = manifestId,
|
||||
["mode"] = mode,
|
||||
["rowsRestoredByTable"] = JsonSerializer.Serialize(result.RowsRestoredByTable),
|
||||
["rowsSkippedByTable"] = JsonSerializer.Serialize(result.RowsSkippedByTable),
|
||||
["inventoryReIncrements"] = result.InventoryReIncrements.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
["durationMs"] = result.DurationMs.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,15 @@ namespace AutopilotMonitor.Functions.Services
|
||||
Task NotifyRuleResultsAvailableAsync(string tenantId, string sessionId, int resultCount);
|
||||
Task NotifyVulnerabilityReportAvailableAsync(string tenantId, string sessionId, string overallRisk);
|
||||
|
||||
/// <summary>
|
||||
/// Emitted by the cascade-deletion worker after a session's tombstone step completes
|
||||
/// (plan §5 PR4). The web UI subscribes to <c>session-{tenantId}-{sessionId}</c> and
|
||||
/// removes the row + closes any open detail pane on receipt. Idempotent — a re-pickup of
|
||||
/// an already-tombstoned session is a no-op at the worker level, so receivers may see
|
||||
/// the event at most once per cascade.
|
||||
/// </summary>
|
||||
Task NotifySessionDeletedAsync(string tenantId, string sessionId);
|
||||
|
||||
Task SendTenantNotificationAsync(string tenantId, NotificationAudience audience, object dto);
|
||||
Task SendTenantNotificationDismissedAsync(string tenantId, string notificationId);
|
||||
Task SendTenantNotificationDismissedAllAsync(string tenantId);
|
||||
|
||||
@@ -95,6 +95,37 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Notify the session group that the cascade-deletion worker has tombstoned the session
|
||||
/// (plan §5 PR4). Receivers (web UI) remove the row and close any open detail pane.
|
||||
/// Failures are logged and swallowed — the SignalR push is observability, not correctness;
|
||||
/// the audit log entry <c>deletion_completed</c> remains the canonical record.
|
||||
/// </summary>
|
||||
public async Task NotifySessionDeletedAsync(string tenantId, string sessionId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var hub = await GetHubContextAsync();
|
||||
var groupName = $"session-{tenantId}-{sessionId}";
|
||||
|
||||
await hub.Clients.Group(groupName).SendCoreAsync("sessionDeleted", new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
sessionId,
|
||||
tenantId,
|
||||
timestamp = DateTime.UtcNow.ToString("o")
|
||||
}
|
||||
});
|
||||
|
||||
_logger.LogDebug("Sent sessionDeleted to group {Group}", groupName);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to send sessionDeleted notification for session {SessionId}", sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Push a newly created tenant notification to the appropriate audience-tier group.
|
||||
/// Member-tier notifications go to the -notify-member group (joined by all tenant members);
|
||||
|
||||
@@ -41,7 +41,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// </summary>
|
||||
public partial class TableStorageService : ISessionDeletionInventoryReader
|
||||
{
|
||||
public async Task<TableEntity?> GetSessionRowAsync(string tenantId, string sessionId, CancellationToken cancellationToken = default)
|
||||
public virtual async Task<TableEntity?> GetSessionRowAsync(string tenantId, string sessionId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Shared.Constants.TableNames.Sessions);
|
||||
try
|
||||
@@ -108,7 +108,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// Returns <see cref="DeletionBatchResult"/> for accurate progress accounting on
|
||||
/// re-runs of partially-completed cascades.
|
||||
/// </summary>
|
||||
public async Task<DeletionBatchResult> DeleteByExactKeysInBatchesAsync(
|
||||
public virtual async Task<DeletionBatchResult> DeleteByExactKeysInBatchesAsync(
|
||||
string tableName,
|
||||
IReadOnlyList<(string Pk, string Rk)> keys,
|
||||
CancellationToken cancellationToken = default)
|
||||
|
||||
@@ -300,7 +300,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// zero. 404 = idempotent no-op. Bounded retry per <see cref="InventoryCasMaxAttempts"/> /
|
||||
/// <see cref="InventoryCasMaxWallClock"/>; throws on exhaustion.
|
||||
/// </summary>
|
||||
public async Task DecrementSoftwareInventoryEntryAsync(
|
||||
public virtual async Task DecrementSoftwareInventoryEntryAsync(
|
||||
string tenantId, string normalizedVendor, string normalizedName, string normalizedVersion,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
@@ -362,6 +362,91 @@ namespace AutopilotMonitor.Functions.Services
|
||||
+ $"{InventoryCasMaxWallClock.TotalSeconds:0}s wall-clock. tenant={tenantId} key={rowKey}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Re-increments a SoftwareInventory counter from a <see cref="DeletionDecrementKey"/>
|
||||
/// (vendor, name, version only — no DisplayName/Publisher). Symmetric inverse of
|
||||
/// <see cref="DecrementSoftwareInventoryEntryAsync"/>; used by the cascade-delete restore
|
||||
/// path (PR4b plan §13.6) where the manifest's side-row only carries the decrement keys.
|
||||
/// <para>
|
||||
/// If the inventory row exists → ETag-CAS <c>SessionCount += 1</c>, preserving DisplayName,
|
||||
/// Publisher, and every other column untouched. If the row is missing (count had dropped
|
||||
/// to zero and the cascade deleted it) → AddEntity with vendor/name/version + count=1 +
|
||||
/// FirstSeenAt + LastSeenAt + empty DisplayName/Publisher. The fidelity loss on display
|
||||
/// metadata is accepted per plan §13.6 ("re-creates the inventory row with SessionCount=1");
|
||||
/// the next live inventory-correlation will fill in DisplayName + Publisher.
|
||||
/// </para>
|
||||
/// Bounded retry per <see cref="InventoryCasMaxAttempts"/> / <see cref="InventoryCasMaxWallClock"/>;
|
||||
/// throws on exhaustion per memory <c>feedback_storage_helpers_fail_soft</c>.
|
||||
/// </summary>
|
||||
public virtual async Task RestoreSoftwareInventoryEntryByKeyAsync(
|
||||
string tenantId, DeletionDecrementKey key, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
if (key == null) throw new ArgumentNullException(nameof(key));
|
||||
var rowKey = BuildSoftwareInventoryRowKey(key.Vendor, key.Name, key.Version);
|
||||
if (string.IsNullOrWhiteSpace(rowKey) || rowKey == "::")
|
||||
{
|
||||
throw new ArgumentException(
|
||||
"DeletionDecrementKey has empty vendor/name/version — cannot restore a SoftwareInventory entry with no identity.",
|
||||
nameof(key));
|
||||
}
|
||||
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.SoftwareInventory);
|
||||
var deadline = DateTime.UtcNow + InventoryCasMaxWallClock;
|
||||
|
||||
for (var attempt = 0; attempt < InventoryCasMaxAttempts && DateTime.UtcNow < deadline; attempt++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
TableEntity existing;
|
||||
try
|
||||
{
|
||||
var response = await tableClient.GetEntityAsync<TableEntity>(tenantId, rowKey, cancellationToken: ct);
|
||||
existing = response.Value;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// Row was deleted by cascade (count reached zero) — re-create with minimal data.
|
||||
var now = DateTime.UtcNow;
|
||||
var fresh = new TableEntity(tenantId, rowKey)
|
||||
{
|
||||
["NormalizedVendor"] = key.Vendor ?? string.Empty,
|
||||
["NormalizedName"] = key.Name ?? string.Empty,
|
||||
["NormalizedVersion"] = key.Version ?? string.Empty,
|
||||
["SessionCount"] = 1,
|
||||
["FirstSeenAt"] = now.ToString("o"),
|
||||
["LastSeenAt"] = now.ToString("o"),
|
||||
};
|
||||
try
|
||||
{
|
||||
await tableClient.AddEntityAsync(fresh, ct);
|
||||
return;
|
||||
}
|
||||
catch (RequestFailedException rfe) when (rfe.Status == 409)
|
||||
{
|
||||
// Race: another writer created the row between our Get and our Add — retry.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
existing["SessionCount"] = (existing.GetInt32("SessionCount") ?? 0) + 1;
|
||||
existing["LastSeenAt"] = DateTime.UtcNow.ToString("o");
|
||||
try
|
||||
{
|
||||
await tableClient.UpdateEntityAsync(existing, existing.ETag, TableUpdateMode.Replace, ct);
|
||||
return;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(
|
||||
$"SoftwareInventory restore ETag-CAS exhausted after {InventoryCasMaxAttempts} attempts or "
|
||||
+ $"{InventoryCasMaxWallClock.TotalSeconds:0}s wall-clock. tenant={tenantId} key={rowKey}");
|
||||
}
|
||||
|
||||
// ============================================================ Helpers ====
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Restore mode for <see cref="TableStorageService.RestoreRowsByExactKeysInBatchesAsync"/>.
|
||||
/// <para>
|
||||
/// <b>PR4c F3b: both values now have identical 409-ignore semantics.</b> The enum is kept for
|
||||
/// API readability — callers still document intent ("I'm doing a full restore" vs "I'm doing
|
||||
/// a partial-poisoned-recovery") — but the implementation no longer throws on row-already-exists.
|
||||
/// This was relaxed because the original "Full mode throws on 409 as corruption signal" intent
|
||||
/// blocked the common case: a previous restore attempt failed mid-flight and left some rows
|
||||
/// behind, and the operator's retry would 409-throw on those leftover rows. With 409-ignore,
|
||||
/// retries are idempotent; <see cref="RestoreBatchResult.Skipped"/> still surfaces the count
|
||||
/// for operator inspection. The plan §13.6 "ghost row corruption signal" check shifts to a
|
||||
/// future post-restore verification pass (out of PR4c scope).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public enum RestoreMode
|
||||
{
|
||||
/// <summary>Restore from a completed cascade (Sessions row absent + progress.CompletedAt set,
|
||||
/// or PR4c F2 tombstone-gap case: Sessions row absent + TombstoneStarted=true).</summary>
|
||||
Full,
|
||||
/// <summary>Restore from a poisoned cascade (Sessions row present + DeletionState=Poisoned).</summary>
|
||||
Partial,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a <see cref="TableStorageService.RestoreRowsByExactKeysInBatchesAsync"/> call.
|
||||
/// <see cref="Restored"/> counts rows newly inserted; <see cref="Skipped"/> counts rows that
|
||||
/// were already present (Partial mode only — Full mode throws on conflict).
|
||||
/// </summary>
|
||||
public class RestoreBatchResult
|
||||
{
|
||||
public int Attempted { get; }
|
||||
public int Restored { get; }
|
||||
public int Skipped { get; }
|
||||
|
||||
public RestoreBatchResult(int attempted, int restored, int skipped)
|
||||
{
|
||||
Attempted = attempted;
|
||||
Restored = restored;
|
||||
Skipped = skipped;
|
||||
}
|
||||
|
||||
public static readonly RestoreBatchResult Empty = new RestoreBatchResult(0, 0, 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cascade-deletion restore primitives (PR4b). Symmetric inverse of the PR2 delete helpers:
|
||||
/// where <c>DeleteByExactKeysInBatchesAsync</c> consumes (Pk, Rk) tuples, this partial
|
||||
/// consumes the full <see cref="DeletionRowDump"/> records (with EDM-tagged Props) and
|
||||
/// re-Inserts them as typed <see cref="TableEntity"/> rows.
|
||||
/// <para>
|
||||
/// EDM round-trip fidelity (plan §13 "manifest IS the backup") is preserved by
|
||||
/// <see cref="ConvertFromPropValue"/>: each <see cref="DeletionPropValue.EdmType"/> tag
|
||||
/// dispatches into the matching <c>JsonElement.Get*</c> + .NET-type conversion so the SDK
|
||||
/// re-serializes with the original column type on the wire. Without the tag a DateTime
|
||||
/// coming out of JSON would be inserted as Edm.String and silently change the column shape.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public partial class TableStorageService
|
||||
{
|
||||
/// <summary>
|
||||
/// Mirror of <see cref="DeleteByExactKeysInBatchesAsync"/>: groups <paramref name="rows"/>
|
||||
/// by PartitionKey (batch transactions are partition-scoped), chunks each group at the
|
||||
/// batch-action limit, and submits a single <c>UpsertReplace</c>/<c>Add</c> transaction
|
||||
/// per chunk. On batch-level 409 (any row exists) the behaviour depends on
|
||||
/// <paramref name="mode"/>:
|
||||
/// <list type="bullet">
|
||||
/// <item><see cref="RestoreMode.Full"/> — falls back to per-row <c>AddEntityAsync</c>;
|
||||
/// any individual 409 throws <see cref="InvalidDataException"/> as a corruption
|
||||
/// signal (the cascade had marked these rows for deletion and they should not exist).</item>
|
||||
/// <item><see cref="RestoreMode.Partial"/> — falls back to per-row <c>AddEntityAsync</c>
|
||||
/// with 409-ignore, counting skipped rows in the returned
|
||||
/// <see cref="RestoreBatchResult.Skipped"/>.</item>
|
||||
/// </list>
|
||||
/// All other storage errors propagate unchanged per memory <c>feedback_storage_helpers_fail_soft</c>.
|
||||
/// </summary>
|
||||
public virtual async Task<RestoreBatchResult> RestoreRowsByExactKeysInBatchesAsync(
|
||||
string tableName,
|
||||
IReadOnlyList<DeletionRowDump> rows,
|
||||
RestoreMode mode,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName is required", nameof(tableName));
|
||||
if (rows == null) throw new ArgumentNullException(nameof(rows));
|
||||
if (rows.Count == 0) return RestoreBatchResult.Empty;
|
||||
|
||||
var tableClient = _tableServiceClient.GetTableClient(tableName);
|
||||
var restored = 0;
|
||||
var skipped = 0;
|
||||
|
||||
foreach (var group in rows.GroupBy(r => r.Pk, StringComparer.Ordinal))
|
||||
{
|
||||
var rowsInGroup = group.ToList();
|
||||
for (var i = 0; i < rowsInGroup.Count; i += RestoreBatchActionLimit)
|
||||
{
|
||||
var chunk = rowsInGroup.Skip(i).Take(RestoreBatchActionLimit).ToList();
|
||||
var actions = chunk
|
||||
.Select(dump => new TableTransactionAction(
|
||||
TableTransactionActionType.Add,
|
||||
ConvertDumpToEntity(dump, tableName)))
|
||||
.ToList();
|
||||
|
||||
try
|
||||
{
|
||||
await tableClient.SubmitTransactionAsync(actions, cancellationToken);
|
||||
restored += chunk.Count;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 400)
|
||||
{
|
||||
// Azure rolls back the entire transaction when any Add hits an existing
|
||||
// row. Fall back to per-row AddEntity so we can distinguish "row already
|
||||
// there" from genuine errors. Some Azure deployments map duplicate-row
|
||||
// conflict to 400 with code EntityAlreadyExists; both surface through
|
||||
// this fallback path.
|
||||
// PR4c F3b: both Full and Partial modes count 409 as Skipped (idempotent
|
||||
// retries after partial-failed restore). The `mode` parameter is kept for
|
||||
// API readability but no longer changes behaviour here.
|
||||
_ = mode; // intentional: relaxation documented above
|
||||
foreach (var dump in chunk)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
await tableClient.AddEntityAsync(ConvertDumpToEntity(dump, tableName), cancellationToken);
|
||||
restored++;
|
||||
}
|
||||
catch (RequestFailedException rfe) when (rfe.Status == 409)
|
||||
{
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new RestoreBatchResult(rows.Count, restored, skipped);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Symmetric inverse of <see cref="Services.Deletion.DeletionManifestBuilder.ConvertToPropValue"/>:
|
||||
/// parses the EDM-tagged <see cref="DeletionPropValue"/> back into a typed .NET object
|
||||
/// suitable for assignment into a <see cref="TableEntity"/>'s dictionary. The SDK then
|
||||
/// re-serializes with the original column type on the wire.
|
||||
/// <para>
|
||||
/// Null handling: <c>EdmType=String</c> + <c>ValueKind=Null</c> returns <c>null</c>
|
||||
/// (mirrors the encode path's null fallback).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static object? ConvertFromPropValue(DeletionPropValue prop)
|
||||
{
|
||||
if (prop == null) throw new ArgumentNullException(nameof(prop));
|
||||
|
||||
// Null is encoded as EdmType=String + JsonNull (no first-class null in Azure Tables).
|
||||
if (prop.Value.ValueKind == JsonValueKind.Null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
switch (prop.EdmType)
|
||||
{
|
||||
case DeletionPropEdmType.String:
|
||||
return prop.Value.GetString();
|
||||
case DeletionPropEdmType.Boolean:
|
||||
return prop.Value.GetBoolean();
|
||||
case DeletionPropEdmType.Int32:
|
||||
return prop.Value.GetInt32();
|
||||
case DeletionPropEdmType.Int64:
|
||||
return prop.Value.GetInt64();
|
||||
case DeletionPropEdmType.Double:
|
||||
return prop.Value.GetDouble();
|
||||
case DeletionPropEdmType.DateTime:
|
||||
{
|
||||
var raw = prop.Value.GetString()
|
||||
?? throw new InvalidDataException("DateTime EdmType has null Value.");
|
||||
// RoundtripKind preserves UTC Kind set by ConvertToPropValue.
|
||||
return DateTime.Parse(raw, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
|
||||
}
|
||||
case DeletionPropEdmType.Guid:
|
||||
{
|
||||
var raw = prop.Value.GetString()
|
||||
?? throw new InvalidDataException("Guid EdmType has null Value.");
|
||||
return Guid.Parse(raw);
|
||||
}
|
||||
case DeletionPropEdmType.Binary:
|
||||
{
|
||||
var raw = prop.Value.GetString()
|
||||
?? throw new InvalidDataException("Binary EdmType has null Value.");
|
||||
return Convert.FromBase64String(raw);
|
||||
}
|
||||
default:
|
||||
throw new InvalidDataException(
|
||||
$"Unsupported EdmType '{prop.EdmType}' in DeletionPropValue. " +
|
||||
"Manifest schema may be from a newer version than this restore endpoint supports.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a <see cref="TableEntity"/> from a <see cref="DeletionRowDump"/> ready for
|
||||
/// <c>AddEntityAsync</c>. For Sessions-table rows the <c>DeletionState</c> +
|
||||
/// <c>PendingDeletionManifestId</c> columns are forcibly overridden to
|
||||
/// <c>None</c> + <c>null</c> — the snapshot captured the row AFTER the producer's
|
||||
/// CAS-Preparing, so a verbatim re-insert would leave the restored session permanently
|
||||
/// locked. This override is the critical correctness invariant of the restore path.
|
||||
/// </summary>
|
||||
public static TableEntity ConvertDumpToEntity(DeletionRowDump dump, string tableName)
|
||||
{
|
||||
if (dump == null) throw new ArgumentNullException(nameof(dump));
|
||||
if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName is required", nameof(tableName));
|
||||
|
||||
var entity = new TableEntity(dump.Pk, dump.Rk);
|
||||
foreach (var kvp in dump.Props)
|
||||
{
|
||||
entity[kvp.Key] = ConvertFromPropValue(kvp.Value);
|
||||
}
|
||||
|
||||
// Sessions-row override: snapshot was captured after CAS-None→Preparing so the row's
|
||||
// DeletionState column carries "Preparing" + the current ManifestId. Restore must
|
||||
// reset both so writers can lock the row again post-restore.
|
||||
if (string.Equals(tableName, Constants.TableNames.Sessions, StringComparison.Ordinal))
|
||||
{
|
||||
entity["DeletionState"] = SessionDeletionState.None;
|
||||
entity["PendingDeletionManifestId"] = null;
|
||||
}
|
||||
|
||||
return entity;
|
||||
}
|
||||
|
||||
// Azure Tables batch transactions cap at 100 actions per submission, all sharing a
|
||||
// PartitionKey. Mirrors the const in the deletion partial.
|
||||
private const int RestoreBatchActionLimit = 100;
|
||||
}
|
||||
}
|
||||
@@ -22,5 +22,42 @@ namespace AutopilotMonitor.Shared.Models.Deletion
|
||||
|
||||
/// <summary>UTC timestamp once the FINAL tombstone step has completed; null while in flight.</summary>
|
||||
public DateTime? CompletedAt { get; set; }
|
||||
|
||||
// ============================================================ PR4c additions ====
|
||||
// Three additive fields close the per-key idempotency + tombstone-gap correctness holes
|
||||
// discovered by Codex review of PR4 + PR4b. All three default to safe values
|
||||
// (null / false) so PR1–PR4 progress blobs without these fields deserialize cleanly.
|
||||
|
||||
/// <summary>
|
||||
/// PR4c F1: Composite keys (<c>{Vendor}:{Name}:{Version}</c>) of SoftwareInventory
|
||||
/// decrements already applied by the cascade worker's AGGREGATE step. Per-key
|
||||
/// persistence so a crash mid-decrement-loop doesn't double-decrement on retry.
|
||||
/// Worker writes one entry after each successful <c>DecrementSoftwareInventoryEntryAsync</c>
|
||||
/// and persists the progress blob with ETag-CAS, then moves to the next key.
|
||||
/// Null on PR1-PR4 progress blobs (worker initializes on first AGGREGATE-step entry).
|
||||
/// </summary>
|
||||
public HashSet<string>? AggregateDecrementsApplied { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// PR4c F4: Composite keys of SoftwareInventory re-increments already applied by the
|
||||
/// partial-restore service. Per-key persistence so a crash between the re-increment
|
||||
/// loop and the final <c>Poisoned → None</c> CAS doesn't double-increment counters on
|
||||
/// retry. Service writes one entry after each successful
|
||||
/// <c>RestoreSoftwareInventoryEntryByKeyAsync</c> and persists with ETag-CAS.
|
||||
/// Null on PR1-PR4 progress blobs (service initializes on first re-increment).
|
||||
/// </summary>
|
||||
public HashSet<string>? RestoreReIncrementsApplied { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// PR4c F2: Set to <c>true</c> by the cascade worker <b>before</b> issuing the first
|
||||
/// FINAL-step row delete. Closes the "tombstone gap": if the worker dies between
|
||||
/// deleting the Sessions row and writing <see cref="CompletedAt"/>, restore can still
|
||||
/// dispatch into full-restore mode by reading this flag (otherwise
|
||||
/// <c>sessions=null + completedAt=null</c> looks like corruption and restore rejects).
|
||||
/// Default <c>false</c> for back-compat — PR1-PR4 progress blobs without this field
|
||||
/// deserialize cleanly to <c>false</c>, preserving the existing "corrupt-state" reject
|
||||
/// behaviour for genuine bugs (Sessions row removed outside the cascade).
|
||||
/// </summary>
|
||||
public bool TombstoneStarted { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user