Files
AutopilotMonitor/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingExpectationsStoreTests.cs
T
Oliver Kieselbach 206388da99 feat(backend,web): tenant offboarding async cascade with customs archive
Replace synchronous DeleteAllTenantDataAsync with a queued async cascade
that wipes all 24 tenant-scoped tables safely, archives custom rules,
and survives crashes at every step.

Phase 1 (TenantOffboardFunction, ~200ms sync):
  - Write History / ByTenantPointer / Marker rows (OffboardingAudit table)
  - Flip TenantConfiguration.Disabled=true (the active auth gate)
  - Enqueue with 6 min visibility-delay (cache-drain barrier =
    TenantConfigurationService.CacheDuration 5 min + 1 min safety)
  - Return 202 with EarliestProcessingAt for the UI countdown

Phase 2 (TenantOffboardingWorker + Handler, async after drain):
  - Enumerate sessions, per-session cascade-enqueue, persist
    Expectations blob (separate offboarding-state container)
  - Drain probe against per-session progress blobs; self re-enqueue
    up to 60x at 2 min visibility-delay
  - 2.D-archive: GatherRules / AnalyzeRules / ImeLogPatterns moved
    to TenantOffboardingCustomsArchive table (3-iter cap + verify)
  - 2.D-wipe: SafeWipe over all tenant tables (4 variants: exact-PK,
    composite-PK range, discriminator+TenantId, property-only) with
    fetch+verify+delete pattern + SafeWipeVerificationException
  - 2.E: blob-prefix wipe deletion-manifests/{tenantId}/
  - 2.D-pass-2: re-wipe AuditLogs (defense-in-depth)
  - 2.G: side-effects FIRST (Marker, Pointer, blob, audit, OpsEvent)
    then History=Completed LAST (DrainCompletedAt as resume anchor)
  - 2.F-final: delete TenantConfiguration row LAST so self-service
    re-onboarding (next /api/auth/me auto-creates default) works

Defense-in-depth:
  - OffboardingMarkerCleanupFunction (2h timer): sweeps Completed
    markers after 15 min grace, conditional TenantConfiguration sweep
    (only when row is still the offboarding tombstone — preserves
    user-driven self-service re-onboards)
  - Worker poison path: durable Failed state MUST land before queue
    moves; transient failure leaves message visible for retry
  - SessionDeletionHandler reorder: Audit + SignalR notify BEFORE
    CompletedAt write so the drain predicate is correct

Web UI:
  - /settings danger zone: typed "OFFBOARD" confirmation dialog,
    post-202 drain-barrier banner with minute-based countdown, hint
    "you can sign out and close this tab — deletion runs in
    background", silent auto-logout when countdown elapses
  - /admin/customs-archive (GA only): list runs + inspect entries +
    typed "DELETE" confirmation modal for run/entry deletion
  - OpsAlertRulesSection: TenantOffboardingFailed event registered
  - lib/api.ts: shared CustomsArchive* response types

Tests: 100+ new tests covering drain contract, handler ordering,
worker poison path, ETag-CAS, SafeWipe variants, OffboardingFilters
prefix-bleed safety, MarkerCleanup conditional sweep, Bootstrap
auth-disabled-tenant gate.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-18 23:06:51 +02:00

371 lines
15 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging.Abstractions;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Rev-6-F1 + Rev-7-F2 + Rev-8-F1: full coverage of the blob-IO contracts on
/// <see cref="BlobOffboardingExpectationsStore"/> via a Dict-backed fake that overrides the
/// four <c>protected virtual</c> seams. The fake reproduces Azure Blob Storage's two
/// "exists" status codes (409 BlobAlreadyExists + 412 ConditionNotMet) so the resume path
/// is exercised under both. POCO/validation cases live next to the IO cases for full
/// surface coverage.
/// </summary>
public class OffboardingExpectationsStoreTests
{
private const string TenantId = "99999999-9999-9999-9999-999999999999";
private const string HistoryRowKey = "20260518091523123_99999999-9999-9999-9999-999999999999";
// ── Validation gates ────────────────────────────────────────────────────────
[Fact]
public async Task TryUploadInitial_RejectsNonGuidTenantId()
{
var store = new FakeStore();
var payload = BuildPayload(tenantId: "not-a-guid");
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
}
[Fact]
public async Task TryUploadInitial_RejectsEmptyHistoryRowKey()
{
var store = new FakeStore();
var payload = BuildPayload(historyRowKey: "");
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
}
[Fact]
public async Task TryUploadInitial_RejectsSchemaVersionZero()
{
var store = new FakeStore();
var payload = BuildPayload();
payload.SchemaVersion = 0;
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
}
[Fact]
public async Task UpdateWithEtagCas_RequiresIfMatchEtag()
{
var store = new FakeStore();
await Assert.ThrowsAsync<ArgumentException>(() => store.UpdateWithEtagCasAsync(BuildPayload(), ""));
}
[Fact]
public async Task TryDownload_RejectsNonGuidTenantId()
{
var store = new FakeStore();
await Assert.ThrowsAsync<ArgumentException>(() => store.TryDownloadAsync("not-a-guid", HistoryRowKey));
}
// ── Initial upload (If-None-Match=*) ────────────────────────────────────────
[Fact]
public async Task TryUploadInitial_FirstWrite_ReturnsTrue()
{
var store = new FakeStore();
var inserted = await store.TryUploadInitialAsync(BuildPayload());
Assert.True(inserted);
Assert.Single(store.Blobs);
Assert.Equal(
BlobOffboardingExpectationsStore.BuildBlobName(TenantId, HistoryRowKey),
store.Blobs.Keys.Single());
}
[Fact]
public async Task TryUploadInitial_Duplicate_409BlobAlreadyExists_ReturnsFalse()
{
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
// Simulate the "older" Azure response code for the same condition.
store.NextOverwriteRefusal = OverwriteRefusal.Status409;
var inserted = await store.TryUploadInitialAsync(BuildPayload());
Assert.False(inserted);
}
[Fact]
public async Task TryUploadInitial_Duplicate_412ConditionNotMet_ReturnsFalse()
{
// High-finding: Azure Blob Storage emits 412 ConditionNotMet for the SAME
// "If-None-Match=* on existing blob" case the SDK historically returned 409 for.
// Without the 412-branch the resume path breaks and the worker poisons.
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
store.NextOverwriteRefusal = OverwriteRefusal.Status412ConditionNotMet;
var inserted = await store.TryUploadInitialAsync(BuildPayload());
Assert.False(inserted);
}
[Fact]
public async Task TryUploadInitial_Duplicate_BlobErrorCodeOnly_ReturnsFalse()
{
// Some SDK paths surface the failure as Status=0 with a populated ErrorCode.
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
store.NextOverwriteRefusal = OverwriteRefusal.ErrorCodeBlobAlreadyExists;
var inserted = await store.TryUploadInitialAsync(BuildPayload());
Assert.False(inserted);
}
[Fact]
public async Task TryUploadInitial_Propagates500_NotASilentResume()
{
var store = new FakeStore { NextOverwriteRefusal = OverwriteRefusal.Status500 };
await Assert.ThrowsAsync<RequestFailedException>(() => store.TryUploadInitialAsync(BuildPayload()));
}
// ── Download (404 → null) ──────────────────────────────────────────────────
[Fact]
public async Task TryDownload_404_Returns_NullPayload_AndNullEtag()
{
var store = new FakeStore();
var (payload, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
Assert.Null(payload);
Assert.Null(etag);
}
[Fact]
public async Task TryDownload_AfterUpload_RoundTripsAllFields()
{
var store = new FakeStore();
var original = BuildPayload();
original.EnumerationCompleted = true;
original.EnumeratedSessionCount = 2;
original.Expectations.Add(new OffboardingExpectation { SessionId = "s1", ManifestId = "m1", Outcome = "Enqueued" });
original.Expectations.Add(new OffboardingExpectation { SessionId = "s2", ManifestId = null, Outcome = "SessionNotFound" });
await store.TryUploadInitialAsync(original);
var (round, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
Assert.NotNull(round);
Assert.NotNull(etag);
Assert.Equal(2, round!.EnumeratedSessionCount);
Assert.True(round.EnumerationCompleted);
Assert.Equal(2, round.Expectations.Count);
Assert.Equal("Enqueued", round.Expectations[0].Outcome);
Assert.Null(round.Expectations[1].ManifestId);
}
// ── Update with ETag CAS ───────────────────────────────────────────────────
[Fact]
public async Task UpdateWithEtagCas_MatchingEtag_ReturnsNewEtag()
{
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
var (round, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
round!.Expectations.Add(new OffboardingExpectation { SessionId = "s1", Outcome = "Enqueued", ManifestId = "m1" });
var newEtag = await store.UpdateWithEtagCasAsync(round, etag!);
Assert.NotNull(newEtag);
Assert.NotEqual(etag, newEtag);
var (refetched, _) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
Assert.Single(refetched!.Expectations);
}
[Fact]
public async Task UpdateWithEtagCas_StaleEtag_Throws412()
{
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
var ex = await Assert.ThrowsAsync<RequestFailedException>(
() => store.UpdateWithEtagCasAsync(BuildPayload(), ifMatchEtag: "\"0xSTALE\""));
Assert.Equal(412, ex.Status);
}
// ── Delete ─────────────────────────────────────────────────────────────────
[Fact]
public async Task Delete_RemovesBlob_WhenPresent()
{
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
Assert.Single(store.Blobs);
await store.DeleteAsync(TenantId, HistoryRowKey);
Assert.Empty(store.Blobs);
}
[Fact]
public async Task Delete_404_IsIdempotent()
{
var store = new FakeStore();
// No upload first — blob doesn't exist.
await store.DeleteAsync(TenantId, HistoryRowKey); // must not throw
Assert.Empty(store.Blobs);
}
// ── Container choice (Rev-8-F1) ────────────────────────────────────────────
[Fact]
public void Container_IsOffboardingState_NotDeletionManifests()
{
// Container choice is part of the resilience design — wiping the wrong container
// (Rev-8-F1: deletion-manifests would orphan Expectations during Phase 2.E).
Assert.Equal("offboarding-state", AutopilotMonitor.Shared.Constants.BlobContainers.OffboardingState);
Assert.NotEqual(
AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests,
AutopilotMonitor.Shared.Constants.BlobContainers.OffboardingState);
}
[Fact]
public void BlobName_FollowsTenantPrefixSchemaForLifecycleSweep()
{
var name = BlobOffboardingExpectationsStore.BuildBlobName(TenantId, HistoryRowKey);
Assert.StartsWith($"{TenantId}/", name);
Assert.EndsWith(".expectations.json", name);
Assert.Contains(HistoryRowKey, name);
}
[Fact]
public void Defaults_ForceCallersToSetEnumerationCompletedExplicitly()
{
// Rev-7-F2 disambiguation lives at the POCO level — the drain probe relies on
// EnumerationCompleted defaulting to false so a half-built payload cannot
// accidentally pass the "0 expectations + completed=true" happy path.
var payload = new OffboardingExpectations();
Assert.False(payload.EnumerationCompleted);
Assert.Equal(0, payload.EnumeratedSessionCount);
Assert.Equal(1, payload.SchemaVersion);
Assert.Empty(payload.Expectations);
}
// ── Helpers ─────────────────────────────────────────────────────────────────
private static OffboardingExpectations BuildPayload(
string tenantId = TenantId,
string historyRowKey = HistoryRowKey)
=> new()
{
SchemaVersion = 1,
TenantId = tenantId,
HistoryRowKey = historyRowKey,
CreatedAt = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc),
EnumerationCompleted = false,
EnumeratedSessionCount = 0,
};
/// <summary>
/// Selects how the next call to <see cref="FakeStore.WriteIfNoneMatchAsync"/> simulates an
/// "exists" refusal — needed because Azure switches between codes depending on tier/SDK.
/// </summary>
private enum OverwriteRefusal
{
Auto,
Status409,
Status412ConditionNotMet,
ErrorCodeBlobAlreadyExists,
Status500,
}
/// <summary>
/// Dict-backed fake that overrides the four protected virtual seams. Mirrors the
/// production Azure semantics for If-None-Match=*, If-Match, 404, and DeleteIfExists.
/// </summary>
private sealed class FakeStore : BlobOffboardingExpectationsStore
{
public Dictionary<string, (byte[] Bytes, string ETag)> Blobs { get; } = new();
public OverwriteRefusal NextOverwriteRefusal { get; set; } = OverwriteRefusal.Auto;
private int _etagCounter;
public FakeStore()
: base(
new BlobStorageService(
new BlobServiceClient("UseDevelopmentStorage=true"),
NullLogger<BlobStorageService>.Instance,
usesManagedIdentity: false),
NullLogger<BlobOffboardingExpectationsStore>.Instance)
{
}
protected override Task<(bool Inserted, string? ETag)> WriteIfNoneMatchAsync(
string blobName, byte[] payload, CancellationToken ct)
{
// Allow tests to inject a specific refusal even though the underlying state would
// already make a real Azure call fail — needed to pin BOTH the 409 and 412 paths.
if (NextOverwriteRefusal == OverwriteRefusal.Status500)
{
NextOverwriteRefusal = OverwriteRefusal.Auto;
throw new RequestFailedException(500, "ServerError", "InternalError", null);
}
if (Blobs.ContainsKey(blobName))
{
var refusal = NextOverwriteRefusal == OverwriteRefusal.Auto
? OverwriteRefusal.Status409
: NextOverwriteRefusal;
NextOverwriteRefusal = OverwriteRefusal.Auto;
RequestFailedException ex = refusal switch
{
OverwriteRefusal.Status409 =>
new RequestFailedException(409, "Conflict", BlobErrorCode.BlobAlreadyExists.ToString(), null),
OverwriteRefusal.Status412ConditionNotMet =>
new RequestFailedException(412, "ConditionNotMet", BlobErrorCode.ConditionNotMet.ToString(), null),
OverwriteRefusal.ErrorCodeBlobAlreadyExists =>
new RequestFailedException(0, "ErrorCodeOnly", BlobErrorCode.BlobAlreadyExists.ToString(), null),
_ => throw new InvalidOperationException("Unhandled refusal mode"),
};
if (IsAlreadyExists(ex)) return Task.FromResult<(bool, string?)>((false, null));
throw ex;
}
var etag = $"\"0xFAKE_{++_etagCounter}\"";
Blobs[blobName] = (payload, etag);
return Task.FromResult<(bool, string?)>((true, etag));
}
protected override Task<(byte[]? Payload, string? ETag)> ReadBlobAsync(
string blobName, CancellationToken ct)
{
if (!Blobs.TryGetValue(blobName, out var entry))
return Task.FromResult<(byte[]?, string?)>((null, null));
return Task.FromResult<(byte[]?, string?)>((entry.Bytes, entry.ETag));
}
protected override Task<string> WriteWithIfMatchAsync(
string blobName, byte[] payload, string ifMatchEtag, CancellationToken ct)
{
if (!Blobs.TryGetValue(blobName, out var entry))
throw new RequestFailedException(404, "NotFound", "BlobNotFound", null);
if (entry.ETag != ifMatchEtag)
throw new RequestFailedException(412, "ConditionNotMet", BlobErrorCode.ConditionNotMet.ToString(), null);
var etag = $"\"0xFAKE_{++_etagCounter}\"";
Blobs[blobName] = (payload, etag);
return Task.FromResult(etag);
}
protected override Task DeleteBlobIfExistsAsync(string blobName, CancellationToken ct)
{
Blobs.Remove(blobName);
return Task.CompletedTask;
}
}
}