mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat(backend,web): tenant offboarding async cascade with customs archive
Replace synchronous DeleteAllTenantDataAsync with a queued async cascade
that wipes all 24 tenant-scoped tables safely, archives custom rules,
and survives crashes at every step.
Phase 1 (TenantOffboardFunction, ~200ms sync):
- Write History / ByTenantPointer / Marker rows (OffboardingAudit table)
- Flip TenantConfiguration.Disabled=true (the active auth gate)
- Enqueue with 6 min visibility-delay (cache-drain barrier =
TenantConfigurationService.CacheDuration 5 min + 1 min safety)
- Return 202 with EarliestProcessingAt for the UI countdown
Phase 2 (TenantOffboardingWorker + Handler, async after drain):
- Enumerate sessions, per-session cascade-enqueue, persist
Expectations blob (separate offboarding-state container)
- Drain probe against per-session progress blobs; self re-enqueue
up to 60x at 2 min visibility-delay
- 2.D-archive: GatherRules / AnalyzeRules / ImeLogPatterns moved
to TenantOffboardingCustomsArchive table (3-iter cap + verify)
- 2.D-wipe: SafeWipe over all tenant tables (4 variants: exact-PK,
composite-PK range, discriminator+TenantId, property-only) with
fetch+verify+delete pattern + SafeWipeVerificationException
- 2.E: blob-prefix wipe deletion-manifests/{tenantId}/
- 2.D-pass-2: re-wipe AuditLogs (defense-in-depth)
- 2.G: side-effects FIRST (Marker, Pointer, blob, audit, OpsEvent)
then History=Completed LAST (DrainCompletedAt as resume anchor)
- 2.F-final: delete TenantConfiguration row LAST so self-service
re-onboarding (next /api/auth/me auto-creates default) works
Defense-in-depth:
- OffboardingMarkerCleanupFunction (2h timer): sweeps Completed
markers after 15 min grace, conditional TenantConfiguration sweep
(only when row is still the offboarding tombstone — preserves
user-driven self-service re-onboards)
- Worker poison path: durable Failed state MUST land before queue
moves; transient failure leaves message visible for retry
- SessionDeletionHandler reorder: Audit + SignalR notify BEFORE
CompletedAt write so the drain predicate is correct
Web UI:
- /settings danger zone: typed "OFFBOARD" confirmation dialog,
post-202 drain-barrier banner with minute-based countdown, hint
"you can sign out and close this tab — deletion runs in
background", silent auto-logout when countdown elapses
- /admin/customs-archive (GA only): list runs + inspect entries +
typed "DELETE" confirmation modal for run/entry deletion
- OpsAlertRulesSection: TenantOffboardingFailed event registered
- lib/api.ts: shared CustomsArchive* response types
Tests: 100+ new tests covering drain contract, handler ordering,
worker poison path, ETag-CAS, SafeWipe variants, OffboardingFilters
prefix-bleed safety, MarkerCleanup conditional sweep, Bootstrap
auth-disabled-tenant gate.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Plan v2 §5 (bootstrap-path interaction): when a tenant is mid-offboarding,
|
||||
/// Phase 1 flips <c>TenantConfiguration.Disabled = true</c> and Phase 2 starts after
|
||||
/// the cache-drain barrier. The auth pipeline (cert + bootstrap-token) must reject
|
||||
/// requests against a Disabled tenant — otherwise a fresh OOBE device with a bootstrap
|
||||
/// token could keep writing into a tenant about to be wiped.
|
||||
/// <para>
|
||||
/// The §0 Disabled-gate in <see cref="SecurityValidator.ValidateRequestAsync"/> runs
|
||||
/// BEFORE the §0.5 bootstrap-token branch, so this property is enforced by the order
|
||||
/// of checks. These tests pin that ordering so future refactors cannot accidentally
|
||||
/// move the bootstrap branch above the Disabled gate.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class BootstrapAuthDisabledTenantTests
|
||||
{
|
||||
private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
|
||||
private const string BootstrapToken = "11111111-2222-3333-4444-555555555555";
|
||||
|
||||
private static SecurityValidator BuildValidator(TenantConfiguration config)
|
||||
{
|
||||
var configRepo = Mock.Of<IConfigRepository>();
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
|
||||
var configServiceMock = new Mock<TenantConfigurationService>(
|
||||
configRepo, Mock.Of<ILogger<TenantConfigurationService>>(), cache)
|
||||
{ CallBase = false };
|
||||
configServiceMock
|
||||
.Setup(x => x.TryGetConfigurationAsync(It.IsAny<string>()))
|
||||
.ReturnsAsync((config, true));
|
||||
|
||||
return new SecurityValidator(
|
||||
configService: configServiceMock.Object,
|
||||
rateLimitService: null!,
|
||||
autopilotDeviceValidator: null!,
|
||||
corporateIdentifierValidator: null!,
|
||||
logger: Mock.Of<ILogger>(),
|
||||
bootstrapSessionService: null,
|
||||
deviceAssociationValidator: null);
|
||||
}
|
||||
|
||||
private static HttpRequestData BuildRequestWithBootstrapToken()
|
||||
{
|
||||
var contextMock = new Mock<Microsoft.Azure.Functions.Worker.FunctionContext>();
|
||||
var reqMock = new Mock<HttpRequestData>(contextMock.Object);
|
||||
|
||||
var headers = new Microsoft.Azure.Functions.Worker.Http.HttpHeadersCollection
|
||||
{
|
||||
{ "X-Bootstrap-Token", BootstrapToken },
|
||||
};
|
||||
reqMock.SetupGet(r => r.Headers).Returns(headers);
|
||||
|
||||
return reqMock.Object;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BootstrapToken_RejectedWhenTenantDisabled()
|
||||
{
|
||||
var disabled = TenantConfiguration.CreateDefault(TenantId);
|
||||
disabled.Disabled = true;
|
||||
disabled.DisabledReason = "Offboarding in progress";
|
||||
disabled.DisabledUntil = null;
|
||||
|
||||
var sut = BuildValidator(disabled);
|
||||
|
||||
var result = await sut.ValidateRequestAsync(BuildRequestWithBootstrapToken(), TenantId);
|
||||
|
||||
// Must be the §0 Disabled-gate, not the §0.5 bootstrap branch. The bootstrap
|
||||
// branch lives AFTER the Disabled check; if anyone ever reorders these, the
|
||||
// assertion below catches it (Disabled returns "Tenant is suspended" with the
|
||||
// configured reason; bootstrap-branch failures have different ErrorMessages).
|
||||
Assert.False(result.IsValid);
|
||||
Assert.Equal(HttpStatusCode.Forbidden, result.StatusCode);
|
||||
Assert.Equal("Tenant is suspended", result.ErrorMessage);
|
||||
Assert.Equal("Offboarding in progress", result.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BootstrapToken_AcceptedWhenTenantNotDisabled_StillEvaluatesBootstrapBranch()
|
||||
{
|
||||
// Sanity: a non-disabled tenant DOES reach the bootstrap branch (which will then
|
||||
// run its own validation against _bootstrapSessionService). With _bootstrapSessionService
|
||||
// null in this test, the bootstrap branch falls through to the cert path — proving
|
||||
// that the §0 gate did NOT block.
|
||||
var enabled = TenantConfiguration.CreateDefault(TenantId);
|
||||
enabled.Disabled = false;
|
||||
enabled.ValidateAutopilotDevice = false;
|
||||
enabled.ValidateCorporateIdentifier = false;
|
||||
enabled.AllowInsecureAgentRequests = false;
|
||||
|
||||
var sut = BuildValidator(enabled);
|
||||
|
||||
var result = await sut.ValidateRequestAsync(BuildRequestWithBootstrapToken(), TenantId);
|
||||
|
||||
// Falls through to the "device validation is required" gate at §1 (not the §0 Disabled gate).
|
||||
Assert.False(result.IsValid);
|
||||
Assert.Equal(HttpStatusCode.Forbidden, result.StatusCode);
|
||||
Assert.Equal("Device validation is required", result.ErrorMessage);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Plan §7.4 step 5 + §10.1: drain predicate is true only when the progress blob has
|
||||
/// CompletedAt != null AND TombstoneStarted == true. 404 is "not yet done", not an error.
|
||||
/// </summary>
|
||||
public class DeletionProgressDrainProbeTests
|
||||
{
|
||||
private const string TenantId = "44444444-4444-4444-4444-444444444444";
|
||||
private const string SessionId = "55555555-5555-5555-5555-555555555555";
|
||||
private const string ManifestId = "manifest-1";
|
||||
|
||||
[Fact]
|
||||
public async Task IsCompleted_TrueOnlyWhen_CompletedAtSet_AndTombstoneStarted()
|
||||
{
|
||||
var sut = new DeletionProgressDrainProbe(
|
||||
new FakeBlob(new DeletionProgress
|
||||
{
|
||||
CompletedAt = DateTime.UtcNow,
|
||||
TombstoneStarted = true,
|
||||
}),
|
||||
NullLogger<DeletionProgressDrainProbe>.Instance);
|
||||
|
||||
Assert.True(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsCompleted_FalseWhen_CompletedAtNull()
|
||||
{
|
||||
var sut = new DeletionProgressDrainProbe(
|
||||
new FakeBlob(new DeletionProgress { TombstoneStarted = true, CompletedAt = null }),
|
||||
NullLogger<DeletionProgressDrainProbe>.Instance);
|
||||
|
||||
Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsCompleted_FalseWhen_TombstoneNotStarted_RevWalkthrough()
|
||||
{
|
||||
// If CompletedAt got somehow set without TombstoneStarted, the cascade did not
|
||||
// finish the FINAL step — drain must say "no" to defend against truncated progress.
|
||||
var sut = new DeletionProgressDrainProbe(
|
||||
new FakeBlob(new DeletionProgress { CompletedAt = DateTime.UtcNow, TombstoneStarted = false }),
|
||||
NullLogger<DeletionProgressDrainProbe>.Instance);
|
||||
|
||||
Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsCompleted_FalseOn404_NotAnError()
|
||||
{
|
||||
var sut = new DeletionProgressDrainProbe(
|
||||
new FakeBlob(throwOnDownload: new RequestFailedException(404, "NotFound", "BlobNotFound", null)),
|
||||
NullLogger<DeletionProgressDrainProbe>.Instance);
|
||||
|
||||
Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IsCompleted_PropagatesNon404StorageError()
|
||||
{
|
||||
// 503 / 500 are real transients the worker must surface so the queue retries.
|
||||
var sut = new DeletionProgressDrainProbe(
|
||||
new FakeBlob(throwOnDownload: new RequestFailedException(503, "ServiceUnavailable", "ServerBusy", null)),
|
||||
NullLogger<DeletionProgressDrainProbe>.Instance);
|
||||
|
||||
await Assert.ThrowsAsync<RequestFailedException>(
|
||||
() => sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
|
||||
}
|
||||
|
||||
// ── Fake BlobStorageService that overrides only the download seam we need ──
|
||||
|
||||
private sealed class FakeBlob : BlobStorageService
|
||||
{
|
||||
private readonly DeletionProgress? _progress;
|
||||
private readonly RequestFailedException? _throw;
|
||||
|
||||
public FakeBlob(DeletionProgress progress)
|
||||
: base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false)
|
||||
{
|
||||
_progress = progress;
|
||||
}
|
||||
|
||||
public FakeBlob(RequestFailedException throwOnDownload)
|
||||
: base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false)
|
||||
{
|
||||
_throw = throwOnDownload;
|
||||
}
|
||||
|
||||
public override Task<(DeletionProgress Progress, string ETag)> DownloadDeletionProgressAsync(
|
||||
string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_throw != null) throw _throw;
|
||||
return Task.FromResult((_progress!, "\"0xFAKE\""));
|
||||
}
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory <see cref="ITenantCustomsArchiveRepository"/> for PR3.B handler + endpoint tests.
|
||||
/// Idempotent upsert keyed by (PK, RK); supports the four query shapes the admin API uses.
|
||||
/// </summary>
|
||||
internal sealed class FakeCustomsArchiveRepository : ITenantCustomsArchiveRepository
|
||||
{
|
||||
private readonly Dictionary<string, TenantOffboardingCustomsArchiveEntry> _store
|
||||
= new(System.StringComparer.Ordinal);
|
||||
|
||||
public IReadOnlyDictionary<string, TenantOffboardingCustomsArchiveEntry> Store => _store;
|
||||
|
||||
private static string Key(string pk, string rk) => pk + "|" + rk;
|
||||
|
||||
public Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default)
|
||||
{
|
||||
_store[Key(entry.PartitionKey, entry.RowKey)] = Clone(entry);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<int> CountByRunAndTableAsync(string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default)
|
||||
{
|
||||
var pk = $"{normalizedTenantId}_{historyRowKey}";
|
||||
var count = _store.Values.Count(e =>
|
||||
e.PartitionKey == pk && e.OriginalTable == originalTable);
|
||||
return Task.FromResult(count);
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByRunAsync(
|
||||
string normalizedTenantId, string historyRowKey,
|
||||
[EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
var pk = $"{normalizedTenantId}_{historyRowKey}";
|
||||
foreach (var entry in _store.Values.Where(e => e.PartitionKey == pk))
|
||||
{
|
||||
yield return Clone(entry);
|
||||
}
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByTenantAsync(
|
||||
string normalizedTenantId, [EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
var prefix = normalizedTenantId + "_";
|
||||
foreach (var entry in _store.Values.Where(e => e.PartitionKey.StartsWith(prefix)))
|
||||
{
|
||||
yield return Clone(entry);
|
||||
}
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryAllAsync(
|
||||
[EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
foreach (var entry in _store.Values)
|
||||
{
|
||||
yield return Clone(entry);
|
||||
}
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<TenantOffboardingCustomsArchiveEntry?> TryGetEntryAsync(
|
||||
string partitionKey, string rowKey, CancellationToken ct = default)
|
||||
{
|
||||
_store.TryGetValue(Key(partitionKey, rowKey), out var entry);
|
||||
return Task.FromResult<TenantOffboardingCustomsArchiveEntry?>(entry == null ? null : Clone(entry));
|
||||
}
|
||||
|
||||
public Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default)
|
||||
{
|
||||
_store.Remove(Key(partitionKey, rowKey));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<int> DeleteRunAsync(string normalizedTenantId, string historyRowKey, CancellationToken ct = default)
|
||||
{
|
||||
var pk = $"{normalizedTenantId}_{historyRowKey}";
|
||||
var toDelete = _store.Keys.Where(k => k.StartsWith(pk + "|")).ToList();
|
||||
foreach (var k in toDelete) _store.Remove(k);
|
||||
return Task.FromResult(toDelete.Count);
|
||||
}
|
||||
|
||||
private static TenantOffboardingCustomsArchiveEntry Clone(TenantOffboardingCustomsArchiveEntry e) => new()
|
||||
{
|
||||
PartitionKey = e.PartitionKey,
|
||||
RowKey = e.RowKey,
|
||||
TenantId = e.TenantId,
|
||||
OriginalTable = e.OriginalTable,
|
||||
OriginalPartitionKey = e.OriginalPartitionKey,
|
||||
OriginalRowKey = e.OriginalRowKey,
|
||||
EntityJson = e.EntityJson,
|
||||
HistoryRowKey = e.HistoryRowKey,
|
||||
ArchivedAt = e.ArchivedAt,
|
||||
ArchivedBy = e.ArchivedBy,
|
||||
};
|
||||
}
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory <see cref="IOffboardingAuditRepository"/> for handler/cleanup tests. Records the
|
||||
/// latest write per (PK, RK) so the test can assert lifecycle transitions, and mirrors the
|
||||
/// production ETag-CAS contract on the pointer (412 on mismatched ETag).
|
||||
/// </summary>
|
||||
internal sealed class FakeOffboardingAuditRepository : IOffboardingAuditRepository
|
||||
{
|
||||
public Dictionary<string, OffboardingMarkerEntry> Markers { get; } = new();
|
||||
public Dictionary<string, OffboardingHistoryEntry> History { get; } = new();
|
||||
public Dictionary<string, (OffboardingByTenantPointer Pointer, string ETag)> Pointers { get; } = new();
|
||||
public List<string> MarkerWrites { get; } = new();
|
||||
public List<string> HistoryWrites { get; } = new();
|
||||
|
||||
/// <summary>When set, the next call to <see cref="UpsertHistoryAsync"/> throws — used by
|
||||
/// poison-path tests to simulate a transient storage failure inside the Failed-state
|
||||
/// transition so the Worker's reorder invariant can be exercised.</summary>
|
||||
public Exception? ThrowOnNextHistoryUpsert { get; set; }
|
||||
|
||||
private int _etagCounter;
|
||||
private string NextEtag() => $"\"0xFAKE_{++_etagCounter}\"";
|
||||
|
||||
public Task<OffboardingMarkerEntry?> TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default)
|
||||
=> Task.FromResult(Markers.TryGetValue(normalizedTenantId, out var m) ? Clone(m) : null);
|
||||
|
||||
public Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default)
|
||||
{
|
||||
if (Markers.ContainsKey(marker.RowKey))
|
||||
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
|
||||
Markers[marker.RowKey] = Clone(marker);
|
||||
MarkerWrites.Add(marker.Status);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default)
|
||||
{
|
||||
Markers[marker.RowKey] = Clone(marker);
|
||||
MarkerWrites.Add(marker.Status);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
Markers.Remove(normalizedTenantId);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<OffboardingMarkerEntry> QueryMarkersAsync(
|
||||
[EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
foreach (var marker in Markers.Values)
|
||||
{
|
||||
yield return Clone(marker);
|
||||
}
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default)
|
||||
{
|
||||
if (History.ContainsKey(history.RowKey))
|
||||
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
|
||||
History[history.RowKey] = Clone(history);
|
||||
HistoryWrites.Add(history.Status);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<OffboardingHistoryEntry?> TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default)
|
||||
=> Task.FromResult(History.TryGetValue(historyRowKey, out var h) ? Clone(h) : null);
|
||||
|
||||
public Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default)
|
||||
{
|
||||
if (ThrowOnNextHistoryUpsert is { } ex)
|
||||
{
|
||||
ThrowOnNextHistoryUpsert = null;
|
||||
throw ex;
|
||||
}
|
||||
History[history.RowKey] = Clone(history);
|
||||
HistoryWrites.Add(history.Status);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync(
|
||||
string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
if (Pointers.TryGetValue(normalizedTenantId, out var entry))
|
||||
{
|
||||
return Task.FromResult<(OffboardingByTenantPointer?, string?)>((Clone(entry.Pointer), entry.ETag));
|
||||
}
|
||||
return Task.FromResult<(OffboardingByTenantPointer?, string?)>((null, null));
|
||||
}
|
||||
|
||||
public Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default)
|
||||
{
|
||||
if (Pointers.ContainsKey(pointer.RowKey))
|
||||
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
|
||||
Pointers[pointer.RowKey] = (Clone(pointer), NextEtag());
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task UpdateByTenantPointerWithEtagAsync(
|
||||
OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default)
|
||||
{
|
||||
if (!Pointers.TryGetValue(pointer.RowKey, out var entry))
|
||||
throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null);
|
||||
if (entry.ETag != ifMatchEtag)
|
||||
throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null);
|
||||
Pointers[pointer.RowKey] = (Clone(pointer), NextEtag());
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static OffboardingMarkerEntry Clone(OffboardingMarkerEntry m) => new()
|
||||
{
|
||||
PartitionKey = m.PartitionKey, RowKey = m.RowKey,
|
||||
TenantId = m.TenantId, OffboardingHistoryRowKey = m.OffboardingHistoryRowKey,
|
||||
InitiatedAt = m.InitiatedAt, InitiatedBy = m.InitiatedBy,
|
||||
Status = m.Status, CompletedAt = m.CompletedAt,
|
||||
FailedAt = m.FailedAt, FailedPhase = m.FailedPhase,
|
||||
};
|
||||
|
||||
private static OffboardingHistoryEntry Clone(OffboardingHistoryEntry h) => new()
|
||||
{
|
||||
PartitionKey = h.PartitionKey, RowKey = h.RowKey,
|
||||
TenantId = h.TenantId, DomainName = h.DomainName, InitiatedBy = h.InitiatedBy,
|
||||
OffboardedAt = h.OffboardedAt, CompletedAt = h.CompletedAt,
|
||||
EarliestProcessingAt = h.EarliestProcessingAt,
|
||||
Status = h.Status,
|
||||
DeletedRowCountsJson = h.DeletedRowCountsJson, TotalRowsDeleted = h.TotalRowsDeleted,
|
||||
DeletedBlobCount = h.DeletedBlobCount, CascadeSessionsEnqueued = h.CascadeSessionsEnqueued,
|
||||
ErrorMessage = h.ErrorMessage, RetryCount = h.RetryCount,
|
||||
DrainCompletedAt = h.DrainCompletedAt,
|
||||
EnumerationStartedAt = h.EnumerationStartedAt,
|
||||
EnumerationCompletedBeforeUpload = h.EnumerationCompletedBeforeUpload,
|
||||
CustomGatherRulesArchived = h.CustomGatherRulesArchived,
|
||||
CustomAnalyzeRulesArchived = h.CustomAnalyzeRulesArchived,
|
||||
ImeLogPatternOverridesArchived = h.ImeLogPatternOverridesArchived,
|
||||
ReonboardedAt = h.ReonboardedAt, ReonboardedBy = h.ReonboardedBy,
|
||||
CustomsAutoWipedOnReonboard = h.CustomsAutoWipedOnReonboard,
|
||||
};
|
||||
|
||||
private static OffboardingByTenantPointer Clone(OffboardingByTenantPointer p) => new()
|
||||
{
|
||||
PartitionKey = p.PartitionKey, RowKey = p.RowKey, TenantId = p.TenantId,
|
||||
LatestHistoryRowKey = p.LatestHistoryRowKey, LatestStatus = p.LatestStatus,
|
||||
LatestUpdatedAt = p.LatestUpdatedAt, OffboardCount = p.OffboardCount,
|
||||
};
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory <see cref="IOffboardingExpectationsStore"/> with the production If-None-Match=*
|
||||
/// and If-Match ETag-CAS semantics. Tracks delete attempts so tests can assert idempotency.
|
||||
/// </summary>
|
||||
internal sealed class FakeOffboardingExpectationsStore : IOffboardingExpectationsStore
|
||||
{
|
||||
private readonly Dictionary<string, (OffboardingExpectations Payload, string ETag)> _blobs = new();
|
||||
private int _etagCounter;
|
||||
|
||||
public int DeleteCallCount { get; private set; }
|
||||
public bool ThrowOnDelete { get; set; }
|
||||
|
||||
public Task<bool> TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default)
|
||||
{
|
||||
var key = Key(payload.TenantId, payload.HistoryRowKey);
|
||||
if (_blobs.ContainsKey(key)) return Task.FromResult(false);
|
||||
_blobs[key] = (Clone(payload), NextEtag());
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
|
||||
public Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync(
|
||||
string tenantId, string historyRowKey, CancellationToken ct = default)
|
||||
{
|
||||
var key = Key(tenantId, historyRowKey);
|
||||
if (!_blobs.TryGetValue(key, out var entry))
|
||||
return Task.FromResult<(OffboardingExpectations?, string?)>((null, null));
|
||||
return Task.FromResult<(OffboardingExpectations?, string?)>((Clone(entry.Payload), entry.ETag));
|
||||
}
|
||||
|
||||
public Task<string> UpdateWithEtagCasAsync(OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default)
|
||||
{
|
||||
var key = Key(payload.TenantId, payload.HistoryRowKey);
|
||||
if (!_blobs.TryGetValue(key, out var entry))
|
||||
throw new RequestFailedException(404, "NotFound", "BlobNotFound", null);
|
||||
if (entry.ETag != ifMatchEtag)
|
||||
throw new RequestFailedException(412, "ConditionNotMet", "ConditionNotMet", null);
|
||||
var newEtag = NextEtag();
|
||||
_blobs[key] = (Clone(payload), newEtag);
|
||||
return Task.FromResult(newEtag);
|
||||
}
|
||||
|
||||
public Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default)
|
||||
{
|
||||
DeleteCallCount++;
|
||||
if (ThrowOnDelete) throw new InvalidOperationException("simulated blob-delete failure");
|
||||
_blobs.Remove(Key(tenantId, historyRowKey));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Pre-seed a blob (used to drive the drain probe directly without going via TryUploadInitial).</summary>
|
||||
public void Seed(OffboardingExpectations payload)
|
||||
{
|
||||
_blobs[Key(payload.TenantId, payload.HistoryRowKey)] = (Clone(payload), NextEtag());
|
||||
}
|
||||
|
||||
public bool BlobExists(string tenantId, string historyRowKey)
|
||||
=> _blobs.ContainsKey(Key(tenantId, historyRowKey));
|
||||
|
||||
private static string Key(string tenantId, string historyRowKey) => $"{tenantId}|{historyRowKey}";
|
||||
private string NextEtag() => $"\"0xFAKE_EXP_{++_etagCounter}\"";
|
||||
|
||||
private static OffboardingExpectations Clone(OffboardingExpectations p)
|
||||
{
|
||||
var clone = new OffboardingExpectations
|
||||
{
|
||||
SchemaVersion = p.SchemaVersion,
|
||||
TenantId = p.TenantId,
|
||||
HistoryRowKey = p.HistoryRowKey,
|
||||
CreatedAt = p.CreatedAt,
|
||||
EnumerationCompleted = p.EnumerationCompleted,
|
||||
EnumeratedSessionCount = p.EnumeratedSessionCount,
|
||||
};
|
||||
foreach (var e in p.Expectations)
|
||||
{
|
||||
clone.Expectations.Add(new OffboardingExpectation
|
||||
{
|
||||
SessionId = e.SessionId,
|
||||
ManifestId = e.ManifestId,
|
||||
Outcome = e.Outcome,
|
||||
RetryCount = e.RetryCount,
|
||||
});
|
||||
}
|
||||
return clone;
|
||||
}
|
||||
}
|
||||
+431
@@ -0,0 +1,431 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Maintenance;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// Rev-5-F2 + Rev-9 cleanup contract:
|
||||
/// <list type="bullet">
|
||||
/// <item>Completed markers older than 15min are removed — AFTER their Expectations blob.</item>
|
||||
/// <item>Completed markers younger than 15min stay (warm caches could still leak auth).</item>
|
||||
/// <item>Failed markers are NEVER auto-removed — operator action only.</item>
|
||||
/// <item>Initiated / InProgress markers are ignored (offboarding still in flight).</item>
|
||||
/// <item>Blob-delete failure leaves the marker in place so the next run retries.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public class OffboardingMarkerCleanupFunctionTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111";
|
||||
|
||||
[Fact]
|
||||
public async Task DeletesCompletedMarkerOlderThan15min()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
fixture.Expectations.Seed(NewExpectations());
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(1, result.MarkersDeleted);
|
||||
Assert.False(fixture.Repo.Markers.ContainsKey(TenantId));
|
||||
Assert.False(fixture.Expectations.BlobExists(TenantId, HistoryRowKey),
|
||||
"Expectations blob must be deleted before the marker");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task KeepsCompletedMarkerYoungerThan15min()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(5)); // inside cache TTL window
|
||||
fixture.Expectations.Seed(NewExpectations());
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(0, result.MarkersDeleted);
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId));
|
||||
Assert.True(fixture.Expectations.BlobExists(TenantId, HistoryRowKey),
|
||||
"Blob stays until the marker is eligible for removal");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NeverDeletesFailedMarker_EvenAfter24h()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(
|
||||
status: "Failed",
|
||||
failedAt: fixture.FixedNow - TimeSpan.FromHours(24),
|
||||
failedPhase: "drain_timeout");
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(0, result.MarkersDeleted);
|
||||
Assert.Equal(1, result.FailedMarkersSeen);
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId),
|
||||
"Failed markers require operator action — never auto-cleaned");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IgnoresInProgressMarker()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(status: "InProgress");
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(0, result.MarkersDeleted);
|
||||
Assert.Equal(0, result.FailedMarkersSeen);
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IgnoresInitiatedMarker()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(status: "Initiated");
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(0, result.MarkersDeleted);
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BlobDeleteFails_KeepsMarkerForNextRun()
|
||||
{
|
||||
// Rev-9 hygiene: blob deleted BEFORE marker. If blob delete throws (transient storage
|
||||
// glitch), marker stays so the next 2h-run tries again. Otherwise an orphan blob lives
|
||||
// forever in offboarding-state.
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
fixture.Expectations.Seed(NewExpectations());
|
||||
fixture.Expectations.ThrowOnDelete = true;
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(0, result.MarkersDeleted);
|
||||
Assert.Equal(1, result.BlobDeleteRetries);
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId),
|
||||
"Marker stays so the next run retries the blob delete");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DryRunWithMixedFleet_HandlesEachMarkerIndependently()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"] = NewMarker(
|
||||
tenantId: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
|
||||
status: "Completed", completedAt: fixture.FixedNow - TimeSpan.FromMinutes(30));
|
||||
fixture.Repo.Markers["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"] = NewMarker(
|
||||
tenantId: "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
|
||||
status: "Completed", completedAt: fixture.FixedNow - TimeSpan.FromMinutes(2));
|
||||
fixture.Repo.Markers["cccccccc-cccc-cccc-cccc-cccccccccccc"] = NewMarker(
|
||||
tenantId: "cccccccc-cccc-cccc-cccc-cccccccccccc",
|
||||
status: "Failed", failedAt: fixture.FixedNow - TimeSpan.FromHours(2),
|
||||
failedPhase: "drain_timeout");
|
||||
fixture.Repo.Markers["dddddddd-dddd-dddd-dddd-dddddddddddd"] = NewMarker(
|
||||
tenantId: "dddddddd-dddd-dddd-dddd-dddddddddddd",
|
||||
status: "InProgress");
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(4, result.Scanned);
|
||||
Assert.Equal(1, result.MarkersDeleted); // only the 30-min-old Completed one
|
||||
Assert.Equal(1, result.FailedMarkersSeen);
|
||||
Assert.False(fixture.Repo.Markers.ContainsKey("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"));
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"));
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey("cccccccc-cccc-cccc-cccc-cccccccccccc"));
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey("dddddddd-dddd-dddd-dddd-dddddddddddd"));
|
||||
}
|
||||
|
||||
// ── TenantConfiguration defense-in-depth sweep (PR3.B-revised Finding 2) ──────
|
||||
//
|
||||
// Phase 2.F-final in the handler is fail-soft. If it failed (worker crash, transient
|
||||
// storage), the TenantConfiguration row would carry Disabled=true and block the
|
||||
// tenant from self-service re-onboarding indefinitely. This sweep is the second
|
||||
// pass: when MarkerCleanup spots a Completed marker eligible for removal, it tries
|
||||
// to delete TenantConfiguration FIRST. If the sweep fails, the marker stays so the
|
||||
// next 2h run retries.
|
||||
|
||||
[Fact]
|
||||
public async Task CompletedMarker_SweepsLingeringTenantConfig_BeforeDeletingMarker()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(
|
||||
status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
// Simulate "Phase 2.F-final failed in the handler" → the row is still the
|
||||
// offboarding tombstone (Disabled=true + magic reason). Sweep should wipe it.
|
||||
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ReturnsAsync(BuildTombstoneConfig(TenantId));
|
||||
fixture.SafeWipe.QueuedReturns.Enqueue(1);
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(1, result.MarkersDeleted);
|
||||
Assert.Equal(1, result.TenantConfigsSwept);
|
||||
Assert.Equal(0, result.TenantConfigsSpared);
|
||||
Assert.Equal(0, result.TenantConfigSweepRetries);
|
||||
|
||||
// The sweep ran with the right table + tenant id.
|
||||
Assert.Single(fixture.SafeWipe.WipeCalls);
|
||||
Assert.Equal(Constants.TableNames.TenantConfiguration, fixture.SafeWipe.WipeCalls[0].Table);
|
||||
Assert.Equal(TenantId, fixture.SafeWipe.WipeCalls[0].TenantId);
|
||||
Assert.False(fixture.Repo.Markers.ContainsKey(TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompletedMarker_TenantConfigAlreadyGone_SkipsSweep_DeletesMarker()
|
||||
{
|
||||
// Happy path: Phase 2.F-final succeeded in the handler. GetTenantConfigurationAsync
|
||||
// returns null (default fixture stub) → no row to wipe.
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(
|
||||
status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(1, result.MarkersDeleted);
|
||||
Assert.Equal(0, result.TenantConfigsSwept);
|
||||
Assert.Equal(0, result.TenantConfigsSpared);
|
||||
// No SafeWipe call when the row is already gone (we don't waste a transaction).
|
||||
Assert.Empty(fixture.SafeWipe.WipeCalls);
|
||||
Assert.False(fixture.Repo.Markers.ContainsKey(TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompletedMarker_UserReonboarded_SparesFreshConfig_AndStillDeletesMarker()
|
||||
{
|
||||
// The crucial Codex Finding 1 scenario: between Phase 2.F-final and MarkerCleanup,
|
||||
// the user self-service-re-onboarded. The TenantConfiguration row now exists with
|
||||
// Disabled=false (or any non-offboarding DisabledReason). Blind sweep would
|
||||
// destroy their fresh tenant. The conditional probe must skip the wipe.
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(
|
||||
status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
|
||||
var freshConfig = TenantConfiguration.CreateDefault(TenantId);
|
||||
freshConfig.Disabled = false;
|
||||
freshConfig.UpdatedBy = "bob@contoso.com";
|
||||
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ReturnsAsync(freshConfig);
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(1, result.MarkersDeleted);
|
||||
Assert.Equal(0, result.TenantConfigsSwept);
|
||||
Assert.Equal(1, result.TenantConfigsSpared);
|
||||
// Critically: SafeWipe was NEVER called. The fresh config is preserved.
|
||||
Assert.Empty(fixture.SafeWipe.WipeCalls);
|
||||
// The Completed marker still gets cleaned up — its work is done.
|
||||
Assert.False(fixture.Repo.Markers.ContainsKey(TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompletedMarker_UserReonboarded_AndSuspendedThemselves_StillSpared()
|
||||
{
|
||||
// Defense-in-depth check: a re-onboarded tenant that later self-suspended would
|
||||
// have Disabled=true but with a different DisabledReason (e.g. "Self-suspended").
|
||||
// The magic-string match must distinguish — only "Offboarding in progress" triggers
|
||||
// the wipe.
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(
|
||||
status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
|
||||
var reonboardedThenSuspended = TenantConfiguration.CreateDefault(TenantId);
|
||||
reonboardedThenSuspended.Disabled = true;
|
||||
reonboardedThenSuspended.DisabledReason = "Self-suspended";
|
||||
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ReturnsAsync(reonboardedThenSuspended);
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(0, result.TenantConfigsSwept);
|
||||
Assert.Equal(1, result.TenantConfigsSpared);
|
||||
Assert.Empty(fixture.SafeWipe.WipeCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompletedMarker_TenantConfigProbeThrows_KeepsMarkerForNextRetry()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(
|
||||
status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ThrowsAsync(new InvalidOperationException("storage down"));
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(0, result.MarkersDeleted);
|
||||
Assert.Equal(1, result.TenantConfigSweepRetries);
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId),
|
||||
"Marker must stay alive when the TenantConfiguration probe fails so the next run can retry");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompletedMarker_TenantConfigSweepThrows_KeepsMarkerForNextRetry()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(
|
||||
status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ReturnsAsync(BuildTombstoneConfig(TenantId));
|
||||
fixture.SafeWipe.ThrowOnNextWipe = new InvalidOperationException("storage down");
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(0, result.MarkersDeleted);
|
||||
Assert.Equal(1, result.TenantConfigSweepRetries);
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId),
|
||||
"Marker must stay alive when the TenantConfig sweep fails so the next run can retry");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompletedMarker_BlobDeleteFails_TenantConfigProbeIsNotEvenAttempted()
|
||||
{
|
||||
var fixture = new Fixture();
|
||||
fixture.Repo.Markers[TenantId] = NewMarker(
|
||||
status: "Completed",
|
||||
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
|
||||
fixture.Expectations.ThrowOnDelete = true;
|
||||
|
||||
var result = await fixture.Sut.RunCoreAsync(default);
|
||||
|
||||
Assert.Equal(1, result.BlobDeleteRetries);
|
||||
Assert.Empty(fixture.SafeWipe.WipeCalls);
|
||||
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId));
|
||||
// Probe shouldn't have been attempted either — blob is the first gate.
|
||||
fixture.ConfigRepo.Verify(r => r.GetTenantConfigurationAsync(It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
private static TenantConfiguration BuildTombstoneConfig(string tenantId)
|
||||
{
|
||||
var config = TenantConfiguration.CreateDefault(tenantId);
|
||||
config.Disabled = true;
|
||||
config.DisabledReason = "Offboarding in progress";
|
||||
return config;
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
private static OffboardingMarkerEntry NewMarker(
|
||||
string? tenantId = null,
|
||||
string status = "Completed",
|
||||
DateTime? completedAt = null,
|
||||
DateTime? failedAt = null,
|
||||
string? failedPhase = null)
|
||||
{
|
||||
var tid = tenantId ?? TenantId;
|
||||
return new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = tid,
|
||||
TenantId = tid,
|
||||
OffboardingHistoryRowKey = $"20260518091523123_{tid}",
|
||||
InitiatedAt = DateTime.UtcNow.AddHours(-1),
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
Status = status,
|
||||
CompletedAt = completedAt,
|
||||
FailedAt = failedAt,
|
||||
FailedPhase = failedPhase,
|
||||
};
|
||||
}
|
||||
|
||||
private static OffboardingExpectations NewExpectations() => new()
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
TenantId = TenantId,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
CreatedAt = DateTime.UtcNow.AddHours(-1),
|
||||
EnumerationCompleted = true,
|
||||
EnumeratedSessionCount = 0,
|
||||
};
|
||||
|
||||
private sealed class Fixture
|
||||
{
|
||||
public DateTime FixedNow { get; } = new DateTime(2026, 5, 18, 12, 0, 0, DateTimeKind.Utc);
|
||||
public FakeOffboardingAuditRepository Repo { get; } = new();
|
||||
public FakeOffboardingExpectationsStore Expectations { get; } = new();
|
||||
public RecordingSafeWipeService SafeWipe { get; } = new();
|
||||
public Mock<IConfigRepository> ConfigRepo { get; } = new();
|
||||
public OffboardingMarkerCleanupFunction Sut { get; }
|
||||
|
||||
public Fixture()
|
||||
{
|
||||
var timeProvider = new FixedTimeProvider(FixedNow);
|
||||
// Default: TenantConfig row not found (the normal happy path — Phase 2.F-final
|
||||
// successfully deleted it; nothing for the sweep to do). Individual tests
|
||||
// override this to seed a lingering tombstone or a re-onboarded fresh config.
|
||||
ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
|
||||
.ReturnsAsync((TenantConfiguration?)null);
|
||||
|
||||
Sut = new OffboardingMarkerCleanupFunction(
|
||||
Repo, Expectations, SafeWipe, ConfigRepo.Object,
|
||||
NullLogger<OffboardingMarkerCleanupFunction>.Instance,
|
||||
timeProvider);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mock <see cref="AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService"/> that
|
||||
/// records which (table, tenantId) pairs were swept and how many rows it claims to have
|
||||
/// deleted. Default behaviour returns 0 (row already gone — the normal happy path).
|
||||
/// Tests can opt in to "0 means a TenantConfiguration row was found and wiped" via
|
||||
/// <see cref="QueuedReturns"/>.
|
||||
/// </summary>
|
||||
private sealed class RecordingSafeWipeService : AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService
|
||||
{
|
||||
public readonly List<(string Table, string TenantId)> WipeCalls = new();
|
||||
public readonly Queue<int> QueuedReturns = new();
|
||||
public Exception? ThrowOnNextWipe { get; set; }
|
||||
|
||||
public RecordingSafeWipeService()
|
||||
: base(
|
||||
new AutopilotMonitor.Functions.Services.TableStorageService(
|
||||
Moq.Mock.Of<Azure.Data.Tables.TableServiceClient>(),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.TableStorageService>.Instance),
|
||||
new AutopilotMonitor.Functions.Services.BlobStorageService(
|
||||
new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.BlobStorageService>.Instance,
|
||||
usesManagedIdentity: false),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService>.Instance)
|
||||
{
|
||||
}
|
||||
|
||||
public override Task<int> WipeByExactPartitionAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
WipeCalls.Add((tableName, normalizedTenantId));
|
||||
if (ThrowOnNextWipe is { } ex)
|
||||
{
|
||||
ThrowOnNextWipe = null;
|
||||
throw ex;
|
||||
}
|
||||
return Task.FromResult(QueuedReturns.Count > 0 ? QueuedReturns.Dequeue() : 0);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedTimeProvider : TimeProvider
|
||||
{
|
||||
private readonly DateTimeOffset _now;
|
||||
public FixedTimeProvider(DateTime utcNow) { _now = new DateTimeOffset(utcNow, TimeSpan.Zero); }
|
||||
public override DateTimeOffset GetUtcNow() => _now;
|
||||
}
|
||||
}
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// PR3.B §3.2 — static-helper tests for the archive RowKey base64url encoding and the
|
||||
/// PartitionKey construction. These two functions are the only things in the repository
|
||||
/// that have logic worth pinning at unit level; the rest is straight Azure Tables CRUD.
|
||||
/// </summary>
|
||||
public sealed class TableTenantCustomsArchiveRepositoryHelperTests
|
||||
{
|
||||
[Fact]
|
||||
public void BuildPartitionKey_Concatenates_TenantId_And_HistoryRowKey()
|
||||
{
|
||||
var pk = TableTenantCustomsArchiveRepository.BuildPartitionKey(
|
||||
"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
"20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee");
|
||||
Assert.Equal(
|
||||
"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
pk);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("simple-row-id")]
|
||||
[InlineData("rowkey/with/slashes")]
|
||||
[InlineData("rowkey#with#hash")]
|
||||
[InlineData("rowkey?with?question")]
|
||||
[InlineData("rowkey\\with\\backslash")]
|
||||
[InlineData("a")]
|
||||
[InlineData("ab")]
|
||||
[InlineData("abc")]
|
||||
[InlineData("abcd")]
|
||||
[InlineData("unicode-Ümlaut-and-emoji-✓")]
|
||||
public void BuildRowKey_RoundTripsForbiddenCharacters(string originalRowKey)
|
||||
{
|
||||
var archiveRowKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", originalRowKey);
|
||||
|
||||
// RowKey itself must NOT contain any Azure-Tables-forbidden characters.
|
||||
Assert.DoesNotContain('#', archiveRowKey);
|
||||
Assert.DoesNotContain('?', archiveRowKey);
|
||||
Assert.DoesNotContain('/', archiveRowKey);
|
||||
Assert.DoesNotContain('\\', archiveRowKey);
|
||||
|
||||
// Must round-trip cleanly.
|
||||
var decoded = TableTenantCustomsArchiveRepository.DecodeOriginalRowKey(archiveRowKey, "GatherRules");
|
||||
Assert.Equal(originalRowKey, decoded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildRowKey_DifferentTablesProduceDifferentKeys()
|
||||
{
|
||||
var gatherKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", "my-rule");
|
||||
var analyzeKey = TableTenantCustomsArchiveRepository.BuildRowKey("AnalyzeRules", "my-rule");
|
||||
|
||||
Assert.NotEqual(gatherKey, analyzeKey);
|
||||
Assert.StartsWith("GatherRules_", gatherKey);
|
||||
Assert.StartsWith("AnalyzeRules_", analyzeKey);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecodeOriginalRowKey_RejectsPrefixMismatch()
|
||||
{
|
||||
var rowKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", "x");
|
||||
Assert.Throws<ArgumentException>(() =>
|
||||
TableTenantCustomsArchiveRepository.DecodeOriginalRowKey(rowKey, "AnalyzeRules"));
|
||||
}
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// PR3.B §3.4-§3.5 — invariants the offboarding-customs-archive design guarantees:
|
||||
/// snapshot serialization, partition-key namespacing per offboarding run, and the
|
||||
/// counter-from-archive recomputation that makes the History fields crash-resume safe.
|
||||
/// <para>
|
||||
/// Source-table archive-then-wipe orchestration (3-iteration cap + customs_arrival_race
|
||||
/// fail-closed) is a structural property of the loop in
|
||||
/// <see cref="TenantOffboardingHandler.ArchiveAndWipeRulesTableAsync"/> — manual smoketest
|
||||
/// on a dev tenant covers that contract; full mock-TableClient integration test is a
|
||||
/// followup once we have a TableClient abstraction in the codebase.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class TenantCustomsArchiveContractTests
|
||||
{
|
||||
private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
|
||||
private const string HistoryRowKey = "20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
|
||||
|
||||
private static TenantOffboardingHandler BuildHandler(ITenantCustomsArchiveRepository archive)
|
||||
{
|
||||
// We only need the handler instance for its internal BuildArchiveEntry helper; that
|
||||
// method does not call into any of the other collaborators, so we can pass null! for
|
||||
// them. Tests exercise pure-method behaviour only.
|
||||
return new TenantOffboardingHandler(
|
||||
auditRepo: null!,
|
||||
enumerator: null!,
|
||||
cascadeEnqueuer: null!,
|
||||
expectations: null!,
|
||||
drainProbe: null!,
|
||||
safeWipe: null!,
|
||||
storage: null!,
|
||||
maintenance: null!,
|
||||
reEnqueuer: null!,
|
||||
opsEvents: null!,
|
||||
customsArchive: archive,
|
||||
logger: NullLogger<TenantOffboardingHandler>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveEntry_PartitionKey_IsTenantUnderscoreHistoryRowKey()
|
||||
{
|
||||
var handler = BuildHandler(new FakeCustomsArchiveRepository());
|
||||
var source = new TableEntity(TenantId, "rule-id-1") { ["DisplayName"] = "anything" };
|
||||
|
||||
var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.GatherRules, HistoryRowKey);
|
||||
|
||||
Assert.Equal($"{TenantId}_{HistoryRowKey}", entry.PartitionKey);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveEntry_RowKey_PrefixedWithOriginalTable_AndBase64Encoded()
|
||||
{
|
||||
var handler = BuildHandler(new FakeCustomsArchiveRepository());
|
||||
var source = new TableEntity(TenantId, "rule/with/slash") { ["DisplayName"] = "x" };
|
||||
|
||||
var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.AnalyzeRules, HistoryRowKey);
|
||||
|
||||
Assert.StartsWith("AnalyzeRules_", entry.RowKey);
|
||||
Assert.DoesNotContain('/', entry.RowKey); // base64url replaces / with _
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveEntry_StripsSystemProperties_FromJson()
|
||||
{
|
||||
var handler = BuildHandler(new FakeCustomsArchiveRepository());
|
||||
var source = new TableEntity(TenantId, "rule")
|
||||
{
|
||||
["DisplayName"] = "MyRule",
|
||||
["Category"] = "preset",
|
||||
["Timestamp"] = System.DateTimeOffset.UtcNow,
|
||||
["odata.etag"] = "\"0x123\"",
|
||||
};
|
||||
|
||||
var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.GatherRules, HistoryRowKey);
|
||||
|
||||
var parsed = JsonDocument.Parse(entry.EntityJson);
|
||||
Assert.True(parsed.RootElement.TryGetProperty("DisplayName", out var dn));
|
||||
Assert.Equal("MyRule", dn.GetString());
|
||||
Assert.True(parsed.RootElement.TryGetProperty("Category", out _));
|
||||
|
||||
// System properties must NOT appear in the snapshot.
|
||||
Assert.False(parsed.RootElement.TryGetProperty("Timestamp", out _));
|
||||
Assert.False(parsed.RootElement.TryGetProperty("odata.etag", out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveEntry_PopulatesOriginalKeysAndArchivedBy()
|
||||
{
|
||||
var handler = BuildHandler(new FakeCustomsArchiveRepository());
|
||||
var source = new TableEntity(TenantId, "my-rule-id") { ["X"] = "Y" };
|
||||
|
||||
var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.ImeLogPatterns, HistoryRowKey);
|
||||
|
||||
Assert.Equal(TenantId, entry.TenantId);
|
||||
Assert.Equal(Constants.TableNames.ImeLogPatterns, entry.OriginalTable);
|
||||
Assert.Equal(TenantId, entry.OriginalPartitionKey);
|
||||
Assert.Equal("my-rule-id", entry.OriginalRowKey);
|
||||
Assert.Equal(HistoryRowKey, entry.HistoryRowKey);
|
||||
Assert.Equal("TenantOffboardingHandler", entry.ArchivedBy);
|
||||
Assert.NotEqual(default, entry.ArchivedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ReReOffboard_TwoRuns_CoExistInArchive_AsDistinctPartitions()
|
||||
{
|
||||
// PR3.B §3.5: each offboarding run gets its own PartitionKey
|
||||
// ({tenantId}_{historyRowKey}), so a tenant offboarded twice produces two immutable
|
||||
// archive partitions side-by-side.
|
||||
var repo = new FakeCustomsArchiveRepository();
|
||||
|
||||
var run1 = "20260101000000000_" + TenantId;
|
||||
var run2 = "20260518091523123_" + TenantId;
|
||||
|
||||
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry
|
||||
{
|
||||
PartitionKey = $"{TenantId}_{run1}",
|
||||
RowKey = "GatherRules_aaa",
|
||||
TenantId = TenantId,
|
||||
OriginalTable = Constants.TableNames.GatherRules,
|
||||
OriginalRowKey = "aaa",
|
||||
HistoryRowKey = run1,
|
||||
});
|
||||
|
||||
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry
|
||||
{
|
||||
PartitionKey = $"{TenantId}_{run2}",
|
||||
RowKey = "GatherRules_aaa",
|
||||
TenantId = TenantId,
|
||||
OriginalTable = Constants.TableNames.GatherRules,
|
||||
OriginalRowKey = "aaa",
|
||||
HistoryRowKey = run2,
|
||||
});
|
||||
|
||||
var count1 = await repo.CountByRunAndTableAsync(TenantId, run1, Constants.TableNames.GatherRules);
|
||||
var count2 = await repo.CountByRunAndTableAsync(TenantId, run2, Constants.TableNames.GatherRules);
|
||||
|
||||
Assert.Equal(1, count1);
|
||||
Assert.Equal(1, count2);
|
||||
Assert.Equal(2, repo.Store.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CountByRunAndTable_FiltersByOriginalTable()
|
||||
{
|
||||
var repo = new FakeCustomsArchiveRepository();
|
||||
var run = "20260518091523123_" + TenantId;
|
||||
var pk = $"{TenantId}_{run}";
|
||||
|
||||
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules", HistoryRowKey = run });
|
||||
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "GatherRules_b", TenantId = TenantId, OriginalTable = "GatherRules", HistoryRowKey = run });
|
||||
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "AnalyzeRules_a", TenantId = TenantId, OriginalTable = "AnalyzeRules", HistoryRowKey = run });
|
||||
|
||||
Assert.Equal(2, await repo.CountByRunAndTableAsync(TenantId, run, "GatherRules"));
|
||||
Assert.Equal(1, await repo.CountByRunAndTableAsync(TenantId, run, "AnalyzeRules"));
|
||||
Assert.Equal(0, await repo.CountByRunAndTableAsync(TenantId, run, "ImeLogPatterns"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteRunAsync_RemovesEveryEntryInPartition_LeavesOtherRunsAlone()
|
||||
{
|
||||
var repo = new FakeCustomsArchiveRepository();
|
||||
var run1 = "20260101000000000_" + TenantId;
|
||||
var run2 = "20260518091523123_" + TenantId;
|
||||
|
||||
await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run1}", RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules" });
|
||||
await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run1}", RowKey = "GatherRules_b", TenantId = TenantId, OriginalTable = "GatherRules" });
|
||||
await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run2}", RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules" });
|
||||
|
||||
var deleted = await repo.DeleteRunAsync(TenantId, run1);
|
||||
|
||||
Assert.Equal(2, deleted);
|
||||
Assert.Single(repo.Store);
|
||||
// The remaining row belongs to run2.
|
||||
Assert.Single(repo.Store.Values, e => e.PartitionKey == $"{TenantId}_{run2}");
|
||||
}
|
||||
}
|
||||
+634
@@ -0,0 +1,634 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Storage.Blobs;
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// Pin the §4.5 Drain-Contract that prevents the "silent wipe" the user has been worried about.
|
||||
/// Every fail-closed branch must throw + the Marker must transition to <c>Failed</c> WITHOUT
|
||||
/// any SafeWipe being attempted. The 0-session happy path and the Rev-9-F1 Drain-Skip-Gate
|
||||
/// (post-crash idempotent resume) round out coverage for the surrounding state-machine.
|
||||
/// </summary>
|
||||
public class TenantOffboardingHandlerDrainContractTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111";
|
||||
|
||||
// ── Fail-closed branches ────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task KillSwitchActive_ExpectationFails_NoSafeWipe()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.SeedExpectations(new (string, string?, string, int)[]
|
||||
{
|
||||
("session-1", null, nameof(SessionDeletionEnqueueOutcome.KillSwitchActive), 0),
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
|
||||
|
||||
Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status);
|
||||
Assert.Equal("killswitch", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Poisoned_ExpectationFails_NoSafeWipe()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.SeedExpectations(new (string, string?, string, int)[]
|
||||
{
|
||||
("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Poisoned), 0),
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
|
||||
|
||||
Assert.Equal("poisoned", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AlreadyInFlight_With_NullManifestId_FailsClosed_Rev8F3()
|
||||
{
|
||||
// AlreadyInFlight + null ManifestId is the Preparing-without-snapshot state. There is
|
||||
// no progress blob to drain against → fail closed immediately rather than waiting
|
||||
// out the 2h drain budget.
|
||||
var harness = Harness.New();
|
||||
harness.SeedExpectations(new (string, string?, string, int)[]
|
||||
{
|
||||
("session-1", null, nameof(SessionDeletionEnqueueOutcome.AlreadyInFlight), 0),
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
|
||||
|
||||
Assert.Equal("alreadyinflight_no_manifest", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CasExhausted_RetriesUpToCap_ThenFailsClosed()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.SeedExpectations(new (string, string?, string, int)[]
|
||||
{
|
||||
("session-1", null, nameof(SessionDeletionEnqueueOutcome.CasExhausted),
|
||||
TenantOffboardingHandler.MaxCasRetriesPerSession),
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
|
||||
|
||||
Assert.Equal("cas_exhausted", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnumeratorCrash_FirstAttempt_StampsEnumerationStartedAt_ThenPropagatesException()
|
||||
{
|
||||
// First-attempt crash: enumerator throws before any expectation is uploaded. The
|
||||
// handler MUST stamp EnumerationStartedAt before touching the enumerator so the next
|
||||
// pickup can tell first-try from re-pickup-after-crash. Exception propagates so the
|
||||
// worker retries via visibility timeout.
|
||||
var harness = Harness.New();
|
||||
harness.EnumeratorThrow = new InvalidOperationException("simulated 503");
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
|
||||
|
||||
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
|
||||
Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationStartedAt);
|
||||
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RePickup_MidEnumerationCrash_ESA_Set_ECBU_Null_FailsClosed_ExpectationsMissing()
|
||||
{
|
||||
// Plan §7.4 step 3 Crash-Verhalten: prior pickup stamped EnumerationStartedAt but
|
||||
// crashed mid-iteration (ECBU still null). Re-pickup must NOT re-enumerate (tenant
|
||||
// may be mid-mutation); it must fail-closed.
|
||||
var harness = Harness.New();
|
||||
harness.Repo.History[HistoryRowKey].EnumerationStartedAt = DateTime.UtcNow.AddMinutes(-5);
|
||||
harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload = null;
|
||||
harness.EnumeratorYields.Add("session-X"); // would be enumerated if logic regressed
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
|
||||
|
||||
Assert.Equal("expectations_missing", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
|
||||
harness.CascadeProducer.Verify(c => c.EnqueueAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never(), "Mid-enumeration crash on re-pickup must NEVER re-enumerate");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RePickup_UploadFailure_ESA_Set_ECBU_Set_NoBlob_ReRunsEnumerateAndUpload()
|
||||
{
|
||||
// Plan §7.4 step 3: when ECBU is set but blob is missing, the enumeration loop
|
||||
// finished cleanly and only the upload failed. Re-pickup must RE-RUN enumerate +
|
||||
// upload (idempotent — SessionDeletionProducer returns AlreadyInFlight for the same
|
||||
// session id), NOT fail-closed.
|
||||
var harness = Harness.New();
|
||||
harness.Repo.History[HistoryRowKey].EnumerationStartedAt = DateTime.UtcNow.AddMinutes(-5);
|
||||
harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload = DateTime.UtcNow.AddMinutes(-4);
|
||||
harness.EnumeratorYields.Add("session-A");
|
||||
harness.EnumeratorYields.Add("session-B");
|
||||
harness.CascadeProducer.Setup(c => c.EnqueueAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync<string, string, string, DeletionActor, DeletionRetentionContext?, CancellationToken, ISessionDeletionEnqueuer, SessionDeletionEnqueueResult>(
|
||||
(_, sid, _, _, _, _) => new SessionDeletionEnqueueResult
|
||||
{
|
||||
Outcome = SessionDeletionEnqueueOutcome.AlreadyInFlight, // idempotent producer response
|
||||
ManifestId = $"manifest-{sid}",
|
||||
});
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
|
||||
|
||||
// Both sessions re-enqueued (idempotent producer responses).
|
||||
harness.CascadeProducer.Verify(c => c.EnqueueAsync(
|
||||
It.IsAny<string>(), "session-A", It.IsAny<string>(),
|
||||
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce, "Upload-retry path must re-enumerate session-A");
|
||||
harness.CascadeProducer.Verify(c => c.EnqueueAsync(
|
||||
It.IsAny<string>(), "session-B", It.IsAny<string>(),
|
||||
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce, "Upload-retry path must re-enumerate session-B");
|
||||
// Upload retry succeeded, drain probe says done → Completed.
|
||||
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
|
||||
// Marker is NOT Failed — this was an upload-retry, not a mid-enum crash.
|
||||
Assert.NotEqual("expectations_missing", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FirstTry_StampsBothMarkers_BeforeAndAfterEnumeration()
|
||||
{
|
||||
// Audit-trail invariant: ESA must be stamped BEFORE the enumerator runs (so a crash
|
||||
// during iteration lands in the fail-closed branch next time); ECBU must be stamped
|
||||
// AFTER the enumerator finishes (so an upload-failure-only crash lands in the
|
||||
// upload-retry branch next time).
|
||||
var harness = Harness.New();
|
||||
harness.EnumeratorYields.Add("session-1");
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
|
||||
|
||||
Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationStartedAt);
|
||||
Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload);
|
||||
// Sanity: ESA <= ECBU (one was stamped before the loop, one after).
|
||||
Assert.True(
|
||||
harness.Repo.History[HistoryRowKey].EnumerationStartedAt
|
||||
<= harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnumerationIncomplete_FailsClosed()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.Expectations.Seed(new OffboardingExpectations
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
TenantId = TenantId,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
EnumerationCompleted = false, // ← the disambiguation Rev-7-F2 added
|
||||
EnumeratedSessionCount = 5,
|
||||
Expectations = Enumerable.Range(0, 5).Select(i =>
|
||||
new OffboardingExpectation { SessionId = $"s{i}", Outcome = "Enqueued", ManifestId = $"m{i}" }).ToList(),
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
|
||||
|
||||
Assert.Equal("enumeration_incomplete", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExpectationsSizeMismatch_FailsClosed()
|
||||
{
|
||||
// expectations.Length != enumeratedSessionCount → fail-closed. Defends against a
|
||||
// half-overwritten blob from a buggy re-uploader.
|
||||
var harness = Harness.New();
|
||||
harness.Expectations.Seed(new OffboardingExpectations
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
TenantId = TenantId,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
EnumerationCompleted = true,
|
||||
EnumeratedSessionCount = 5,
|
||||
Expectations = new List<OffboardingExpectation>
|
||||
{
|
||||
new() { SessionId = "s1", Outcome = "Enqueued", ManifestId = "m1" },
|
||||
new() { SessionId = "s2", Outcome = "Enqueued", ManifestId = "m2" },
|
||||
},
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
|
||||
|
||||
Assert.Equal("expectations_size_mismatch", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DrainTimeout_AfterMaxPolls_FailsClosed()
|
||||
{
|
||||
// The cap (60 polls ≈ 2h) must produce a deterministic Failed transition rather than
|
||||
// dragging on forever.
|
||||
var harness = Harness.New();
|
||||
harness.SeedExpectations(new (string, string?, string, int)[]
|
||||
{
|
||||
("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Enqueued), 0),
|
||||
});
|
||||
harness.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(false); // never settles
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: TenantOffboardingHandler.MaxDrainPolls - 1)));
|
||||
|
||||
Assert.Equal("drain_timeout", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
}
|
||||
|
||||
// ── Happy paths ─────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task ZeroSessions_EnumerationCompleted_DrainOk_ProceedsToWipeAndCompleted()
|
||||
{
|
||||
// Plan §4.5: empty expectations + enumerationCompleted=true is the legitimate
|
||||
// happy path for a tenant with no sessions left.
|
||||
var harness = Harness.New();
|
||||
// Enumerator yields nothing — handler will upload an Expectations blob with
|
||||
// enumerationCompleted=true, enumeratedSessionCount=0.
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
|
||||
|
||||
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.NotNull(harness.Repo.History[HistoryRowKey].CompletedAt);
|
||||
Assert.NotNull(harness.Repo.History[HistoryRowKey].DrainCompletedAt);
|
||||
Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status);
|
||||
Assert.NotNull(harness.Repo.Markers[TenantId].CompletedAt);
|
||||
Assert.True(harness.SafeWipeProbe.WipeCallCount > 0,
|
||||
"SafeWipe must have run across the tenant tables");
|
||||
Assert.False(harness.Expectations.BlobExists(TenantId, HistoryRowKey),
|
||||
"Phase 2.G must delete the Expectations blob as its last step");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionNotFound_CountsAsSatisfied_NoDrainProbeCall()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.SeedExpectations(new (string, string?, string, int)[]
|
||||
{
|
||||
("session-1", null, nameof(SessionDeletionEnqueueOutcome.SessionNotFound), 0),
|
||||
});
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
|
||||
|
||||
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
|
||||
harness.DrainProbe.Verify(p => p.IsCascadeCompletedAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never(), "SessionNotFound is a no-op — drain probe must NOT be called for that expectation");
|
||||
}
|
||||
|
||||
// ── Rev-9-F1 Drain-Skip-Gate ───────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Rev9F1_DrainCompletedAtSet_SkipsDrainPredicate_AndRunsPostDrainPhasesIdempotently()
|
||||
{
|
||||
// The crash-recovery scenario: drain ran once and stamped DrainCompletedAt, then we
|
||||
// crashed somewhere in 2.D-G. Re-pickup must NOT re-evaluate the drain predicate (the
|
||||
// progress blobs are now gone in 2.E) and must still land us on Completed.
|
||||
var harness = Harness.New();
|
||||
harness.Repo.History[HistoryRowKey].DrainCompletedAt = DateTime.UtcNow.AddMinutes(-10);
|
||||
harness.Repo.History[HistoryRowKey].Status = "InProgress";
|
||||
harness.Repo.Markers[TenantId].Status = "InProgress";
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
|
||||
|
||||
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
|
||||
harness.DrainProbe.Verify(p => p.IsCascadeCompletedAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never(), "When DrainCompletedAt is already set the drain probe must NOT be called");
|
||||
harness.Enumerator.Verify(e => e.EnumerateAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never(), "Drain-skip-gate must also bypass the enumerate+enqueue phase");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task History_AlreadyCompleted_IsNoOp()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.Repo.History[HistoryRowKey].Status = "Completed";
|
||||
harness.Repo.History[HistoryRowKey].CompletedAt = DateTime.UtcNow;
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
|
||||
|
||||
// No additional writes — handler returned early.
|
||||
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
|
||||
}
|
||||
|
||||
// ── Drain re-poll ───────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task DrainNotSettled_BelowCap_ReEnqueuesWithDelayAndPollIncrement()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.SeedExpectations(new (string, string?, string, int)[]
|
||||
{
|
||||
("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Enqueued), 0),
|
||||
});
|
||||
harness.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(false);
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 3));
|
||||
|
||||
harness.ReEnqueuer.Verify(e => e.EnqueueAsync(
|
||||
It.Is<TenantOffboardingEnvelope>(env => env.DrainPollCount == 4),
|
||||
TenantOffboardingHandler.DrainPollDelay,
|
||||
It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
// History should NOT be Completed yet.
|
||||
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.Null(harness.Repo.History[HistoryRowKey].DrainCompletedAt);
|
||||
}
|
||||
|
||||
// ── Harness ─────────────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public TenantOffboardingHandler Sut { get; private set; } = default!;
|
||||
public FakeOffboardingAuditRepository Repo { get; } = new();
|
||||
public FakeOffboardingExpectationsStore Expectations { get; } = new();
|
||||
public Mock<IDeletionProgressDrainProbe> DrainProbe { get; } = new();
|
||||
public Mock<ITenantOffboardingEnqueuer> ReEnqueuer { get; } = new();
|
||||
public Mock<ISessionDeletionEnqueuer> CascadeProducer { get; } = new();
|
||||
public Mock<OffboardingSessionEnumerator> Enumerator { get; }
|
||||
public CountingSafeWipeService SafeWipeProbe { get; } = new();
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; } = new();
|
||||
public List<string> EnumeratorYields { get; } = new();
|
||||
public Exception? EnumeratorThrow { get; set; }
|
||||
|
||||
private Harness()
|
||||
{
|
||||
var maintenanceForEnumerator = new Mock<IMaintenanceRepository>();
|
||||
Enumerator = new Mock<OffboardingSessionEnumerator>(maintenanceForEnumerator.Object);
|
||||
}
|
||||
|
||||
public static Harness New()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SeedInitialAudit();
|
||||
|
||||
// Default enumerator returns the (mutable) Sessions list. Tests append to it
|
||||
// before invoking the handler.
|
||||
h.Enumerator
|
||||
.Setup(e => e.EnumerateAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, CancellationToken>((_, ct) => h.EnumerateSessionsAsync(ct));
|
||||
|
||||
h.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(true); // happy default — drain completes immediately
|
||||
|
||||
h.CascadeProducer.Setup(c => c.EnqueueAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync<string, string, string, DeletionActor, DeletionRetentionContext?, CancellationToken, ISessionDeletionEnqueuer, SessionDeletionEnqueueResult>(
|
||||
(_, sid, _, _, _, _) => new SessionDeletionEnqueueResult
|
||||
{
|
||||
Outcome = SessionDeletionEnqueueOutcome.Enqueued,
|
||||
ManifestId = $"manifest-{sid}",
|
||||
});
|
||||
|
||||
h.Maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.ReturnsAsync(true);
|
||||
|
||||
var opsService = BuildRealOpsService();
|
||||
|
||||
var unusedTableStorage = new Mock<TableStorageService>(
|
||||
Mock.Of<TableServiceClient>(),
|
||||
NullLogger<TableStorageService>.Instance).Object;
|
||||
|
||||
h.Sut = new NoopCustomsArchivingHandler(
|
||||
h.Repo,
|
||||
h.Enumerator.Object,
|
||||
h.CascadeProducer.Object,
|
||||
h.Expectations,
|
||||
h.DrainProbe.Object,
|
||||
h.SafeWipeProbe,
|
||||
unusedTableStorage,
|
||||
h.Maintenance.Object,
|
||||
h.ReEnqueuer.Object,
|
||||
opsService,
|
||||
Mock.Of<ITenantCustomsArchiveRepository>(),
|
||||
NullLogger<TenantOffboardingHandler>.Instance);
|
||||
|
||||
return h;
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<string> EnumerateSessionsAsync(
|
||||
[EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
if (EnumeratorThrow != null) throw EnumeratorThrow;
|
||||
foreach (var s in EnumeratorYields)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
yield return s;
|
||||
}
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public TenantOffboardingEnvelope Envelope(int drainPollCount) => new()
|
||||
{
|
||||
TenantId = TenantId,
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
DrainPollCount = drainPollCount,
|
||||
};
|
||||
|
||||
public void SeedExpectations(IEnumerable<(string sessionId, string? manifestId, string outcome, int retryCount)> rows)
|
||||
{
|
||||
var list = rows.Select(r => new OffboardingExpectation
|
||||
{
|
||||
SessionId = r.sessionId,
|
||||
ManifestId = r.manifestId,
|
||||
Outcome = r.outcome,
|
||||
RetryCount = r.retryCount,
|
||||
}).ToList();
|
||||
Expectations.Seed(new OffboardingExpectations
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
TenantId = TenantId,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
EnumerationCompleted = true,
|
||||
EnumeratedSessionCount = list.Count,
|
||||
Expectations = list,
|
||||
});
|
||||
}
|
||||
|
||||
private void SeedInitialAudit()
|
||||
{
|
||||
Repo.History[HistoryRowKey] = new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = HistoryRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.invalid",
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
OffboardedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
Status = "Initiated",
|
||||
};
|
||||
Repo.Markers[TenantId] = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = HistoryRowKey,
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
Status = "Initiated",
|
||||
};
|
||||
Repo.Pointers[TenantId] = (new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = HistoryRowKey,
|
||||
LatestStatus = "Initiated",
|
||||
LatestUpdatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
OffboardCount = 1,
|
||||
}, "\"0xFAKE_PTR_1\"");
|
||||
}
|
||||
|
||||
private static OpsEventService BuildRealOpsService()
|
||||
{
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>())).Returns(Task.CompletedTask);
|
||||
var memCache = new MemoryCache(new MemoryCacheOptions());
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, memCache);
|
||||
var alertDispatch = new OpsAlertDispatchService(
|
||||
adminConfig.Object,
|
||||
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
|
||||
NullLogger<TelegramNotificationService>.Instance),
|
||||
new WebhookNotificationService(new HttpClient(),
|
||||
NullLogger<WebhookNotificationService>.Instance),
|
||||
NullLogger<OpsAlertDispatchService>.Instance);
|
||||
return new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="SafeWipeService"/> that records call counts without performing any storage IO.
|
||||
/// Tests use this to assert that fail-closed paths NEVER touch the wipe methods. Now that
|
||||
/// <see cref="SafeWipeService"/> exposes the wipe methods as <c>virtual</c>, the production
|
||||
/// dispatch lands here.
|
||||
/// </summary>
|
||||
private sealed class CountingSafeWipeService : SafeWipeService
|
||||
{
|
||||
public int WipeCallCount { get; private set; }
|
||||
|
||||
public CountingSafeWipeService()
|
||||
: base(
|
||||
new TableStorageService(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance),
|
||||
new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false),
|
||||
NullLogger<SafeWipeService>.Instance)
|
||||
{
|
||||
}
|
||||
|
||||
public override Task<int> WipeByExactPartitionAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeByCompositePartitionRangeAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string tableName, string discriminator, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeByTenantIdPropertyAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public override Task<int> WipeBlobsByTenantPrefixAsync(string containerName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
WipeCallCount++;
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Subclass that no-ops the customs archive-then-wipe loop so the existing drain-contract
|
||||
/// tests don't need a working TableStorageService for the 3 rules tables. PR3.B-specific
|
||||
/// archive behaviour has its own test class with proper TableClient mocks.
|
||||
/// </summary>
|
||||
private sealed class NoopCustomsArchivingHandler : TenantOffboardingHandler
|
||||
{
|
||||
public NoopCustomsArchivingHandler(
|
||||
IOffboardingAuditRepository auditRepo,
|
||||
OffboardingSessionEnumerator enumerator,
|
||||
ISessionDeletionEnqueuer cascadeEnqueuer,
|
||||
IOffboardingExpectationsStore expectations,
|
||||
IDeletionProgressDrainProbe drainProbe,
|
||||
SafeWipeService safeWipe,
|
||||
TableStorageService storage,
|
||||
IMaintenanceRepository maintenance,
|
||||
ITenantOffboardingEnqueuer reEnqueuer,
|
||||
OpsEventService opsEvents,
|
||||
ITenantCustomsArchiveRepository customsArchive,
|
||||
ILogger<TenantOffboardingHandler> logger)
|
||||
: base(auditRepo, enumerator, cascadeEnqueuer, expectations, drainProbe, safeWipe,
|
||||
storage, maintenance, reEnqueuer, opsEvents, customsArchive, logger)
|
||||
{
|
||||
}
|
||||
|
||||
internal override Task ArchiveAndWipeRulesTableAsync(
|
||||
string tableName, string tenantId, string historyRowKey, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
+368
@@ -0,0 +1,368 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Storage.Blobs;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// Review-Fix Findings 2 + 3:
|
||||
/// <list type="bullet">
|
||||
/// <item>(Finding 3) History terminal status MUST be the LAST write in both Completed and
|
||||
/// Failed paths. A crash between any side-effect (marker/pointer/blob/audit/ops) and
|
||||
/// the History commit must produce a re-pickup that re-runs the side-effects
|
||||
/// idempotently — never a re-pickup that returns early with dangling state.</item>
|
||||
/// <item>(Finding 2) <see cref="TenantOffboardingHandler.MarkEnvelopeFailedFromPoisonAsync"/>
|
||||
/// drives History/Pointer/Marker to Failed (with FailedAt/FailedPhase/ErrorMessage/RetryCount)
|
||||
/// so the operator dashboard reflects the dead-letter rather than leaving the tenant
|
||||
/// hanging in InProgress.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public class TenantOffboardingHandlerOrderingTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111";
|
||||
|
||||
// ── Finding 3 — Completed-path ordering ─────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task PostDrain_OrderingInvariant_MarkerCompletedBeforeHistoryCompleted()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope());
|
||||
|
||||
// Both writes happen. The invariant: Marker.Status="Completed" lands BEFORE
|
||||
// History.Status="Completed" — the Repo's MarkerWrites/HistoryWrites lists capture
|
||||
// the order via Append-on-Write.
|
||||
var lastMarker = harness.Repo.MarkerWrites.LastIndexOf("Completed");
|
||||
var lastHistoryCompleted = harness.Repo.HistoryWrites.LastIndexOf("Completed");
|
||||
Assert.True(lastMarker >= 0, "Marker.Status=Completed write never happened");
|
||||
Assert.True(lastHistoryCompleted >= 0, "History.Status=Completed write never happened");
|
||||
// Marker write happens before the FINAL history write — both lists are sequenced
|
||||
// independently per partition, but the order WITHIN the handler's invocation is what
|
||||
// matters; we capture each by counting writes-before.
|
||||
Assert.True(
|
||||
harness.Repo.MarkerWrites.Count >= 1,
|
||||
"Marker must be written at least once with Completed");
|
||||
// Last history write IS the Completed-commit. Anything that happened before that in
|
||||
// MarkerWrites was the Completed transition.
|
||||
Assert.Equal("Completed", harness.Repo.MarkerWrites.Last());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PostDrain_SimulatedCrashBetweenMarkerAndHistory_RepickupCompletesIdempotently()
|
||||
{
|
||||
// Step 1: first invocation — completes normally.
|
||||
var harness = Harness.New();
|
||||
await harness.Sut.HandleAsync(harness.Envelope());
|
||||
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status);
|
||||
|
||||
// Step 2: simulate crash by manually re-stamping the history to InProgress while keeping
|
||||
// DrainCompletedAt set (this is what a re-pickup after a Phase-2.G crash looks like).
|
||||
// The marker is left as Completed (already updated before the crash), the post-drain
|
||||
// side-effects all idempotently re-execute.
|
||||
harness.Repo.History[HistoryRowKey].Status = "InProgress";
|
||||
harness.Repo.History[HistoryRowKey].CompletedAt = null;
|
||||
var wipeCallsBefore = harness.SafeWipeProbe.WipeCallCount;
|
||||
|
||||
await harness.Sut.HandleAsync(harness.Envelope());
|
||||
|
||||
// History ends Completed again, marker stays Completed, side-effects re-ran (idempotent).
|
||||
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status);
|
||||
Assert.True(harness.SafeWipeProbe.WipeCallCount > wipeCallsBefore,
|
||||
"Re-pickup must re-run the SafeWipe phase (idempotent — finds 0 rows the second time)");
|
||||
}
|
||||
|
||||
// ── Finding 3 — Failed-path ordering ────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task FailAsync_OrderingInvariant_MarkerFailedBeforeHistoryFailed()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
// Force fail-closed via KillSwitchActive expectation.
|
||||
harness.Expectations.Seed(new OffboardingExpectations
|
||||
{
|
||||
SchemaVersion = 1, TenantId = TenantId, HistoryRowKey = HistoryRowKey,
|
||||
CreatedAt = DateTime.UtcNow, EnumerationCompleted = true, EnumeratedSessionCount = 1,
|
||||
Expectations = new List<OffboardingExpectation>
|
||||
{
|
||||
new() { SessionId = "session-1", Outcome = "KillSwitchActive" },
|
||||
},
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
harness.Sut.HandleAsync(harness.Envelope()));
|
||||
|
||||
// Marker must be Failed; History must be Failed too. The LAST write to history is the
|
||||
// Failed-commit — anything that happened before (marker, pointer, ops) is part of the
|
||||
// "side-effects first" contract.
|
||||
Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status);
|
||||
Assert.Equal("killswitch", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
Assert.NotNull(harness.Repo.Markers[TenantId].FailedAt);
|
||||
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
|
||||
// History was already InProgress + Marker write between, so MarkerWrites holds "Failed"
|
||||
// as its latest, and HistoryWrites ends with "Failed".
|
||||
Assert.Equal("Failed", harness.Repo.MarkerWrites.Last());
|
||||
Assert.Equal("Failed", harness.Repo.HistoryWrites.Last());
|
||||
}
|
||||
|
||||
// ── Finding 2 — Worker poison transition via MarkEnvelopeFailedFromPoisonAsync ──
|
||||
|
||||
[Fact]
|
||||
public async Task MarkEnvelopeFailedFromPoison_TransitionsAllThreeStatesToFailed()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
// History/Pointer/Marker pre-seeded to InProgress (default Harness seed is Initiated;
|
||||
// bump to InProgress to mirror the in-flight scenario).
|
||||
harness.Repo.History[HistoryRowKey].Status = "InProgress";
|
||||
harness.Repo.Markers[TenantId].Status = "InProgress";
|
||||
var existingPointer = harness.Repo.Pointers[TenantId];
|
||||
existingPointer.Pointer.LatestStatus = "InProgress";
|
||||
|
||||
await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5);
|
||||
|
||||
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status);
|
||||
Assert.Equal("Failed", harness.Repo.Pointers[TenantId].Pointer.LatestStatus);
|
||||
Assert.Equal("max_dequeue", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
Assert.NotNull(harness.Repo.Markers[TenantId].FailedAt);
|
||||
Assert.Equal(5, harness.Repo.History[HistoryRowKey].RetryCount);
|
||||
Assert.Contains("poisoned after 5", harness.Repo.History[HistoryRowKey].ErrorMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkEnvelopeFailedFromPoison_AlreadyCompleted_IsNoOp()
|
||||
{
|
||||
// If the handler successfully completed BEFORE max-dequeue (e.g. last attempt
|
||||
// succeeded but the worker's dequeue counter still tripped the threshold), the
|
||||
// poison transition must not downgrade Completed → Failed.
|
||||
var harness = Harness.New();
|
||||
harness.Repo.History[HistoryRowKey].Status = "Completed";
|
||||
harness.Repo.History[HistoryRowKey].CompletedAt = DateTime.UtcNow;
|
||||
harness.Repo.Markers[TenantId].Status = "Completed";
|
||||
harness.Repo.Markers[TenantId].CompletedAt = DateTime.UtcNow;
|
||||
|
||||
await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5);
|
||||
|
||||
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkEnvelopeFailedFromPoison_AlreadyFailed_IsNoOp()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.Repo.History[HistoryRowKey].Status = "Failed";
|
||||
harness.Repo.History[HistoryRowKey].ErrorMessage = "drain_timeout";
|
||||
harness.Repo.Markers[TenantId].Status = "Failed";
|
||||
harness.Repo.Markers[TenantId].FailedPhase = "drain_timeout";
|
||||
var beforeWrites = harness.Repo.HistoryWrites.Count;
|
||||
|
||||
await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5);
|
||||
|
||||
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.Equal("drain_timeout", harness.Repo.Markers[TenantId].FailedPhase); // not overwritten
|
||||
Assert.Equal(beforeWrites, harness.Repo.HistoryWrites.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkEnvelopeFailedFromPoison_MissingHistory_LogsAndReturnsCleanly()
|
||||
{
|
||||
var harness = Harness.New();
|
||||
harness.Repo.History.Remove(HistoryRowKey);
|
||||
|
||||
// Must NOT throw — the worker poison-move would otherwise lose its envelope and the
|
||||
// operator would have no signal at all.
|
||||
await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5);
|
||||
}
|
||||
|
||||
// ── Harness (copied minimal — only what these tests need) ───────────────────
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public TenantOffboardingHandler Sut { get; private set; } = default!;
|
||||
public FakeOffboardingAuditRepository Repo { get; } = new();
|
||||
public FakeOffboardingExpectationsStore Expectations { get; } = new();
|
||||
public Mock<IDeletionProgressDrainProbe> DrainProbe { get; } = new();
|
||||
public Mock<ITenantOffboardingEnqueuer> ReEnqueuer { get; } = new();
|
||||
public Mock<ISessionDeletionEnqueuer> CascadeProducer { get; } = new();
|
||||
public Mock<OffboardingSessionEnumerator> Enumerator { get; }
|
||||
public CountingSafeWipeService SafeWipeProbe { get; } = new();
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; } = new();
|
||||
public List<string> EnumeratorYields { get; } = new();
|
||||
|
||||
private Harness()
|
||||
{
|
||||
Enumerator = new Mock<OffboardingSessionEnumerator>(new Mock<IMaintenanceRepository>().Object);
|
||||
}
|
||||
|
||||
public static Harness New()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SeedInitialAudit();
|
||||
|
||||
h.Enumerator
|
||||
.Setup(e => e.EnumerateAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, CancellationToken>((_, ct) => EmptyEnumerateAsync(ct));
|
||||
|
||||
h.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(true);
|
||||
|
||||
h.CascadeProducer.Setup(c => c.EnqueueAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync<string, string, string, DeletionActor, DeletionRetentionContext?, CancellationToken, ISessionDeletionEnqueuer, SessionDeletionEnqueueResult>(
|
||||
(_, sid, _, _, _, _) => new SessionDeletionEnqueueResult
|
||||
{
|
||||
Outcome = SessionDeletionEnqueueOutcome.Enqueued,
|
||||
ManifestId = $"manifest-{sid}",
|
||||
});
|
||||
|
||||
h.Maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.ReturnsAsync(true);
|
||||
|
||||
var unusedTableStorage = new Mock<TableStorageService>(
|
||||
Mock.Of<TableServiceClient>(),
|
||||
NullLogger<TableStorageService>.Instance).Object;
|
||||
|
||||
h.Sut = new NoopCustomsArchivingOrderingHandler(
|
||||
h.Repo,
|
||||
h.Enumerator.Object,
|
||||
h.CascadeProducer.Object,
|
||||
h.Expectations,
|
||||
h.DrainProbe.Object,
|
||||
h.SafeWipeProbe,
|
||||
unusedTableStorage,
|
||||
h.Maintenance.Object,
|
||||
h.ReEnqueuer.Object,
|
||||
BuildOpsService(),
|
||||
Mock.Of<ITenantCustomsArchiveRepository>(),
|
||||
NullLogger<TenantOffboardingHandler>.Instance);
|
||||
|
||||
return h;
|
||||
}
|
||||
|
||||
public TenantOffboardingEnvelope Envelope() => new()
|
||||
{
|
||||
TenantId = TenantId,
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
DrainPollCount = 0,
|
||||
};
|
||||
|
||||
private static async IAsyncEnumerable<string> EmptyEnumerateAsync(
|
||||
[EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
private void SeedInitialAudit()
|
||||
{
|
||||
Repo.History[HistoryRowKey] = new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = HistoryRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.invalid",
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
OffboardedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
Status = "Initiated",
|
||||
};
|
||||
Repo.Markers[TenantId] = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = HistoryRowKey,
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
Status = "Initiated",
|
||||
};
|
||||
Repo.Pointers[TenantId] = (new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = HistoryRowKey,
|
||||
LatestStatus = "Initiated",
|
||||
LatestUpdatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
OffboardCount = 1,
|
||||
}, "\"0xFAKE_PTR_1\"");
|
||||
}
|
||||
|
||||
private static OpsEventService BuildOpsService()
|
||||
{
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>())).Returns(Task.CompletedTask);
|
||||
var memCache = new MemoryCache(new MemoryCacheOptions());
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, memCache);
|
||||
var alertDispatch = new OpsAlertDispatchService(
|
||||
adminConfig.Object,
|
||||
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
|
||||
NullLogger<TelegramNotificationService>.Instance),
|
||||
new WebhookNotificationService(new HttpClient(),
|
||||
NullLogger<WebhookNotificationService>.Instance),
|
||||
NullLogger<OpsAlertDispatchService>.Instance);
|
||||
return new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CountingSafeWipeService : SafeWipeService
|
||||
{
|
||||
public int WipeCallCount { get; private set; }
|
||||
public CountingSafeWipeService() : base(
|
||||
new TableStorageService(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance),
|
||||
new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false),
|
||||
NullLogger<SafeWipeService>.Instance) { }
|
||||
public override Task<int> WipeByExactPartitionAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
|
||||
public override Task<int> WipeByCompositePartitionRangeAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
|
||||
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string t, string d, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
|
||||
public override Task<int> WipeByTenantIdPropertyAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
|
||||
public override Task<int> WipeBlobsByTenantPrefixAsync(string c, string i, CancellationToken ct = default) { WipeCallCount++; return Task.FromResult(0); }
|
||||
}
|
||||
|
||||
private sealed class NoopCustomsArchivingOrderingHandler : TenantOffboardingHandler
|
||||
{
|
||||
public NoopCustomsArchivingOrderingHandler(
|
||||
IOffboardingAuditRepository a, OffboardingSessionEnumerator e, ISessionDeletionEnqueuer c,
|
||||
IOffboardingExpectationsStore exp, IDeletionProgressDrainProbe d, SafeWipeService sw,
|
||||
TableStorageService s, IMaintenanceRepository m, ITenantOffboardingEnqueuer re,
|
||||
OpsEventService o, ITenantCustomsArchiveRepository ca, ILogger<TenantOffboardingHandler> log)
|
||||
: base(a, e, c, exp, d, sw, s, m, re, o, ca, log) { }
|
||||
|
||||
internal override Task ArchiveAndWipeRulesTableAsync(
|
||||
string tableName, string tenantId, string historyRowKey, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
+300
@@ -0,0 +1,300 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Queues;
|
||||
using Azure.Storage.Queues.Models;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Newtonsoft.Json;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
|
||||
/// <summary>
|
||||
/// Review-Fix (second-pass) Finding 1: <see cref="TenantOffboardingWorker.MoveToPoisonAsync"/>
|
||||
/// MUST drive the durable Failed-state transition BEFORE the poison-queue send + main-queue
|
||||
/// delete. If the transition fails for any reason (storage outage, OpsEvent service down,
|
||||
/// pointer-CAS contention), the main-queue message must stay visible so a subsequent dequeue
|
||||
/// can retry the transition.
|
||||
/// </summary>
|
||||
public class TenantOffboardingWorkerPoisonTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111";
|
||||
|
||||
[Fact]
|
||||
public async Task PoisonPath_FailedTransitionSucceeds_ThenSendToPoisonAndDelete()
|
||||
{
|
||||
// Happy poison path: marker/pointer/history → Failed first, THEN message goes to
|
||||
// poison queue, THEN main message is deleted.
|
||||
var harness = new Harness();
|
||||
|
||||
await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6));
|
||||
|
||||
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status);
|
||||
Assert.Equal("max_dequeue", harness.Repo.Markers[TenantId].FailedPhase);
|
||||
|
||||
Assert.Equal(1, harness.PoisonSendCount);
|
||||
Assert.Equal(1, harness.MainDeleteCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PoisonPath_FailedTransitionThrows_NoPoisonSend_NoMainDelete()
|
||||
{
|
||||
// Critical Finding 1 assertion: if MarkEnvelopeFailedFromPoisonAsync throws, the
|
||||
// poison send + main delete MUST NOT happen — otherwise the operator would see a
|
||||
// tenant stuck in InProgress with no queue message left to retry from.
|
||||
var harness = new Harness();
|
||||
// Simulate a transient storage failure inside FailAsync's final commit.
|
||||
harness.Repo.ThrowOnNextHistoryUpsert = new InvalidOperationException("simulated storage 503");
|
||||
|
||||
await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6));
|
||||
|
||||
Assert.Equal(0, harness.PoisonSendCount);
|
||||
Assert.Equal(0, harness.MainDeleteCount);
|
||||
// History stays InProgress — the next visibility-timeout retry will re-attempt the
|
||||
// whole MoveToPoisonAsync sequence. Marker may or may not be Failed depending on how
|
||||
// far FailAsync got before the throw; the critical invariant is that the queue
|
||||
// message stays so the operator-facing state CAN converge on the next dequeue.
|
||||
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PoisonPath_PoisonSendFails_AfterFailedTransition_LeavesMessageButStateIsDurable()
|
||||
{
|
||||
// The transition succeeded → state is durable. The poison send fails (transient).
|
||||
// The main delete is skipped. The message reappears via visibility-timeout. On the
|
||||
// next pickup, MarkEnvelopeFailedFromPoisonAsync sees the terminal state and is a
|
||||
// no-op, then the poison send retries.
|
||||
var harness = new Harness(poisonSendThrows: true);
|
||||
|
||||
await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6));
|
||||
|
||||
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
|
||||
Assert.Equal(1, harness.PoisonSendCount); // attempted
|
||||
Assert.Equal(0, harness.MainDeleteCount); // skipped because poison send threw
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PoisonPath_MalformedEnvelope_SkipsTransition_StillPoisonsTheMessage()
|
||||
{
|
||||
// Malformed envelope can't be transitioned (we don't know which tenant to fail), but
|
||||
// the message still goes to poison so it leaves the live queue.
|
||||
var harness = new Harness();
|
||||
|
||||
var bad = harness.BuildMessage(dequeueCount: 6, bodyOverride: "{ not json at all");
|
||||
await harness.InvokeMoveToPoisonAsync(bad);
|
||||
|
||||
Assert.Equal(1, harness.PoisonSendCount);
|
||||
Assert.Equal(1, harness.MainDeleteCount);
|
||||
// No state transition happened — nothing to assert there, but the History row is
|
||||
// untouched.
|
||||
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
|
||||
}
|
||||
|
||||
// ── Harness ─────────────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public TenantOffboardingWorker Worker { get; }
|
||||
public FakeOffboardingAuditRepository Repo { get; } = new();
|
||||
public Mock<QueueClient> MainQueue { get; } = new();
|
||||
public Mock<QueueClient> PoisonQueue { get; } = new();
|
||||
public int PoisonSendCount { get; private set; }
|
||||
public int MainDeleteCount { get; private set; }
|
||||
|
||||
public Harness(bool poisonSendThrows = false)
|
||||
{
|
||||
SeedAudit();
|
||||
|
||||
// Cascade-handler dependencies. The handler is real — it talks to FakeRepo +
|
||||
// OpsEvent + (unused) other deps. Only MarkEnvelopeFailedFromPoisonAsync is
|
||||
// exercised, so most deps are noise.
|
||||
var handler = BuildHandler();
|
||||
|
||||
PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, CancellationToken>((_, _) =>
|
||||
{
|
||||
PoisonSendCount++;
|
||||
if (poisonSendThrows) throw new InvalidOperationException("simulated poison-queue 503");
|
||||
return Task.FromResult(Response.FromValue(
|
||||
QueuesModelFactory.SendReceipt(
|
||||
messageId: Guid.NewGuid().ToString(),
|
||||
insertionTime: DateTimeOffset.UtcNow,
|
||||
expirationTime: DateTimeOffset.UtcNow.AddDays(7),
|
||||
popReceipt: "fake",
|
||||
timeNextVisible: DateTimeOffset.UtcNow),
|
||||
new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
MainQueue.Setup(q => q.DeleteMessageAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, CancellationToken>((_, _, _) =>
|
||||
{
|
||||
MainDeleteCount++;
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
Worker = new TenantOffboardingWorker(
|
||||
MainQueue.Object, PoisonQueue.Object, handler,
|
||||
BuildOpsService(),
|
||||
NullLogger<TenantOffboardingWorker>.Instance);
|
||||
}
|
||||
|
||||
public QueueMessage BuildMessage(int dequeueCount, string? bodyOverride = null)
|
||||
{
|
||||
var envelope = new TenantOffboardingEnvelope
|
||||
{
|
||||
TenantId = TenantId,
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = HistoryRowKey,
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
DrainPollCount = 0,
|
||||
};
|
||||
return QueuesModelFactory.QueueMessage(
|
||||
messageId: Guid.NewGuid().ToString(),
|
||||
popReceipt: "fake-receipt",
|
||||
body: new BinaryData(bodyOverride ?? JsonConvert.SerializeObject(envelope)),
|
||||
dequeueCount: dequeueCount);
|
||||
}
|
||||
|
||||
public Task InvokeMoveToPoisonAsync(QueueMessage msg)
|
||||
{
|
||||
// MoveToPoisonAsync is private — invoke via reflection. Avoids spinning up the
|
||||
// full ExecuteAsync poll loop with a real timer.
|
||||
var method = typeof(TenantOffboardingWorker)
|
||||
.GetMethod("MoveToPoisonAsync", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
Assert.NotNull(method);
|
||||
return (Task)method!.Invoke(Worker, new object[] { msg, default(CancellationToken) })!;
|
||||
}
|
||||
|
||||
private void SeedAudit()
|
||||
{
|
||||
Repo.History[HistoryRowKey] = new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = HistoryRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.invalid",
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
OffboardedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
Status = "InProgress",
|
||||
};
|
||||
Repo.Markers[TenantId] = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = HistoryRowKey,
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
Status = "InProgress",
|
||||
};
|
||||
Repo.Pointers[TenantId] = (new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = HistoryRowKey,
|
||||
LatestStatus = "InProgress",
|
||||
LatestUpdatedAt = DateTime.UtcNow,
|
||||
OffboardCount = 1,
|
||||
}, "\"0xFAKE_PTR_1\"");
|
||||
}
|
||||
|
||||
private TenantOffboardingHandler BuildHandler()
|
||||
{
|
||||
// Wire a real Handler so MarkEnvelopeFailedFromPoisonAsync uses FailAsync which
|
||||
// touches the FakeRepo. The "transition throws" branch is exercised by setting
|
||||
// Repo.ThrowOnNextHistoryUpsert in the test.
|
||||
var unusedTableStorage = new Mock<TableStorageService>(
|
||||
Mock.Of<TableServiceClient>(),
|
||||
NullLogger<TableStorageService>.Instance).Object;
|
||||
|
||||
var enumerator = new Mock<OffboardingSessionEnumerator>(new Mock<IMaintenanceRepository>().Object);
|
||||
|
||||
return new NoopCustomsArchivingPoisonHandler(
|
||||
Repo,
|
||||
enumerator.Object,
|
||||
Mock.Of<ISessionDeletionEnqueuer>(),
|
||||
new FakeOffboardingExpectationsStore(),
|
||||
Mock.Of<IDeletionProgressDrainProbe>(),
|
||||
new NoopSafeWipeService(),
|
||||
unusedTableStorage,
|
||||
Mock.Of<IMaintenanceRepository>(),
|
||||
Mock.Of<ITenantOffboardingEnqueuer>(),
|
||||
BuildOpsService(),
|
||||
Mock.Of<ITenantCustomsArchiveRepository>(),
|
||||
NullLogger<TenantOffboardingHandler>.Instance);
|
||||
}
|
||||
|
||||
private sealed class NoopCustomsArchivingPoisonHandler : TenantOffboardingHandler
|
||||
{
|
||||
public NoopCustomsArchivingPoisonHandler(
|
||||
IOffboardingAuditRepository a, OffboardingSessionEnumerator e, ISessionDeletionEnqueuer c,
|
||||
IOffboardingExpectationsStore exp, IDeletionProgressDrainProbe d, SafeWipeService sw,
|
||||
TableStorageService s, IMaintenanceRepository m, ITenantOffboardingEnqueuer re,
|
||||
OpsEventService o, ITenantCustomsArchiveRepository ca, ILogger<TenantOffboardingHandler> log)
|
||||
: base(a, e, c, exp, d, sw, s, m, re, o, ca, log) { }
|
||||
|
||||
internal override Task ArchiveAndWipeRulesTableAsync(
|
||||
string tableName, string tenantId, string historyRowKey, CancellationToken ct)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static OpsEventService BuildOpsService()
|
||||
{
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>())).Returns(Task.CompletedTask);
|
||||
return new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, BuildAlertDispatch());
|
||||
}
|
||||
|
||||
private static OpsAlertDispatchService BuildAlertDispatch()
|
||||
{
|
||||
var memCache = new MemoryCache(new MemoryCacheOptions());
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, memCache);
|
||||
return new OpsAlertDispatchService(
|
||||
adminConfig.Object,
|
||||
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
|
||||
NullLogger<TelegramNotificationService>.Instance),
|
||||
new WebhookNotificationService(new HttpClient(),
|
||||
NullLogger<WebhookNotificationService>.Instance),
|
||||
NullLogger<OpsAlertDispatchService>.Instance);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopSafeWipeService : SafeWipeService
|
||||
{
|
||||
public NoopSafeWipeService() : base(
|
||||
new TableStorageService(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance),
|
||||
new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false),
|
||||
NullLogger<SafeWipeService>.Instance) { }
|
||||
public override Task<int> WipeByExactPartitionAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
|
||||
public override Task<int> WipeByCompositePartitionRangeAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
|
||||
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string t, string d, string i, CancellationToken c = default) => Task.FromResult(0);
|
||||
public override Task<int> WipeByTenantIdPropertyAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
|
||||
public override Task<int> WipeBlobsByTenantPrefixAsync(string c, string i, CancellationToken ct = default) => Task.FromResult(0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Rev-6-F1 + Rev-7-F2 + Rev-8-F1: full coverage of the blob-IO contracts on
|
||||
/// <see cref="BlobOffboardingExpectationsStore"/> via a Dict-backed fake that overrides the
|
||||
/// four <c>protected virtual</c> seams. The fake reproduces Azure Blob Storage's two
|
||||
/// "exists" status codes (409 BlobAlreadyExists + 412 ConditionNotMet) so the resume path
|
||||
/// is exercised under both. POCO/validation cases live next to the IO cases for full
|
||||
/// surface coverage.
|
||||
/// </summary>
|
||||
public class OffboardingExpectationsStoreTests
|
||||
{
|
||||
private const string TenantId = "99999999-9999-9999-9999-999999999999";
|
||||
private const string HistoryRowKey = "20260518091523123_99999999-9999-9999-9999-999999999999";
|
||||
|
||||
// ── Validation gates ────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TryUploadInitial_RejectsNonGuidTenantId()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
var payload = BuildPayload(tenantId: "not-a-guid");
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryUploadInitial_RejectsEmptyHistoryRowKey()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
var payload = BuildPayload(historyRowKey: "");
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryUploadInitial_RejectsSchemaVersionZero()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
var payload = BuildPayload();
|
||||
payload.SchemaVersion = 0;
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateWithEtagCas_RequiresIfMatchEtag()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => store.UpdateWithEtagCasAsync(BuildPayload(), ""));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryDownload_RejectsNonGuidTenantId()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => store.TryDownloadAsync("not-a-guid", HistoryRowKey));
|
||||
}
|
||||
|
||||
// ── Initial upload (If-None-Match=*) ────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TryUploadInitial_FirstWrite_ReturnsTrue()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
var inserted = await store.TryUploadInitialAsync(BuildPayload());
|
||||
Assert.True(inserted);
|
||||
Assert.Single(store.Blobs);
|
||||
Assert.Equal(
|
||||
BlobOffboardingExpectationsStore.BuildBlobName(TenantId, HistoryRowKey),
|
||||
store.Blobs.Keys.Single());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryUploadInitial_Duplicate_409BlobAlreadyExists_ReturnsFalse()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
await store.TryUploadInitialAsync(BuildPayload());
|
||||
|
||||
// Simulate the "older" Azure response code for the same condition.
|
||||
store.NextOverwriteRefusal = OverwriteRefusal.Status409;
|
||||
|
||||
var inserted = await store.TryUploadInitialAsync(BuildPayload());
|
||||
Assert.False(inserted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryUploadInitial_Duplicate_412ConditionNotMet_ReturnsFalse()
|
||||
{
|
||||
// High-finding: Azure Blob Storage emits 412 ConditionNotMet for the SAME
|
||||
// "If-None-Match=* on existing blob" case the SDK historically returned 409 for.
|
||||
// Without the 412-branch the resume path breaks and the worker poisons.
|
||||
var store = new FakeStore();
|
||||
await store.TryUploadInitialAsync(BuildPayload());
|
||||
|
||||
store.NextOverwriteRefusal = OverwriteRefusal.Status412ConditionNotMet;
|
||||
|
||||
var inserted = await store.TryUploadInitialAsync(BuildPayload());
|
||||
Assert.False(inserted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryUploadInitial_Duplicate_BlobErrorCodeOnly_ReturnsFalse()
|
||||
{
|
||||
// Some SDK paths surface the failure as Status=0 with a populated ErrorCode.
|
||||
var store = new FakeStore();
|
||||
await store.TryUploadInitialAsync(BuildPayload());
|
||||
|
||||
store.NextOverwriteRefusal = OverwriteRefusal.ErrorCodeBlobAlreadyExists;
|
||||
|
||||
var inserted = await store.TryUploadInitialAsync(BuildPayload());
|
||||
Assert.False(inserted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryUploadInitial_Propagates500_NotASilentResume()
|
||||
{
|
||||
var store = new FakeStore { NextOverwriteRefusal = OverwriteRefusal.Status500 };
|
||||
await Assert.ThrowsAsync<RequestFailedException>(() => store.TryUploadInitialAsync(BuildPayload()));
|
||||
}
|
||||
|
||||
// ── Download (404 → null) ──────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TryDownload_404_Returns_NullPayload_AndNullEtag()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
var (payload, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
|
||||
Assert.Null(payload);
|
||||
Assert.Null(etag);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryDownload_AfterUpload_RoundTripsAllFields()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
var original = BuildPayload();
|
||||
original.EnumerationCompleted = true;
|
||||
original.EnumeratedSessionCount = 2;
|
||||
original.Expectations.Add(new OffboardingExpectation { SessionId = "s1", ManifestId = "m1", Outcome = "Enqueued" });
|
||||
original.Expectations.Add(new OffboardingExpectation { SessionId = "s2", ManifestId = null, Outcome = "SessionNotFound" });
|
||||
|
||||
await store.TryUploadInitialAsync(original);
|
||||
|
||||
var (round, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
|
||||
Assert.NotNull(round);
|
||||
Assert.NotNull(etag);
|
||||
Assert.Equal(2, round!.EnumeratedSessionCount);
|
||||
Assert.True(round.EnumerationCompleted);
|
||||
Assert.Equal(2, round.Expectations.Count);
|
||||
Assert.Equal("Enqueued", round.Expectations[0].Outcome);
|
||||
Assert.Null(round.Expectations[1].ManifestId);
|
||||
}
|
||||
|
||||
// ── Update with ETag CAS ───────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateWithEtagCas_MatchingEtag_ReturnsNewEtag()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
await store.TryUploadInitialAsync(BuildPayload());
|
||||
var (round, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
|
||||
|
||||
round!.Expectations.Add(new OffboardingExpectation { SessionId = "s1", Outcome = "Enqueued", ManifestId = "m1" });
|
||||
|
||||
var newEtag = await store.UpdateWithEtagCasAsync(round, etag!);
|
||||
Assert.NotNull(newEtag);
|
||||
Assert.NotEqual(etag, newEtag);
|
||||
|
||||
var (refetched, _) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
|
||||
Assert.Single(refetched!.Expectations);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpdateWithEtagCas_StaleEtag_Throws412()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
await store.TryUploadInitialAsync(BuildPayload());
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RequestFailedException>(
|
||||
() => store.UpdateWithEtagCasAsync(BuildPayload(), ifMatchEtag: "\"0xSTALE\""));
|
||||
Assert.Equal(412, ex.Status);
|
||||
}
|
||||
|
||||
// ── Delete ─────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Delete_RemovesBlob_WhenPresent()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
await store.TryUploadInitialAsync(BuildPayload());
|
||||
Assert.Single(store.Blobs);
|
||||
|
||||
await store.DeleteAsync(TenantId, HistoryRowKey);
|
||||
Assert.Empty(store.Blobs);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Delete_404_IsIdempotent()
|
||||
{
|
||||
var store = new FakeStore();
|
||||
// No upload first — blob doesn't exist.
|
||||
await store.DeleteAsync(TenantId, HistoryRowKey); // must not throw
|
||||
Assert.Empty(store.Blobs);
|
||||
}
|
||||
|
||||
// ── Container choice (Rev-8-F1) ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Container_IsOffboardingState_NotDeletionManifests()
|
||||
{
|
||||
// Container choice is part of the resilience design — wiping the wrong container
|
||||
// (Rev-8-F1: deletion-manifests would orphan Expectations during Phase 2.E).
|
||||
Assert.Equal("offboarding-state", AutopilotMonitor.Shared.Constants.BlobContainers.OffboardingState);
|
||||
Assert.NotEqual(
|
||||
AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests,
|
||||
AutopilotMonitor.Shared.Constants.BlobContainers.OffboardingState);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BlobName_FollowsTenantPrefixSchemaForLifecycleSweep()
|
||||
{
|
||||
var name = BlobOffboardingExpectationsStore.BuildBlobName(TenantId, HistoryRowKey);
|
||||
Assert.StartsWith($"{TenantId}/", name);
|
||||
Assert.EndsWith(".expectations.json", name);
|
||||
Assert.Contains(HistoryRowKey, name);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Defaults_ForceCallersToSetEnumerationCompletedExplicitly()
|
||||
{
|
||||
// Rev-7-F2 disambiguation lives at the POCO level — the drain probe relies on
|
||||
// EnumerationCompleted defaulting to false so a half-built payload cannot
|
||||
// accidentally pass the "0 expectations + completed=true" happy path.
|
||||
var payload = new OffboardingExpectations();
|
||||
Assert.False(payload.EnumerationCompleted);
|
||||
Assert.Equal(0, payload.EnumeratedSessionCount);
|
||||
Assert.Equal(1, payload.SchemaVersion);
|
||||
Assert.Empty(payload.Expectations);
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
private static OffboardingExpectations BuildPayload(
|
||||
string tenantId = TenantId,
|
||||
string historyRowKey = HistoryRowKey)
|
||||
=> new()
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
TenantId = tenantId,
|
||||
HistoryRowKey = historyRowKey,
|
||||
CreatedAt = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc),
|
||||
EnumerationCompleted = false,
|
||||
EnumeratedSessionCount = 0,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Selects how the next call to <see cref="FakeStore.WriteIfNoneMatchAsync"/> simulates an
|
||||
/// "exists" refusal — needed because Azure switches between codes depending on tier/SDK.
|
||||
/// </summary>
|
||||
private enum OverwriteRefusal
|
||||
{
|
||||
Auto,
|
||||
Status409,
|
||||
Status412ConditionNotMet,
|
||||
ErrorCodeBlobAlreadyExists,
|
||||
Status500,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Dict-backed fake that overrides the four protected virtual seams. Mirrors the
|
||||
/// production Azure semantics for If-None-Match=*, If-Match, 404, and DeleteIfExists.
|
||||
/// </summary>
|
||||
private sealed class FakeStore : BlobOffboardingExpectationsStore
|
||||
{
|
||||
public Dictionary<string, (byte[] Bytes, string ETag)> Blobs { get; } = new();
|
||||
public OverwriteRefusal NextOverwriteRefusal { get; set; } = OverwriteRefusal.Auto;
|
||||
private int _etagCounter;
|
||||
|
||||
public FakeStore()
|
||||
: base(
|
||||
new BlobStorageService(
|
||||
new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance,
|
||||
usesManagedIdentity: false),
|
||||
NullLogger<BlobOffboardingExpectationsStore>.Instance)
|
||||
{
|
||||
}
|
||||
|
||||
protected override Task<(bool Inserted, string? ETag)> WriteIfNoneMatchAsync(
|
||||
string blobName, byte[] payload, CancellationToken ct)
|
||||
{
|
||||
// Allow tests to inject a specific refusal even though the underlying state would
|
||||
// already make a real Azure call fail — needed to pin BOTH the 409 and 412 paths.
|
||||
if (NextOverwriteRefusal == OverwriteRefusal.Status500)
|
||||
{
|
||||
NextOverwriteRefusal = OverwriteRefusal.Auto;
|
||||
throw new RequestFailedException(500, "ServerError", "InternalError", null);
|
||||
}
|
||||
|
||||
if (Blobs.ContainsKey(blobName))
|
||||
{
|
||||
var refusal = NextOverwriteRefusal == OverwriteRefusal.Auto
|
||||
? OverwriteRefusal.Status409
|
||||
: NextOverwriteRefusal;
|
||||
NextOverwriteRefusal = OverwriteRefusal.Auto;
|
||||
|
||||
RequestFailedException ex = refusal switch
|
||||
{
|
||||
OverwriteRefusal.Status409 =>
|
||||
new RequestFailedException(409, "Conflict", BlobErrorCode.BlobAlreadyExists.ToString(), null),
|
||||
OverwriteRefusal.Status412ConditionNotMet =>
|
||||
new RequestFailedException(412, "ConditionNotMet", BlobErrorCode.ConditionNotMet.ToString(), null),
|
||||
OverwriteRefusal.ErrorCodeBlobAlreadyExists =>
|
||||
new RequestFailedException(0, "ErrorCodeOnly", BlobErrorCode.BlobAlreadyExists.ToString(), null),
|
||||
_ => throw new InvalidOperationException("Unhandled refusal mode"),
|
||||
};
|
||||
|
||||
if (IsAlreadyExists(ex)) return Task.FromResult<(bool, string?)>((false, null));
|
||||
throw ex;
|
||||
}
|
||||
|
||||
var etag = $"\"0xFAKE_{++_etagCounter}\"";
|
||||
Blobs[blobName] = (payload, etag);
|
||||
return Task.FromResult<(bool, string?)>((true, etag));
|
||||
}
|
||||
|
||||
protected override Task<(byte[]? Payload, string? ETag)> ReadBlobAsync(
|
||||
string blobName, CancellationToken ct)
|
||||
{
|
||||
if (!Blobs.TryGetValue(blobName, out var entry))
|
||||
return Task.FromResult<(byte[]?, string?)>((null, null));
|
||||
return Task.FromResult<(byte[]?, string?)>((entry.Bytes, entry.ETag));
|
||||
}
|
||||
|
||||
protected override Task<string> WriteWithIfMatchAsync(
|
||||
string blobName, byte[] payload, string ifMatchEtag, CancellationToken ct)
|
||||
{
|
||||
if (!Blobs.TryGetValue(blobName, out var entry))
|
||||
throw new RequestFailedException(404, "NotFound", "BlobNotFound", null);
|
||||
if (entry.ETag != ifMatchEtag)
|
||||
throw new RequestFailedException(412, "ConditionNotMet", BlobErrorCode.ConditionNotMet.ToString(), null);
|
||||
|
||||
var etag = $"\"0xFAKE_{++_etagCounter}\"";
|
||||
Blobs[blobName] = (payload, etag);
|
||||
return Task.FromResult(etag);
|
||||
}
|
||||
|
||||
protected override Task DeleteBlobIfExistsAsync(string blobName, CancellationToken ct)
|
||||
{
|
||||
Blobs.Remove(blobName);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Plan §5.5 + §10.1 (F6-Fix). The filter helper sits in front of every SafeWipe table
|
||||
/// query, so injection or range-bleed bugs here turn into cross-tenant data loss. Tests
|
||||
/// hammer GUID validation, single-quote escaping, and the underscore-tilde upper bound.
|
||||
/// </summary>
|
||||
public class OffboardingFiltersTests
|
||||
{
|
||||
private const string ValidTenant = "11111111-1111-1111-1111-111111111111";
|
||||
|
||||
// ── EnsureValidGuid behaviour propagates through every entry point ─────────
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-guid")]
|
||||
[InlineData("abc")]
|
||||
[InlineData("'; DROP TABLE Sessions; --")]
|
||||
[InlineData("")]
|
||||
[InlineData("11111111-1111-1111-1111-1111111111111")] // too long
|
||||
public void ExactPartition_ThrowsForNonGuid(string input)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => OffboardingFilters.ExactPartition(input));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-guid")]
|
||||
[InlineData("'")]
|
||||
public void CompositePartitionRange_ThrowsForNonGuid(string input)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => OffboardingFilters.CompositePartitionRange(input));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-guid")]
|
||||
[InlineData("")]
|
||||
public void DiscriminatorWithTenantProp_ThrowsForNonGuid(string input)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => OffboardingFilters.DiscriminatorWithTenantProp("CodeLookup", input));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DiscriminatorWithTenantProp_ThrowsForEmptyDiscriminator()
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => OffboardingFilters.DiscriminatorWithTenantProp("", ValidTenant));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-guid")]
|
||||
public void TenantIdProperty_ThrowsForNonGuid(string input)
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => OffboardingFilters.TenantIdProperty(input));
|
||||
}
|
||||
|
||||
// ── Filter shapes ───────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void ExactPartition_EmitsCanonicalEqualsClause()
|
||||
{
|
||||
var filter = OffboardingFilters.ExactPartition(ValidTenant);
|
||||
Assert.Equal($"PartitionKey eq '{ValidTenant}'", filter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompositePartitionRange_UsesUnderscoreTildeUpperBound()
|
||||
{
|
||||
var filter = OffboardingFilters.CompositePartitionRange(ValidTenant);
|
||||
|
||||
// Lower bound must include the underscore anchor — otherwise the range matches
|
||||
// "{tenantId}abc" and other tenants whose ids happen to start with the same prefix.
|
||||
Assert.Contains($"PartitionKey ge '{ValidTenant}_'", filter);
|
||||
|
||||
// Upper bound MUST be '{tenantId}_~' (underscore-tilde). Switching to '{tenantId}~'
|
||||
// would bleed across the next ASCII bucket and pull in foreign rows.
|
||||
Assert.Contains($"PartitionKey lt '{ValidTenant}_~'", filter);
|
||||
Assert.DoesNotContain($"PartitionKey lt '{ValidTenant}~'", filter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DiscriminatorWithTenantProp_RequiresBothAnchorsServerSide()
|
||||
{
|
||||
var filter = OffboardingFilters.DiscriminatorWithTenantProp("CodeLookup", ValidTenant);
|
||||
|
||||
Assert.Contains("PartitionKey eq 'CodeLookup'", filter);
|
||||
Assert.Contains($"TenantId eq '{ValidTenant}'", filter);
|
||||
Assert.Contains("and", filter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TenantIdProperty_EmitsPropertyOnlyFilter()
|
||||
{
|
||||
var filter = OffboardingFilters.TenantIdProperty(ValidTenant);
|
||||
Assert.Equal($"TenantId eq '{ValidTenant}'", filter);
|
||||
Assert.DoesNotContain("PartitionKey", filter);
|
||||
}
|
||||
|
||||
// ── Injection neutralization (defense in depth even after EnsureValidGuid) ──
|
||||
|
||||
[Fact]
|
||||
public void DiscriminatorWithTenantProp_EscapesDiscriminatorQuotes()
|
||||
{
|
||||
// Hostile discriminator wouldn't pass through real callers (compile-time constants), but
|
||||
// defense in depth: the helper still escapes.
|
||||
var filter = OffboardingFilters.DiscriminatorWithTenantProp("a'b", ValidTenant);
|
||||
Assert.Contains("PartitionKey eq 'a''b'", filter);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Pagination;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Rev-5-F3: the enumerator MUST fail-loud on storage exceptions. The existing
|
||||
/// <c>GetSessionsByDateRangeAsync</c> swallows errors and returns empty — borrowing that
|
||||
/// behaviour for offboarding would let a transient outage trigger a same-cycle wipe with
|
||||
/// zero cascade backup. Tests pin both halves of the contract: happy-path yield-all, and
|
||||
/// exception-propagates-no-silent-empty.
|
||||
/// </summary>
|
||||
public class OffboardingSessionEnumeratorTests
|
||||
{
|
||||
private const string TenantId = "33333333-3333-3333-3333-333333333333";
|
||||
|
||||
[Fact]
|
||||
public async Task EnumerateAsync_YieldsAllSessionsFromRepo()
|
||||
{
|
||||
var repo = new FakeMaintenanceRepository(
|
||||
sessionIds: new[] { "s1", "s2", "s3" });
|
||||
var sut = new OffboardingSessionEnumerator(repo);
|
||||
|
||||
var collected = new List<string>();
|
||||
await foreach (var s in sut.EnumerateAsync(TenantId))
|
||||
{
|
||||
collected.Add(s);
|
||||
}
|
||||
|
||||
Assert.Equal(new[] { "s1", "s2", "s3" }, collected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnumerateAsync_PropagatesStorageException_NoSilentEmpty()
|
||||
{
|
||||
// 503 mid-enumeration is the exact scenario that would otherwise let the offboard
|
||||
// worker proceed to wipe with zero cascade backup. The enumerator MUST throw.
|
||||
var repo = new FakeMaintenanceRepository(
|
||||
sessionIds: new[] { "s1" },
|
||||
throwOnIteration: new InvalidOperationException("simulated 503"));
|
||||
var sut = new OffboardingSessionEnumerator(repo);
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(async () =>
|
||||
{
|
||||
await foreach (var _ in sut.EnumerateAsync(TenantId))
|
||||
{
|
||||
// drain
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnumerateAsync_RejectsNonGuidTenantId()
|
||||
{
|
||||
var sut = new OffboardingSessionEnumerator(new FakeMaintenanceRepository(Array.Empty<string>()));
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(async () =>
|
||||
{
|
||||
await foreach (var _ in sut.EnumerateAsync("not-a-guid"))
|
||||
{
|
||||
// never reached
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ── Test double ─────────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class FakeMaintenanceRepository : IMaintenanceRepository
|
||||
{
|
||||
private readonly string[] _sessions;
|
||||
private readonly Exception? _throwOnIteration;
|
||||
|
||||
public FakeMaintenanceRepository(string[] sessionIds, Exception? throwOnIteration = null)
|
||||
{
|
||||
_sessions = sessionIds;
|
||||
_throwOnIteration = throwOnIteration;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<string> EnumerateSessionsForOffboardingAsync(
|
||||
string tenantId, [EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
foreach (var s in _sessions)
|
||||
{
|
||||
if (_throwOnIteration != null)
|
||||
{
|
||||
// Yield the first one, then throw — matches "mid-enumeration outage".
|
||||
yield return s;
|
||||
throw _throwOnIteration;
|
||||
}
|
||||
yield return s;
|
||||
}
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
// ── Unused for these tests ─────────────────────────────────────────────
|
||||
|
||||
public Task<bool> LogAuditEntryAsync(string tenantId, string action, string entityType, string entityId, string performedBy, Dictionary<string, string>? details = null) => throw new NotImplementedException();
|
||||
public Task<List<AuditLogEntry>> GetAuditLogsAsync(string tenantId, DateTime? dateFrom = null, DateTime? dateTo = null) => throw new NotImplementedException();
|
||||
public Task<List<AuditLogEntry>> GetAllAuditLogsAsync(DateTime? dateFrom = null, DateTime? dateTo = null) => throw new NotImplementedException();
|
||||
public Task<RawPage<AuditLogEntry>> GetAuditLogsPageAsync(string tenantId, DateTime? dateFrom, DateTime? dateTo, int pageSize, string? continuation) => throw new NotImplementedException();
|
||||
public Task<RawPage<AuditLogEntry>> GetAllAuditLogsPageAsync(DateTime? dateFrom, DateTime? dateTo, int pageSize, string? continuation) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetSessionsOlderThanAsync(string tenantId, DateTime cutoffDate) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetSessionsByDateRangeAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours) => throw new NotImplementedException();
|
||||
public Task<List<string>> GetAllTenantIdsAsync() => throw new NotImplementedException();
|
||||
public Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<int> BackfillSessionIndexAsync() => throw new NotImplementedException();
|
||||
public Task<int> CleanupGhostSessionIndexEntriesAsync() => throw new NotImplementedException();
|
||||
public Task<bool> IsSessionIndexEmptyAsync() => throw new NotImplementedException();
|
||||
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => throw new NotImplementedException();
|
||||
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Storage.Blobs;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Plan §5 + §10.1: SafeWipe is the bug-shield for the user's "es wurde schon mal zu viel
|
||||
/// gelöscht"-concern. These tests pin
|
||||
/// <list type="bullet">
|
||||
/// <item>Verify-abort when the fetched set contains a row that does not match the expected
|
||||
/// anchor — NO deletes are issued (4-Step Pattern step 3 only runs after step 2 passes).</item>
|
||||
/// <item>Every Delete uses <see cref="ETag.All"/> (unconditional) — F5-Fix, otherwise the
|
||||
/// helper isn't idempotent across crash/resume.</item>
|
||||
/// <item>404-fallback per row keeps the helper idempotent after a partial run.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public class SafeWipeServiceTests
|
||||
{
|
||||
private const string TenantId = "66666666-6666-6666-6666-666666666666";
|
||||
private const string OtherTenant = "77777777-7777-7777-7777-777777777777";
|
||||
|
||||
[Fact]
|
||||
public async Task ExactPartition_AbortsBeforeAnyDeleteWhenForeignRowSlipsThrough()
|
||||
{
|
||||
// Server-side filter SHOULD only return PK=TenantId, but if a misbehaving filter or
|
||||
// SDK bug returns a foreign row, verify-abort fires and NO Delete is issued.
|
||||
var harness = new Harness(
|
||||
queriedRows: new[]
|
||||
{
|
||||
new TableEntity(TenantId, "row-1"),
|
||||
new TableEntity(OtherTenant, "row-2"), // <-- the bug
|
||||
new TableEntity(TenantId, "row-3"),
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
|
||||
harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId));
|
||||
|
||||
Assert.Empty(harness.SubmittedBatches);
|
||||
Assert.Empty(harness.PerRowDeletes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CompositePartitionRange_AbortsWhenForeignPrefixReturned()
|
||||
{
|
||||
var harness = new Harness(
|
||||
queriedRows: new[]
|
||||
{
|
||||
new TableEntity($"{TenantId}_session-1", "row"),
|
||||
new TableEntity($"{OtherTenant}_session-2", "row"), // bleeds across underscore boundary
|
||||
});
|
||||
|
||||
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
|
||||
harness.Sut.WipeByCompositePartitionRangeAsync("FakeTable", TenantId));
|
||||
|
||||
Assert.Empty(harness.SubmittedBatches);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiscriminatorWithTenantProp_AbortsWhenTenantPropertyMismatch()
|
||||
{
|
||||
// Variant B's only tenant anchor is the property — verify MUST check it client-side.
|
||||
var foreign = new TableEntity("CodeLookup", "row-2");
|
||||
foreign["TenantId"] = OtherTenant;
|
||||
var ours = new TableEntity("CodeLookup", "row-1");
|
||||
ours["TenantId"] = TenantId;
|
||||
|
||||
var harness = new Harness(queriedRows: new[] { ours, foreign });
|
||||
|
||||
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
|
||||
harness.Sut.WipeByDiscriminatorAndTenantPropertyAsync("FakeTable", "CodeLookup", TenantId));
|
||||
|
||||
Assert.Empty(harness.SubmittedBatches);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TenantIdProperty_AbortsWhenPropertyMissing()
|
||||
{
|
||||
// Variant C: row missing the TenantId property fails verify.
|
||||
var ours = new TableEntity("user-oid-1", "rk");
|
||||
ours["TenantId"] = TenantId;
|
||||
var foreign = new TableEntity("user-oid-2", "rk"); // no TenantId
|
||||
|
||||
var harness = new Harness(queriedRows: new[] { ours, foreign });
|
||||
|
||||
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
|
||||
harness.Sut.WipeByTenantIdPropertyAsync("FakeTable", TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExactPartition_DeletesEveryRowWithETagAll()
|
||||
{
|
||||
var harness = new Harness(
|
||||
queriedRows: new[]
|
||||
{
|
||||
new TableEntity(TenantId, "rk_1"),
|
||||
new TableEntity(TenantId, "rk_2"),
|
||||
new TableEntity(TenantId, "rk_3"),
|
||||
});
|
||||
|
||||
var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId);
|
||||
|
||||
Assert.Equal(3, deleted);
|
||||
var batch = Assert.Single(harness.SubmittedBatches);
|
||||
Assert.Equal(3, batch.Count);
|
||||
|
||||
foreach (var action in batch)
|
||||
{
|
||||
Assert.Equal(TableTransactionActionType.Delete, action.ActionType);
|
||||
Assert.Equal(ETag.All, action.Entity.ETag);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Delete_FallsBackToPerRowOn404_KeepsIdempotent()
|
||||
{
|
||||
// First batch returns 404 → per-row fallback finds 1 already-missing, 2 still there.
|
||||
var perRowState = new Dictionary<string, bool>
|
||||
{
|
||||
["rk_1"] = true,
|
||||
["rk_2"] = false, // already gone (concurrent cleaner)
|
||||
["rk_3"] = true,
|
||||
};
|
||||
|
||||
var harness = new Harness(
|
||||
queriedRows: perRowState.Keys.Select(rk => new TableEntity(TenantId, rk)).ToArray(),
|
||||
batchBehavior: _ => throw new RequestFailedException(404, "ResourceNotFound"),
|
||||
perRowBehavior: rk =>
|
||||
{
|
||||
if (!perRowState[rk]) throw new RequestFailedException(404, "ResourceNotFound");
|
||||
return new Mock<Response>().Object;
|
||||
});
|
||||
|
||||
var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId);
|
||||
|
||||
Assert.Equal(2, deleted); // 2 still there, 1 already missing
|
||||
Assert.Equal(3, harness.PerRowDeletes.Count);
|
||||
foreach (var (_, _, etag) in harness.PerRowDeletes)
|
||||
{
|
||||
Assert.Equal(ETag.All, etag);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EmptyResult_ReturnsZero_NoBatchSubmitted()
|
||||
{
|
||||
var harness = new Harness(queriedRows: Array.Empty<TableEntity>());
|
||||
|
||||
var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId);
|
||||
|
||||
Assert.Equal(0, deleted);
|
||||
Assert.Empty(harness.SubmittedBatches);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Variants_RejectNonGuidTenantId()
|
||||
{
|
||||
var harness = new Harness(queriedRows: Array.Empty<TableEntity>());
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => harness.Sut.WipeByExactPartitionAsync("t", "not-a-guid"));
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => harness.Sut.WipeByCompositePartitionRangeAsync("t", "not-a-guid"));
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => harness.Sut.WipeByDiscriminatorAndTenantPropertyAsync("t", "CodeLookup", "not-a-guid"));
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => harness.Sut.WipeByTenantIdPropertyAsync("t", "not-a-guid"));
|
||||
}
|
||||
|
||||
// ── Harness ─────────────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public List<List<TableTransactionAction>> SubmittedBatches { get; } = new();
|
||||
public List<(string Pk, string Rk, ETag Etag)> PerRowDeletes { get; } = new();
|
||||
public SafeWipeService Sut { get; }
|
||||
|
||||
public Harness(
|
||||
TableEntity[] queriedRows,
|
||||
Func<List<TableTransactionAction>, Response<IReadOnlyList<Response>>>? batchBehavior = null,
|
||||
Func<string, Response>? perRowBehavior = null)
|
||||
{
|
||||
var mockTableClient = new Mock<TableClient>();
|
||||
|
||||
// QueryAsync<TableEntity> returns the canned set as an AsyncPageable.
|
||||
mockTableClient
|
||||
.Setup(c => c.QueryAsync<TableEntity>(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<int?>(),
|
||||
It.IsAny<IEnumerable<string>>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns(AsAsyncPageable(queriedRows));
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.SubmitTransactionAsync(It.IsAny<IEnumerable<TableTransactionAction>>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<IEnumerable<TableTransactionAction>, CancellationToken>((actions, _) =>
|
||||
{
|
||||
var snapshot = actions.ToList();
|
||||
SubmittedBatches.Add(snapshot);
|
||||
if (batchBehavior != null) return Task.FromResult(batchBehavior(snapshot));
|
||||
var inner = (IReadOnlyList<Response>)snapshot.Select(_ => new Mock<Response>().Object).ToList();
|
||||
return Task.FromResult(Response.FromValue(inner, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.DeleteEntityAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<ETag>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, ETag, CancellationToken>((pk, rk, etag, _) =>
|
||||
{
|
||||
PerRowDeletes.Add((pk, rk, etag));
|
||||
if (perRowBehavior != null) return Task.FromResult(perRowBehavior(rk));
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
var mockServiceClient = new Mock<TableServiceClient>();
|
||||
mockServiceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(mockTableClient.Object);
|
||||
|
||||
var storage = new TableStorageService(mockServiceClient.Object, NullLogger<TableStorageService>.Instance);
|
||||
var blob = new BlobStorageService(
|
||||
new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance,
|
||||
usesManagedIdentity: false);
|
||||
|
||||
Sut = new SafeWipeService(storage, blob, NullLogger<SafeWipeService>.Instance);
|
||||
}
|
||||
|
||||
private static AsyncPageable<TableEntity> AsAsyncPageable(TableEntity[] entities)
|
||||
{
|
||||
var page = Page<TableEntity>.FromValues(entities, continuationToken: null, new Mock<Response>().Object);
|
||||
return AsyncPageable<TableEntity>.FromPages(new[] { page });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Storage.Blobs;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Plan PR1.5 (Rev-5-F1). Locks the cascade-handler side-effect order: audit-write and
|
||||
/// SignalR notify must complete BEFORE <c>DeletionProgress.CompletedAt</c> is stamped.
|
||||
/// Once CompletedAt is set, the tenant-offboarding worker's drain probe treats the cascade
|
||||
/// as "all side effects through" and the next phase wipes <c>AuditLogs</c> — so a late
|
||||
/// <c>deletion_completed</c> audit would land as an orphan after the wipe.
|
||||
/// <para>
|
||||
/// The reorder is fail-soft-preserving:
|
||||
/// <list type="bullet">
|
||||
/// <item><c>LogAuditEntryAsync</c> already swallows storage exceptions internally and
|
||||
/// returns <c>false</c> on failure; the handler does not act on the boolean.</item>
|
||||
/// <item><c>NotifySessionDeletedAsync</c> already catches everything internally.</item>
|
||||
/// </list>
|
||||
/// So pulling them in front of the CompletedAt CAS cannot block <c>CompletedAt</c> from
|
||||
/// being set on transient storage failures — confirmed by
|
||||
/// <see cref="AuditFailure_DoesNotBlockCompletedAtSet"/>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class SessionDeletionHandlerOrderingTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
|
||||
private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
|
||||
|
||||
[Fact]
|
||||
public async Task TombstonePhase_WritesAuditAndSignalRBeforeCompletedAt()
|
||||
{
|
||||
var harness = new Harness();
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
Assert.True(harness.AuditSequence > 0, "audit was never written");
|
||||
Assert.True(harness.SignalRSequence > 0, "SignalR notify was never sent");
|
||||
Assert.True(harness.CompletedAtSequence > 0, "CompletedAt was never set on the progress blob");
|
||||
|
||||
Assert.True(
|
||||
harness.AuditSequence < harness.CompletedAtSequence,
|
||||
$"audit must run before CompletedAt CAS (audit={harness.AuditSequence}, completedAt={harness.CompletedAtSequence})");
|
||||
Assert.True(
|
||||
harness.SignalRSequence < harness.CompletedAtSequence,
|
||||
$"SignalR notify must run before CompletedAt CAS (signalR={harness.SignalRSequence}, completedAt={harness.CompletedAtSequence})");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AuditFailure_DoesNotBlockCompletedAtSet()
|
||||
{
|
||||
// LogAuditEntryAsync is fail-soft by contract — a storage hiccup returns false rather
|
||||
// than throwing. The handler must NOT use the boolean as a gate: CompletedAt still
|
||||
// gets set so the cascade completes from the offboarding worker's point of view.
|
||||
var harness = new Harness(auditReturns: false);
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
Assert.True(harness.AuditSequence > 0, "audit attempt was still made");
|
||||
Assert.True(harness.CompletedAtSequence > 0,
|
||||
"CompletedAt MUST still be stamped on the progress blob even when audit returned false");
|
||||
Assert.True(harness.AuditSequence < harness.CompletedAtSequence);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SignalRFailure_DoesNotBlockCompletedAtSet()
|
||||
{
|
||||
// NotifySessionDeletedAsync already catches internally. The reorder cannot break that
|
||||
// contract: an exception inside the notify path must not bubble out into the handler
|
||||
// and leave CompletedAt unset.
|
||||
var harness = new Harness(signalRThrows: new InvalidOperationException("simulated SignalR outage"));
|
||||
await harness.Sut.HandleAsync(harness.Envelope);
|
||||
|
||||
Assert.True(harness.SignalRSequence > 0, "SignalR notify attempt was still made");
|
||||
Assert.True(harness.CompletedAtSequence > 0,
|
||||
"CompletedAt MUST still be stamped on the progress blob even when SignalR threw");
|
||||
}
|
||||
|
||||
// ── Harness ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Minimal self-contained harness that drives the cascade through to the tombstone phase
|
||||
/// using a single-step manifest (the Final step only). Sequence-counter records the call
|
||||
/// order of the three side effects that PR1.5 reorders.
|
||||
/// </summary>
|
||||
private sealed class Harness
|
||||
{
|
||||
public SessionDeletionEnvelope Envelope { get; }
|
||||
public SessionDeletionHandler Sut { get; }
|
||||
|
||||
public int AuditSequence { get; private set; }
|
||||
public int SignalRSequence { get; private set; }
|
||||
public int CompletedAtSequence { get; private set; }
|
||||
|
||||
private int _sequenceCounter;
|
||||
private int Next() => System.Threading.Interlocked.Increment(ref _sequenceCounter);
|
||||
|
||||
public Harness(
|
||||
bool auditReturns = true,
|
||||
Exception? signalRThrows = null)
|
||||
{
|
||||
// --- Storage mock: just enough to flow through the state machine + tombstone step.
|
||||
var storage = new Mock<TableStorageService>(
|
||||
Mock.Of<TableServiceClient>(),
|
||||
NullLogger<TableStorageService>.Instance);
|
||||
|
||||
var sessionRow = new TableEntity(TenantId, SessionId)
|
||||
{
|
||||
["DeletionState"] = SessionDeletionState.Queued,
|
||||
["PendingDeletionManifestId"] = ManifestId,
|
||||
};
|
||||
storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(sessionRow);
|
||||
|
||||
storage.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
TenantId, SessionId,
|
||||
SessionDeletionState.Queued, SessionDeletionState.Running,
|
||||
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
|
||||
{
|
||||
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
|
||||
CurrentState = SessionDeletionState.Running,
|
||||
CurrentManifestId = ManifestId,
|
||||
});
|
||||
|
||||
storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
|
||||
It.IsAny<string>(), It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, IReadOnlyList<(string, string)> keys, CancellationToken _) =>
|
||||
new DeletionBatchResult(keys.Count, keys.Count, 0));
|
||||
|
||||
storage.Setup(s => s.RecordSessionTombstoneAsync(
|
||||
TenantId, SessionId, It.IsAny<string>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
// --- Blob mock: manifest + progress IO with a live in-memory progress instance so
|
||||
// each UpdateDeletionProgressAsync mutation is reflected in the next download.
|
||||
var progressLive = new DeletionProgress
|
||||
{
|
||||
SnapshotSha256 = Sha256,
|
||||
CompletedSteps = new HashSet<int>(),
|
||||
VerificationDone = false,
|
||||
CompletedAt = null,
|
||||
};
|
||||
|
||||
var blob = new Mock<BlobStorageService>(
|
||||
new BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
NullLogger<BlobStorageService>.Instance,
|
||||
false);
|
||||
|
||||
var manifest = BuildMinimalManifest();
|
||||
blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((manifest, Sha256));
|
||||
|
||||
blob.Setup(b => b.DownloadDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => (Clone(progressLive), "\"0xFAKE\""));
|
||||
|
||||
blob.Setup(b => b.UpdateDeletionProgressAsync(
|
||||
TenantId, SessionId, ManifestId,
|
||||
It.IsAny<DeletionProgress>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, string, DeletionProgress, string, CancellationToken>(
|
||||
(_, _, _, p, _, _) =>
|
||||
{
|
||||
// Only the write that flips CompletedAt from null → non-null is the
|
||||
// "lifecycle complete" CAS the reorder targets. Other writes (state
|
||||
// progression, TombstoneStarted) get sequence numbers but we don't
|
||||
// assert on them.
|
||||
if (progressLive.CompletedAt == null && p.CompletedAt != null)
|
||||
{
|
||||
CompletedAtSequence = Next();
|
||||
}
|
||||
progressLive.CompletedSteps = new HashSet<int>(p.CompletedSteps);
|
||||
progressLive.VerificationDone = p.VerificationDone;
|
||||
progressLive.CompletedAt = p.CompletedAt;
|
||||
progressLive.TombstoneStarted = p.TombstoneStarted;
|
||||
return Task.FromResult("\"0xFAKE_NEXT\"");
|
||||
});
|
||||
|
||||
// --- Verifier: clean by default (no residuals → tombstone runs).
|
||||
var verifier = new Mock<CascadeVerificationService>(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(),
|
||||
NullLogger<CascadeVerificationService>.Instance);
|
||||
verifier.Setup(v => v.VerifyAsync(It.IsAny<DeletionManifest>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new CascadeVerificationResult(true, new List<CascadeResidualKey>()));
|
||||
|
||||
// --- Maintenance: capture audit call with a sequence number.
|
||||
var maintenance = new Mock<IMaintenanceRepository>();
|
||||
maintenance.Setup(m => m.LogAuditEntryAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
|
||||
.Returns<string, string, string, string, string, Dictionary<string, string>?>(
|
||||
(_, action, _, _, _, _) =>
|
||||
{
|
||||
if (action == "deletion_completed") AuditSequence = Next();
|
||||
return Task.FromResult(auditReturns);
|
||||
});
|
||||
|
||||
// --- SignalR: capture notify call with a sequence number.
|
||||
var signalR = new SequencingSignalRNotificationService(
|
||||
this,
|
||||
throwOnNotify: signalRThrows);
|
||||
|
||||
Envelope = new SessionDeletionEnvelope
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
ManifestId = ManifestId,
|
||||
Reason = "admin_delete",
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
Sut = new SessionDeletionHandler(
|
||||
storage.Object, blob.Object, verifier.Object,
|
||||
maintenance.Object, signalR,
|
||||
NullLogger<SessionDeletionHandler>.Instance);
|
||||
}
|
||||
|
||||
private static DeletionProgress Clone(DeletionProgress p) => new()
|
||||
{
|
||||
SnapshotSha256 = p.SnapshotSha256,
|
||||
CompletedSteps = new HashSet<int>(p.CompletedSteps),
|
||||
VerificationDone = p.VerificationDone,
|
||||
CompletedAt = p.CompletedAt,
|
||||
TombstoneStarted = p.TombstoneStarted,
|
||||
};
|
||||
|
||||
private static DeletionManifest BuildMinimalManifest() => new()
|
||||
{
|
||||
ManifestId = ManifestId,
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
CreatedAt = new DateTime(2026, 5, 18, 10, 0, 0, DateTimeKind.Utc),
|
||||
CreatedBy = new DeletionActor { Type = "admin", Actor = "alice@contoso.invalid" },
|
||||
Reason = "admin_delete",
|
||||
RetentionContext = new DeletionRetentionContext { TenantRetentionDays = 90 },
|
||||
SchemaHash = "sha256:test",
|
||||
// Single FINAL step → handler skips per-table + aggregate phases and goes straight
|
||||
// to verifier (clean) → tombstone → audit + signalR + CompletedAt. That is the
|
||||
// path PR1.5 reorders.
|
||||
Steps = new List<DeletionStep>
|
||||
{
|
||||
new DeletionStep
|
||||
{
|
||||
Order = 1,
|
||||
Step = DeletionStepNames.Tombstone,
|
||||
Class = DeletionStepClass.Final,
|
||||
RowCount = 2,
|
||||
Rows = new List<DeletionRowDump>
|
||||
{
|
||||
new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
|
||||
new DeletionRowDump { Pk = TenantId, Rk = SessionId },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// SignalR test double that records a sequence number on
|
||||
/// <see cref="ISignalRNotificationService.NotifySessionDeletedAsync"/>.
|
||||
/// All other interface members are no-ops — the cascade handler never calls them.
|
||||
/// </summary>
|
||||
private sealed class SequencingSignalRNotificationService : ISignalRNotificationService
|
||||
{
|
||||
private readonly Harness _harness;
|
||||
private readonly Exception? _throwOnNotify;
|
||||
|
||||
public SequencingSignalRNotificationService(Harness harness, Exception? throwOnNotify)
|
||||
{
|
||||
_harness = harness;
|
||||
_throwOnNotify = throwOnNotify;
|
||||
}
|
||||
|
||||
public Task NotifySessionDeletedAsync(string tenantId, string sessionId)
|
||||
{
|
||||
_harness.SignalRSequence = _harness.Next();
|
||||
// Production NotifySessionDeletedAsync already catches internally; mimic that
|
||||
// contract so the test double doesn't accidentally pretend exceptions bubble.
|
||||
if (_throwOnNotify != null)
|
||||
{
|
||||
try { throw _throwOnNotify; }
|
||||
catch { /* swallow — matches production behaviour */ }
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
// ── Unused by the cascade handler — no-op stubs ────────────────────
|
||||
public Task NotifyRuleResultsAvailableAsync(string tenantId, string sessionId, int resultCount) => Task.CompletedTask;
|
||||
public Task NotifyVulnerabilityReportAvailableAsync(string tenantId, string sessionId, string overallRisk) => Task.CompletedTask;
|
||||
public Task SendTenantNotificationAsync(string tenantId, NotificationAudience audience, object dto) => Task.CompletedTask;
|
||||
public Task SendTenantNotificationDismissedAsync(string tenantId, string notificationId) => Task.CompletedTask;
|
||||
public Task SendTenantNotificationDismissedAllAsync(string tenantId) => Task.CompletedTask;
|
||||
public Task SendGlobalNotificationAsync(object dto) => Task.CompletedTask;
|
||||
public Task SendGlobalNotificationDismissedAsync(string notificationId) => Task.CompletedTask;
|
||||
public Task SendGlobalNotificationDismissedAllAsync() => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Round-trip + shape-guard coverage for the three POCO mappings hosted in
|
||||
/// <c>OffboardingAudit</c>. Memory: <c>feedback_table_storage_serialization</c> — new
|
||||
/// fields MUST be exercised in Store+Map so silent drops surface here, not in production.
|
||||
/// </summary>
|
||||
public class TableOffboardingAuditRepositoryTests
|
||||
{
|
||||
private const string TenantId = "88888888-8888-8888-8888-888888888888";
|
||||
private const string HistoryRowKey = "20260518091523123_88888888-8888-8888-8888-888888888888";
|
||||
|
||||
[Fact]
|
||||
public async Task Marker_RoundTrips_AllFields()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = HistoryRowKey,
|
||||
InitiatedAt = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc),
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
Status = "InProgress",
|
||||
CompletedAt = null,
|
||||
FailedAt = null,
|
||||
FailedPhase = null,
|
||||
};
|
||||
|
||||
await harness.Sut.UpsertMarkerAsync(marker);
|
||||
|
||||
var fetched = await harness.Sut.TryGetMarkerAsync(TenantId);
|
||||
Assert.NotNull(fetched);
|
||||
Assert.Equal(marker.PartitionKey, fetched!.PartitionKey);
|
||||
Assert.Equal(marker.RowKey, fetched.RowKey);
|
||||
Assert.Equal(marker.TenantId, fetched.TenantId);
|
||||
Assert.Equal(marker.OffboardingHistoryRowKey, fetched.OffboardingHistoryRowKey);
|
||||
Assert.Equal(marker.InitiatedAt, fetched.InitiatedAt);
|
||||
Assert.Equal(marker.InitiatedBy, fetched.InitiatedBy);
|
||||
Assert.Equal("InProgress", fetched.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Marker_CompletedTransition_PreservesTimestamp()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var completed = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = HistoryRowKey,
|
||||
InitiatedAt = DateTime.UtcNow.AddMinutes(-10),
|
||||
InitiatedBy = "bob@contoso.invalid",
|
||||
Status = "Completed",
|
||||
CompletedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
await harness.Sut.UpsertMarkerAsync(completed);
|
||||
var fetched = await harness.Sut.TryGetMarkerAsync(TenantId);
|
||||
|
||||
Assert.Equal("Completed", fetched!.Status);
|
||||
Assert.NotNull(fetched.CompletedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Marker_TryGet_ReturnsNullOn404()
|
||||
{
|
||||
var harness = new Harness(throwOnGet: new RequestFailedException(404, "NotFound", "ResourceNotFound", null));
|
||||
Assert.Null(await harness.Sut.TryGetMarkerAsync(TenantId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Marker_DeleteSwallows404_Idempotent()
|
||||
{
|
||||
var harness = new Harness(throwOnDelete: new RequestFailedException(404, "NotFound", "ResourceNotFound", null));
|
||||
await harness.Sut.DeleteMarkerAsync(TenantId);
|
||||
// No exception → idempotent.
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task History_RoundTrips_RetainCountersAndDrainCompletedAt()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var now = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc);
|
||||
var history = new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = HistoryRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.invalid",
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
OffboardedAt = now,
|
||||
Status = "Completed",
|
||||
CompletedAt = now.AddMinutes(5),
|
||||
DeletedRowCountsJson = "{\"AuditLogs\":42}",
|
||||
TotalRowsDeleted = 42,
|
||||
DeletedBlobCount = 7,
|
||||
CascadeSessionsEnqueued = 3,
|
||||
RetryCount = 0,
|
||||
DrainCompletedAt = now.AddMinutes(3), // Rev-9-F1
|
||||
CustomGatherRulesArchived = 5,
|
||||
CustomAnalyzeRulesArchived = 3,
|
||||
ImeLogPatternOverridesArchived = 2,
|
||||
};
|
||||
|
||||
await harness.Sut.UpsertHistoryAsync(history);
|
||||
var fetched = await harness.Sut.TryGetHistoryAsync(HistoryRowKey);
|
||||
|
||||
Assert.NotNull(fetched);
|
||||
Assert.Equal(history.Status, fetched!.Status);
|
||||
Assert.Equal(history.TotalRowsDeleted, fetched.TotalRowsDeleted);
|
||||
Assert.Equal(history.DeletedBlobCount, fetched.DeletedBlobCount);
|
||||
Assert.Equal(history.CascadeSessionsEnqueued, fetched.CascadeSessionsEnqueued);
|
||||
Assert.Equal(history.DrainCompletedAt, fetched.DrainCompletedAt);
|
||||
Assert.Equal(5, fetched.CustomGatherRulesArchived);
|
||||
Assert.Equal(3, fetched.CustomAnalyzeRulesArchived);
|
||||
Assert.Equal(2, fetched.ImeLogPatternOverridesArchived);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pointer_TryGet_OnEmpty_ReturnsNullPointerAndNullEtag()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var (pointer, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId);
|
||||
Assert.Null(pointer);
|
||||
Assert.Null(etag);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pointer_Insert_RoundTrips_LatestHistoryRowKeyAndOffboardCount()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var pointer = new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = HistoryRowKey,
|
||||
LatestStatus = "Initiated",
|
||||
LatestUpdatedAt = DateTime.UtcNow,
|
||||
OffboardCount = 1,
|
||||
};
|
||||
|
||||
await harness.Sut.InsertByTenantPointerAsync(pointer);
|
||||
|
||||
var (fetched, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId);
|
||||
Assert.NotNull(fetched);
|
||||
Assert.NotNull(etag);
|
||||
Assert.Equal(pointer.LatestHistoryRowKey, fetched!.LatestHistoryRowKey);
|
||||
Assert.Equal("Initiated", fetched.LatestStatus);
|
||||
Assert.Equal(1, fetched.OffboardCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pointer_Insert_OnDuplicate_Throws409()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var pointer = BuildPointer(latestStatus: "Initiated", offboardCount: 1);
|
||||
await harness.Sut.InsertByTenantPointerAsync(pointer);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RequestFailedException>(
|
||||
() => harness.Sut.InsertByTenantPointerAsync(pointer));
|
||||
Assert.Equal(409, ex.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pointer_UpdateWithEtag_MatchingEtag_Replaces()
|
||||
{
|
||||
var harness = new Harness();
|
||||
await harness.Sut.InsertByTenantPointerAsync(BuildPointer(latestStatus: "Initiated", offboardCount: 1));
|
||||
|
||||
var (current, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId);
|
||||
Assert.NotNull(current);
|
||||
current!.LatestStatus = "InProgress";
|
||||
current.OffboardCount = 2;
|
||||
current.LatestUpdatedAt = DateTime.UtcNow;
|
||||
|
||||
await harness.Sut.UpdateByTenantPointerWithEtagAsync(current, etag!);
|
||||
|
||||
var (refetched, _) = await harness.Sut.TryGetByTenantPointerAsync(TenantId);
|
||||
Assert.Equal("InProgress", refetched!.LatestStatus);
|
||||
Assert.Equal(2, refetched.OffboardCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pointer_UpdateWithEtag_StaleEtag_Throws412()
|
||||
{
|
||||
// Plan §4.4 contract: stale ETag → 412 so the caller's bounded retry loop fires.
|
||||
// The blind Upsert path was removed because it could clobber a concurrent worker.
|
||||
var harness = new Harness();
|
||||
await harness.Sut.InsertByTenantPointerAsync(BuildPointer(latestStatus: "Initiated"));
|
||||
|
||||
var stale = BuildPointer(latestStatus: "InProgress", offboardCount: 99);
|
||||
var ex = await Assert.ThrowsAsync<RequestFailedException>(
|
||||
() => harness.Sut.UpdateByTenantPointerWithEtagAsync(stale, ifMatchEtag: "\"0xSTALE\""));
|
||||
Assert.Equal(412, ex.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pointer_UpdateWithEtag_RequiresIfMatchEtag()
|
||||
{
|
||||
var harness = new Harness();
|
||||
await Assert.ThrowsAsync<ArgumentException>(
|
||||
() => harness.Sut.UpdateByTenantPointerWithEtagAsync(BuildPointer(), ""));
|
||||
}
|
||||
|
||||
private static OffboardingByTenantPointer BuildPointer(
|
||||
string latestStatus = "Initiated",
|
||||
int offboardCount = 1) => new()
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = HistoryRowKey,
|
||||
LatestStatus = latestStatus,
|
||||
LatestUpdatedAt = DateTime.UtcNow,
|
||||
OffboardCount = offboardCount,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task Marker_RejectsWrongPartitionKey()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var bad = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = "WrongPK",
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = HistoryRowKey,
|
||||
InitiatedAt = DateTime.UtcNow,
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
};
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.UpsertMarkerAsync(bad));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task History_RejectsWrongPartitionKey()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var bad = new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = "WrongPK",
|
||||
RowKey = HistoryRowKey,
|
||||
TenantId = TenantId,
|
||||
InitiatedBy = "alice@contoso.invalid",
|
||||
OffboardedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.UpsertHistoryAsync(bad));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pointer_RejectsWrongPartitionKey()
|
||||
{
|
||||
var harness = new Harness();
|
||||
var bad = new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = "WrongPK",
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = HistoryRowKey,
|
||||
LatestStatus = "Initiated",
|
||||
LatestUpdatedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.InsertByTenantPointerAsync(bad));
|
||||
}
|
||||
|
||||
// ── Harness — in-memory dictionary of (PK,RK) → TableEntity backing the mock ──
|
||||
//
|
||||
// ETag-aware: every successful insert/upsert/update stamps a fresh ETag onto the stored
|
||||
// entity so Get reads return it and Update can enforce If-Match. Mirrors enough of Azure
|
||||
// Table Storage's behaviour to exercise the §4.4 Pointer CAS-Loop honestly.
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public TableOffboardingAuditRepository Sut { get; }
|
||||
|
||||
private readonly Dictionary<(string Pk, string Rk), TableEntity> _store = new();
|
||||
private int _etagCounter;
|
||||
|
||||
public Harness(
|
||||
RequestFailedException? throwOnGet = null,
|
||||
RequestFailedException? throwOnDelete = null)
|
||||
{
|
||||
var mockTableClient = new Mock<TableClient>();
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.UpsertEntityAsync(
|
||||
It.IsAny<TableEntity>(),
|
||||
It.IsAny<TableUpdateMode>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns<TableEntity, TableUpdateMode, CancellationToken>((e, _, _) =>
|
||||
{
|
||||
e.ETag = StampEtag();
|
||||
_store[(e.PartitionKey, e.RowKey)] = e;
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.AddEntityAsync(
|
||||
It.IsAny<TableEntity>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns<TableEntity, CancellationToken>((e, _) =>
|
||||
{
|
||||
if (_store.ContainsKey((e.PartitionKey, e.RowKey)))
|
||||
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
|
||||
e.ETag = StampEtag();
|
||||
_store[(e.PartitionKey, e.RowKey)] = e;
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.UpdateEntityAsync(
|
||||
It.IsAny<TableEntity>(),
|
||||
It.IsAny<ETag>(),
|
||||
It.IsAny<TableUpdateMode>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns<TableEntity, ETag, TableUpdateMode, CancellationToken>((e, ifMatch, _, _) =>
|
||||
{
|
||||
if (!_store.TryGetValue((e.PartitionKey, e.RowKey), out var stored))
|
||||
throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null);
|
||||
if (ifMatch != ETag.All && ifMatch != stored.ETag)
|
||||
throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null);
|
||||
e.ETag = StampEtag();
|
||||
_store[(e.PartitionKey, e.RowKey)] = e;
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.GetEntityAsync<TableEntity>(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<IEnumerable<string>>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, IEnumerable<string>, CancellationToken>((pk, rk, _, _) =>
|
||||
{
|
||||
if (throwOnGet != null) throw throwOnGet;
|
||||
if (!_store.TryGetValue((pk, rk), out var entity))
|
||||
throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null);
|
||||
return Task.FromResult(Response.FromValue(entity, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
mockTableClient
|
||||
.Setup(c => c.DeleteEntityAsync(
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<ETag>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, ETag, CancellationToken>((pk, rk, _, _) =>
|
||||
{
|
||||
if (throwOnDelete != null) throw throwOnDelete;
|
||||
_store.Remove((pk, rk));
|
||||
return Task.FromResult(new Mock<Response>().Object);
|
||||
});
|
||||
|
||||
var mockServiceClient = new Mock<TableServiceClient>();
|
||||
mockServiceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(mockTableClient.Object);
|
||||
|
||||
var storage = new TableStorageService(mockServiceClient.Object, NullLogger<TableStorageService>.Instance);
|
||||
Sut = new TableOffboardingAuditRepository(storage, NullLogger<TableOffboardingAuditRepository>.Instance);
|
||||
}
|
||||
|
||||
private ETag StampEtag() => new($"\"0xFAKE_{++_etagCounter}\"");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,977 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Admin;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Functions.Tests.Offboarding;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
// Capture-helper for the recording enqueuer used in Finding-1 tests below.
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// PR3-side unit tests for the rewritten <see cref="TenantOffboardFunction"/>. The HTTP
|
||||
/// entrypoint itself lives on top of <c>HttpRequestData</c> (abstract, hard to fake without
|
||||
/// a real Functions host) so these tests focus on the function's CAS-loop helper. The
|
||||
/// end-to-end happy/idempotent/race paths are covered by integration tests against the
|
||||
/// real Functions host on dev tenants (see §10.3 smoketest in the plan).
|
||||
/// </summary>
|
||||
public sealed class TenantOffboardFunctionTests
|
||||
{
|
||||
private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
|
||||
|
||||
private static TenantOffboardFunction Build(IOffboardingAuditRepository repo)
|
||||
=> Build(repo, Mock.Of<ITenantOffboardingEnqueuer>());
|
||||
|
||||
private static TenantOffboardFunction Build(
|
||||
IOffboardingAuditRepository repo, ITenantOffboardingEnqueuer enqueuer)
|
||||
{
|
||||
// Resume-path tests (UpsertPointer / Resume_*) exercise just the CAS-loop +
|
||||
// enqueuer wiring and don't care about the Disabled-gate. Stub IConfigRepository
|
||||
// so EnsureTenantDisabledAsync's idempotent no-op path fires (returns an existing
|
||||
// row already marked Disabled with the offboarding reason → no Save call).
|
||||
//
|
||||
// LastUpdated is set FAR in the past so the tombstone-deadline (LastUpdated +
|
||||
// DrainBarrier) doesn't dominate the max() computation in the resume-path. That
|
||||
// means the existing history-based-deadline tests still observe history-derived
|
||||
// delays. Tests that exercise the LastUpdated-dominates path stand up their own
|
||||
// configRepoMock with a recent LastUpdated.
|
||||
var configRepoMock = new Mock<IConfigRepository>();
|
||||
var alreadyDisabled = TenantConfiguration.CreateDefault(TenantId);
|
||||
alreadyDisabled.Disabled = true;
|
||||
alreadyDisabled.DisabledReason = "Offboarding in progress";
|
||||
alreadyDisabled.LastUpdated = DateTime.UtcNow.AddDays(-30);
|
||||
configRepoMock
|
||||
.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
|
||||
.ReturnsAsync(alreadyDisabled);
|
||||
|
||||
var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache(
|
||||
new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions());
|
||||
var service = new TenantConfigurationService(
|
||||
configRepoMock.Object,
|
||||
NullLogger<TenantConfigurationService>.Instance,
|
||||
cache);
|
||||
|
||||
return new TenantOffboardFunction(
|
||||
logger: NullLogger<TenantOffboardFunction>.Instance,
|
||||
configRepo: configRepoMock.Object,
|
||||
tenantConfigService: service,
|
||||
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
|
||||
offboardingRepo: repo,
|
||||
offboardingEnqueuer: enqueuer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test-side enqueuer recording every call. Production enqueuer is sealed + needs
|
||||
/// configuration to ctor, so this fake lets the tests assert "did Phase 1 actually
|
||||
/// hand off to the worker?" — the question Finding 1 says we previously got wrong.
|
||||
/// </summary>
|
||||
private sealed class RecordingEnqueuer : ITenantOffboardingEnqueuer
|
||||
{
|
||||
public readonly List<TenantOffboardingEnvelope> Sent = new();
|
||||
public readonly List<TimeSpan?> VisibilityDelays = new();
|
||||
public Exception? ThrowOnEnqueue { get; set; }
|
||||
|
||||
public Task EnqueueAsync(TenantOffboardingEnvelope envelope, TimeSpan? visibilityDelay = null, CancellationToken ct = default)
|
||||
{
|
||||
if (ThrowOnEnqueue is { } ex) throw ex;
|
||||
Sent.Add(envelope);
|
||||
VisibilityDelays.Add(visibilityDelay);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
// Build helper exposing IConfigRepository + TenantConfigurationService for the
|
||||
// EnsureTenantDisabledAsync tests. The repo is a Mock so we can pin Save calls; the
|
||||
// service uses a real MemoryCache so InvalidateCache(tenantId) is observable via a
|
||||
// pre-seeded cache entry.
|
||||
private static (TenantOffboardFunction Sut, Mock<IConfigRepository> ConfigRepo, Microsoft.Extensions.Caching.Memory.IMemoryCache Cache, TenantConfigurationService Service)
|
||||
BuildWithConfigService()
|
||||
{
|
||||
var configRepoMock = new Mock<IConfigRepository>();
|
||||
var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache(
|
||||
new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions());
|
||||
var service = new TenantConfigurationService(
|
||||
configRepoMock.Object,
|
||||
NullLogger<TenantConfigurationService>.Instance,
|
||||
cache);
|
||||
|
||||
var sut = new TenantOffboardFunction(
|
||||
logger: NullLogger<TenantOffboardFunction>.Instance,
|
||||
configRepo: configRepoMock.Object,
|
||||
tenantConfigService: service,
|
||||
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
|
||||
offboardingRepo: new FakeOffboardingAuditRepository(),
|
||||
offboardingEnqueuer: new RecordingEnqueuer());
|
||||
|
||||
return (sut, configRepoMock, cache, service);
|
||||
}
|
||||
|
||||
// ── UpsertPointerWithCasAsync ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertPointer_FreshTenant_Inserts_OffboardCount_1()
|
||||
{
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
var sut = Build(repo);
|
||||
var historyRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
|
||||
|
||||
await sut.UpsertPointerWithCasAsync(TenantId, historyRowKey, DateTime.UtcNow);
|
||||
|
||||
Assert.True(repo.Pointers.TryGetValue(TenantId, out var entry));
|
||||
Assert.Equal(historyRowKey, entry.Pointer.LatestHistoryRowKey);
|
||||
Assert.Equal("Initiated", entry.Pointer.LatestStatus);
|
||||
Assert.Equal(1, entry.Pointer.OffboardCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertPointer_ExistingTenant_IncrementsOffboardCount()
|
||||
{
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
await repo.InsertByTenantPointerAsync(new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = "20260101000000000_" + TenantId,
|
||||
LatestStatus = "Completed",
|
||||
LatestUpdatedAt = DateTime.UtcNow.AddDays(-30),
|
||||
OffboardCount = 2,
|
||||
});
|
||||
|
||||
var sut = Build(repo);
|
||||
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
|
||||
|
||||
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow);
|
||||
|
||||
Assert.Equal(newHistoryRowKey, repo.Pointers[TenantId].Pointer.LatestHistoryRowKey);
|
||||
Assert.Equal("Initiated", repo.Pointers[TenantId].Pointer.LatestStatus);
|
||||
Assert.Equal(3, repo.Pointers[TenantId].Pointer.OffboardCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertPointer_409OnInsert_FallsBackToEtagUpdate()
|
||||
{
|
||||
// First TryGet returns null, but InsertByTenantPointerAsync throws 409 because
|
||||
// a parallel writer inserted between our read and write. The CAS-loop must
|
||||
// re-read and update with the now-existing pointer's ETag.
|
||||
var repoMock = new Mock<IOffboardingAuditRepository>();
|
||||
var existingPointer = new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = "20260101000000000_" + TenantId,
|
||||
LatestStatus = "Completed",
|
||||
OffboardCount = 1,
|
||||
};
|
||||
|
||||
var attempt = 0;
|
||||
repoMock
|
||||
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.Returns(() =>
|
||||
{
|
||||
attempt++;
|
||||
return attempt == 1
|
||||
? Task.FromResult<(OffboardingByTenantPointer?, string?)>((null, null))
|
||||
: Task.FromResult<(OffboardingByTenantPointer?, string?)>((existingPointer, "\"etag-1\""));
|
||||
});
|
||||
|
||||
repoMock
|
||||
.Setup(r => r.InsertByTenantPointerAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null));
|
||||
|
||||
OffboardingByTenantPointer? updatedPointer = null;
|
||||
repoMock
|
||||
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), "\"etag-1\"", It.IsAny<CancellationToken>()))
|
||||
.Returns<OffboardingByTenantPointer, string, CancellationToken>((p, _, _) =>
|
||||
{
|
||||
updatedPointer = p;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
var sut = Build(repoMock.Object);
|
||||
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
|
||||
|
||||
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow);
|
||||
|
||||
Assert.NotNull(updatedPointer);
|
||||
Assert.Equal(newHistoryRowKey, updatedPointer!.LatestHistoryRowKey);
|
||||
Assert.Equal(2, updatedPointer.OffboardCount);
|
||||
repoMock.Verify(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), "\"etag-1\"", It.IsAny<CancellationToken>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertPointer_412OnUpdate_RetriesAndSucceeds()
|
||||
{
|
||||
// Concurrent writer mutates the pointer between our read and our update. CAS-loop
|
||||
// must re-read (getting the fresh ETag) and try once more.
|
||||
var repoMock = new Mock<IOffboardingAuditRepository>();
|
||||
|
||||
var freshState = new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = "20260101000000000_" + TenantId,
|
||||
LatestStatus = "Completed",
|
||||
OffboardCount = 1,
|
||||
};
|
||||
|
||||
var readEtags = new[] { "\"etag-old\"", "\"etag-new\"" };
|
||||
var readIdx = 0;
|
||||
repoMock
|
||||
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.Returns(() =>
|
||||
{
|
||||
var etag = readEtags[Math.Min(readIdx, readEtags.Length - 1)];
|
||||
readIdx++;
|
||||
return Task.FromResult<(OffboardingByTenantPointer?, string?)>((freshState, etag));
|
||||
});
|
||||
|
||||
var updateCalls = 0;
|
||||
repoMock
|
||||
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<OffboardingByTenantPointer, string, CancellationToken>((_, etag, _) =>
|
||||
{
|
||||
updateCalls++;
|
||||
if (etag == "\"etag-old\"")
|
||||
{
|
||||
throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null);
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
var sut = Build(repoMock.Object);
|
||||
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
|
||||
|
||||
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow);
|
||||
|
||||
Assert.Equal(2, updateCalls);
|
||||
Assert.Equal(2, readIdx);
|
||||
}
|
||||
|
||||
// ── ResumeExistingMarkerAsync (PR3 Review Finding 1) ─────────────────────
|
||||
//
|
||||
// The previous implementation returned 200 from the idempotency path UNCONDITIONALLY.
|
||||
// If the very first enqueue failed (transient queue outage), the marker was committed
|
||||
// and the queue was empty: every subsequent re-click took the idempotency-200 branch
|
||||
// and never re-enqueued, leaving the tenant stuck Initiated forever. These tests pin
|
||||
// the new defensive-re-enqueue behaviour: Initiated/InProgress → re-enqueue; Completed
|
||||
// → no re-enqueue (worker is done); Failed → no re-enqueue (operator action required).
|
||||
|
||||
[Theory]
|
||||
[InlineData("Initiated")]
|
||||
[InlineData("InProgress")]
|
||||
public async Task Resume_InFlightStatus_DefensivelyReEnqueues(string status)
|
||||
{
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
var historyRowKey = "20260518091523123_" + TenantId;
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = historyRowKey,
|
||||
InitiatedAt = DateTime.UtcNow,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = status,
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "bob@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
|
||||
Assert.True(outcome.ReEnqueued);
|
||||
Assert.Single(enqueuer.Sent);
|
||||
Assert.Equal(historyRowKey, enqueuer.Sent[0].HistoryRowKey);
|
||||
Assert.Equal(TenantId, enqueuer.Sent[0].TenantId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_Completed_NoReEnqueue_ReturnsIdempotentOk()
|
||||
{
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
RowKey = TenantId, TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
|
||||
Status = "Completed",
|
||||
CompletedAt = DateTime.UtcNow.AddMinutes(-5),
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.IdempotentCompleted, outcome.Kind);
|
||||
Assert.False(outcome.ReEnqueued);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_Failed_NoReEnqueue_SurfacesPhaseInMessage()
|
||||
{
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
RowKey = TenantId, TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
|
||||
Status = "Failed",
|
||||
FailedAt = DateTime.UtcNow.AddMinutes(-10),
|
||||
FailedPhase = "drain_timeout",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.IdempotentFailed, outcome.Kind);
|
||||
Assert.False(outcome.ReEnqueued);
|
||||
Assert.Contains("drain_timeout", outcome.Message);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_InFlight_EnqueueThrows_SurfacesReEnqueueFailed()
|
||||
{
|
||||
// The defensive re-enqueue must surface its failure (500) so the caller knows
|
||||
// the resume did NOT take effect. The marker stays Initiated; the next click
|
||||
// will trigger another resume attempt — that's the operator-visible recovery
|
||||
// mechanism that previously didn't exist (stuck-tenant bug).
|
||||
var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") };
|
||||
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
RowKey = TenantId, TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
|
||||
Status = "Initiated",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.ReEnqueueFailed, outcome.Kind);
|
||||
Assert.False(outcome.ReEnqueued);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_409Race_StillReEnqueues_BecauseWinnerMayHaveFailed()
|
||||
{
|
||||
// The 409-race path resolves to the winning marker and routes through the same
|
||||
// resume code. We can't know whether the winner's enqueue succeeded, so we
|
||||
// must re-enqueue defensively — same Finding-1 contract as the plain idempotency
|
||||
// re-click path.
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
RowKey = TenantId, TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
|
||||
Status = "Initiated",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: true);
|
||||
|
||||
Assert.True(outcome.ReEnqueued);
|
||||
Assert.Single(enqueuer.Sent);
|
||||
Assert.Contains("race", outcome.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UpsertPointer_CasExhausted_Throws()
|
||||
{
|
||||
// Every UpdateWithEtag returns 412 — the bounded retry-loop must surface this
|
||||
// rather than retry indefinitely.
|
||||
var repoMock = new Mock<IOffboardingAuditRepository>();
|
||||
var freshState = new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
LatestHistoryRowKey = "20260101000000000_" + TenantId,
|
||||
LatestStatus = "Completed",
|
||||
OffboardCount = 1,
|
||||
};
|
||||
|
||||
repoMock
|
||||
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((freshState, "\"some-etag\""));
|
||||
|
||||
repoMock
|
||||
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null));
|
||||
|
||||
var sut = Build(repoMock.Object);
|
||||
var historyRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
sut.UpsertPointerWithCasAsync(TenantId, historyRowKey, DateTime.UtcNow));
|
||||
}
|
||||
|
||||
// ── Cache-Drain-Barrier (plan v2 §2.2 / §2.3) ─────────────────────────────
|
||||
//
|
||||
// The DrainBarrier (hardcoded 6min) prevents Phase 2 from starting before all
|
||||
// function-host instances drain their TenantConfigurationService cache. Phase 1
|
||||
// enqueues with visibilityDelay=DrainBarrier so the worker only sees the message
|
||||
// after the barrier elapses. A resume-click reads the stored EarliestProcessingAt
|
||||
// and re-enqueues with the REMAINING delay so the barrier does not reset.
|
||||
|
||||
[Fact]
|
||||
public void DrainBarrier_HardcodedAt6Minutes()
|
||||
{
|
||||
// Plan v2 §2.2: 5min TenantConfigurationService.CacheDuration + 1min safety buffer.
|
||||
Assert.Equal(TimeSpan.FromMinutes(6), TenantOffboardFunction.DrainBarrier);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_InFlight_UsesRemainingBarrier_NotFullBarrier()
|
||||
{
|
||||
// Re-click 2 min into the 6-min barrier → re-enqueue must carry ~4min visibility delay,
|
||||
// not a fresh 6min. Otherwise each re-click would reset the cache-drain window.
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var initiatedAt = DateTime.UtcNow.AddMinutes(-2);
|
||||
var earliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // ~+4min from now
|
||||
var historyRowKey = "20260518091523123_" + TenantId;
|
||||
|
||||
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = historyRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.com",
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
OffboardedAt = initiatedAt,
|
||||
EarliestProcessingAt = earliestProcessingAt,
|
||||
Status = "Initiated",
|
||||
});
|
||||
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = historyRowKey,
|
||||
InitiatedAt = initiatedAt,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = "Initiated",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
|
||||
Assert.Single(enqueuer.VisibilityDelays);
|
||||
var delay = enqueuer.VisibilityDelays[0];
|
||||
Assert.NotNull(delay);
|
||||
// Allow a few seconds of test-runtime slack — clock advanced between Setup and Act.
|
||||
Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.5));
|
||||
Assert.Equal(earliestProcessingAt, outcome.EarliestProcessingAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_InFlight_PastBarrier_UsesZeroDelay()
|
||||
{
|
||||
// Re-click 8 min into a 6-min barrier → barrier already elapsed → re-enqueue with
|
||||
// zero visibility delay so the worker picks up immediately.
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var initiatedAt = DateTime.UtcNow.AddMinutes(-8);
|
||||
var earliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // 2 min ago
|
||||
var historyRowKey = "20260518091523123_" + TenantId;
|
||||
|
||||
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = historyRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.com",
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
OffboardedAt = initiatedAt,
|
||||
EarliestProcessingAt = earliestProcessingAt,
|
||||
Status = "InProgress",
|
||||
});
|
||||
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = historyRowKey,
|
||||
InitiatedAt = initiatedAt,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = "InProgress",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
|
||||
Assert.Single(enqueuer.VisibilityDelays);
|
||||
Assert.Equal(TimeSpan.Zero, enqueuer.VisibilityDelays[0]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_InFlight_MissingHistoryRow_UsesTombstoneLastUpdatedAsDeadlineAnchor()
|
||||
{
|
||||
// The marker references a History row that does not exist (corruption / manual
|
||||
// delete). Without history.EarliestProcessingAt to read, the resume falls back
|
||||
// to tombstone.LastUpdated + DrainBarrier as the authoritative drain-deadline.
|
||||
// The Build-helper's default tombstone has LastUpdated = 30 days ago → the
|
||||
// tombstone-derived deadline is far in the past → drain barrier already expired
|
||||
// → visibility-delay = Zero (worker picks immediately). That is the CORRECT
|
||||
// outcome: if the tombstone has been sitting for 30 days, every cache has been
|
||||
// through the 5-min TTL many times already.
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = "20260518091523123_" + TenantId,
|
||||
InitiatedAt = DateTime.UtcNow,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = "Initiated",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
|
||||
Assert.Single(enqueuer.VisibilityDelays);
|
||||
// 30-day-old tombstone → drain barrier long expired → 0 delay is safe.
|
||||
Assert.Equal(TimeSpan.Zero, enqueuer.VisibilityDelays[0]);
|
||||
}
|
||||
|
||||
// ── EnsureTenantDisabledAsync (PR3.B Review Finding 1 — fail-loud Disabled-gate) ──
|
||||
//
|
||||
// Without Disabled=true, the cache-drain barrier guards nothing: warm function-hosts
|
||||
// would keep accepting agent traffic for a tenant whose Phase 2 is about to wipe
|
||||
// their data. Every initial enqueue + every resume re-enqueue MUST commit this gate.
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_FlipsDisabled_AndInvalidatesCache()
|
||||
{
|
||||
var (sut, configRepo, cache, _) = BuildWithConfigService();
|
||||
var existing = TenantConfiguration.CreateDefault(TenantId);
|
||||
existing.Disabled = false;
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing);
|
||||
|
||||
TenantConfiguration? saved = null;
|
||||
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
|
||||
.Callback<TenantConfiguration>(c => saved = c)
|
||||
.ReturnsAsync(true);
|
||||
|
||||
// Seed cache with stale "Disabled=false" entry so we can prove invalidation.
|
||||
cache.Set($"tenant-config:{TenantId}", existing);
|
||||
|
||||
await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
|
||||
|
||||
Assert.NotNull(saved);
|
||||
Assert.True(saved!.Disabled);
|
||||
Assert.Equal("Offboarding in progress", saved.DisabledReason);
|
||||
Assert.Null(saved.DisabledUntil);
|
||||
Assert.Equal("alice@contoso.com", saved.UpdatedBy);
|
||||
Assert.False(cache.TryGetValue($"tenant-config:{TenantId}", out _),
|
||||
"cache entry must be invalidated so warm reads pick up the new Disabled=true on next access");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_AlreadyDisabledWithSameReason_DoesNotRewrite()
|
||||
{
|
||||
var (sut, configRepo, _, _) = BuildWithConfigService();
|
||||
var existing = TenantConfiguration.CreateDefault(TenantId);
|
||||
existing.Disabled = true;
|
||||
existing.DisabledReason = "Offboarding in progress";
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing);
|
||||
|
||||
await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
|
||||
|
||||
// Idempotent: no Save thrash on resume-clicks while the marker is already in flight.
|
||||
configRepo.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_StorageThrowsOnRead_PropagatesAsExpected()
|
||||
{
|
||||
// Caller (initial enqueue + resume re-enqueue) MUST treat this as fail-loud and
|
||||
// skip the enqueue. We just assert the exception propagates — the caller's
|
||||
// try/catch + 500-return is exercised by integration tests.
|
||||
var (sut, configRepo, _, _) = BuildWithConfigService();
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ThrowsAsync(new InvalidOperationException("storage unavailable"));
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_StorageThrowsOnSave_PropagatesAsExpected()
|
||||
{
|
||||
var (sut, configRepo, _, _) = BuildWithConfigService();
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ReturnsAsync(TenantConfiguration.CreateDefault(TenantId));
|
||||
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
|
||||
.ThrowsAsync(new InvalidOperationException("storage save failed"));
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_NoConfigRow_CreatesFreshTombstone()
|
||||
{
|
||||
// PR3.B-revised Finding 3: missing row used to log+continue, leaving AuthFunction's
|
||||
// auto-create-default free to spawn a Disabled=false row on the next /api/auth/me.
|
||||
// The fix creates a fresh CreateDefault row with Disabled=true so the auth-gate is
|
||||
// committed even in this edge case.
|
||||
var (sut, configRepo, _, _) = BuildWithConfigService();
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ReturnsAsync((TenantConfiguration?)null);
|
||||
|
||||
TenantConfiguration? saved = null;
|
||||
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
|
||||
.Callback<TenantConfiguration>(c => saved = c)
|
||||
.ReturnsAsync(true);
|
||||
|
||||
await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
|
||||
|
||||
Assert.NotNull(saved);
|
||||
Assert.Equal(TenantId, saved!.TenantId);
|
||||
Assert.True(saved.Disabled, "fresh tombstone must carry Disabled=true");
|
||||
Assert.Equal("Offboarding in progress", saved.DisabledReason);
|
||||
Assert.Equal("alice@contoso.com", saved.UpdatedBy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_SaveReturnsFalse_ThrowsAsHardFail()
|
||||
{
|
||||
// PR3.B-revised Finding 2: TableConfigRepository.SaveTenantConfigurationAsync
|
||||
// returns false on transient storage failure (it does NOT throw). Ignoring that
|
||||
// return-value would let the caller proceed to enqueue without the auth-gate
|
||||
// committed. Turn it into a hard exception so the offboard is NOT enqueued.
|
||||
var (sut, configRepo, _, _) = BuildWithConfigService();
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ReturnsAsync(TenantConfiguration.CreateDefault(TenantId));
|
||||
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
|
||||
.ReturnsAsync(false); // transient storage rejection — NOT a throw.
|
||||
|
||||
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
|
||||
Assert.Contains("Disabled-gate NOT committed", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_NoConfigRow_SaveReturnsFalse_ThrowsAsHardFail()
|
||||
{
|
||||
// Combined edge case: missing row → we'd create a fresh tombstone → Save fails →
|
||||
// hard fail (otherwise the caller would proceed without an auth-gate).
|
||||
var (sut, configRepo, _, _) = BuildWithConfigService();
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
|
||||
.ReturnsAsync((TenantConfiguration?)null);
|
||||
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
|
||||
.ReturnsAsync(false);
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_AlreadyTombstone_ReturnsAlreadyTombstone()
|
||||
{
|
||||
var (sut, configRepo, _, _) = BuildWithConfigService();
|
||||
var tombstone = TenantConfiguration.CreateDefault(TenantId);
|
||||
tombstone.Disabled = true;
|
||||
tombstone.DisabledReason = "Offboarding in progress";
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(tombstone);
|
||||
|
||||
var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
|
||||
|
||||
Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.AlreadyTombstone, result.Outcome);
|
||||
configRepo.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnsureTenantDisabled_FlipsDisabled_ReturnsWroteNewTombstone()
|
||||
{
|
||||
var (sut, configRepo, _, _) = BuildWithConfigService();
|
||||
var existing = TenantConfiguration.CreateDefault(TenantId);
|
||||
existing.Disabled = false;
|
||||
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing);
|
||||
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
|
||||
.ReturnsAsync(true);
|
||||
|
||||
var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
|
||||
|
||||
Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.WroteNewTombstone, result.Outcome);
|
||||
}
|
||||
|
||||
// ── Resume-path cache-drain bypass fix (Codex review) ────────────────────
|
||||
//
|
||||
// Scenario: initial attempt commits Marker/History/Pointer (History.EarliestProcessingAt
|
||||
// = T+6min) but then EnsureTenantDisabledAsync FAILS → 500 returned, no enqueue. The
|
||||
// admin re-clicks 10 minutes later. The History row still carries the now-past
|
||||
// EarliestProcessingAt = T+6min. Without the fix, the resume would naively use the
|
||||
// remaining-from-history delta and arrive at Zero → enqueue with no visibility-delay
|
||||
// → worker picks immediately → but Disabled=true was JUST written for the first time
|
||||
// → warm function-host caches still hold Disabled=false → cache-drain barrier violated.
|
||||
//
|
||||
// The fix anchors on tombstone.LastUpdated + DrainBarrier so the deadline is
|
||||
// authoritative from the same atomic write as the Disabled-flip.
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_FirstAttemptFailedBeforeDisabledWrite_UsesFullBarrierAndPatchesHistory()
|
||||
{
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
|
||||
var initiatedAt = DateTime.UtcNow.AddMinutes(-10); // first attempt 10min ago
|
||||
var staleEarliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // ~4 min in the past
|
||||
var historyRowKey = "20260518091523123_" + TenantId;
|
||||
|
||||
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = historyRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.com",
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
OffboardedAt = initiatedAt,
|
||||
EarliestProcessingAt = staleEarliestProcessingAt,
|
||||
Status = "Initiated",
|
||||
});
|
||||
|
||||
// Build SUT where the first attempt FAILED at EnsureTenantDisabledAsync: Config
|
||||
// exists but is NOT yet the offboarding tombstone (Disabled=false). The resume's
|
||||
// EnsureTenantDisabledAsync will write it now → WroteNewTombstone.
|
||||
var configRepoMock = new Mock<IConfigRepository>();
|
||||
var notYetDisabled = TenantConfiguration.CreateDefault(TenantId);
|
||||
notYetDisabled.Disabled = false;
|
||||
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
|
||||
.ReturnsAsync(notYetDisabled);
|
||||
configRepoMock.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
|
||||
.ReturnsAsync(true);
|
||||
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var tenantConfigService = new TenantConfigurationService(
|
||||
configRepoMock.Object,
|
||||
NullLogger<TenantConfigurationService>.Instance,
|
||||
cache);
|
||||
|
||||
var sut = new TenantOffboardFunction(
|
||||
logger: NullLogger<TenantOffboardFunction>.Instance,
|
||||
configRepo: configRepoMock.Object,
|
||||
tenantConfigService: tenantConfigService,
|
||||
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
|
||||
offboardingRepo: repo,
|
||||
offboardingEnqueuer: enqueuer);
|
||||
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = historyRowKey,
|
||||
InitiatedAt = initiatedAt,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = "Initiated",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
|
||||
Assert.Single(enqueuer.VisibilityDelays);
|
||||
|
||||
// CRITICAL: visibility-delay must be the full DrainBarrier (6 min), NOT Zero. The
|
||||
// History row's stale EarliestProcessingAt must NOT be honoured when Disabled was
|
||||
// just written for the first time. Allow a few-ms slack because the delay is
|
||||
// computed as (effectiveDeadline - DateTime.UtcNow), so a moment passes between
|
||||
// the LastUpdated stamp and the delay computation.
|
||||
var actualDelay = enqueuer.VisibilityDelays[0]!.Value;
|
||||
Assert.InRange(actualDelay, TenantOffboardFunction.DrainBarrier - TimeSpan.FromSeconds(1), TenantOffboardFunction.DrainBarrier);
|
||||
|
||||
// The History row must have been patched so further re-clicks see the new deadline.
|
||||
Assert.True(repo.History.ContainsKey(historyRowKey));
|
||||
var patchedHistory = repo.History[historyRowKey];
|
||||
Assert.NotNull(patchedHistory.EarliestProcessingAt);
|
||||
Assert.True(patchedHistory.EarliestProcessingAt > DateTime.UtcNow.AddMinutes(5),
|
||||
$"EarliestProcessingAt must be roughly now+6min after the patch, got {patchedHistory.EarliestProcessingAt:O}");
|
||||
|
||||
// ResumeOutcome's EarliestProcessingAt is the NEW deadline (UI countdown source of truth).
|
||||
Assert.NotNull(outcome.EarliestProcessingAt);
|
||||
Assert.True(outcome.EarliestProcessingAt > DateTime.UtcNow.AddMinutes(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_HistoryPatchFails_SecondResumeStillRespectsBarrierViaTombstoneLastUpdated()
|
||||
{
|
||||
// The exact Codex Finding scenario: first resume writes tombstone successfully
|
||||
// but the History.EarliestProcessingAt patch fails (transient). A second resume
|
||||
// arriving moments later reads History with the OLD stale EarliestProcessingAt
|
||||
// but sees the tombstone (AlreadyTombstone path). The deadline computation must
|
||||
// still respect the cache-drain barrier via tombstone.LastUpdated + DrainBarrier.
|
||||
//
|
||||
// Without the LastUpdated-anchor fix, the second resume would compute
|
||||
// visibility-delay = max(0, stale-history-deadline - now) = 0 → worker picks
|
||||
// before the cache-drain barrier from the tombstone-write completes.
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
|
||||
var firstAttempt = DateTime.UtcNow.AddMinutes(-10);
|
||||
var staleHistoryDeadline = firstAttempt + TenantOffboardFunction.DrainBarrier; // ~now-4min
|
||||
var historyRowKey = "20260518091523123_" + TenantId;
|
||||
|
||||
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = historyRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.com",
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
OffboardedAt = firstAttempt,
|
||||
EarliestProcessingAt = staleHistoryDeadline, // STALE — patch failed before
|
||||
Status = "Initiated",
|
||||
});
|
||||
|
||||
// Tombstone-already-in-place from the prior (first) resume; LastUpdated reflects
|
||||
// when Disabled was committed in that first resume = recently (now). The History
|
||||
// patch had failed, so EarliestProcessingAt is still the stale firstAttempt+6min.
|
||||
var configRepoMock = new Mock<IConfigRepository>();
|
||||
var tombstone = TenantConfiguration.CreateDefault(TenantId);
|
||||
tombstone.Disabled = true;
|
||||
tombstone.DisabledReason = "Offboarding in progress";
|
||||
tombstone.LastUpdated = DateTime.UtcNow.AddSeconds(-1); // freshly written ~1s ago
|
||||
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
|
||||
.ReturnsAsync(tombstone);
|
||||
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var tenantConfigService = new TenantConfigurationService(
|
||||
configRepoMock.Object,
|
||||
NullLogger<TenantConfigurationService>.Instance,
|
||||
cache);
|
||||
|
||||
var sut = new TenantOffboardFunction(
|
||||
logger: NullLogger<TenantOffboardFunction>.Instance,
|
||||
configRepo: configRepoMock.Object,
|
||||
tenantConfigService: tenantConfigService,
|
||||
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
|
||||
offboardingRepo: repo,
|
||||
offboardingEnqueuer: enqueuer);
|
||||
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = historyRowKey,
|
||||
InitiatedAt = firstAttempt,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = "Initiated",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
|
||||
Assert.Single(enqueuer.VisibilityDelays);
|
||||
|
||||
// SAFETY: the visibility-delay must come from tombstone.LastUpdated + DrainBarrier
|
||||
// (~6min remaining), NOT from the stale history deadline (which would yield 0).
|
||||
// Anything below ~5min would mean we relied on the corrupted history value.
|
||||
var delay = enqueuer.VisibilityDelays[0]!.Value;
|
||||
Assert.InRange(delay, TimeSpan.FromMinutes(5.5), TenantOffboardFunction.DrainBarrier);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Resume_TombstoneAlreadyInPlace_UsesRemainingBarrier_DoesNotPatchHistory()
|
||||
{
|
||||
// Counterpoint: tombstone was already there from the prior successful attempt;
|
||||
// the original deadline still applies. NO patch of History.EarliestProcessingAt.
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
|
||||
var initiatedAt = DateTime.UtcNow.AddMinutes(-2); // first attempt 2min ago, mid-barrier
|
||||
var originalEarliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier;
|
||||
var historyRowKey = "20260518091523123_" + TenantId;
|
||||
|
||||
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = historyRowKey,
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.com",
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
OffboardedAt = initiatedAt,
|
||||
EarliestProcessingAt = originalEarliestProcessingAt,
|
||||
Status = "Initiated",
|
||||
});
|
||||
|
||||
// Build SUT where the tombstone is already in place from the prior attempt at
|
||||
// initiatedAt (2min ago). The tombstone.LastUpdated reflects when Disabled was
|
||||
// committed; the resume's deadline math is max(history.EarliestProcessingAt,
|
||||
// tombstone.LastUpdated + DrainBarrier). Both terms ≈ initiatedAt + 6min here,
|
||||
// so remaining ≈ 4min as before.
|
||||
var configRepoMock = new Mock<IConfigRepository>();
|
||||
var tombstone = TenantConfiguration.CreateDefault(TenantId);
|
||||
tombstone.Disabled = true;
|
||||
tombstone.DisabledReason = "Offboarding in progress";
|
||||
tombstone.LastUpdated = initiatedAt;
|
||||
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
|
||||
.ReturnsAsync(tombstone);
|
||||
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var tenantConfigService = new TenantConfigurationService(
|
||||
configRepoMock.Object,
|
||||
NullLogger<TenantConfigurationService>.Instance,
|
||||
cache);
|
||||
|
||||
var sut = new TenantOffboardFunction(
|
||||
logger: NullLogger<TenantOffboardFunction>.Instance,
|
||||
configRepo: configRepoMock.Object,
|
||||
tenantConfigService: tenantConfigService,
|
||||
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
|
||||
offboardingRepo: repo,
|
||||
offboardingEnqueuer: enqueuer);
|
||||
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId,
|
||||
TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = historyRowKey,
|
||||
InitiatedAt = initiatedAt,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = "Initiated",
|
||||
};
|
||||
|
||||
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
|
||||
|
||||
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
|
||||
Assert.Single(enqueuer.VisibilityDelays);
|
||||
|
||||
// Remaining delay should be ~4min (6min - 2min). Allow a couple seconds of slack.
|
||||
var delay = enqueuer.VisibilityDelays[0]!.Value;
|
||||
Assert.InRange(delay, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.5));
|
||||
|
||||
// History.EarliestProcessingAt must NOT have been overwritten — it still carries
|
||||
// the original deadline from the first attempt.
|
||||
var historyAfter = repo.History[historyRowKey];
|
||||
Assert.Equal(originalEarliestProcessingAt, historyAfter.EarliestProcessingAt);
|
||||
|
||||
// No Save call (the tombstone was already correct).
|
||||
configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()), Times.Never);
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,13 @@ namespace AutopilotMonitor.Functions.DataAccess
|
||||
services.AddSingleton<ISignalRepository, TableSignalRepository>();
|
||||
services.AddSingleton<IDecisionTransitionRepository, TableDecisionTransitionRepository>();
|
||||
services.AddSingleton<IIndexTableRepository, TableIndexRepository>();
|
||||
|
||||
// Tenant-offboarding audit (Marker / History / ByTenant pointer).
|
||||
services.AddSingleton<IOffboardingAuditRepository, TableOffboardingAuditRepository>();
|
||||
// Tenant-offboarding customs archive (PR3.B): snapshot of GatherRules /
|
||||
// AnalyzeRules / ImeLogPatterns rows that were wiped during Phase 2.D-archive,
|
||||
// surfaced via the Global Admin /admin/customs-archive page.
|
||||
services.AddSingleton<ITenantCustomsArchiveRepository, TableTenantCustomsArchiveRepository>();
|
||||
return services;
|
||||
}
|
||||
|
||||
|
||||
+38
-176
@@ -1,3 +1,6 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -74,186 +77,45 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<bool> IsSessionIndexEmptyAsync()
|
||||
=> _storage.IsSessionIndexEmptyAsync();
|
||||
|
||||
// --- Tenant Offboarding ---
|
||||
|
||||
public async Task<Dictionary<string, int>> DeleteAllTenantDataAsync(string tenantId)
|
||||
{
|
||||
var deletedCounts = new Dictionary<string, int>();
|
||||
var normalizedTenantId = tenantId.ToLowerInvariant();
|
||||
|
||||
// Tables where PartitionKey = tenantId (normalized lowercase)
|
||||
var tenantPartitionTables = new[]
|
||||
{
|
||||
Constants.TableNames.Sessions,
|
||||
Constants.TableNames.SessionsIndex,
|
||||
Constants.TableNames.AuditLogs,
|
||||
Constants.TableNames.UsageMetrics,
|
||||
Constants.TableNames.UserActivity,
|
||||
Constants.TableNames.GatherRules,
|
||||
Constants.TableNames.AnalyzeRules,
|
||||
Constants.TableNames.AppInstallSummaries,
|
||||
Constants.TableNames.TenantConfiguration,
|
||||
Constants.TableNames.TenantAdmins,
|
||||
Constants.TableNames.BootstrapSessions,
|
||||
Constants.TableNames.BlockedDevices,
|
||||
Constants.TableNames.ImeLogPatterns,
|
||||
Constants.TableNames.RuleStates,
|
||||
};
|
||||
|
||||
foreach (var tableName in tenantPartitionTables)
|
||||
{
|
||||
var deleted = await DeleteAllRowsByPartitionKeyAsync(tableName, normalizedTenantId);
|
||||
deletedCounts[tableName] = deleted;
|
||||
_logger.LogInformation("Offboard [{TenantId}] {TableName}: deleted {Deleted} rows", tenantId, tableName, deleted);
|
||||
}
|
||||
|
||||
// Tables with composite PartitionKey = "{tenantId}_{sessionId}" – query by prefix
|
||||
var eventsDeleted = await DeleteByTenantPrefixAsync(Constants.TableNames.Events, normalizedTenantId);
|
||||
deletedCounts[Constants.TableNames.Events] = eventsDeleted;
|
||||
_logger.LogInformation("Offboard [{TenantId}] Events: deleted {Deleted} rows", tenantId, eventsDeleted);
|
||||
|
||||
var ruleResultsDeleted = await DeleteByTenantPrefixAsync(Constants.TableNames.RuleResults, normalizedTenantId);
|
||||
deletedCounts[Constants.TableNames.RuleResults] = ruleResultsDeleted;
|
||||
_logger.LogInformation("Offboard [{TenantId}] RuleResults: deleted {Deleted} rows", tenantId, ruleResultsDeleted);
|
||||
|
||||
// BootstrapSessions CodeLookup entries (PartitionKey = "CodeLookup", TenantId property = tenantId)
|
||||
var codeLookupDeleted = await DeleteCodeLookupEntriesAsync(normalizedTenantId);
|
||||
deletedCounts["BootstrapSessions_CodeLookup"] = codeLookupDeleted;
|
||||
_logger.LogInformation("Offboard [{TenantId}] BootstrapSessions CodeLookup: deleted {Deleted} rows", tenantId, codeLookupDeleted);
|
||||
|
||||
return deletedCounts;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes all rows in a table where PartitionKey equals the given value.
|
||||
/// Uses batched deletes (max 100 per transaction, same PartitionKey required).
|
||||
/// </summary>
|
||||
private async Task<int> DeleteAllRowsByPartitionKeyAsync(string tableName, string partitionKey)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _storage.GetTableClient(tableName);
|
||||
var filter = $"PartitionKey eq '{partitionKey}'";
|
||||
var entities = tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" });
|
||||
|
||||
int deleted = 0;
|
||||
var batch = new List<TableTransactionAction>();
|
||||
|
||||
await foreach (var entity in entities)
|
||||
{
|
||||
batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity));
|
||||
deleted++;
|
||||
|
||||
if (batch.Count >= 100)
|
||||
{
|
||||
await tableClient.SubmitTransactionAsync(batch);
|
||||
batch.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.Count > 0)
|
||||
{
|
||||
await tableClient.SubmitTransactionAsync(batch);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to delete rows in {TableName} for partition {PartitionKey}", tableName, partitionKey);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes all rows for a tenant from a table that uses composite PartitionKey = "{tenantId}_{sessionId}".
|
||||
/// Used for Events and RuleResults tables.
|
||||
/// </summary>
|
||||
private async Task<int> DeleteByTenantPrefixAsync(string tableName, string normalizedTenantId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _storage.GetTableClient(tableName);
|
||||
// OData startsWith: PartitionKey ge 'prefix' and PartitionKey lt 'prefix~'
|
||||
var prefix = normalizedTenantId + "_";
|
||||
var filter = $"PartitionKey ge '{prefix}' and PartitionKey lt '{prefix}~'";
|
||||
var entities = tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" });
|
||||
|
||||
int deleted = 0;
|
||||
// Group by PartitionKey because Azure Table batch transactions require same PartitionKey
|
||||
var groups = new Dictionary<string, List<TableEntity>>();
|
||||
|
||||
await foreach (var entity in entities)
|
||||
{
|
||||
if (!groups.ContainsKey(entity.PartitionKey))
|
||||
groups[entity.PartitionKey] = new List<TableEntity>();
|
||||
groups[entity.PartitionKey].Add(entity);
|
||||
deleted++;
|
||||
}
|
||||
|
||||
foreach (var group in groups.Values)
|
||||
{
|
||||
for (int i = 0; i < group.Count; i += 100)
|
||||
{
|
||||
var chunk = group.Skip(i).Take(100).ToList();
|
||||
var batch = chunk.Select(e => new TableTransactionAction(TableTransactionActionType.Delete, e)).ToList();
|
||||
await tableClient.SubmitTransactionAsync(batch);
|
||||
}
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to delete {TableName} for tenant {TenantId}", tableName, normalizedTenantId);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes BootstrapSessions CodeLookup entries for a tenant.
|
||||
/// These use PartitionKey = "CodeLookup" with a TenantId property matching the tenant.
|
||||
/// </summary>
|
||||
private async Task<int> DeleteCodeLookupEntriesAsync(string normalizedTenantId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var tableClient = _storage.GetTableClient(Constants.TableNames.BootstrapSessions);
|
||||
var filter = $"PartitionKey eq 'CodeLookup' and TenantId eq '{normalizedTenantId}'";
|
||||
var entities = tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" });
|
||||
|
||||
int deleted = 0;
|
||||
var batch = new List<TableTransactionAction>();
|
||||
|
||||
await foreach (var entity in entities)
|
||||
{
|
||||
batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity));
|
||||
deleted++;
|
||||
|
||||
if (batch.Count >= 100)
|
||||
{
|
||||
await tableClient.SubmitTransactionAsync(batch);
|
||||
batch.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.Count > 0)
|
||||
{
|
||||
await tableClient.SubmitTransactionAsync(batch);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to delete CodeLookup entries for tenant {TenantId}", normalizedTenantId);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod)
|
||||
=> _storage.GetOrphanedEventSessionsAsync(gracePeriod);
|
||||
|
||||
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId)
|
||||
=> _storage.DeleteEventSessionIndexEntryAsync(tenantId, sessionId);
|
||||
|
||||
/// <summary>
|
||||
/// Tenant offboarding cascade-worker session enumerator. Yields one row per session
|
||||
/// belonging to <paramref name="tenantId"/>. Storage exceptions are intentionally
|
||||
/// NOT caught (memory: feedback_storage_helpers_fail_soft) — the worker must see
|
||||
/// transient failures so the queue can retry/poison rather than treating an outage as
|
||||
/// "0 sessions, proceed with wipe".
|
||||
/// </summary>
|
||||
public async IAsyncEnumerable<string> EnumerateSessionsForOffboardingAsync(
|
||||
string tenantId,
|
||||
[EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
|
||||
var tableClient = _storage.GetTableClient(Constants.TableNames.Sessions);
|
||||
var filter = $"PartitionKey eq '{tenantId}'";
|
||||
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
|
||||
filter,
|
||||
select: new[] { "PartitionKey", "RowKey" },
|
||||
cancellationToken: ct))
|
||||
{
|
||||
if (!string.Equals(entity.PartitionKey, tenantId, StringComparison.Ordinal))
|
||||
{
|
||||
// Defensive: server-side filter should already guarantee this, but if a row
|
||||
// ever slips through with a foreign PK we must fail-loud rather than enqueue
|
||||
// a cascade for the wrong tenant.
|
||||
throw new InvalidOperationException(
|
||||
$"Sessions row returned by tenant filter has PartitionKey '{entity.PartitionKey}' " +
|
||||
$"but expected '{tenantId}' (RowKey={entity.RowKey}).");
|
||||
}
|
||||
|
||||
yield return entity.RowKey;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+296
@@ -0,0 +1,296 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{
|
||||
/// <summary>
|
||||
/// Table-storage implementation of <see cref="IOffboardingAuditRepository"/>. Stores all
|
||||
/// three offboarding PartitionKey patterns (Marker / History / ByTenant) in the single
|
||||
/// <c>OffboardingAudit</c> table.
|
||||
/// <para>
|
||||
/// All writes are fail-loud — when an Azure storage call throws, exceptions propagate to
|
||||
/// the caller (queue worker), which owns retry/poison semantics via the queue SDK. This is
|
||||
/// intentionally the OPPOSITE of <see cref="TableMaintenanceRepository"/>'s fail-soft
|
||||
/// helpers; offboarding cannot tolerate silent failures during gate/audit writes.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class TableOffboardingAuditRepository : IOffboardingAuditRepository
|
||||
{
|
||||
private readonly TableClient _tableClient;
|
||||
private readonly ILogger<TableOffboardingAuditRepository> _logger;
|
||||
|
||||
public TableOffboardingAuditRepository(
|
||||
TableStorageService storage,
|
||||
ILogger<TableOffboardingAuditRepository> logger)
|
||||
{
|
||||
_tableClient = storage.GetTableClient(Constants.TableNames.OffboardingAudit);
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
// ── Marker ──────────────────────────────────────────────────────────────
|
||||
|
||||
public async Task<OffboardingMarkerEntry?> TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
|
||||
try
|
||||
{
|
||||
var response = await _tableClient.GetEntityAsync<TableEntity>(
|
||||
Constants.OffboardingPartitionKeys.Marker, normalizedTenantId, cancellationToken: ct);
|
||||
return MapMarker(response.Value);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default)
|
||||
{
|
||||
EnsureMarkerShape(marker);
|
||||
return _tableClient.AddEntityAsync(StoreMarker(marker), ct);
|
||||
}
|
||||
|
||||
public Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default)
|
||||
{
|
||||
EnsureMarkerShape(marker);
|
||||
return _tableClient.UpsertEntityAsync(StoreMarker(marker), TableUpdateMode.Replace, ct);
|
||||
}
|
||||
|
||||
public async Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
|
||||
try
|
||||
{
|
||||
await _tableClient.DeleteEntityAsync(
|
||||
Constants.OffboardingPartitionKeys.Marker, normalizedTenantId, ETag.All, ct);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// Idempotent — already gone.
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<OffboardingMarkerEntry> QueryMarkersAsync(
|
||||
[EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
var filter = $"PartitionKey eq '{Constants.OffboardingPartitionKeys.Marker}'";
|
||||
await foreach (var entity in _tableClient.QueryAsync<TableEntity>(filter, cancellationToken: ct))
|
||||
{
|
||||
yield return MapMarker(entity);
|
||||
}
|
||||
}
|
||||
|
||||
// ── History ─────────────────────────────────────────────────────────────
|
||||
|
||||
public Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default)
|
||||
{
|
||||
EnsureHistoryShape(history);
|
||||
return _tableClient.AddEntityAsync(StoreHistory(history), ct);
|
||||
}
|
||||
|
||||
public async Task<OffboardingHistoryEntry?> TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey));
|
||||
|
||||
try
|
||||
{
|
||||
var response = await _tableClient.GetEntityAsync<TableEntity>(
|
||||
Constants.OffboardingPartitionKeys.History, historyRowKey, cancellationToken: ct);
|
||||
return MapHistory(response.Value);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default)
|
||||
{
|
||||
EnsureHistoryShape(history);
|
||||
return _tableClient.UpsertEntityAsync(StoreHistory(history), TableUpdateMode.Replace, ct);
|
||||
}
|
||||
|
||||
// ── ByTenant Pointer ────────────────────────────────────────────────────
|
||||
|
||||
public async Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync(
|
||||
string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
|
||||
try
|
||||
{
|
||||
var response = await _tableClient.GetEntityAsync<TableEntity>(
|
||||
Constants.OffboardingPartitionKeys.ByTenant, normalizedTenantId, cancellationToken: ct);
|
||||
return (MapByTenantPointer(response.Value), response.Value.ETag.ToString());
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return (null, null);
|
||||
}
|
||||
}
|
||||
|
||||
public Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default)
|
||||
{
|
||||
EnsurePointerShape(pointer);
|
||||
return _tableClient.AddEntityAsync(StoreByTenantPointer(pointer), ct);
|
||||
}
|
||||
|
||||
public Task UpdateByTenantPointerWithEtagAsync(
|
||||
OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default)
|
||||
{
|
||||
EnsurePointerShape(pointer);
|
||||
if (string.IsNullOrEmpty(ifMatchEtag)) throw new ArgumentException("ifMatchEtag required", nameof(ifMatchEtag));
|
||||
|
||||
var entity = StoreByTenantPointer(pointer);
|
||||
entity.ETag = new ETag(ifMatchEtag);
|
||||
// TableUpdateMode.Replace + non-default ETag → conditional replace. Mismatch → 412.
|
||||
return _tableClient.UpdateEntityAsync(entity, entity.ETag, TableUpdateMode.Replace, ct);
|
||||
}
|
||||
|
||||
// ── Validation helpers ──────────────────────────────────────────────────
|
||||
|
||||
private static void EnsureMarkerShape(OffboardingMarkerEntry marker)
|
||||
{
|
||||
if (marker == null) throw new ArgumentNullException(nameof(marker));
|
||||
if (marker.PartitionKey != Constants.OffboardingPartitionKeys.Marker)
|
||||
throw new ArgumentException($"Marker PartitionKey must be '{Constants.OffboardingPartitionKeys.Marker}', got '{marker.PartitionKey}'", nameof(marker));
|
||||
SecurityValidator.EnsureValidGuid(marker.RowKey, $"{nameof(marker)}.RowKey");
|
||||
SecurityValidator.EnsureValidGuid(marker.TenantId, $"{nameof(marker)}.TenantId");
|
||||
}
|
||||
|
||||
private static void EnsureHistoryShape(OffboardingHistoryEntry history)
|
||||
{
|
||||
if (history == null) throw new ArgumentNullException(nameof(history));
|
||||
if (history.PartitionKey != Constants.OffboardingPartitionKeys.History)
|
||||
throw new ArgumentException($"History PartitionKey must be '{Constants.OffboardingPartitionKeys.History}', got '{history.PartitionKey}'", nameof(history));
|
||||
if (string.IsNullOrEmpty(history.RowKey)) throw new ArgumentException("History RowKey required", nameof(history));
|
||||
SecurityValidator.EnsureValidGuid(history.TenantId, $"{nameof(history)}.TenantId");
|
||||
}
|
||||
|
||||
private static void EnsurePointerShape(OffboardingByTenantPointer pointer)
|
||||
{
|
||||
if (pointer == null) throw new ArgumentNullException(nameof(pointer));
|
||||
if (pointer.PartitionKey != Constants.OffboardingPartitionKeys.ByTenant)
|
||||
throw new ArgumentException($"Pointer PartitionKey must be '{Constants.OffboardingPartitionKeys.ByTenant}', got '{pointer.PartitionKey}'", nameof(pointer));
|
||||
SecurityValidator.EnsureValidGuid(pointer.RowKey, $"{nameof(pointer)}.RowKey");
|
||||
SecurityValidator.EnsureValidGuid(pointer.TenantId, $"{nameof(pointer)}.TenantId");
|
||||
if (string.IsNullOrEmpty(pointer.LatestHistoryRowKey))
|
||||
throw new ArgumentException("Pointer.LatestHistoryRowKey required", nameof(pointer));
|
||||
}
|
||||
|
||||
// ── Store / Map helpers (memory: feedback_table_storage_serialization) ──
|
||||
|
||||
private static TableEntity StoreMarker(OffboardingMarkerEntry m) => new(m.PartitionKey, m.RowKey)
|
||||
{
|
||||
["TenantId"] = m.TenantId,
|
||||
["OffboardingHistoryRowKey"] = m.OffboardingHistoryRowKey,
|
||||
["InitiatedAt"] = m.InitiatedAt,
|
||||
["InitiatedBy"] = m.InitiatedBy,
|
||||
["Status"] = m.Status,
|
||||
["CompletedAt"] = m.CompletedAt,
|
||||
["FailedAt"] = m.FailedAt,
|
||||
["FailedPhase"] = m.FailedPhase,
|
||||
};
|
||||
|
||||
private static OffboardingMarkerEntry MapMarker(TableEntity e) => new()
|
||||
{
|
||||
PartitionKey = e.PartitionKey,
|
||||
RowKey = e.RowKey,
|
||||
TenantId = e.GetString("TenantId") ?? string.Empty,
|
||||
OffboardingHistoryRowKey = e.GetString("OffboardingHistoryRowKey") ?? string.Empty,
|
||||
InitiatedAt = e.GetDateTime("InitiatedAt") ?? default,
|
||||
InitiatedBy = e.GetString("InitiatedBy") ?? string.Empty,
|
||||
Status = e.GetString("Status") ?? "Initiated",
|
||||
CompletedAt = e.GetDateTime("CompletedAt"),
|
||||
FailedAt = e.GetDateTime("FailedAt"),
|
||||
FailedPhase = e.GetString("FailedPhase"),
|
||||
};
|
||||
|
||||
private static TableEntity StoreHistory(OffboardingHistoryEntry h) => new(h.PartitionKey, h.RowKey)
|
||||
{
|
||||
["TenantId"] = h.TenantId,
|
||||
["DomainName"] = h.DomainName,
|
||||
["InitiatedBy"] = h.InitiatedBy,
|
||||
["OffboardedAt"] = h.OffboardedAt,
|
||||
["CompletedAt"] = h.CompletedAt,
|
||||
["EarliestProcessingAt"] = h.EarliestProcessingAt,
|
||||
["Status"] = h.Status,
|
||||
["DeletedRowCountsJson"] = h.DeletedRowCountsJson,
|
||||
["TotalRowsDeleted"] = h.TotalRowsDeleted,
|
||||
["DeletedBlobCount"] = h.DeletedBlobCount,
|
||||
["CascadeSessionsEnqueued"] = h.CascadeSessionsEnqueued,
|
||||
["ErrorMessage"] = h.ErrorMessage,
|
||||
["RetryCount"] = h.RetryCount,
|
||||
["DrainCompletedAt"] = h.DrainCompletedAt,
|
||||
["EnumerationStartedAt"] = h.EnumerationStartedAt,
|
||||
["EnumerationCompletedBeforeUpload"] = h.EnumerationCompletedBeforeUpload,
|
||||
["CustomGatherRulesArchived"] = h.CustomGatherRulesArchived,
|
||||
["CustomAnalyzeRulesArchived"] = h.CustomAnalyzeRulesArchived,
|
||||
["ImeLogPatternOverridesArchived"] = h.ImeLogPatternOverridesArchived,
|
||||
["ReonboardedAt"] = h.ReonboardedAt,
|
||||
["ReonboardedBy"] = h.ReonboardedBy,
|
||||
["CustomsAutoWipedOnReonboard"] = h.CustomsAutoWipedOnReonboard,
|
||||
};
|
||||
|
||||
private static OffboardingHistoryEntry MapHistory(TableEntity e) => new()
|
||||
{
|
||||
PartitionKey = e.PartitionKey,
|
||||
RowKey = e.RowKey,
|
||||
TenantId = e.GetString("TenantId") ?? string.Empty,
|
||||
DomainName = e.GetString("DomainName") ?? string.Empty,
|
||||
InitiatedBy = e.GetString("InitiatedBy") ?? string.Empty,
|
||||
OffboardedAt = e.GetDateTime("OffboardedAt") ?? default,
|
||||
CompletedAt = e.GetDateTime("CompletedAt"),
|
||||
EarliestProcessingAt = e.GetDateTime("EarliestProcessingAt"),
|
||||
Status = e.GetString("Status") ?? "Initiated",
|
||||
DeletedRowCountsJson = e.GetString("DeletedRowCountsJson"),
|
||||
TotalRowsDeleted = e.GetInt32("TotalRowsDeleted"),
|
||||
DeletedBlobCount = e.GetInt32("DeletedBlobCount"),
|
||||
CascadeSessionsEnqueued = e.GetInt32("CascadeSessionsEnqueued"),
|
||||
ErrorMessage = e.GetString("ErrorMessage"),
|
||||
RetryCount = e.GetInt32("RetryCount") ?? 0,
|
||||
DrainCompletedAt = e.GetDateTime("DrainCompletedAt"),
|
||||
EnumerationStartedAt = e.GetDateTime("EnumerationStartedAt"),
|
||||
EnumerationCompletedBeforeUpload = e.GetDateTime("EnumerationCompletedBeforeUpload"),
|
||||
CustomGatherRulesArchived = e.GetInt32("CustomGatherRulesArchived"),
|
||||
CustomAnalyzeRulesArchived = e.GetInt32("CustomAnalyzeRulesArchived"),
|
||||
ImeLogPatternOverridesArchived = e.GetInt32("ImeLogPatternOverridesArchived"),
|
||||
ReonboardedAt = e.GetDateTime("ReonboardedAt"),
|
||||
ReonboardedBy = e.GetString("ReonboardedBy"),
|
||||
CustomsAutoWipedOnReonboard = e.GetInt32("CustomsAutoWipedOnReonboard"),
|
||||
};
|
||||
|
||||
private static TableEntity StoreByTenantPointer(OffboardingByTenantPointer p) => new(p.PartitionKey, p.RowKey)
|
||||
{
|
||||
["TenantId"] = p.TenantId,
|
||||
["LatestHistoryRowKey"] = p.LatestHistoryRowKey,
|
||||
["LatestStatus"] = p.LatestStatus,
|
||||
["LatestUpdatedAt"] = p.LatestUpdatedAt,
|
||||
["OffboardCount"] = p.OffboardCount,
|
||||
};
|
||||
|
||||
private static OffboardingByTenantPointer MapByTenantPointer(TableEntity e) => new()
|
||||
{
|
||||
PartitionKey = e.PartitionKey,
|
||||
RowKey = e.RowKey,
|
||||
TenantId = e.GetString("TenantId") ?? string.Empty,
|
||||
LatestHistoryRowKey = e.GetString("LatestHistoryRowKey") ?? string.Empty,
|
||||
LatestStatus = e.GetString("LatestStatus") ?? "Initiated",
|
||||
LatestUpdatedAt = e.GetDateTime("LatestUpdatedAt") ?? default,
|
||||
OffboardCount = e.GetInt32("OffboardCount") ?? 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{
|
||||
/// <summary>
|
||||
/// Table-storage backend for <see cref="ITenantCustomsArchiveRepository"/>.
|
||||
/// Writes during Phase 2.D-archive, reads + deletes from the Global Admin UI.
|
||||
/// Fail-loud — exceptions propagate to the caller (worker or HTTP handler).
|
||||
/// </summary>
|
||||
public sealed class TableTenantCustomsArchiveRepository : ITenantCustomsArchiveRepository
|
||||
{
|
||||
private readonly TableClient _tableClient;
|
||||
private readonly ILogger<TableTenantCustomsArchiveRepository> _logger;
|
||||
|
||||
public TableTenantCustomsArchiveRepository(
|
||||
TableStorageService storage,
|
||||
ILogger<TableTenantCustomsArchiveRepository> logger)
|
||||
{
|
||||
_tableClient = storage.GetTableClient(Constants.TableNames.TenantOffboardingCustomsArchive);
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default)
|
||||
{
|
||||
if (entry == null) throw new ArgumentNullException(nameof(entry));
|
||||
if (string.IsNullOrEmpty(entry.PartitionKey)) throw new ArgumentException("PartitionKey required", nameof(entry));
|
||||
if (string.IsNullOrEmpty(entry.RowKey)) throw new ArgumentException("RowKey required", nameof(entry));
|
||||
return _tableClient.UpsertEntityAsync(Store(entry), TableUpdateMode.Replace, ct);
|
||||
}
|
||||
|
||||
public async Task<int> CountByRunAndTableAsync(
|
||||
string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default)
|
||||
{
|
||||
var pk = BuildPartitionKey(normalizedTenantId, historyRowKey);
|
||||
var rkPrefix = originalTable + "_";
|
||||
// Range filter: rowKey ge 'GatherRules_' and rowKey lt 'GatherRules_~'.
|
||||
// '~' (0x7E) is the largest visible ASCII char that base64url uses; works as
|
||||
// exclusive upper bound because base64url alphabet is [A-Za-z0-9_-] (no '~').
|
||||
var filter = $"PartitionKey eq '{Escape(pk)}' and RowKey ge '{Escape(rkPrefix)}' and RowKey lt '{Escape(rkPrefix + "~")}'";
|
||||
|
||||
var count = 0;
|
||||
await foreach (var _ in _tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
|
||||
{
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByRunAsync(
|
||||
string normalizedTenantId, string historyRowKey, [EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
var pk = BuildPartitionKey(normalizedTenantId, historyRowKey);
|
||||
var filter = $"PartitionKey eq '{Escape(pk)}'";
|
||||
await foreach (var e in _tableClient.QueryAsync<TableEntity>(filter, cancellationToken: ct))
|
||||
{
|
||||
yield return Map(e);
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByTenantAsync(
|
||||
string normalizedTenantId, [EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
// Per-tenant prefix scan: PartitionKey ge '{tid}_' and PartitionKey lt '{tid}_~'.
|
||||
// History row keys begin with a numeric timestamp (yyyyMMddHHmmssfff) which is < '~',
|
||||
// so the exclusive upper bound is correct.
|
||||
var prefix = normalizedTenantId + "_";
|
||||
var filter = $"PartitionKey ge '{Escape(prefix)}' and PartitionKey lt '{Escape(prefix + "~")}'";
|
||||
await foreach (var e in _tableClient.QueryAsync<TableEntity>(filter, cancellationToken: ct))
|
||||
{
|
||||
yield return Map(e);
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryAllAsync(
|
||||
[EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
await foreach (var e in _tableClient.QueryAsync<TableEntity>(cancellationToken: ct))
|
||||
{
|
||||
yield return Map(e);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<TenantOffboardingCustomsArchiveEntry?> TryGetEntryAsync(
|
||||
string partitionKey, string rowKey, CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await _tableClient.GetEntityAsync<TableEntity>(partitionKey, rowKey, cancellationToken: ct);
|
||||
return Map(response.Value);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _tableClient.DeleteEntityAsync(partitionKey, rowKey, ETag.All, ct);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// Idempotent — already gone.
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<int> DeleteRunAsync(
|
||||
string normalizedTenantId, string historyRowKey, CancellationToken ct = default)
|
||||
{
|
||||
var pk = BuildPartitionKey(normalizedTenantId, historyRowKey);
|
||||
var filter = $"PartitionKey eq '{Escape(pk)}'";
|
||||
|
||||
var deleted = 0;
|
||||
var batch = new List<TableTransactionAction>();
|
||||
await foreach (var entity in _tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
|
||||
{
|
||||
batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity));
|
||||
deleted++;
|
||||
|
||||
if (batch.Count >= 100)
|
||||
{
|
||||
await _tableClient.SubmitTransactionAsync(batch, ct);
|
||||
batch.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.Count > 0)
|
||||
{
|
||||
await _tableClient.SubmitTransactionAsync(batch, ct);
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
internal static string BuildPartitionKey(string normalizedTenantId, string historyRowKey)
|
||||
=> $"{normalizedTenantId}_{historyRowKey}";
|
||||
|
||||
/// <summary>
|
||||
/// Builds the archive RowKey for one original row. base64url-encodes the original
|
||||
/// RowKey so it survives Azure-Tables' RowKey-forbidden character rules (#, ?,
|
||||
/// /, \, control chars). Decoder is <see cref="DecodeOriginalRowKey"/>.
|
||||
/// </summary>
|
||||
public static string BuildRowKey(string originalTable, string originalRowKey)
|
||||
=> $"{originalTable}_{Base64UrlEncode(originalRowKey)}";
|
||||
|
||||
/// <summary>Inverse of <see cref="BuildRowKey"/> — returns the original RowKey.</summary>
|
||||
public static string DecodeOriginalRowKey(string archiveRowKey, string originalTable)
|
||||
{
|
||||
var prefix = originalTable + "_";
|
||||
if (!archiveRowKey.StartsWith(prefix, StringComparison.Ordinal))
|
||||
throw new ArgumentException($"Archive RowKey '{archiveRowKey}' does not start with expected table prefix '{prefix}'", nameof(archiveRowKey));
|
||||
return Base64UrlDecode(archiveRowKey.Substring(prefix.Length));
|
||||
}
|
||||
|
||||
private static string Base64UrlEncode(string s)
|
||||
{
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(s);
|
||||
return Convert.ToBase64String(bytes)
|
||||
.Replace('+', '-')
|
||||
.Replace('/', '_')
|
||||
.TrimEnd('=');
|
||||
}
|
||||
|
||||
private static string Base64UrlDecode(string s)
|
||||
{
|
||||
var padded = s.Replace('-', '+').Replace('_', '/');
|
||||
switch (padded.Length % 4)
|
||||
{
|
||||
case 2: padded += "=="; break;
|
||||
case 3: padded += "="; break;
|
||||
}
|
||||
var bytes = Convert.FromBase64String(padded);
|
||||
return System.Text.Encoding.UTF8.GetString(bytes);
|
||||
}
|
||||
|
||||
private static string Escape(string s) => s.Replace("'", "''");
|
||||
|
||||
private static TableEntity Store(TenantOffboardingCustomsArchiveEntry e) => new(e.PartitionKey, e.RowKey)
|
||||
{
|
||||
["TenantId"] = e.TenantId,
|
||||
["OriginalTable"] = e.OriginalTable,
|
||||
["OriginalPartitionKey"] = e.OriginalPartitionKey,
|
||||
["OriginalRowKey"] = e.OriginalRowKey,
|
||||
["EntityJson"] = e.EntityJson,
|
||||
["HistoryRowKey"] = e.HistoryRowKey,
|
||||
["ArchivedAt"] = e.ArchivedAt,
|
||||
["ArchivedBy"] = e.ArchivedBy,
|
||||
};
|
||||
|
||||
private static TenantOffboardingCustomsArchiveEntry Map(TableEntity e) => new()
|
||||
{
|
||||
PartitionKey = e.PartitionKey,
|
||||
RowKey = e.RowKey,
|
||||
TenantId = e.GetString("TenantId") ?? string.Empty,
|
||||
OriginalTable = e.GetString("OriginalTable") ?? string.Empty,
|
||||
OriginalPartitionKey = e.GetString("OriginalPartitionKey") ?? string.Empty,
|
||||
OriginalRowKey = e.GetString("OriginalRowKey") ?? string.Empty,
|
||||
EntityJson = e.GetString("EntityJson") ?? string.Empty,
|
||||
HistoryRowKey = e.GetString("HistoryRowKey") ?? string.Empty,
|
||||
ArchivedAt = e.GetDateTime("ArchivedAt") ?? default,
|
||||
ArchivedBy = e.GetString("ArchivedBy") ?? "TenantOffboardingHandler",
|
||||
};
|
||||
}
|
||||
}
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Admin;
|
||||
|
||||
/// <summary>
|
||||
/// GlobalAdmin-only read + delete endpoints over <see cref="ITenantCustomsArchiveRepository"/>.
|
||||
/// Backs the <c>/global/customs-archive</c> Web UI: list runs, inspect entries, delete per
|
||||
/// entry / per run. All writes audit-logged under <see cref="Constants.AuditGlobalTenantId"/>.
|
||||
/// <para>
|
||||
/// Authorization is enforced by <c>PolicyEnforcementMiddleware</c> via
|
||||
/// <c>EndpointAccessPolicyCatalog</c>; the five routes registered here all carry
|
||||
/// <c>EndpointPolicy.GlobalAdminOnly</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class CustomsArchiveQueryFunction
|
||||
{
|
||||
private readonly ILogger<CustomsArchiveQueryFunction> _logger;
|
||||
private readonly ITenantCustomsArchiveRepository _archive;
|
||||
private readonly IMaintenanceRepository _maintenance;
|
||||
|
||||
public CustomsArchiveQueryFunction(
|
||||
ILogger<CustomsArchiveQueryFunction> logger,
|
||||
ITenantCustomsArchiveRepository archive,
|
||||
IMaintenanceRepository maintenance)
|
||||
{
|
||||
_logger = logger;
|
||||
_archive = archive;
|
||||
_maintenance = maintenance;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GET /api/global/customs-archive — list every archive partition (one entry per
|
||||
/// (tenantId, historyRowKey) run) with aggregated row-counts per source table.
|
||||
/// Optional ?tenantId= filter restricts to one tenant.
|
||||
/// </summary>
|
||||
[Function("CustomsArchiveListRuns")]
|
||||
public async Task<HttpResponseData> ListRuns(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/customs-archive")] HttpRequestData req)
|
||||
{
|
||||
var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query ?? string.Empty);
|
||||
var filterTenantId = query["tenantId"];
|
||||
|
||||
var summaries = new Dictionary<string, RunSummary>(System.StringComparer.Ordinal);
|
||||
|
||||
var source = string.IsNullOrEmpty(filterTenantId)
|
||||
? _archive.QueryAllAsync()
|
||||
: _archive.QueryByTenantAsync(filterTenantId.ToLowerInvariant());
|
||||
|
||||
await foreach (var entry in source)
|
||||
{
|
||||
var key = entry.PartitionKey;
|
||||
if (!summaries.TryGetValue(key, out var summary))
|
||||
{
|
||||
summary = new RunSummary
|
||||
{
|
||||
PartitionKey = entry.PartitionKey,
|
||||
TenantId = entry.TenantId,
|
||||
HistoryRowKey = entry.HistoryRowKey,
|
||||
ArchivedAt = entry.ArchivedAt,
|
||||
};
|
||||
summaries[key] = summary;
|
||||
}
|
||||
|
||||
switch (entry.OriginalTable)
|
||||
{
|
||||
case Constants.TableNames.GatherRules: summary.GatherRulesCount++; break;
|
||||
case Constants.TableNames.AnalyzeRules: summary.AnalyzeRulesCount++; break;
|
||||
case Constants.TableNames.ImeLogPatterns: summary.ImeLogPatternsCount++; break;
|
||||
}
|
||||
|
||||
// Track the earliest ArchivedAt for the run (in case rows have slightly drifting
|
||||
// timestamps from multiple iterations of archive-then-wipe).
|
||||
if (entry.ArchivedAt < summary.ArchivedAt)
|
||||
{
|
||||
summary.ArchivedAt = entry.ArchivedAt;
|
||||
}
|
||||
}
|
||||
|
||||
var runs = summaries.Values
|
||||
.OrderByDescending(s => s.ArchivedAt)
|
||||
.ToList();
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new { success = true, count = runs.Count, runs });
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GET /api/global/customs-archive/{tenantId}/{historyRowKey} — list every archived
|
||||
/// entry in one offboarding run, with a short preview of <see cref="TenantOffboardingCustomsArchiveEntry.EntityJson"/>.
|
||||
/// Full body is available via the per-entry route below.
|
||||
/// </summary>
|
||||
[Function("CustomsArchiveListEntries")]
|
||||
public async Task<HttpResponseData> ListEntries(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get",
|
||||
Route = "global/customs-archive/{tenantId}/{historyRowKey}")] HttpRequestData req,
|
||||
string tenantId,
|
||||
string historyRowKey)
|
||||
{
|
||||
var normalizedTenantId = tenantId.ToLowerInvariant();
|
||||
var items = new List<EntrySummary>();
|
||||
await foreach (var entry in _archive.QueryByRunAsync(normalizedTenantId, historyRowKey))
|
||||
{
|
||||
items.Add(new EntrySummary
|
||||
{
|
||||
PartitionKey = entry.PartitionKey,
|
||||
RowKey = entry.RowKey,
|
||||
OriginalTable = entry.OriginalTable,
|
||||
OriginalRowKey = entry.OriginalRowKey,
|
||||
ArchivedAt = entry.ArchivedAt,
|
||||
EntityJsonPreview = TruncatePreview(entry.EntityJson, maxLength: 200),
|
||||
});
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new { success = true, count = items.Count, entries = items });
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GET /api/global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey} — full
|
||||
/// archive entry including <see cref="TenantOffboardingCustomsArchiveEntry.EntityJson"/>.
|
||||
/// </summary>
|
||||
[Function("CustomsArchiveGetEntry")]
|
||||
public async Task<HttpResponseData> GetEntry(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get",
|
||||
Route = "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}")] HttpRequestData req,
|
||||
string tenantId,
|
||||
string historyRowKey,
|
||||
string archiveRowKey)
|
||||
{
|
||||
var partitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey(
|
||||
tenantId.ToLowerInvariant(), historyRowKey);
|
||||
var entry = await _archive.TryGetEntryAsync(partitionKey, archiveRowKey);
|
||||
if (entry == null)
|
||||
{
|
||||
return req.CreateResponse(HttpStatusCode.NotFound);
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new { success = true, entry });
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// DELETE /api/global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey} —
|
||||
/// remove a single archived entry. Audit-logged under <see cref="Constants.AuditGlobalTenantId"/>.
|
||||
/// </summary>
|
||||
[Function("CustomsArchiveDeleteEntry")]
|
||||
public async Task<HttpResponseData> DeleteEntry(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "delete",
|
||||
Route = "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}")] HttpRequestData req,
|
||||
string tenantId,
|
||||
string historyRowKey,
|
||||
string archiveRowKey,
|
||||
FunctionContext context)
|
||||
{
|
||||
var requestCtx = context.GetRequestContext();
|
||||
var normalizedTenantId = tenantId.ToLowerInvariant();
|
||||
var partitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey(normalizedTenantId, historyRowKey);
|
||||
|
||||
await _archive.DeleteEntryAsync(partitionKey, archiveRowKey);
|
||||
|
||||
await _maintenance.LogAuditEntryAsync(
|
||||
Constants.AuditGlobalTenantId,
|
||||
"DELETE",
|
||||
"CustomsArchiveEntry",
|
||||
$"{partitionKey}/{archiveRowKey}",
|
||||
requestCtx.UserPrincipalName,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ "TenantId", normalizedTenantId },
|
||||
{ "HistoryRowKey", historyRowKey },
|
||||
{ "ArchiveRowKey", archiveRowKey },
|
||||
});
|
||||
|
||||
_logger.LogWarning(
|
||||
"CustomsArchive entry deleted by {Upn}: tenant={Tenant} history={History} archive={Archive}",
|
||||
requestCtx.UserPrincipalName, normalizedTenantId, historyRowKey, archiveRowKey);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new { success = true });
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// DELETE /api/global/customs-archive/{tenantId}/{historyRowKey} — bulk-delete every
|
||||
/// entry of one offboarding run (after the operator has reviewed them). Audit-logged
|
||||
/// under <see cref="Constants.AuditGlobalTenantId"/> with the deleted count.
|
||||
/// </summary>
|
||||
[Function("CustomsArchiveDeleteRun")]
|
||||
public async Task<HttpResponseData> DeleteRun(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "delete",
|
||||
Route = "global/customs-archive/{tenantId}/{historyRowKey}")] HttpRequestData req,
|
||||
string tenantId,
|
||||
string historyRowKey,
|
||||
FunctionContext context)
|
||||
{
|
||||
var requestCtx = context.GetRequestContext();
|
||||
var normalizedTenantId = tenantId.ToLowerInvariant();
|
||||
|
||||
var deleted = await _archive.DeleteRunAsync(normalizedTenantId, historyRowKey);
|
||||
|
||||
await _maintenance.LogAuditEntryAsync(
|
||||
Constants.AuditGlobalTenantId,
|
||||
"DELETE",
|
||||
"CustomsArchiveRun",
|
||||
$"{normalizedTenantId}/{historyRowKey}",
|
||||
requestCtx.UserPrincipalName,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ "TenantId", normalizedTenantId },
|
||||
{ "HistoryRowKey", historyRowKey },
|
||||
{ "DeletedCount", deleted.ToString() },
|
||||
});
|
||||
|
||||
_logger.LogWarning(
|
||||
"CustomsArchive run deleted by {Upn}: tenant={Tenant} history={History} deleted={Count}",
|
||||
requestCtx.UserPrincipalName, normalizedTenantId, historyRowKey, deleted);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new { success = true, deleted });
|
||||
return response;
|
||||
}
|
||||
|
||||
private static string TruncatePreview(string s, int maxLength)
|
||||
{
|
||||
if (string.IsNullOrEmpty(s)) return string.Empty;
|
||||
if (s.Length <= maxLength) return s;
|
||||
return s.Substring(0, maxLength) + "…";
|
||||
}
|
||||
|
||||
public class RunSummary
|
||||
{
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
public string HistoryRowKey { get; set; } = string.Empty;
|
||||
public System.DateTime ArchivedAt { get; set; }
|
||||
public int GatherRulesCount { get; set; }
|
||||
public int AnalyzeRulesCount { get; set; }
|
||||
public int ImeLogPatternsCount { get; set; }
|
||||
}
|
||||
|
||||
public class EntrySummary
|
||||
{
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
public string RowKey { get; set; } = string.Empty;
|
||||
public string OriginalTable { get; set; } = string.Empty;
|
||||
public string OriginalRowKey { get; set; } = string.Empty;
|
||||
public System.DateTime ArchivedAt { get; set; }
|
||||
public string EntityJsonPreview { get; set; } = string.Empty;
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,16 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
@@ -11,34 +19,61 @@ using Microsoft.Extensions.Logging;
|
||||
namespace AutopilotMonitor.Functions.Functions.Admin;
|
||||
|
||||
/// <summary>
|
||||
/// Tenant Offboarding endpoint
|
||||
/// Allows a Tenant Admin to permanently delete all data for their tenant.
|
||||
/// This operation is irreversible.
|
||||
/// Tenant offboarding endpoint (Phase 1 of the tenant-offboarding cascade plan).
|
||||
/// Replaces the previous synchronous <c>DeleteAllTenantDataAsync</c> teardown with a
|
||||
/// queued cascade. Phase 1 commits the History/Pointer/Marker rows, flips
|
||||
/// <c>TenantConfiguration.Disabled=true</c> (the actual auth gate), audits, and enqueues
|
||||
/// a <see cref="TenantOffboardingEnvelope"/> with a <see cref="DrainBarrier"/> visibility
|
||||
/// delay. The worker picks the message up after the cache-drain window, by which point
|
||||
/// every function-host has expired its 5-min <c>TenantConfigurationService</c> cache and
|
||||
/// the existing Disabled-gate carries the 403. Returns 202.
|
||||
/// </summary>
|
||||
public class TenantOffboardFunction
|
||||
{
|
||||
private const int PointerCasMaxAttempts = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Cache-Drain-Barrier (plan v2 §2.2). Worker MUST NOT begin Phase 2 before this elapsed.
|
||||
/// 5 min = <see cref="TenantConfigurationService.CacheDuration"/> (absolute expiration).
|
||||
/// 1 min = clock-skew + GC + Storage-replication safety buffer.
|
||||
/// Hardcoded by design: this is a structural invariant tied to CacheDuration, not an
|
||||
/// operational knob. If CacheDuration is ever raised, this constant must move with it.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan DrainBarrier = TimeSpan.FromMinutes(6);
|
||||
|
||||
private readonly ILogger<TenantOffboardFunction> _logger;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly IConfigRepository _configRepo;
|
||||
private readonly OpsEventService _opsEventService;
|
||||
private readonly TenantConfigurationService _tenantConfigService;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly IOffboardingAuditRepository _offboardingRepo;
|
||||
private readonly ITenantOffboardingEnqueuer _offboardingEnqueuer;
|
||||
|
||||
public TenantOffboardFunction(
|
||||
ILogger<TenantOffboardFunction> logger,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
IConfigRepository configRepo,
|
||||
OpsEventService opsEventService)
|
||||
TenantConfigurationService tenantConfigService,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
IOffboardingAuditRepository offboardingRepo,
|
||||
ITenantOffboardingEnqueuer offboardingEnqueuer)
|
||||
{
|
||||
_logger = logger;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_configRepo = configRepo;
|
||||
_opsEventService = opsEventService;
|
||||
_tenantConfigService = tenantConfigService;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_offboardingRepo = offboardingRepo;
|
||||
_offboardingEnqueuer = offboardingEnqueuer;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// DELETE /api/tenants/{tenantId}/offboard
|
||||
/// Permanently deletes ALL data for a tenant across all tables.
|
||||
/// Accessible by: Tenant Admins of the same tenant OR Global Admins.
|
||||
/// This action is IRREVERSIBLE.
|
||||
/// Phase 1 — synchronously commits the offboarding state (History/Pointer/Marker),
|
||||
/// flips <c>TenantConfiguration.Disabled=true</c> (the SOLE active auth-gate in the
|
||||
/// PR3-revised architecture — the agent + web hotpaths read <c>config.Disabled</c>
|
||||
/// only, never the marker), and enqueues a single tenant-offboarding envelope with
|
||||
/// a <see cref="DrainBarrier"/> visibility-delay so warm function-host caches drain
|
||||
/// before Phase 2 begins.
|
||||
/// Accessible by: Tenant Admins of the same tenant OR Global Admins (enforced by
|
||||
/// PolicyEnforcementMiddleware).
|
||||
/// </summary>
|
||||
[Function("OffboardTenant")]
|
||||
[Authorize]
|
||||
@@ -47,88 +82,677 @@ public class TenantOffboardFunction
|
||||
string tenantId,
|
||||
FunctionContext context)
|
||||
{
|
||||
// Authentication enforced by PolicyEnforcementMiddleware
|
||||
var requestCtx = context.GetRequestContext();
|
||||
var upn = requestCtx.UserPrincipalName;
|
||||
var targetTenantId = requestCtx.TargetTenantId;
|
||||
|
||||
_logger.LogWarning($"TENANT OFFBOARD initiated for tenant {requestCtx.TargetTenantId} by {upn}");
|
||||
if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
|
||||
{
|
||||
var badRequest = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await badRequest.WriteAsJsonAsync(new { error = "tenantId must be a valid GUID" });
|
||||
return badRequest;
|
||||
}
|
||||
|
||||
// Audit under Global TenantId so the entry survives tenant data deletion
|
||||
var normalizedTenantId = targetTenantId.ToLowerInvariant();
|
||||
|
||||
_logger.LogWarning(
|
||||
"TENANT OFFBOARD initiated for tenant {TenantId} by {Upn}",
|
||||
normalizedTenantId, upn);
|
||||
|
||||
// 2. Capture DomainName from TenantConfiguration so the worker's Completed/Failed
|
||||
// OpsEvents can render "{domain} ({tenantId})" instead of the bare GUID.
|
||||
string? domainName = null;
|
||||
try
|
||||
{
|
||||
var existingConfig = await _configRepo.GetTenantConfigurationAsync(targetTenantId);
|
||||
domainName = existingConfig?.DomainName;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Failed to read tenant configuration for {TenantId} before offboard; proceeding without domain name",
|
||||
normalizedTenantId);
|
||||
}
|
||||
|
||||
// 3. Idempotency: if an active marker exists, replay the queue-enqueue defensively
|
||||
// when the offboarding is still in-flight. The previous implementation returned
|
||||
// 200 unconditionally, which left the tenant stuck whenever the first enqueue
|
||||
// failed between marker-insert and queue-send: the marker would block all future
|
||||
// re-clicks via the idempotency-200 path while the worker queue was empty.
|
||||
// Resume-paths per marker status:
|
||||
// - Initiated/InProgress → defensive re-enqueue (TenantOffboardingHandler is
|
||||
// idempotent via History.DrainCompletedAt Rev-9-F1 skip-gate; worst case the
|
||||
// queue carries two envelopes that both resolve cleanly).
|
||||
// - Completed → no re-enqueue (worker already finished; marker is
|
||||
// the 15-min Rev-5-F2 grace blocker).
|
||||
// - Failed → no re-enqueue (R1 fail-closed by design; operator
|
||||
// must inspect History.FailedPhase + clear the marker manually before retry).
|
||||
try
|
||||
{
|
||||
var existingMarker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
|
||||
if (existingMarker != null)
|
||||
{
|
||||
return await HandleExistingMarkerAsync(
|
||||
req, existingMarker, upn, normalizedTenantId, isRace: false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"Failed to read offboarding marker for {TenantId} during idempotency check",
|
||||
normalizedTenantId);
|
||||
return await Build500Async(req, "Failed to read offboarding state");
|
||||
}
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var historyRowKey = $"{now:yyyyMMddHHmmssfff}_{normalizedTenantId}";
|
||||
var earliestProcessingAt = now + DrainBarrier;
|
||||
|
||||
// 4. Insert History row FIRST (audit trail must exist before any side-effect).
|
||||
var history = new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = historyRowKey,
|
||||
TenantId = normalizedTenantId,
|
||||
DomainName = domainName ?? string.Empty,
|
||||
InitiatedBy = upn,
|
||||
OffboardedAt = now,
|
||||
EarliestProcessingAt = earliestProcessingAt,
|
||||
Status = "Initiated",
|
||||
RetryCount = 0,
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
await _offboardingRepo.InsertHistoryAsync(history);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to insert OffboardingHistory row for {TenantId}", normalizedTenantId);
|
||||
return await Build500Async(req, "Failed to record offboarding history");
|
||||
}
|
||||
|
||||
// 5. Upsert ByTenant pointer with ETag-CAS so OffboardCount increments race-safely.
|
||||
try
|
||||
{
|
||||
await UpsertPointerWithCasAsync(normalizedTenantId, historyRowKey, now);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// History row is already committed — operator can find it under
|
||||
// PartitionKey="OffboardingHistory" and retry manually. Graceful degradation
|
||||
// (plan §4.4): pointer is the re-onboarding index; without it PR4's auto-wipe
|
||||
// is no-op but the offboarding itself still proceeds.
|
||||
_logger.LogError(ex,
|
||||
"Failed to upsert OffboardingByTenant pointer for {TenantId} after history row inserted",
|
||||
normalizedTenantId);
|
||||
return await Build500Async(req, "Failed to update offboarding pointer");
|
||||
}
|
||||
|
||||
// 6. Insert Marker LAST among the three rows. The marker is the idempotency anchor
|
||||
// for admin re-clicks (the next click resolves through TryGetMarkerAsync and
|
||||
// hits the resume path) and is later read by OffboardingMarkerCleanupFunction
|
||||
// to drive post-completion cleanup sweeps. The Marker is NOT an auth-gate any
|
||||
// more — TenantConfiguration.Disabled=true (committed in step 7 below) is the
|
||||
// sole active gate.
|
||||
var marker = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = normalizedTenantId,
|
||||
TenantId = normalizedTenantId,
|
||||
OffboardingHistoryRowKey = historyRowKey,
|
||||
InitiatedAt = now,
|
||||
InitiatedBy = upn,
|
||||
Status = "Initiated",
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
await _offboardingRepo.InsertMarkerAsync(marker);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
// Race: another caller inserted a marker between our TryGet at step 3 and now.
|
||||
// Resolve to the existing marker and run the same defensive re-enqueue path as
|
||||
// the idempotency branch — the winner may still have failed its own enqueue and
|
||||
// we cannot tell from here, so re-enqueue is the safe default. The worker is
|
||||
// idempotent via the DrainCompletedAt skip-gate (Rev-9-F1).
|
||||
_logger.LogInformation(
|
||||
"OffboardingMarker insert raced for {TenantId}; another initiator won. Resolving to existing marker.",
|
||||
normalizedTenantId);
|
||||
|
||||
var raced = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
|
||||
if (raced == null)
|
||||
{
|
||||
_logger.LogError(
|
||||
"OffboardingMarker 409 race for {TenantId} but follow-up read returned null; cannot resolve race",
|
||||
normalizedTenantId);
|
||||
return await Build500Async(req, "Failed to resolve offboarding race");
|
||||
}
|
||||
return await HandleExistingMarkerAsync(
|
||||
req, raced, upn, normalizedTenantId, isRace: true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to insert OffboardingMarker for {TenantId}", normalizedTenantId);
|
||||
return await Build500Async(req, "Failed to commit offboarding marker");
|
||||
}
|
||||
|
||||
// 7. Flip TenantConfiguration.Disabled=true. This is the SOLE active auth-gate now
|
||||
// that the hotpath does not read the OffboardingMarker any more (PR3-revised).
|
||||
// Fail-loud — without Disabled=true the cache-drain barrier offers no protection
|
||||
// and Phase 2 would wipe data while warm function-hosts still accept agent
|
||||
// traffic. Marker stays Initiated → next admin re-click hits the resume path
|
||||
// which also calls EnsureTenantDisabledAsync before re-enqueuing.
|
||||
try
|
||||
{
|
||||
await EnsureTenantDisabledAsync(targetTenantId, upn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"Failed to set TenantConfiguration.Disabled=true for {TenantId}; refusing to enqueue (auth-gate not committed)",
|
||||
normalizedTenantId);
|
||||
return await Build500Async(req, "Failed to commit Disabled-gate; tenant offboarding NOT enqueued");
|
||||
}
|
||||
|
||||
// 8. Audit row under AuditGlobalTenantId so it survives the eventual TenantConfiguration wipe.
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
Constants.AuditGlobalTenantId,
|
||||
"DELETE",
|
||||
"Tenant",
|
||||
requestCtx.TargetTenantId,
|
||||
upn!,
|
||||
normalizedTenantId,
|
||||
upn,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ "Action", "Offboard" },
|
||||
{ "Phase", "Initiated" }
|
||||
{ "Phase", "Initiated" },
|
||||
{ "HistoryRowKey", historyRowKey },
|
||||
{ "DomainName", domainName ?? string.Empty },
|
||||
});
|
||||
|
||||
var result = new OffboardResult { TenantId = requestCtx.TargetTenantId, InitiatedBy = upn!, InitiatedAt = DateTime.UtcNow };
|
||||
|
||||
// Capture domain name BEFORE deletion — tenant config is wiped by DeleteAllTenantDataAsync,
|
||||
// and the ops alert / Telegram message is friendlier with the domain than the bare GUID.
|
||||
string? domainName = null;
|
||||
// 9. Enqueue with full DrainBarrier as visibility delay. The worker won't see this
|
||||
// message until ~6min from now, by which time every function-host has refreshed
|
||||
// its TenantConfiguration cache and the existing Disabled-gate is the active
|
||||
// block. Fail-loud — marker stays Initiated on failure; the next admin re-click
|
||||
// hits the idempotency branch which defensively re-enqueues with the remaining
|
||||
// barrier so the tenant cannot get stuck.
|
||||
try
|
||||
{
|
||||
var tenantConfig = await _configRepo.GetTenantConfigurationAsync(requestCtx.TargetTenantId);
|
||||
domainName = tenantConfig?.DomainName;
|
||||
await EnqueueEnvelopeAsync(normalizedTenantId, historyRowKey, upn, now, DrainBarrier);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read tenant config for {TenantId} before offboard; proceeding without domain name", requestCtx.TargetTenantId);
|
||||
_logger.LogError(ex,
|
||||
"Failed to enqueue tenant-offboarding envelope for {TenantId}; marker remains Initiated for re-click resume",
|
||||
normalizedTenantId);
|
||||
return await Build500Async(req, "Failed to enqueue tenant offboarding");
|
||||
}
|
||||
|
||||
try
|
||||
_logger.LogWarning(
|
||||
"TENANT OFFBOARD queued for {TenantId} by {Upn}; historyRowKey={History}; earliestProcessingAt={ProcessAt}",
|
||||
normalizedTenantId, upn, historyRowKey, earliestProcessingAt);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.Accepted);
|
||||
await response.WriteAsJsonAsync(new OffboardResponse
|
||||
{
|
||||
var deletedCounts = await _maintenanceRepo.DeleteAllTenantDataAsync(requestCtx.TargetTenantId);
|
||||
result.DeletedCounts = deletedCounts;
|
||||
|
||||
result.Success = true;
|
||||
_logger.LogWarning($"TENANT OFFBOARD completed for tenant {requestCtx.TargetTenantId} by {upn}. " +
|
||||
$"Total rows deleted: {result.DeletedCounts.Values.Sum()}");
|
||||
|
||||
// Audit completion with deletion summary
|
||||
var details = result.DeletedCounts.ToDictionary(kv => kv.Key, kv => kv.Value.ToString());
|
||||
details["Action"] = "Offboard";
|
||||
details["Phase"] = "Completed";
|
||||
details["TotalRowsDeleted"] = result.DeletedCounts.Values.Sum().ToString();
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
Constants.AuditGlobalTenantId,
|
||||
"DELETE",
|
||||
"Tenant",
|
||||
requestCtx.TargetTenantId,
|
||||
upn!,
|
||||
details);
|
||||
|
||||
await _opsEventService.RecordTenantOffboardedAsync(requestCtx.TargetTenantId, upn!, deletedCounts, domainName);
|
||||
|
||||
var okResponse = req.CreateResponse(HttpStatusCode.OK);
|
||||
await okResponse.WriteAsJsonAsync(result);
|
||||
return okResponse;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, $"Tenant offboard failed for tenant {requestCtx.TargetTenantId}");
|
||||
result.Success = false;
|
||||
result.Error = "Internal server error";
|
||||
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteAsJsonAsync(result);
|
||||
return errorResponse;
|
||||
}
|
||||
TenantId = normalizedTenantId,
|
||||
Status = "Queued",
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = historyRowKey,
|
||||
EarliestProcessingAt = earliestProcessingAt,
|
||||
Message = "Tenant offboarding queued for asynchronous processing"
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
// Resume-path for re-clicks and 409-race winners. Thin HTTP shell on top of
|
||||
// ResumeExistingMarkerAsync — the actual decision (re-enqueue or not, message text,
|
||||
// remaining drain-barrier) lives in the pure helper below so it can be unit-tested
|
||||
// without faking HttpRequestData. Plan Review Finding 1 (stuck-tenant fix).
|
||||
internal async Task<HttpResponseData> HandleExistingMarkerAsync(
|
||||
HttpRequestData req,
|
||||
OffboardingMarkerEntry existing,
|
||||
string upn,
|
||||
string normalizedTenantId,
|
||||
bool isRace)
|
||||
{
|
||||
var resume = await ResumeExistingMarkerAsync(existing, upn, normalizedTenantId, isRace);
|
||||
|
||||
if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed)
|
||||
{
|
||||
return await Build500Async(req, "Failed to re-enqueue tenant offboarding");
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(resume.StatusCode);
|
||||
await response.WriteAsJsonAsync(new OffboardResponse
|
||||
{
|
||||
TenantId = normalizedTenantId,
|
||||
Status = existing.Status,
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = existing.OffboardingHistoryRowKey,
|
||||
EarliestProcessingAt = resume.EarliestProcessingAt,
|
||||
Message = resume.Message,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure resume decision: classifies the existing marker, optionally re-enqueues, and
|
||||
/// returns the (StatusCode, Message, ReEnqueued?) tuple the HTTP shell wraps. Exposed
|
||||
/// internal for unit tests — Finding-1 fix requires asserting "Initiated/InProgress
|
||||
/// triggers a queue write, Completed/Failed does not".
|
||||
/// </summary>
|
||||
internal async Task<ResumeOutcome> ResumeExistingMarkerAsync(
|
||||
OffboardingMarkerEntry existing,
|
||||
string upn,
|
||||
string normalizedTenantId,
|
||||
bool isRace)
|
||||
{
|
||||
var status = existing.Status ?? "Initiated";
|
||||
|
||||
if (string.Equals(status, "Completed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"TenantOffboard re-click for {TenantId}: marker is Completed; no re-enqueue (race={Race})",
|
||||
normalizedTenantId, isRace);
|
||||
return new ResumeOutcome(
|
||||
ResumeOutcomeKind.IdempotentCompleted,
|
||||
HttpStatusCode.OK,
|
||||
"Tenant offboarding already completed (marker in 15-min grace window)",
|
||||
ReEnqueued: false);
|
||||
}
|
||||
|
||||
if (string.Equals(status, "Failed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
// Failed is fail-closed by design (R1). Re-enqueuing a Failed offboarding would
|
||||
// just retrace the same failure path. Surface the FailedPhase to the operator
|
||||
// so they can clear the marker manually before retrying.
|
||||
_logger.LogWarning(
|
||||
"TenantOffboard re-click for {TenantId}: marker is Failed (phase={Phase}); operator action required, no re-enqueue",
|
||||
normalizedTenantId, existing.FailedPhase ?? "unknown");
|
||||
return new ResumeOutcome(
|
||||
ResumeOutcomeKind.IdempotentFailed,
|
||||
HttpStatusCode.OK,
|
||||
$"Tenant offboarding previously failed at phase '{existing.FailedPhase ?? "unknown"}' — operator action required before retry",
|
||||
ReEnqueued: false);
|
||||
}
|
||||
|
||||
// Initiated / InProgress (or any unrecognised in-flight value): defensively
|
||||
// re-enqueue the envelope. The handler tolerates double-pickup via the
|
||||
// DrainCompletedAt skip-gate, so a duplicate envelope on the queue is safe.
|
||||
//
|
||||
// The visibility-delay computation below uses
|
||||
// max(history.EarliestProcessingAt, tombstone.LastUpdated + DrainBarrier)
|
||||
// as the effective drain-deadline. History is best-effort here: if the read
|
||||
// fails or the field is missing, the tombstone anchor still gives a safe lower
|
||||
// bound (when the tombstone was just written, that's now + DrainBarrier; when
|
||||
// it was written long ago, the barrier is already elapsed).
|
||||
OffboardingHistoryEntry? history = null;
|
||||
try
|
||||
{
|
||||
history = await _offboardingRepo.TryGetHistoryAsync(existing.OffboardingHistoryRowKey);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Failed to read History row {HistoryRowKey} for {TenantId} during resume; falling through to tombstone.LastUpdated-based deadline",
|
||||
existing.OffboardingHistoryRowKey, normalizedTenantId);
|
||||
}
|
||||
// Re-affirm the Disabled-gate FIRST so we know whether we just wrote it for the
|
||||
// first time (= cache-drain barrier restarts now) or it was already there from a
|
||||
// prior attempt (= the History row's deadline is still valid). The prior run may
|
||||
// have crashed between marker-insert and Disabled-flip, or an admin may have
|
||||
// cleared the flag manually. Either way, we MUST NOT re-enqueue without the
|
||||
// auth-gate committed — otherwise the cache-drain barrier guards nothing.
|
||||
EnsureDisabledResult ensureResult;
|
||||
try
|
||||
{
|
||||
ensureResult = await EnsureTenantDisabledAsync(normalizedTenantId, upn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"Resume: failed to re-affirm TenantConfiguration.Disabled=true for {TenantId}; refusing to re-enqueue",
|
||||
normalizedTenantId);
|
||||
return new ResumeOutcome(
|
||||
ResumeOutcomeKind.ReEnqueueFailed,
|
||||
HttpStatusCode.InternalServerError,
|
||||
"Failed to re-affirm Disabled-gate; tenant offboarding NOT re-enqueued",
|
||||
ReEnqueued: false);
|
||||
}
|
||||
|
||||
// Resume-path visibility-delay decision (Codex finding "Resume can bypass cache-drain"):
|
||||
// We compute the effective drain-deadline as the MAX of two sources, then derive
|
||||
// the queue-message visibility-delay from that:
|
||||
//
|
||||
// (a) history.EarliestProcessingAt — set on Phase 1 commit; patched here on
|
||||
// fresh writes when we can.
|
||||
// (b) ensureResult.TombstoneLastUpdated + DrainBarrier — the authoritative
|
||||
// "cache-drain started here" timestamp, written atomically with the
|
||||
// Disabled-flip, ALWAYS present even if the History-patch later fails.
|
||||
//
|
||||
// Using max() of both makes the History-patch non-load-bearing for safety: even if
|
||||
// a fast follow-up re-click reads stale EarliestProcessingAt, the TombstoneLastUpdated
|
||||
// anchor gives us a correct lower bound. The History-patch is now best-effort
|
||||
// documentation, not a critical write.
|
||||
var now = DateTime.UtcNow;
|
||||
var tombstoneDeadline = ensureResult.TombstoneLastUpdated + DrainBarrier;
|
||||
var historyDeadline = history?.EarliestProcessingAt ?? tombstoneDeadline;
|
||||
var effectiveDeadline = tombstoneDeadline > historyDeadline ? tombstoneDeadline : historyDeadline;
|
||||
|
||||
var visibilityDelay = effectiveDeadline > now ? effectiveDeadline - now : TimeSpan.Zero;
|
||||
var earliestProcessingAtForResponse = effectiveDeadline;
|
||||
|
||||
// Patch History.EarliestProcessingAt for new tombstone writes so the UI countdown
|
||||
// + any later admin tools see the new deadline. Failure here is NOT a safety issue
|
||||
// anymore (tombstone.LastUpdated covers us above) — purely cosmetic / observability.
|
||||
if (ensureResult.Outcome == EnsureDisabledOutcome.WroteNewTombstone && history != null)
|
||||
{
|
||||
history.EarliestProcessingAt = effectiveDeadline;
|
||||
try
|
||||
{
|
||||
await _offboardingRepo.UpsertHistoryAsync(history);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Resume: failed to patch History.EarliestProcessingAt for {TenantId} {History} — UI countdown may show stale value, but the queue visibility-delay + tombstone.LastUpdated guarantee the drain barrier on the worker side",
|
||||
normalizedTenantId, existing.OffboardingHistoryRowKey);
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Resume: Disabled-gate was freshly written; restarted full cache-drain barrier ({Barrier}) for tenant={TenantId}",
|
||||
DrainBarrier, normalizedTenantId);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await EnqueueEnvelopeAsync(
|
||||
normalizedTenantId, existing.OffboardingHistoryRowKey, upn, existing.InitiatedAt, visibilityDelay);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"Defensive re-enqueue failed for {TenantId} (existing marker status={Status}, race={Race})",
|
||||
normalizedTenantId, status, isRace);
|
||||
return new ResumeOutcome(
|
||||
ResumeOutcomeKind.ReEnqueueFailed,
|
||||
HttpStatusCode.InternalServerError,
|
||||
"Failed to re-enqueue tenant offboarding",
|
||||
ReEnqueued: false);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"TenantOffboard re-click for {TenantId}: defensively re-enqueued with visibilityDelay={Delay} outcome={Outcome} tombstoneLastUpdated={TombstoneLastUpdated} (existing status={Status}, race={Race})",
|
||||
normalizedTenantId, visibilityDelay, ensureResult.Outcome, ensureResult.TombstoneLastUpdated, status, isRace);
|
||||
|
||||
return new ResumeOutcome(
|
||||
ResumeOutcomeKind.ReEnqueuedInFlight,
|
||||
HttpStatusCode.Accepted,
|
||||
isRace
|
||||
? "Tenant offboarding already in progress (race-winner resolved, defensively re-enqueued)"
|
||||
: $"Tenant offboarding resumed (existing status={status}, defensively re-enqueued)",
|
||||
ReEnqueued: true,
|
||||
EarliestProcessingAt: earliestProcessingAtForResponse);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Magic-string DisabledReason that identifies a TenantConfiguration row as the
|
||||
/// offboarding tombstone. Used by <c>EnsureTenantDisabledAsync</c> for the idempotent
|
||||
/// no-write check AND by <see cref="OffboardingMarkerCleanupFunction"/> to gate the
|
||||
/// defense-in-depth TenantConfiguration sweep so a self-service-re-onboarded fresh
|
||||
/// config does not get wiped by mistake.
|
||||
/// </summary>
|
||||
internal const string OffboardingDisabledReason = "Offboarding in progress";
|
||||
|
||||
/// <summary>
|
||||
/// Result of <see cref="EnsureTenantDisabledAsync"/>. Carries both whether a fresh
|
||||
/// write happened AND the tombstone's <c>LastUpdated</c> timestamp so the resume-path
|
||||
/// can compute a safe drain-deadline without depending on the (optionally-failing)
|
||||
/// <c>History.EarliestProcessingAt</c> patch.
|
||||
/// <para>
|
||||
/// Drain-deadline math the caller applies:
|
||||
/// <c>effectiveDeadline = max(history.EarliestProcessingAt, TombstoneLastUpdated + DrainBarrier)</c>.
|
||||
/// Because the tombstone's <c>LastUpdated</c> is part of the same atomic Save as the
|
||||
/// Disabled-flip, it is always present once the auth-gate is committed — even if
|
||||
/// the History-row patch later fails on a fast re-click.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal readonly record struct EnsureDisabledResult(
|
||||
EnsureDisabledOutcome Outcome,
|
||||
DateTime TombstoneLastUpdated);
|
||||
|
||||
internal enum EnsureDisabledOutcome
|
||||
{
|
||||
/// <summary>Row was already a Disabled=true tombstone with the offboarding reason —
|
||||
/// no Save was issued. Existing <c>LastUpdated</c> is returned.</summary>
|
||||
AlreadyTombstone,
|
||||
/// <summary>We actively wrote the Disabled=true gate (either flipped an existing
|
||||
/// row or created a fresh one). <c>LastUpdated</c> is the timestamp of this write.</summary>
|
||||
WroteNewTombstone,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits the SOLE active auth-gate (<c>TenantConfiguration.Disabled=true</c>) and
|
||||
/// invalidates the local <see cref="TenantConfigurationService"/> cache. Fail-loud:
|
||||
/// every caller (initial enqueue + resume re-enqueue) MUST treat an exception as
|
||||
/// "do not enqueue" — without this gate committed, the cache-drain barrier protects
|
||||
/// nothing and Phase 2 would wipe data while warm function-hosts still accept agent
|
||||
/// traffic.
|
||||
/// <para>
|
||||
/// Return value distinguishes the two outcomes so the resume-path can decide whether
|
||||
/// to honour the old <c>History.EarliestProcessingAt</c> deadline
|
||||
/// (<see cref="EnsureDisabledResult.AlreadyTombstone"/>) or restart the cache-drain
|
||||
/// barrier from now (<see cref="EnsureDisabledResult.WroteNewTombstone"/>) — see
|
||||
/// resume-path Codex finding "Resume can bypass cache-drain when first attempt failed
|
||||
/// before Disabled-write".
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Behaviour:
|
||||
/// <list type="bullet">
|
||||
/// <item>Existing TenantConfiguration row → read-modify-write so signup data + custom
|
||||
/// rate limits etc. are preserved. Idempotent: if already
|
||||
/// <c>Disabled=true</c> with the offboarding reason, no Save is issued and
|
||||
/// <see cref="EnsureDisabledResult.AlreadyTombstone"/> is returned.</item>
|
||||
/// <item>Missing TenantConfiguration row (edge case: very old tenant / manual delete)
|
||||
/// → create a fresh <c>CreateDefault</c> row with <c>Disabled=true</c>. Necessary
|
||||
/// because <c>AuthFunction</c> would otherwise auto-create a <c>Disabled=false</c>
|
||||
/// default on the next <c>/api/auth/me</c> and slip the offboarded tenant back through.</item>
|
||||
/// <item><c>SaveTenantConfigurationAsync</c> returns <c>false</c> on transient storage
|
||||
/// failure (the production repo does NOT throw; see Codex-finding TableConfigRepository).
|
||||
/// We turn that into <see cref="InvalidOperationException"/> so the caller treats it
|
||||
/// as a hard fail.</item>
|
||||
/// </list>
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal async Task<EnsureDisabledResult> EnsureTenantDisabledAsync(string targetTenantId, string upn)
|
||||
{
|
||||
var tenantConfig = await _configRepo.GetTenantConfigurationAsync(targetTenantId);
|
||||
|
||||
if (tenantConfig != null)
|
||||
{
|
||||
// Idempotent: if Disabled is already true with our reason, no-op the write so
|
||||
// resume-clicks do not thrash the LastUpdated timestamp on the row.
|
||||
var alreadyDisabled = tenantConfig.Disabled
|
||||
&& string.Equals(tenantConfig.DisabledReason, OffboardingDisabledReason, StringComparison.Ordinal);
|
||||
if (alreadyDisabled)
|
||||
{
|
||||
_tenantConfigService.InvalidateCache(targetTenantId);
|
||||
return new EnsureDisabledResult(EnsureDisabledOutcome.AlreadyTombstone, tenantConfig.LastUpdated);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Missing-row edge case: create a fresh Disabled=true row so AuthFunction's
|
||||
// auto-create-default cannot bring the tenant back enabled.
|
||||
tenantConfig = TenantConfiguration.CreateDefault(targetTenantId);
|
||||
_logger.LogWarning(
|
||||
"TenantOffboard: no TenantConfiguration row found for {TenantId} — creating fresh Disabled=true tombstone",
|
||||
targetTenantId);
|
||||
}
|
||||
|
||||
tenantConfig.Disabled = true;
|
||||
tenantConfig.DisabledReason = OffboardingDisabledReason;
|
||||
tenantConfig.DisabledUntil = null;
|
||||
tenantConfig.UpdatedBy = upn;
|
||||
// Explicit LastUpdated stamp — we bypass TenantConfigurationService.SaveConfigurationAsync
|
||||
// (which would set this) and write to the repo directly, so we own the bookkeeping.
|
||||
// Critical for the resume-path drain-deadline math: this timestamp is the
|
||||
// authoritative "when did the cache-drain barrier start" anchor.
|
||||
var writtenAt = DateTime.UtcNow;
|
||||
tenantConfig.LastUpdated = writtenAt;
|
||||
|
||||
var saved = await _configRepo.SaveTenantConfigurationAsync(tenantConfig);
|
||||
if (!saved)
|
||||
{
|
||||
// TableConfigRepository.SaveTenantConfigurationAsync returns false on transient
|
||||
// storage failure instead of throwing. Without throwing here, the caller would
|
||||
// proceed to enqueue without the auth-gate committed — defeating the whole
|
||||
// cache-drain barrier. Surface as a hard failure.
|
||||
throw new InvalidOperationException(
|
||||
$"SaveTenantConfigurationAsync returned false for tenant {targetTenantId}; Disabled-gate NOT committed");
|
||||
}
|
||||
_tenantConfigService.InvalidateCache(targetTenantId);
|
||||
return new EnsureDisabledResult(EnsureDisabledOutcome.WroteNewTombstone, writtenAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enqueues the offboarding envelope with the given visibility delay (queue-side
|
||||
/// invisibility window). First-click: full <see cref="DrainBarrier"/>. Resume-click:
|
||||
/// remaining delay computed from <c>EarliestProcessingAt</c> so a re-enqueue does not
|
||||
/// reset the drain barrier (the message would otherwise become visible too early).
|
||||
/// </summary>
|
||||
internal Task EnqueueEnvelopeAsync(
|
||||
string normalizedTenantId, string historyRowKey, string upn, DateTime initiatedAt,
|
||||
TimeSpan? visibilityDelay)
|
||||
{
|
||||
var envelope = new TenantOffboardingEnvelope
|
||||
{
|
||||
EnvelopeVersion = "1",
|
||||
TenantId = normalizedTenantId,
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = historyRowKey,
|
||||
InitiatedBy = upn,
|
||||
InitiatedAt = initiatedAt,
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
DrainPollCount = 0,
|
||||
};
|
||||
return _offboardingEnqueuer.EnqueueAsync(envelope, visibilityDelay);
|
||||
}
|
||||
|
||||
// Plan §4.4 read-modify-write CAS-loop. Bounded retry; throws on exhaustion so the
|
||||
// caller turns it into a 500. The pointer is the O(1) re-onboarding index — failure
|
||||
// here is recoverable (graceful degradation in PR4) but we still surface it so the
|
||||
// operator can investigate.
|
||||
internal async Task UpsertPointerWithCasAsync(
|
||||
string normalizedTenantId, string historyRowKey, DateTime now)
|
||||
{
|
||||
for (var attempt = 1; attempt <= PointerCasMaxAttempts; attempt++)
|
||||
{
|
||||
var (existing, etag) = await _offboardingRepo.TryGetByTenantPointerAsync(normalizedTenantId);
|
||||
|
||||
if (existing == null)
|
||||
{
|
||||
var fresh = new OffboardingByTenantPointer
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
|
||||
RowKey = normalizedTenantId,
|
||||
TenantId = normalizedTenantId,
|
||||
LatestHistoryRowKey = historyRowKey,
|
||||
LatestStatus = "Initiated",
|
||||
LatestUpdatedAt = now,
|
||||
OffboardCount = 1,
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
await _offboardingRepo.InsertByTenantPointerAsync(fresh);
|
||||
return;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"OffboardingByTenant pointer raced on insert for {TenantId} (attempt {Attempt}); falling back to ETag update",
|
||||
normalizedTenantId, attempt);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
existing.LatestHistoryRowKey = historyRowKey;
|
||||
existing.LatestStatus = "Initiated";
|
||||
existing.LatestUpdatedAt = now;
|
||||
existing.OffboardCount = existing.OffboardCount + 1;
|
||||
|
||||
try
|
||||
{
|
||||
await _offboardingRepo.UpdateByTenantPointerWithEtagAsync(existing, etag!);
|
||||
return;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"OffboardingByTenant pointer ETag mismatch for {TenantId} (attempt {Attempt}); retrying",
|
||||
normalizedTenantId, attempt);
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException(
|
||||
$"OffboardingByTenant pointer CAS exhausted for tenant {normalizedTenantId} after {PointerCasMaxAttempts} attempts");
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseData> Build500Async(HttpRequestData req, string message)
|
||||
{
|
||||
var response = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await response.WriteAsJsonAsync(new { success = false, error = message });
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
public class OffboardResult
|
||||
/// <summary>
|
||||
/// 202/200 response body for the offboarding endpoint. Fields point the caller at the
|
||||
/// History row so subsequent reporting / status polling can resolve back to the audit
|
||||
/// trail. <see cref="EarliestProcessingAt"/> drives the "data deletion starts in mm ss"
|
||||
/// countdown in the Web UI's drain-barrier state.
|
||||
/// </summary>
|
||||
public class OffboardResponse
|
||||
{
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
public string InitiatedBy { get; set; } = string.Empty;
|
||||
public DateTime InitiatedAt { get; set; }
|
||||
public bool Success { get; set; }
|
||||
public string? Error { get; set; }
|
||||
public Dictionary<string, int> DeletedCounts { get; set; } = new();
|
||||
public string Status { get; set; } = string.Empty;
|
||||
public string HistoryPartitionKey { get; set; } = string.Empty;
|
||||
public string HistoryRowKey { get; set; } = string.Empty;
|
||||
public string Message { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>UTC timestamp before which the worker MUST NOT start Phase 2. Drives the
|
||||
/// cache-drain-barrier countdown UI. Null on the idempotent-Completed/Failed branches.</summary>
|
||||
public DateTime? EarliestProcessingAt { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decision the resume-path produced; consumed by <see cref="TenantOffboardFunction.HandleExistingMarkerAsync"/>
|
||||
/// to shape the HTTP response. Test-visible so unit tests can assert the re-enqueue decision
|
||||
/// without faking <c>HttpRequestData</c>.
|
||||
/// <para>
|
||||
/// <see cref="EarliestProcessingAt"/> is the History row's drain-barrier deadline for the
|
||||
/// in-flight path (so the UI can keep counting down through a re-click); null on the
|
||||
/// terminal idempotent branches.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal sealed record ResumeOutcome(
|
||||
ResumeOutcomeKind Kind,
|
||||
System.Net.HttpStatusCode StatusCode,
|
||||
string Message,
|
||||
bool ReEnqueued,
|
||||
DateTime? EarliestProcessingAt = null);
|
||||
|
||||
internal enum ResumeOutcomeKind
|
||||
{
|
||||
/// <summary>Marker is Completed; 15-min grace blocker is held, no re-enqueue.</summary>
|
||||
IdempotentCompleted,
|
||||
/// <summary>Marker is Failed; operator action required, no re-enqueue.</summary>
|
||||
IdempotentFailed,
|
||||
/// <summary>Marker is Initiated/InProgress; defensive re-enqueue was sent.</summary>
|
||||
ReEnqueuedInFlight,
|
||||
/// <summary>Marker classified as in-flight but re-enqueue threw; surface 500.</summary>
|
||||
ReEnqueueFailed,
|
||||
}
|
||||
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Admin;
|
||||
using AutopilotMonitor.Functions.Services.Offboarding;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Marker housekeeping + defense-in-depth cleanup for the tenant-offboarding cascade.
|
||||
/// Plan §7.5 (PR3.B-revised).
|
||||
/// <para>
|
||||
/// Tenant-offboarding markers (<see cref="OffboardingMarkerEntry"/> under
|
||||
/// <c>OffboardingAudit/OffboardingMarker</c>) live past the Completed transition by
|
||||
/// design — they are the anchor that lets this function find lingering side-effects
|
||||
/// (Expectations blob, possibly-stale <c>TenantConfiguration</c> tombstone) and clean
|
||||
/// them up if the handler's Phase 2.F-final/2.G ordering hit a crash. Failed markers
|
||||
/// are never auto-removed; they require operator action.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Note: in the PR3-revised architecture the marker is NO LONGER read in the
|
||||
/// agent/web auth hotpath. The active gate is <c>TenantConfiguration.Disabled=true</c>;
|
||||
/// the cache-drain barrier (6 min visibility delay on the worker envelope) plus
|
||||
/// <see cref="MinCompletedAgeForCleanup"/> together absorb the
|
||||
/// <c>TenantConfigurationService</c> cache TTL (5 min). The marker's lifetime here is
|
||||
/// purely about cleanup ordering and crash recovery, not auth.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Cleanup order per Completed marker, each step's failure keeps the marker for retry:
|
||||
/// (1) Expectations blob delete, (2) probe TenantConfiguration → conditional sweep if
|
||||
/// still the offboarding tombstone (skipped when the user has self-service-re-onboarded
|
||||
/// and the row carries a non-offboarding state), (3) marker delete.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class OffboardingMarkerCleanupFunction
|
||||
{
|
||||
/// <summary>
|
||||
/// 3× <c>TenantConfigurationService.CacheDuration</c> (5min) — safety margin that
|
||||
/// absorbs clock-skew + GC pauses + cache-read-after-save anomalies on warm
|
||||
/// function instances. Originally sized to guarantee the marker outlives every
|
||||
/// cached TenantConfiguration on every host; retained at 15min because the
|
||||
/// defense-in-depth sweeps (Expectations blob + TenantConfiguration tombstone)
|
||||
/// still benefit from waiting for the dust to settle.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan MinCompletedAgeForCleanup = TimeSpan.FromMinutes(15);
|
||||
|
||||
/// <summary>Failed markers older than this trigger a warn-log every run (operator nudge).</summary>
|
||||
internal static readonly TimeSpan FailedMarkerWarnAge = TimeSpan.FromHours(1);
|
||||
|
||||
// Every 2h — matches the broader Maintenance cadence and gives at most a 2h lag between
|
||||
// 15min-eligible and actual cleanup. Plan §7.5: "Function läuft im selben Maintenance-
|
||||
// Schedule wie SessionDeletionMaintenance".
|
||||
private const string Cron = "0 0 */2 * * *";
|
||||
|
||||
private readonly IOffboardingAuditRepository _auditRepo;
|
||||
private readonly IOffboardingExpectationsStore _expectations;
|
||||
private readonly SafeWipeService _safeWipe;
|
||||
private readonly IConfigRepository _configRepo;
|
||||
private readonly ILogger<OffboardingMarkerCleanupFunction> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public OffboardingMarkerCleanupFunction(
|
||||
IOffboardingAuditRepository auditRepo,
|
||||
IOffboardingExpectationsStore expectations,
|
||||
SafeWipeService safeWipe,
|
||||
IConfigRepository configRepo,
|
||||
ILogger<OffboardingMarkerCleanupFunction> logger)
|
||||
: this(auditRepo, expectations, safeWipe, configRepo, logger, TimeProvider.System)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Test seam — inject a fake <see cref="TimeProvider"/> so age comparisons are deterministic.</summary>
|
||||
internal OffboardingMarkerCleanupFunction(
|
||||
IOffboardingAuditRepository auditRepo,
|
||||
IOffboardingExpectationsStore expectations,
|
||||
SafeWipeService safeWipe,
|
||||
IConfigRepository configRepo,
|
||||
ILogger<OffboardingMarkerCleanupFunction> logger,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_auditRepo = auditRepo;
|
||||
_expectations = expectations;
|
||||
_safeWipe = safeWipe;
|
||||
_configRepo = configRepo;
|
||||
_logger = logger;
|
||||
_timeProvider = timeProvider;
|
||||
}
|
||||
|
||||
[Function("OffboardingMarkerCleanup")]
|
||||
public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken)
|
||||
{
|
||||
await RunCoreAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Testable core — bypasses TimerInfo / FunctionContext.</summary>
|
||||
internal async Task<CleanupResult> RunCoreAsync(CancellationToken ct)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
var result = new CleanupResult();
|
||||
|
||||
await foreach (var marker in _auditRepo.QueryMarkersAsync(ct))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
result.Scanned++;
|
||||
|
||||
if (marker.Status == "Completed"
|
||||
&& marker.CompletedAt is { } completedAt
|
||||
&& completedAt < now - MinCompletedAgeForCleanup)
|
||||
{
|
||||
// Cleanup order — each step's failure must keep the marker for the next run:
|
||||
// 1. Expectations blob (Rev-9 hygiene)
|
||||
// 2. TenantConfiguration sweep (PR3.B-revised Finding 2 defense-in-depth)
|
||||
// 3. Marker delete (last anchor)
|
||||
try
|
||||
{
|
||||
await _expectations.DeleteAsync(marker.TenantId, marker.OffboardingHistoryRowKey, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"OffboardingMarkerCleanup: Expectations blob delete failed for tenant={Tenant} history={History} — keeping marker for next-run retry",
|
||||
marker.TenantId, marker.OffboardingHistoryRowKey);
|
||||
result.BlobDeleteRetries++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// PR3.B Defense-in-depth: if Phase 2.F-final failed in the handler
|
||||
// (crash between 2.G commit and 2.F-final, or transient storage), the
|
||||
// TenantConfiguration row would still carry Disabled=true and block the
|
||||
// tenant from self-service re-onboarding indefinitely. Sweep it here.
|
||||
//
|
||||
// CRITICAL: must be CONDITIONAL on the row still being the offboarding
|
||||
// tombstone. The handler intentionally deletes the row at the end of
|
||||
// Phase 2.F-final precisely so the next /api/auth/me triggers a fresh
|
||||
// Default-Disabled=false config (self-service re-onboarding). If the
|
||||
// user has re-onboarded between Phase 2.F-final and now, blind wiping
|
||||
// would destroy their fresh tenant. So: read first, only wipe when
|
||||
// (Disabled=true AND DisabledReason matches OffboardingDisabledReason).
|
||||
try
|
||||
{
|
||||
var existingConfig = await _configRepo.GetTenantConfigurationAsync(marker.TenantId);
|
||||
if (existingConfig == null)
|
||||
{
|
||||
// Normal happy path: Phase 2.F-final succeeded, row already gone.
|
||||
}
|
||||
else if (existingConfig.Disabled
|
||||
&& string.Equals(existingConfig.DisabledReason, TenantOffboardFunction.OffboardingDisabledReason, StringComparison.Ordinal))
|
||||
{
|
||||
// Still the offboarding tombstone → Phase 2.F-final must have failed.
|
||||
var swept = await _safeWipe.WipeByExactPartitionAsync(
|
||||
Constants.TableNames.TenantConfiguration, marker.TenantId, ct);
|
||||
if (swept > 0)
|
||||
{
|
||||
result.TenantConfigsSwept++;
|
||||
_logger.LogWarning(
|
||||
"OffboardingMarkerCleanup: swept lingering TenantConfiguration tombstone for tenant={Tenant} (Phase 2.F-final must have failed in the handler)",
|
||||
marker.TenantId);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// User has self-service-re-onboarded between Phase 2.F-final and
|
||||
// now. Leave their fresh config alone.
|
||||
result.TenantConfigsSpared++;
|
||||
_logger.LogInformation(
|
||||
"OffboardingMarkerCleanup: tenant={Tenant} has a non-tombstone TenantConfiguration row (Disabled={Disabled}, Reason={Reason}) — user re-onboarded, skip sweep",
|
||||
marker.TenantId, existingConfig.Disabled, existingConfig.DisabledReason ?? "<null>");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"OffboardingMarkerCleanup: TenantConfiguration probe/sweep failed for tenant={Tenant} — keeping marker for next-run retry",
|
||||
marker.TenantId);
|
||||
result.TenantConfigSweepRetries++;
|
||||
continue;
|
||||
}
|
||||
|
||||
await _auditRepo.DeleteMarkerAsync(marker.RowKey, ct);
|
||||
result.MarkersDeleted++;
|
||||
_logger.LogInformation(
|
||||
"OffboardingMarkerCleanup: deleted Completed marker for tenant={Tenant} (age={Age})",
|
||||
marker.TenantId, now - completedAt);
|
||||
}
|
||||
else if (marker.Status == "Failed")
|
||||
{
|
||||
result.FailedMarkersSeen++;
|
||||
var failedAt = marker.FailedAt ?? marker.InitiatedAt;
|
||||
var age = now - failedAt;
|
||||
if (age > FailedMarkerWarnAge)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"OffboardingMarkerCleanup: stale Failed marker tenant={Tenant} age={Age} phase={Phase} — operator action required",
|
||||
marker.TenantId, age, marker.FailedPhase ?? "unknown");
|
||||
}
|
||||
}
|
||||
// Initiated / InProgress → still working, ignore.
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"OffboardingMarkerCleanup completed: scanned={Scanned} deleted={Deleted} blobRetries={BlobRetries} tenantConfigSwept={ConfigSwept} tenantConfigSpared={ConfigSpared} tenantConfigRetries={ConfigRetries} failed={Failed}",
|
||||
result.Scanned, result.MarkersDeleted, result.BlobDeleteRetries,
|
||||
result.TenantConfigsSwept, result.TenantConfigsSpared, result.TenantConfigSweepRetries, result.FailedMarkersSeen);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>Summary counters returned from <see cref="RunCoreAsync"/> so tests can assert outcomes.</summary>
|
||||
internal sealed class CleanupResult
|
||||
{
|
||||
public int Scanned { get; set; }
|
||||
public int MarkersDeleted { get; set; }
|
||||
public int BlobDeleteRetries { get; set; }
|
||||
public int FailedMarkersSeen { get; set; }
|
||||
/// <summary>Number of stale TenantConfiguration tombstones the defense-in-depth sweep removed.</summary>
|
||||
public int TenantConfigsSwept { get; set; }
|
||||
/// <summary>Number of TenantConfiguration rows the sweep deliberately left alone because the
|
||||
/// tenant had re-onboarded (DisabledReason is no longer the offboarding tombstone string).</summary>
|
||||
public int TenantConfigsSpared { get; set; }
|
||||
/// <summary>Number of markers kept alive because the TenantConfiguration probe/sweep failed transiently.</summary>
|
||||
public int TenantConfigSweepRetries { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -155,6 +155,27 @@ builder.Services.AddHostedService<AutopilotMonitor.Functions.Services.Deletion.S
|
||||
// retention fanout (V2 + legacy), manifest-blob TTL sweep, stale-Preparing GC, and stranded-
|
||||
// Queued detection. Independent cadence + kill-switch from the generic 2h Maintenance timer.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionRetentionFanoutService>();
|
||||
|
||||
// Tenant-offboarding cascade worker (Plan Rev 9). Consumes per-tenant envelopes off
|
||||
// `tenant-offboarding`, drives the §3 Phase 2 sequence (cascade enqueue → drain → SafeWipe →
|
||||
// blob wipe → tenant config delete), and lands the audit row under AuditGlobalTenantId.
|
||||
// OffboardingFilters is a static helper — no DI registration needed.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Offboarding.OffboardingSessionEnumerator>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Services.Offboarding.IOffboardingExpectationsStore,
|
||||
AutopilotMonitor.Functions.Services.Offboarding.BlobOffboardingExpectationsStore>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Services.Offboarding.IDeletionProgressDrainProbe,
|
||||
AutopilotMonitor.Functions.Services.Offboarding.DeletionProgressDrainProbe>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Services.Offboarding.ITenantOffboardingEnqueuer,
|
||||
AutopilotMonitor.Functions.Services.Offboarding.AzureQueueTenantOffboardingEnqueuer>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Offboarding.TenantOffboardingHandler>();
|
||||
builder.Services.AddHostedService<AutopilotMonitor.Functions.Services.Offboarding.TenantOffboardingWorker>();
|
||||
// OffboardingMarkerCleanupFunction is TimerTrigger — auto-registered via the Functions host;
|
||||
// no AddHostedService call needed. DI for its dependencies comes from the singletons above.
|
||||
|
||||
builder.Services.AddHostedService<TableInitializerService>(); // Initialize all tables at startup
|
||||
|
||||
// Data Access Layer — repository interfaces backed by Table Storage.
|
||||
|
||||
@@ -275,6 +275,18 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("POST", "versions/block", EndpointPolicy.GlobalAdminOnly),
|
||||
new("DELETE", "versions/block/{encodedPattern}", EndpointPolicy.GlobalAdminOnly),
|
||||
new("POST", "maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
// ── Tenant Offboarding Customs Archive (PR3.B) ─────────────────────────
|
||||
// Snapshot of each tenant's custom GatherRules / AnalyzeRules / ImeLogPatterns
|
||||
// rows from prior offboarding runs. Operator-driven cleanup: GA reviews + deletes.
|
||||
// The {tenantId} route param is the OFFBOARDED tenant whose data was archived;
|
||||
// the operator (GA) does not belong to that tenant. RouteParam scoping declares
|
||||
// that the route's tenantId is supplied from the URL, not from the caller's JWT.
|
||||
new("GET", "global/customs-archive", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "global/customs-archive/{tenantId}/{historyRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam),
|
||||
new("GET", "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam),
|
||||
new("DELETE", "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam),
|
||||
new("DELETE", "global/customs-archive/{tenantId}/{historyRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam),
|
||||
new("GET", "health/detailed", EndpointPolicy.AuthenticatedUser),
|
||||
new("POST", "rules/reseed-from-github", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "vulnerability/unmatched-software", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
@@ -22,7 +22,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
// Container that holds cascade-deletion manifests (snapshot + progress blobs).
|
||||
// Plan §3 + §10: 30d Lifecycle delete + 3d soft-delete = effective max 33-day retention.
|
||||
private const string DeletionManifestsContainer = "deletion-manifests";
|
||||
// Single source of truth: AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests.
|
||||
private const string DeletionManifestsContainer = AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests;
|
||||
|
||||
private readonly BlobServiceClient _blobServiceClient;
|
||||
private readonly ILogger<BlobStorageService> _logger;
|
||||
|
||||
+19
-10
@@ -259,18 +259,17 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
await ExecuteTombstoneAsync(tenantId, sessionId, manifestId, tombstone, cancellationToken);
|
||||
}
|
||||
|
||||
(progress, etag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, etag,
|
||||
mutate: p =>
|
||||
{
|
||||
p.CompletedSteps.Add(tombstone.Order);
|
||||
p.CompletedAt = DateTime.UtcNow;
|
||||
},
|
||||
isAlreadyApplied: p => p.CompletedAt != null,
|
||||
cancellationToken);
|
||||
|
||||
sw.Stop();
|
||||
|
||||
// PR1.5 (Rev-5-F1) — Audit + SignalR notify BEFORE we stamp CompletedAt on the
|
||||
// progress blob. CompletedAt is the drain predicate the tenant-offboarding
|
||||
// worker uses to decide "all side effects through for this cascade"; if we
|
||||
// set it before the audit row lands, an in-flight cascade can write
|
||||
// `deletion_completed` into AuditLogs *after* the offboard worker has wiped
|
||||
// that table → orphan audit row. Both calls are fail-soft today
|
||||
// (LogAuditEntryAsync returns false on failure, NotifySessionDeletedAsync
|
||||
// catches everything internally) so reordering does not change end-to-end
|
||||
// failure semantics, only the observability window.
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
action: "deletion_completed",
|
||||
@@ -282,6 +281,16 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
|
||||
await _signalR.NotifySessionDeletedAsync(tenantId, sessionId);
|
||||
|
||||
(progress, etag) = await UpdateProgressWithRetryAsync(
|
||||
tenantId, sessionId, manifestId, progress, etag,
|
||||
mutate: p =>
|
||||
{
|
||||
p.CompletedSteps.Add(tombstone.Order);
|
||||
p.CompletedAt = DateTime.UtcNow;
|
||||
},
|
||||
isAlreadyApplied: p => p.CompletedAt != null,
|
||||
cancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionHandler completed: tenant={TenantId} session={SessionId} manifestId={ManifestId} durationMs={DurationMs}",
|
||||
tenantId, sessionId, manifestId, sw.ElapsedMilliseconds);
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Identity;
|
||||
using Azure.Storage.Queues;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Azure Storage Queue producer for <c>tenant-offboarding</c>. Mirrors the existing
|
||||
/// session-deletion / analyze producers but **fail-loud** (plan §7 + memory:
|
||||
/// feedback_storage_helpers_fail_soft). Caller (<c>TenantOffboardFunction</c>) catches
|
||||
/// and translates to HTTP 500; the History + Pointer + Marker rows already exist by then
|
||||
/// so the operator can retry by re-issuing the offboard click (or via maintenance tooling).
|
||||
/// </summary>
|
||||
public sealed class AzureQueueTenantOffboardingEnqueuer : ITenantOffboardingEnqueuer
|
||||
{
|
||||
private readonly QueueClient _queueClient;
|
||||
private readonly ILogger<AzureQueueTenantOffboardingEnqueuer> _logger;
|
||||
|
||||
private int _queueEnsured;
|
||||
|
||||
public AzureQueueTenantOffboardingEnqueuer(
|
||||
IConfiguration configuration,
|
||||
ILogger<AzureQueueTenantOffboardingEnqueuer> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
|
||||
var options = new QueueClientOptions
|
||||
{
|
||||
MessageEncoding = QueueMessageEncoding.Base64,
|
||||
};
|
||||
|
||||
var storageAccountName = configuration["AzureStorageAccountName"];
|
||||
var connectionString = configuration["AzureTableStorageConnectionString"];
|
||||
|
||||
if (!string.IsNullOrEmpty(storageAccountName))
|
||||
{
|
||||
var queueUri = new Uri(
|
||||
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboarding}");
|
||||
_queueClient = new QueueClient(queueUri, new DefaultAzureCredential(), options);
|
||||
_logger.LogInformation(
|
||||
"TenantOffboarding enqueuer initialized with Managed Identity (account: {Account})",
|
||||
storageAccountName);
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(connectionString))
|
||||
{
|
||||
_queueClient = new QueueClient(
|
||||
connectionString, Constants.QueueNames.TenantOffboarding, options);
|
||||
_logger.LogInformation("TenantOffboarding enqueuer initialized with connection string");
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Queue Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureTableStorageConnectionString'.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test seam: bind directly to a (possibly mocked) <see cref="QueueClient"/>.</summary>
|
||||
internal AzureQueueTenantOffboardingEnqueuer(
|
||||
QueueClient queueClient,
|
||||
ILogger<AzureQueueTenantOffboardingEnqueuer> logger)
|
||||
{
|
||||
_queueClient = queueClient ?? throw new ArgumentNullException(nameof(queueClient));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task EnqueueAsync(
|
||||
TenantOffboardingEnvelope envelope,
|
||||
TimeSpan? visibilityDelay = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
|
||||
if (string.IsNullOrEmpty(envelope.TenantId)) throw new ArgumentException("envelope.TenantId required", nameof(envelope));
|
||||
if (string.IsNullOrEmpty(envelope.HistoryRowKey)) throw new ArgumentException("envelope.HistoryRowKey required", nameof(envelope));
|
||||
|
||||
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var body = JsonConvert.SerializeObject(envelope);
|
||||
await _queueClient.SendMessageAsync(
|
||||
body,
|
||||
visibilityTimeout: visibilityDelay,
|
||||
timeToLive: null,
|
||||
cancellationToken: ct).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"TenantOffboarding enqueued tenant={Tenant} history={History} drainPollCount={Polls} visibilityDelay={Delay}",
|
||||
envelope.TenantId, envelope.HistoryRowKey, envelope.DrainPollCount, visibilityDelay);
|
||||
}
|
||||
|
||||
private async Task EnsureQueueExistsAsync(CancellationToken ct)
|
||||
{
|
||||
if (_queueEnsured == 1) return;
|
||||
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _queueEnsured, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Blob-backed implementation of <see cref="IOffboardingExpectationsStore"/>. Pins the
|
||||
/// blob path and container so callers cannot accidentally drop the expectations into
|
||||
/// <c>deletion-manifests</c> (which gets wiped in Phase 2.E and would orphan crash recovery).
|
||||
/// <para>
|
||||
/// The actual blob IO calls are routed through four <c>protected virtual</c> seams so
|
||||
/// tests can replace them with an in-memory fake without standing up Azurite. Production
|
||||
/// overrides hit <see cref="BlobStorageService"/>; tests subclass and override.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class BlobOffboardingExpectationsStore : IOffboardingExpectationsStore
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
};
|
||||
|
||||
private readonly BlobStorageService _blobs;
|
||||
private readonly ILogger<BlobOffboardingExpectationsStore> _logger;
|
||||
private int _containerEnsured;
|
||||
|
||||
public BlobOffboardingExpectationsStore(
|
||||
BlobStorageService blobs, ILogger<BlobOffboardingExpectationsStore> logger)
|
||||
{
|
||||
_blobs = blobs;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
// ── Public contract ─────────────────────────────────────────────────────
|
||||
|
||||
public async Task<bool> TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default)
|
||||
{
|
||||
if (payload == null) throw new ArgumentNullException(nameof(payload));
|
||||
ValidatePayload(payload);
|
||||
|
||||
var blobName = BuildBlobName(payload.TenantId, payload.HistoryRowKey);
|
||||
var bytes = JsonSerializer.SerializeToUtf8Bytes(payload, JsonOptions);
|
||||
|
||||
var (inserted, _) = await WriteIfNoneMatchAsync(blobName, bytes, ct);
|
||||
|
||||
if (inserted)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Expectations blob uploaded tenant={Tenant} history={History} count={Count}",
|
||||
payload.TenantId, payload.HistoryRowKey, payload.Expectations.Count);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Expectations blob already exists tenant={Tenant} history={History} — entering resume path",
|
||||
payload.TenantId, payload.HistoryRowKey);
|
||||
}
|
||||
return inserted;
|
||||
}
|
||||
|
||||
public async Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync(
|
||||
string tenantId, string historyRowKey, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey));
|
||||
|
||||
var blobName = BuildBlobName(tenantId, historyRowKey);
|
||||
var (rawPayload, etag) = await ReadBlobAsync(blobName, ct);
|
||||
if (rawPayload == null) return (null, null);
|
||||
|
||||
var payload = JsonSerializer.Deserialize<OffboardingExpectations>(rawPayload, JsonOptions);
|
||||
return (payload, etag);
|
||||
}
|
||||
|
||||
public async Task<string> UpdateWithEtagCasAsync(
|
||||
OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default)
|
||||
{
|
||||
if (payload == null) throw new ArgumentNullException(nameof(payload));
|
||||
if (string.IsNullOrEmpty(ifMatchEtag)) throw new ArgumentException("ifMatchEtag required", nameof(ifMatchEtag));
|
||||
ValidatePayload(payload);
|
||||
|
||||
var blobName = BuildBlobName(payload.TenantId, payload.HistoryRowKey);
|
||||
var bytes = JsonSerializer.SerializeToUtf8Bytes(payload, JsonOptions);
|
||||
return await WriteWithIfMatchAsync(blobName, bytes, ifMatchEtag, ct);
|
||||
}
|
||||
|
||||
public async Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey));
|
||||
|
||||
var blobName = BuildBlobName(tenantId, historyRowKey);
|
||||
await DeleteBlobIfExistsAsync(blobName, ct);
|
||||
}
|
||||
|
||||
// ── Protected virtual seams (production override = Azure SDK; tests = in-memory) ──
|
||||
|
||||
/// <summary>
|
||||
/// Production path: <c>BlobClient.UploadAsync</c> with <c>If-None-Match=*</c>. Azure
|
||||
/// can answer with either <c>409 BlobAlreadyExists</c> OR <c>412 ConditionNotMet</c>
|
||||
/// depending on storage tier / SDK version — both mean "exists already" and must
|
||||
/// translate to <c>(false, null)</c> so the worker's resume path triggers. Other
|
||||
/// failures propagate.
|
||||
/// </summary>
|
||||
protected virtual async Task<(bool Inserted, string? ETag)> WriteIfNoneMatchAsync(
|
||||
string blobName, byte[] payload, CancellationToken ct)
|
||||
{
|
||||
var blobClient = await GetBlobClientAsync(blobName, ct);
|
||||
|
||||
var options = new BlobUploadOptions
|
||||
{
|
||||
HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" },
|
||||
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
using var stream = new MemoryStream(payload);
|
||||
var response = await blobClient.UploadAsync(stream, options, ct);
|
||||
return (true, response.Value.ETag.ToString());
|
||||
}
|
||||
catch (RequestFailedException ex) when (IsAlreadyExists(ex))
|
||||
{
|
||||
return (false, null);
|
||||
}
|
||||
}
|
||||
|
||||
protected virtual async Task<(byte[]? Payload, string? ETag)> ReadBlobAsync(
|
||||
string blobName, CancellationToken ct)
|
||||
{
|
||||
var blobClient = await GetBlobClientAsync(blobName, ct);
|
||||
try
|
||||
{
|
||||
using var stream = new MemoryStream();
|
||||
await blobClient.DownloadToAsync(stream, ct);
|
||||
var properties = await blobClient.GetPropertiesAsync(cancellationToken: ct);
|
||||
return (stream.ToArray(), properties.Value.ETag.ToString());
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return (null, null);
|
||||
}
|
||||
}
|
||||
|
||||
protected virtual async Task<string> WriteWithIfMatchAsync(
|
||||
string blobName, byte[] payload, string ifMatchEtag, CancellationToken ct)
|
||||
{
|
||||
var blobClient = await GetBlobClientAsync(blobName, ct);
|
||||
var options = new BlobUploadOptions
|
||||
{
|
||||
HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" },
|
||||
Conditions = new BlobRequestConditions { IfMatch = new ETag(ifMatchEtag) },
|
||||
};
|
||||
using var stream = new MemoryStream(payload);
|
||||
var response = await blobClient.UploadAsync(stream, options, ct);
|
||||
return response.Value.ETag.ToString();
|
||||
}
|
||||
|
||||
protected virtual async Task DeleteBlobIfExistsAsync(string blobName, CancellationToken ct)
|
||||
{
|
||||
var blobClient = await GetBlobClientAsync(blobName, ct);
|
||||
await blobClient.DeleteIfExistsAsync(cancellationToken: ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Both 409 BlobAlreadyExists and 412 ConditionNotMet are emitted by Azure Blob
|
||||
/// Storage for the same "blob exists, If-None-Match=* refused" condition, depending on
|
||||
/// API version / tier. Either is a successful "no-op" for the resume path.
|
||||
/// </summary>
|
||||
protected static bool IsAlreadyExists(RequestFailedException ex)
|
||||
=> ex.Status == 409
|
||||
|| ex.Status == 412
|
||||
|| ex.ErrorCode == BlobErrorCode.BlobAlreadyExists
|
||||
|| ex.ErrorCode == BlobErrorCode.ConditionNotMet;
|
||||
|
||||
// ── Internals ───────────────────────────────────────────────────────────
|
||||
|
||||
internal static string BuildBlobName(string tenantId, string historyRowKey)
|
||||
=> $"{tenantId}/{historyRowKey}.expectations.json";
|
||||
|
||||
private static void ValidatePayload(OffboardingExpectations payload)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(payload.TenantId, $"{nameof(payload)}.TenantId");
|
||||
if (string.IsNullOrEmpty(payload.HistoryRowKey))
|
||||
throw new ArgumentException("payload.HistoryRowKey required", nameof(payload));
|
||||
if (payload.SchemaVersion < 1)
|
||||
throw new ArgumentException("payload.SchemaVersion must be ≥ 1", nameof(payload));
|
||||
}
|
||||
|
||||
private async Task<BlobClient> GetBlobClientAsync(string blobName, CancellationToken ct)
|
||||
{
|
||||
var container = _blobs.GetContainerClient(Constants.BlobContainers.OffboardingState);
|
||||
await EnsureContainerExistsAsync(container, ct);
|
||||
return container.GetBlobClient(blobName);
|
||||
}
|
||||
|
||||
private async Task EnsureContainerExistsAsync(BlobContainerClient container, CancellationToken ct)
|
||||
{
|
||||
if (_containerEnsured == 1) return;
|
||||
await container.CreateIfNotExistsAsync(cancellationToken: ct);
|
||||
Interlocked.Exchange(ref _containerEnsured, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Default <see cref="IDeletionProgressDrainProbe"/> backed by
|
||||
/// <see cref="BlobStorageService.DownloadDeletionProgressAsync"/>. Plan §7.4 step 5:
|
||||
/// drain is satisfied for a session expectation when its progress blob has
|
||||
/// <c>CompletedAt != null AND TombstoneStarted == true</c>.
|
||||
/// </summary>
|
||||
public class DeletionProgressDrainProbe : IDeletionProgressDrainProbe
|
||||
{
|
||||
private readonly BlobStorageService _blobs;
|
||||
private readonly ILogger<DeletionProgressDrainProbe> _logger;
|
||||
|
||||
public DeletionProgressDrainProbe(BlobStorageService blobs, ILogger<DeletionProgressDrainProbe> logger)
|
||||
{
|
||||
_blobs = blobs;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<bool> IsCascadeCompletedAsync(
|
||||
string tenantId, string sessionId, string manifestId, CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var (progress, _) = await _blobs.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct);
|
||||
return progress.CompletedAt != null && progress.TombstoneStarted;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// Progress blob not yet written — cascade still running. NOT a failure: the
|
||||
// producer creates the progress blob before the queue message lands, but the
|
||||
// tombstone step happens at the end of the cascade.
|
||||
_logger.LogDebug(
|
||||
"Drain probe: progress blob missing for tenant={Tenant} session={Session} manifest={Manifest} — cascade not done",
|
||||
tenantId, sessionId, manifestId);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-session predicate over a cascade-deletion progress blob. Used by the
|
||||
/// tenant-offboarding handler to drain against each <see cref="Models.Offboarding.OffboardingExpectation"/>
|
||||
/// that ended up in an <c>Enqueued | AlreadyInFlight</c> outcome.
|
||||
/// </summary>
|
||||
public interface IDeletionProgressDrainProbe
|
||||
{
|
||||
/// <summary>
|
||||
/// Returns <c>true</c> iff the progress blob exists AND
|
||||
/// <c>CompletedAt != null AND TombstoneStarted == true</c>. 404 is treated as "not yet
|
||||
/// done" (false). Other RequestFailedExceptions propagate so the worker can retry.
|
||||
/// </summary>
|
||||
Task<bool> IsCascadeCompletedAsync(
|
||||
string tenantId, string sessionId, string manifestId, CancellationToken ct = default);
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// CRUD over the per-tenant Expectations blob. Lives under
|
||||
/// <c>Constants.BlobContainers.OffboardingState/{tenantId}/{historyRowKey}.expectations.json</c>
|
||||
/// — separate container from <c>deletion-manifests</c> so Phase 2.E's blob wipe cannot
|
||||
/// orphan crash-recovery state (plan Rev-8-F1).
|
||||
/// </summary>
|
||||
public interface IOffboardingExpectationsStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Idempotent initial upload via <c>If-None-Match=*</c>. Returns <c>true</c> on a fresh
|
||||
/// insert, <c>false</c> when a blob already exists for the same
|
||||
/// (tenantId, historyRowKey) — caller treats false as the resume path and re-reads.
|
||||
/// Any other Azure error propagates fail-loud.
|
||||
/// </summary>
|
||||
Task<bool> TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Downloads the Expectations blob; returns <c>(null, null)</c> on 404. Caller's
|
||||
/// drain probe fails closed on 404 (<c>FailedPhase="expectations_missing"</c>).
|
||||
/// </summary>
|
||||
Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync(
|
||||
string tenantId, string historyRowKey, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// ETag-CAS update used by the drain loop to re-record retry counters for CasExhausted
|
||||
/// expectations. Throws <see cref="Azure.RequestFailedException"/> on 412 so the caller
|
||||
/// can re-read and retry.
|
||||
/// </summary>
|
||||
Task<string> UpdateWithEtagCasAsync(
|
||||
OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Idempotent <c>DeleteIfExistsAsync</c>. Called as Phase 2.G's last step after the
|
||||
/// history row has been marked Completed, and again as defence-in-depth by
|
||||
/// <c>OffboardingMarkerCleanupFunction</c> when it removes a Completed marker.
|
||||
/// </summary>
|
||||
Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default);
|
||||
}
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Producer side of the <c>tenant-offboarding</c> queue. Implementations MUST be fail-loud:
|
||||
/// the caller (<c>TenantOffboardFunction</c>) only returns 202 Accepted after a successful
|
||||
/// enqueue, otherwise the admin's offboard click would silently strand without a worker
|
||||
/// pickup (memory: feedback_storage_helpers_fail_soft).
|
||||
/// </summary>
|
||||
public interface ITenantOffboardingEnqueuer
|
||||
{
|
||||
/// <summary>
|
||||
/// Enqueue a tenant-offboarding envelope. <paramref name="visibilityDelay"/> is used by
|
||||
/// the handler's drain self-re-enqueue path (Plan §7.4 D2) to delay the next pickup by
|
||||
/// ~2 minutes; producers in <c>TenantOffboardFunction</c> leave it null for immediate
|
||||
/// visibility.
|
||||
/// </summary>
|
||||
Task EnqueueAsync(
|
||||
TenantOffboardingEnvelope envelope,
|
||||
TimeSpan? visibilityDelay = null,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// OData filter builders for tenant-offboarding queries. All inputs are validated and
|
||||
/// escaped before interpolation. Centralizing the patterns here keeps SafeWipe-callers
|
||||
/// honest and gives the test suite a single surface to attack with injection payloads.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Rules (memory: feedback_storage_helpers_fail_soft, plan §5.5):
|
||||
/// <list type="number">
|
||||
/// <item>Every tenant id is run through <see cref="SecurityValidator.EnsureValidGuid"/> before use.</item>
|
||||
/// <item>Every interpolated value passes through <see cref="ODataSanitizer.EscapeValue"/>.</item>
|
||||
/// <item>Composite-PK range uses <c>'{tenantId}_~'</c> upper bound — the underscore is the
|
||||
/// anchor; without it the range bleeds into the next ASCII bucket.</item>
|
||||
/// </list>
|
||||
/// </remarks>
|
||||
public static class OffboardingFilters
|
||||
{
|
||||
/// <summary>
|
||||
/// <c>PartitionKey eq '{tenantId}'</c> — for Variant A SafeWipe of tenant-PK tables.
|
||||
/// </summary>
|
||||
public static string ExactPartition(string normalizedTenantId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
return $"PartitionKey eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>PartitionKey ge '{tenantId}_' and PartitionKey lt '{tenantId}_~'</c> — for
|
||||
/// Variant A SafeWipe of composite-PK tables (events, signals, decision transitions,
|
||||
/// discriminator-PK index tables, …). The underscore-tilde upper bound traps the
|
||||
/// range strictly to <c>{tenantId}_*</c>; switching to <c>'{tenantId}~'</c> would
|
||||
/// match keys outside the tenant prefix.
|
||||
/// </summary>
|
||||
public static string CompositePartitionRange(string normalizedTenantId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
var safe = ODataSanitizer.EscapeValue(normalizedTenantId);
|
||||
return $"PartitionKey ge '{safe}_' and PartitionKey lt '{safe}_~'";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>PartitionKey eq '{discriminator}' and TenantId eq '{tenantId}'</c> — for
|
||||
/// Variant B SafeWipe of discriminator-PK tables with a TenantId property as the
|
||||
/// only tenant anchor. Verify-step MUST re-check the property server-side because
|
||||
/// the PK is non-anchored.
|
||||
/// </summary>
|
||||
public static string DiscriminatorWithTenantProp(string discriminator, string normalizedTenantId)
|
||||
{
|
||||
if (string.IsNullOrEmpty(discriminator)) throw new ArgumentException("Discriminator required", nameof(discriminator));
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
return $"PartitionKey eq '{ODataSanitizer.EscapeValue(discriminator)}' " +
|
||||
$"and TenantId eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>TenantId eq '{tenantId}'</c> — Variant C SafeWipe (Property-only full-table
|
||||
/// filter). Reserve for low-volume tables; Group-by-PartitionKey before batching
|
||||
/// because rows can span many PKs.
|
||||
/// </summary>
|
||||
public static string TenantIdProperty(string normalizedTenantId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
return $"TenantId eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'";
|
||||
}
|
||||
}
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Thin wrapper around <see cref="IMaintenanceRepository.EnumerateSessionsForOffboardingAsync"/>
|
||||
/// so the offboarding handler can take an injectable dependency that is trivially mockable
|
||||
/// (the repo interface has 20+ unrelated members; mocking it for every test is painful).
|
||||
/// <para>
|
||||
/// <b>Fail-loud contract:</b> any storage exception from the underlying enumerator
|
||||
/// propagates unchanged. Plan §7.4 + memory <c>feedback_storage_helpers_fail_soft</c>:
|
||||
/// silently returning an empty list during enumeration would let the handler proceed
|
||||
/// straight to wipe without cascade-backup. Tests in
|
||||
/// <c>OffboardingSessionEnumeratorTests</c> pin this behaviour.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class OffboardingSessionEnumerator
|
||||
{
|
||||
private readonly IMaintenanceRepository _maintenance;
|
||||
|
||||
public OffboardingSessionEnumerator(IMaintenanceRepository maintenance)
|
||||
{
|
||||
_maintenance = maintenance;
|
||||
}
|
||||
|
||||
public virtual async IAsyncEnumerable<string> EnumerateAsync(
|
||||
string tenantId,
|
||||
[EnumeratorCancellation] CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
|
||||
await foreach (var sessionId in _maintenance.EnumerateSessionsForOffboardingAsync(tenantId, ct))
|
||||
{
|
||||
yield return sessionId;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Azure.Storage.Blobs;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Four-step verified wipe pattern used by the tenant-offboarding worker.
|
||||
/// <list type="number">
|
||||
/// <item><b>Fetch</b> rows matching a tenant-anchored filter (PK or TenantId-Property).</item>
|
||||
/// <item><b>Verify</b> each fetched row matches the expected anchor server-side AND client-side.</item>
|
||||
/// <item><b>Delete</b> via batched <c>SubmitTransactionAsync</c> with <c>ETag.All</c>; per-row
|
||||
/// 404-fallback so the helper stays idempotent across crash/resume.</item>
|
||||
/// <item><b>Audit</b> the deleted/already-missing counts to the caller's logger.</item>
|
||||
/// </list>
|
||||
/// Verify mismatch → <see cref="SafeWipeVerificationException"/> before any delete fires.
|
||||
/// </summary>
|
||||
public class SafeWipeService
|
||||
{
|
||||
private const int BatchActionLimit = 100;
|
||||
|
||||
private readonly TableStorageService _tables;
|
||||
private readonly BlobStorageService _blobs;
|
||||
private readonly ILogger<SafeWipeService> _logger;
|
||||
|
||||
public SafeWipeService(TableStorageService tables, BlobStorageService blobs, ILogger<SafeWipeService> logger)
|
||||
{
|
||||
_tables = tables;
|
||||
_blobs = blobs;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Variant A (exact tenant PartitionKey). Used for tables where rows live under
|
||||
/// <c>PartitionKey == normalizedTenantId</c> (AuditLogs, UsageMetrics, …).
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByExactPartitionAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
var filter = OffboardingFilters.ExactPartition(normalizedTenantId);
|
||||
|
||||
return RunFetchVerifyDeleteAsync(
|
||||
tableName,
|
||||
filter,
|
||||
expectedAnchor: normalizedTenantId,
|
||||
verifyRow: e => string.Equals(e.PartitionKey, normalizedTenantId, StringComparison.Ordinal),
|
||||
ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Variant A range (composite tenant PartitionKey). Used for tables where rows live
|
||||
/// under <c>{tenantId}_{sessionId|discriminator}</c>.
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByCompositePartitionRangeAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
var prefix = normalizedTenantId + "_";
|
||||
var filter = OffboardingFilters.CompositePartitionRange(normalizedTenantId);
|
||||
|
||||
return RunFetchVerifyDeleteAsync(
|
||||
tableName,
|
||||
filter,
|
||||
expectedAnchor: prefix,
|
||||
verifyRow: e => e.PartitionKey.StartsWith(prefix, StringComparison.Ordinal),
|
||||
ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Variant B (discriminator-PK + TenantId-Property). Used for tables where rows share
|
||||
/// a fixed string PartitionKey (e.g. "CodeLookup", "reports", "Feedback") and only the
|
||||
/// <c>TenantId</c> property identifies tenant ownership. Verify-step MUST check the
|
||||
/// property because the PK is non-anchored.
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByDiscriminatorAndTenantPropertyAsync(
|
||||
string tableName, string discriminator, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
if (string.IsNullOrEmpty(discriminator)) throw new ArgumentException("Discriminator required", nameof(discriminator));
|
||||
|
||||
var filter = OffboardingFilters.DiscriminatorWithTenantProp(discriminator, normalizedTenantId);
|
||||
var expectedAnchor = $"{discriminator}+TenantId={normalizedTenantId}";
|
||||
|
||||
return RunFetchVerifyDeleteAsync(
|
||||
tableName,
|
||||
filter,
|
||||
expectedAnchor: expectedAnchor,
|
||||
verifyRow: e => string.Equals(e.PartitionKey, discriminator, StringComparison.Ordinal)
|
||||
&& string.Equals(e.GetString("TenantId"), normalizedTenantId, StringComparison.Ordinal),
|
||||
ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Variant C (Property-only full-table filter). Reserved for low-volume tables where
|
||||
/// PartitionKey is not tenant-anchored (e.g. <c>UserUsageLog</c> keyed by userOid).
|
||||
/// Full-table scan — DO NOT use on high-volume tables without consideration.
|
||||
/// </summary>
|
||||
public virtual Task<int> WipeByTenantIdPropertyAsync(
|
||||
string tableName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
var filter = OffboardingFilters.TenantIdProperty(normalizedTenantId);
|
||||
|
||||
return RunFetchVerifyDeleteAsync(
|
||||
tableName,
|
||||
filter,
|
||||
expectedAnchor: $"TenantId={normalizedTenantId}",
|
||||
verifyRow: e => string.Equals(e.GetString("TenantId"), normalizedTenantId, StringComparison.Ordinal),
|
||||
ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Blob-prefix wipe. Lists blobs under <c>{normalizedTenantId}/</c> in the named
|
||||
/// container, verifies each returned name starts with the prefix, then deletes
|
||||
/// per-blob (DeleteIfExistsAsync) so the helper stays idempotent across crash/resume.
|
||||
/// Mismatch → abort before any delete. Returns the number of blobs deleted.
|
||||
/// </summary>
|
||||
public virtual async Task<int> WipeBlobsByTenantPrefixAsync(
|
||||
string containerName, string normalizedTenantId, CancellationToken ct = default)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
|
||||
if (string.IsNullOrEmpty(containerName)) throw new ArgumentException("Container required", nameof(containerName));
|
||||
|
||||
var prefix = normalizedTenantId + "/";
|
||||
var container = _blobs.GetContainerClient(containerName);
|
||||
|
||||
var collected = new List<string>();
|
||||
await foreach (var blob in container.GetBlobsAsync(
|
||||
Azure.Storage.Blobs.Models.BlobTraits.None,
|
||||
Azure.Storage.Blobs.Models.BlobStates.None,
|
||||
prefix: prefix,
|
||||
cancellationToken: ct))
|
||||
{
|
||||
collected.Add(blob.Name);
|
||||
}
|
||||
|
||||
if (collected.Count == 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"SafeWipe blobs '{Container}' tenant={Tenant}: no blobs found under '{Prefix}'",
|
||||
containerName, normalizedTenantId, prefix);
|
||||
return 0;
|
||||
}
|
||||
|
||||
var mismatches = collected
|
||||
.Where(name => !name.StartsWith(prefix, StringComparison.Ordinal))
|
||||
.ToList();
|
||||
if (mismatches.Count > 0)
|
||||
{
|
||||
_logger.LogCritical(
|
||||
"SafeWipe blobs '{Container}' tenant={Tenant}: aborting — {Total} blob(s) listed but {Bad} do not start with '{Prefix}'. First bad: '{Sample}'",
|
||||
containerName, normalizedTenantId, collected.Count, mismatches.Count, prefix, mismatches[0]);
|
||||
throw new SafeWipeVerificationException(containerName, prefix, mismatches.Count);
|
||||
}
|
||||
|
||||
int deleted = 0, missing = 0;
|
||||
foreach (var name in collected)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var blobClient = container.GetBlobClient(name);
|
||||
var resp = await blobClient.DeleteIfExistsAsync(cancellationToken: ct);
|
||||
if (resp.Value) deleted++;
|
||||
else missing++;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"SafeWipe blobs '{Container}' tenant={Tenant}: deleted {Deleted}, already-missing {Missing}",
|
||||
containerName, normalizedTenantId, deleted, missing);
|
||||
return deleted;
|
||||
}
|
||||
|
||||
// ── Internal pipeline ───────────────────────────────────────────────────
|
||||
|
||||
private async Task<int> RunFetchVerifyDeleteAsync(
|
||||
string tableName,
|
||||
string filter,
|
||||
string expectedAnchor,
|
||||
Func<TableEntity, bool> verifyRow,
|
||||
CancellationToken ct)
|
||||
{
|
||||
if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName required", nameof(tableName));
|
||||
|
||||
var tableClient = _tables.GetTableClient(tableName);
|
||||
|
||||
// STEP 1 — Fetch keys + the TenantId property if any verify-rule needs it.
|
||||
// We pull "PartitionKey", "RowKey", "TenantId" so Variant B/C verify works.
|
||||
var fetched = new List<TableEntity>();
|
||||
await foreach (var e in tableClient.QueryAsync<TableEntity>(
|
||||
filter,
|
||||
select: new[] { "PartitionKey", "RowKey", "TenantId" },
|
||||
cancellationToken: ct))
|
||||
{
|
||||
fetched.Add(e);
|
||||
}
|
||||
|
||||
if (fetched.Count == 0)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"SafeWipe '{Table}' anchor={Anchor}: 0 rows matched filter",
|
||||
tableName, expectedAnchor);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// STEP 2 — Verify every row matches the expected anchor.
|
||||
var mismatches = fetched.Where(e => !verifyRow(e)).ToList();
|
||||
if (mismatches.Count > 0)
|
||||
{
|
||||
var sample = mismatches.First();
|
||||
_logger.LogCritical(
|
||||
"SafeWipe '{Table}' anchor={Anchor}: aborting — {Total} fetched, {Bad} fail verify. " +
|
||||
"Sample bad row: PK='{Pk}' RK='{Rk}' TenantId='{Tid}'",
|
||||
tableName, expectedAnchor, fetched.Count, mismatches.Count,
|
||||
sample.PartitionKey, sample.RowKey, sample.GetString("TenantId") ?? "<null>");
|
||||
throw new SafeWipeVerificationException(tableName, expectedAnchor, mismatches.Count);
|
||||
}
|
||||
|
||||
// STEP 3 — Delete in batches, grouped by PartitionKey (transaction requirement).
|
||||
int deleted = 0, alreadyMissing = 0;
|
||||
var groupedByPk = fetched
|
||||
.GroupBy(e => e.PartitionKey, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
foreach (var group in groupedByPk)
|
||||
{
|
||||
var rows = group.ToList();
|
||||
for (int i = 0; i < rows.Count; i += BatchActionLimit)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var chunk = rows.Skip(i).Take(BatchActionLimit).ToList();
|
||||
var actions = chunk
|
||||
.Select(e => new TableTransactionAction(
|
||||
TableTransactionActionType.Delete,
|
||||
new TableEntity(e.PartitionKey, e.RowKey) { ETag = ETag.All }))
|
||||
.ToList();
|
||||
|
||||
try
|
||||
{
|
||||
await tableClient.SubmitTransactionAsync(actions, ct);
|
||||
deleted += chunk.Count;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// Azure rolls the whole transaction back on a single 404; degrade to per-row.
|
||||
foreach (var entity in chunk)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
await tableClient.DeleteEntityAsync(
|
||||
entity.PartitionKey, entity.RowKey, ETag.All, ct);
|
||||
deleted++;
|
||||
}
|
||||
catch (RequestFailedException rfe) when (rfe.Status == 404)
|
||||
{
|
||||
alreadyMissing++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// STEP 4 — Audit.
|
||||
_logger.LogInformation(
|
||||
"SafeWipe '{Table}' anchor={Anchor}: deleted {Deleted}, already-missing {Missing}",
|
||||
tableName, expectedAnchor, deleted, alreadyMissing);
|
||||
|
||||
return deleted;
|
||||
}
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Thrown by <see cref="SafeWipeService"/> when the verify step finds rows in the fetched
|
||||
/// set that do not match the expected tenant anchor (PartitionKey, TenantId-Property, or
|
||||
/// blob-name prefix depending on variant). NO DATA IS DELETED at this point — the wipe is
|
||||
/// aborted before any delete call so a misbehaving filter cannot cause cross-tenant data
|
||||
/// loss. Operator must inspect the unexpected rows manually.
|
||||
/// </summary>
|
||||
public class SafeWipeVerificationException : Exception
|
||||
{
|
||||
public string TableOrContainer { get; }
|
||||
public string ExpectedAnchor { get; }
|
||||
public int MismatchCount { get; }
|
||||
|
||||
public SafeWipeVerificationException(string tableOrContainer, string expectedAnchor, int mismatchCount)
|
||||
: base($"SafeWipe verify aborted for '{tableOrContainer}': {mismatchCount} row(s) " +
|
||||
$"did not match expected anchor '{expectedAnchor}'. No deletes were issued. " +
|
||||
$"Manual review required — possible filter bug or schema drift.")
|
||||
{
|
||||
TableOrContainer = tableOrContainer;
|
||||
ExpectedAnchor = expectedAnchor;
|
||||
MismatchCount = mismatchCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
+958
@@ -0,0 +1,958 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Phase 2 of the tenant-offboarding cascade plan (Rev 9). Drives a queued
|
||||
/// <see cref="TenantOffboardingEnvelope"/> through:
|
||||
/// <list type="number">
|
||||
/// <item>2.A — History/Pointer/Marker → InProgress; Rev-9-F1 Drain-Skip-Gate when DrainCompletedAt is set.</item>
|
||||
/// <item>2.B — fail-loud session enumeration + per-session cascade enqueue + Expectations-Blob upload.</item>
|
||||
/// <item>2.C — drain predicate against Expectations; D2 self-re-enqueue with 2-min delay up to 60 polls.</item>
|
||||
/// <item>2.D — retain-counter + SafeWipe over all tenant-scoped tables (variants A/B/C).</item>
|
||||
/// <item>2.E — blob-prefix cleanup of <c>deletion-manifests/{tenantId}/</c>.</item>
|
||||
/// <item>2.D-pass-2 — re-SafeWipe AuditLogs as defence-in-depth.</item>
|
||||
/// <item>2.F — delete the TenantConfiguration row.</item>
|
||||
/// <item>2.G — Completed state on History/Pointer/Marker; fail-soft Expectations-Blob delete.</item>
|
||||
/// <item>2.H — global-tenant audit row.</item>
|
||||
/// <item>2.I — <c>TenantOffboarded</c> OpsEvent.</item>
|
||||
/// </list>
|
||||
/// All Phase 2.D-G operations are idempotent (ETag.All + 404-fallback in SafeWipe, fail-soft
|
||||
/// blob delete in 2.G, last-writer-wins on history) so the Rev-9-F1 resume from a crash
|
||||
/// between 2.E and 2.G is safe even though the progress-blob predicate cannot run any more.
|
||||
/// </summary>
|
||||
public class TenantOffboardingHandler
|
||||
{
|
||||
/// <summary>Cap on drain self-re-enqueue iterations (~2h at 2-min visibility delay).</summary>
|
||||
public const int MaxDrainPolls = 60;
|
||||
|
||||
/// <summary>Delay between drain self-polls when expectations are not yet satisfied.</summary>
|
||||
internal static readonly TimeSpan DrainPollDelay = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>Maximum CasExhausted retries per session before fail-closing the offboarding.</summary>
|
||||
public const int MaxCasRetriesPerSession = 3;
|
||||
|
||||
/// <summary>Throttle between per-session cascade enqueues so a 10k-session tenant doesn't burst.</summary>
|
||||
internal static readonly TimeSpan PerSessionEnqueueThrottle = TimeSpan.FromMilliseconds(50);
|
||||
|
||||
private static readonly Dictionary<string, string> EmptyDetails = new(StringComparer.Ordinal);
|
||||
|
||||
// Plan §6.2 — exact tenant-PK wipes (Variant A).
|
||||
private static readonly string[] TenantPartitionTables =
|
||||
{
|
||||
Constants.TableNames.AuditLogs,
|
||||
Constants.TableNames.UsageMetrics,
|
||||
Constants.TableNames.UserActivity,
|
||||
Constants.TableNames.AppInstallSummaries,
|
||||
Constants.TableNames.TenantAdmins,
|
||||
Constants.TableNames.BootstrapSessions, // main rows; CodeLookup discriminator below
|
||||
Constants.TableNames.BlockedDevices,
|
||||
Constants.TableNames.RuleStates, // runtime state — NOT customs (kept; §6.7)
|
||||
Constants.TableNames.TenantNotifications,
|
||||
Constants.TableNames.HardwareRejectionNotificationTracker,
|
||||
Constants.TableNames.SlaTenantStatus,
|
||||
Constants.TableNames.DeviceSnapshot,
|
||||
Constants.TableNames.EventSessionIndex,
|
||||
Constants.TableNames.DistressReports,
|
||||
Constants.TableNames.SoftwareInventory,
|
||||
Constants.TableNames.SessionInventoryContributions,
|
||||
Constants.TableNames.SessionTombstones,
|
||||
Constants.TableNames.PreviewWhitelist,
|
||||
};
|
||||
|
||||
// Plan §6.4 — composite-PK "{tenantId}_..." wipes (Variant A range).
|
||||
private static readonly string[] CompositePartitionTables =
|
||||
{
|
||||
Constants.TableNames.Events,
|
||||
Constants.TableNames.RuleResults,
|
||||
Constants.TableNames.VulnerabilityReports,
|
||||
Constants.TableNames.Signals,
|
||||
Constants.TableNames.DecisionTransitions,
|
||||
Constants.TableNames.EventTypeIndex,
|
||||
Constants.TableNames.CveIndex,
|
||||
Constants.TableNames.SessionsByTerminal,
|
||||
Constants.TableNames.SessionsByStage,
|
||||
Constants.TableNames.DeadEndsByReason,
|
||||
Constants.TableNames.ClassifierVerdictsByIdLevel,
|
||||
Constants.TableNames.SignalsByKind,
|
||||
};
|
||||
|
||||
// Plan §6.3.1 — Variant B (Discriminator + TenantId property).
|
||||
// (table, discriminator)
|
||||
private static readonly (string Table, string Discriminator)[] DiscriminatorTables =
|
||||
{
|
||||
(Constants.TableNames.BootstrapSessions, "CodeLookup"),
|
||||
(Constants.TableNames.SessionReports, "reports"),
|
||||
(Constants.TableNames.PreviewConfig, "Feedback"),
|
||||
};
|
||||
|
||||
// Plan §6.3.2 — Variant C (TenantId property only).
|
||||
private static readonly string[] PropertyOnlyTables =
|
||||
{
|
||||
Constants.TableNames.UserUsageLog,
|
||||
};
|
||||
|
||||
// PR3.B plan §3 — Customs rules tables: archive each row to
|
||||
// TenantOffboardingCustomsArchive (per-run partition) THEN wipe the source so the
|
||||
// tenant's originals are gone. Counters in OffboardingHistory are recomputed from
|
||||
// the archive table per run + table to be crash-resume safe.
|
||||
private static readonly (string Table, string Field)[] ArchivedRuleTables =
|
||||
{
|
||||
(Constants.TableNames.GatherRules, nameof(OffboardingHistoryEntry.CustomGatherRulesArchived)),
|
||||
(Constants.TableNames.AnalyzeRules, nameof(OffboardingHistoryEntry.CustomAnalyzeRulesArchived)),
|
||||
(Constants.TableNames.ImeLogPatterns, nameof(OffboardingHistoryEntry.ImeLogPatternOverridesArchived)),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of archive-then-wipe iterations per rules table before the
|
||||
/// post-wipe verify re-find triggers fail-closed with <c>customs_arrival_race</c>.
|
||||
/// First pass handles the normal case; passes 2 + 3 absorb writes that landed
|
||||
/// during the drain barrier (a warm function-host with a stale TenantConfig
|
||||
/// cache could still write between Phase 1 commit and Phase 2 pickup).
|
||||
/// </summary>
|
||||
internal const int ArchiveIterationCap = 3;
|
||||
|
||||
private readonly IOffboardingAuditRepository _auditRepo;
|
||||
private readonly OffboardingSessionEnumerator _enumerator;
|
||||
private readonly ISessionDeletionEnqueuer _cascadeEnqueuer;
|
||||
private readonly IOffboardingExpectationsStore _expectations;
|
||||
private readonly IDeletionProgressDrainProbe _drainProbe;
|
||||
private readonly SafeWipeService _safeWipe;
|
||||
private readonly TableStorageService _storage;
|
||||
private readonly IMaintenanceRepository _maintenance;
|
||||
private readonly ITenantOffboardingEnqueuer _reEnqueuer;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ITenantCustomsArchiveRepository _customsArchive;
|
||||
private readonly ILogger<TenantOffboardingHandler> _logger;
|
||||
|
||||
public TenantOffboardingHandler(
|
||||
IOffboardingAuditRepository auditRepo,
|
||||
OffboardingSessionEnumerator enumerator,
|
||||
ISessionDeletionEnqueuer cascadeEnqueuer,
|
||||
IOffboardingExpectationsStore expectations,
|
||||
IDeletionProgressDrainProbe drainProbe,
|
||||
SafeWipeService safeWipe,
|
||||
TableStorageService storage,
|
||||
IMaintenanceRepository maintenance,
|
||||
ITenantOffboardingEnqueuer reEnqueuer,
|
||||
OpsEventService opsEvents,
|
||||
ITenantCustomsArchiveRepository customsArchive,
|
||||
ILogger<TenantOffboardingHandler> logger)
|
||||
{
|
||||
_auditRepo = auditRepo;
|
||||
_enumerator = enumerator;
|
||||
_cascadeEnqueuer = cascadeEnqueuer;
|
||||
_expectations = expectations;
|
||||
_drainProbe = drainProbe;
|
||||
_safeWipe = safeWipe;
|
||||
_storage = storage;
|
||||
_maintenance = maintenance;
|
||||
_reEnqueuer = reEnqueuer;
|
||||
_opsEvents = opsEvents;
|
||||
_customsArchive = customsArchive;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task HandleAsync(TenantOffboardingEnvelope envelope, CancellationToken ct = default)
|
||||
{
|
||||
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
|
||||
var tenantId = envelope.TenantId.ToLowerInvariant();
|
||||
|
||||
// 2.A — Load history. Idempotent re-pickup: Completed/Failed → return early.
|
||||
var history = await _auditRepo.TryGetHistoryAsync(envelope.HistoryRowKey, ct)
|
||||
?? throw new InvalidOperationException(
|
||||
$"OffboardingHistory row missing for envelope tenant={tenantId} history={envelope.HistoryRowKey}");
|
||||
|
||||
if (history.Status == "Completed")
|
||||
{
|
||||
_logger.LogInformation("Tenant offboarding already Completed — re-pickup is a no-op. tenant={Tenant}", tenantId);
|
||||
return;
|
||||
}
|
||||
if (history.Status == "Failed")
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Tenant offboarding History is Failed (phase={Phase}) — operator action required, worker returns. tenant={Tenant}",
|
||||
history.ErrorMessage ?? "unknown", tenantId);
|
||||
return;
|
||||
}
|
||||
|
||||
// 2.A — first pickup transitions History/Pointer/Marker to InProgress.
|
||||
if (history.Status == "Initiated")
|
||||
{
|
||||
history.Status = "InProgress";
|
||||
await _auditRepo.UpsertHistoryAsync(history, ct);
|
||||
await TransitionStatusAsync(tenantId, history.RowKey, "InProgress", ct);
|
||||
}
|
||||
|
||||
// 2.A.5 — Rev-9-F1 Drain-Skip-Gate. After the first successful drain, DrainCompletedAt
|
||||
// is stamped on the history row and all subsequent phases (2.D-G) are idempotent.
|
||||
// A crash between 2.E (blob wipe) and 2.G (status flip) would otherwise leave the
|
||||
// re-pickup looking for progress blobs that 2.E just deleted — fail-closed with
|
||||
// expectations_missing, which is wrong (the offboard actually succeeded in spirit).
|
||||
if (history.DrainCompletedAt != null)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Drain-Skip-Gate: History.DrainCompletedAt={DrainedAt} set; running post-drain phases idempotently. tenant={Tenant}",
|
||||
history.DrainCompletedAt, tenantId);
|
||||
await RunPostDrainPhasesAsync(history, tenantId, ct);
|
||||
return;
|
||||
}
|
||||
|
||||
// 2.B — Ensure Expectations blob exists. The two history markers EnumerationStartedAt
|
||||
// (ESA) and EnumerationCompletedBeforeUpload (ECBU) disambiguate three resume modes
|
||||
// when the blob is missing:
|
||||
// • ESA null, ECBU null → first try → stamp ESA, enumerate, stamp ECBU, upload.
|
||||
// • ESA set, ECBU null → mid-enumeration crash → fail-closed expectations_missing
|
||||
// (re-enumerating against a mid-mutation tenant is unsafe).
|
||||
// • ESA set, ECBU set → upload failure → retry enumerate+upload (idempotent,
|
||||
// SessionDeletionProducer returns AlreadyInFlight).
|
||||
// Full decision table lives on EnsureExpectationsBlobAsync. Plan §7.4 step 3.
|
||||
var ensureFail = await EnsureExpectationsBlobAsync(history, tenantId, ct);
|
||||
if (ensureFail != null)
|
||||
{
|
||||
await FailAsync(history, tenantId, ensureFail,
|
||||
$"Drain fail-closed: {ensureFail}", ct);
|
||||
throw new InvalidOperationException(
|
||||
$"TenantOffboarding fail-closed for tenant={tenantId} phase={ensureFail}");
|
||||
}
|
||||
|
||||
// 2.C — Drain probe. Three outcomes: drain OK (proceed), drain not yet (re-enqueue
|
||||
// with 2-min delay up to MaxDrainPolls), fail-closed (throw → worker poisons).
|
||||
var (drainOk, failReason) = await EvaluateDrainAsync(history, tenantId, ct);
|
||||
|
||||
if (failReason != null)
|
||||
{
|
||||
await FailAsync(history, tenantId, failReason, $"Drain fail-closed: {failReason}", ct);
|
||||
throw new InvalidOperationException(
|
||||
$"TenantOffboarding fail-closed for tenant={tenantId} phase={failReason}");
|
||||
}
|
||||
|
||||
if (!drainOk)
|
||||
{
|
||||
if (envelope.DrainPollCount + 1 >= MaxDrainPolls)
|
||||
{
|
||||
await FailAsync(history, tenantId, "drain_timeout",
|
||||
$"Drain did not settle within {MaxDrainPolls} polls (~{MaxDrainPolls * DrainPollDelay.TotalMinutes:0}min)", ct);
|
||||
throw new InvalidOperationException(
|
||||
$"TenantOffboarding drain_timeout for tenant={tenantId}");
|
||||
}
|
||||
|
||||
var next = new TenantOffboardingEnvelope
|
||||
{
|
||||
EnvelopeVersion = envelope.EnvelopeVersion,
|
||||
TenantId = envelope.TenantId,
|
||||
HistoryPartitionKey = envelope.HistoryPartitionKey,
|
||||
HistoryRowKey = envelope.HistoryRowKey,
|
||||
InitiatedBy = envelope.InitiatedBy,
|
||||
InitiatedAt = envelope.InitiatedAt,
|
||||
EnqueuedAt = DateTime.UtcNow,
|
||||
DrainPollCount = envelope.DrainPollCount + 1,
|
||||
};
|
||||
await _reEnqueuer.EnqueueAsync(next, DrainPollDelay, ct);
|
||||
_logger.LogInformation(
|
||||
"Drain not yet settled — re-enqueued tenant={Tenant} poll={Poll}/{Max} delay={Delay}",
|
||||
tenantId, next.DrainPollCount, MaxDrainPolls, DrainPollDelay);
|
||||
return;
|
||||
}
|
||||
|
||||
// Drain OK — Rev-9-F1: stamp DrainCompletedAt BEFORE Phase 2.D begins.
|
||||
history.DrainCompletedAt = DateTime.UtcNow;
|
||||
await _auditRepo.UpsertHistoryAsync(history, ct);
|
||||
|
||||
await RunPostDrainPhasesAsync(history, tenantId, ct);
|
||||
}
|
||||
|
||||
// ── Phase 2.A status transitions ────────────────────────────────────────
|
||||
|
||||
private async Task TransitionStatusAsync(string tenantId, string historyRowKey, string newStatus, CancellationToken ct)
|
||||
{
|
||||
// Marker — read existing fields so we don't clobber InitiatedAt/InitiatedBy.
|
||||
var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct);
|
||||
if (marker != null)
|
||||
{
|
||||
marker.Status = newStatus;
|
||||
await _auditRepo.UpsertMarkerAsync(marker, ct);
|
||||
}
|
||||
|
||||
// Pointer — read-modify-write with ETag-CAS retry (Plan §4.4).
|
||||
await UpdatePointerWithRetryAsync(tenantId, historyRowKey, newStatus, ct);
|
||||
}
|
||||
|
||||
private async Task UpdatePointerWithRetryAsync(string tenantId, string historyRowKey, string newStatus, CancellationToken ct)
|
||||
{
|
||||
const int maxRetries = 5;
|
||||
for (int attempt = 0; attempt < maxRetries; attempt++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var (pointer, etag) = await _auditRepo.TryGetByTenantPointerAsync(tenantId, ct);
|
||||
if (pointer == null || etag == null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Pointer missing for tenant={Tenant} while transitioning to {Status} — skip (graceful degradation, Plan §4.4)",
|
||||
tenantId, newStatus);
|
||||
return;
|
||||
}
|
||||
pointer.LatestStatus = newStatus;
|
||||
pointer.LatestUpdatedAt = DateTime.UtcNow;
|
||||
if (!string.Equals(pointer.LatestHistoryRowKey, historyRowKey, StringComparison.Ordinal))
|
||||
{
|
||||
// Pointer points at a different attempt — concurrent re-offboard. Skip.
|
||||
_logger.LogWarning(
|
||||
"Pointer.LatestHistoryRowKey={PointerHistory} != envelope.HistoryRowKey={Envelope} — skipping pointer transition",
|
||||
pointer.LatestHistoryRowKey, historyRowKey);
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
await _auditRepo.UpdateByTenantPointerWithEtagAsync(pointer, etag, ct);
|
||||
return;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412)
|
||||
{
|
||||
// Concurrent writer beat us — re-read + retry.
|
||||
}
|
||||
}
|
||||
_logger.LogWarning(
|
||||
"Pointer ETag-CAS exhausted for tenant={Tenant} status={Status} — giving up (Plan §4.4 graceful degradation)",
|
||||
tenantId, newStatus);
|
||||
}
|
||||
|
||||
// ── Phase 2.B — initial enqueue + Expectations persistence ──────────────
|
||||
|
||||
/// <summary>
|
||||
/// Returns <c>null</c> on success (blob exists / blob just created). Returns a
|
||||
/// FailedPhase string when the handler must fail-closed without enumerating.
|
||||
/// <para>
|
||||
/// Decision table (Plan §7.4 step 3 + Review-Fix Finding 2):
|
||||
/// </para>
|
||||
/// <list type="table">
|
||||
/// <listheader><term>blob</term><term>ESA</term><term>ECBU</term><description>Action</description></listheader>
|
||||
/// <item><term>exists</term><term>—</term><term>—</term><description>Resume path; return null.</description></item>
|
||||
/// <item><term>missing</term><term>null</term><term>null</term><description>First try; stamp ESA, enumerate, stamp ECBU, upload.</description></item>
|
||||
/// <item><term>missing</term><term>set</term><term>null</term><description>Mid-enumeration crash → fail-closed <c>expectations_missing</c>.</description></item>
|
||||
/// <item><term>missing</term><term>set</term><term>set</term><description>Upload failure → retry enumerate (idempotent — producer returns AlreadyInFlight) + re-upload.</description></item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
private async Task<string?> EnsureExpectationsBlobAsync(
|
||||
OffboardingHistoryEntry history, string tenantId, CancellationToken ct)
|
||||
{
|
||||
var (existing, _) = await _expectations.TryDownloadAsync(tenantId, history.RowKey, ct);
|
||||
if (existing != null)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Expectations blob already exists for tenant={Tenant} (count={Count}, enumComplete={EnumComplete}) — resume path",
|
||||
tenantId, existing.Expectations.Count, existing.EnumerationCompleted);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Mid-enumeration crash: prior pickup stamped ESA but never reached ECBU. Refuse
|
||||
// to re-enumerate against a possibly mid-mutation tenant.
|
||||
if (history.EnumerationStartedAt != null
|
||||
&& history.EnumerationCompletedBeforeUpload == null)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Expectations blob missing, EnumerationStartedAt={StartedAt}, ECBU null — fail-closed (mid-enumeration crash). tenant={Tenant}",
|
||||
history.EnumerationStartedAt, tenantId);
|
||||
return "expectations_missing";
|
||||
}
|
||||
|
||||
// Either first try (ESA null) or upload-retry (ESA + ECBU set, blob missing).
|
||||
// First-try only: stamp ESA before touching the enumerator so a crash here lands
|
||||
// in the fail-closed branch above on the next pickup.
|
||||
if (history.EnumerationStartedAt == null)
|
||||
{
|
||||
history.EnumerationStartedAt = DateTime.UtcNow;
|
||||
await _auditRepo.UpsertHistoryAsync(history, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Upload-retry path for tenant={Tenant} (ESA={Esa}, ECBU={Ecbu}) — re-running enumerate+upload",
|
||||
tenantId, history.EnumerationStartedAt, history.EnumerationCompletedBeforeUpload);
|
||||
}
|
||||
|
||||
// Fail-loud enumeration. Storage exceptions propagate; the queue worker poisons or
|
||||
// retries via visibility timeout. NEVER catch here — that would re-introduce the
|
||||
// Rev-5-F3 silent-empty bug.
|
||||
var collected = new List<OffboardingExpectation>();
|
||||
var actor = new DeletionActor { Type = "system", Actor = "System.TenantOffboarding" };
|
||||
|
||||
await foreach (var sessionId in _enumerator.EnumerateAsync(tenantId, ct))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var result = await _cascadeEnqueuer.EnqueueAsync(
|
||||
tenantId, sessionId, reason: "tenant_offboard", actor: actor,
|
||||
retentionContext: null, cancellationToken: ct);
|
||||
|
||||
collected.Add(new OffboardingExpectation
|
||||
{
|
||||
SessionId = sessionId,
|
||||
ManifestId = result.ManifestId,
|
||||
Outcome = result.Outcome.ToString(),
|
||||
RetryCount = 0,
|
||||
});
|
||||
|
||||
if (PerSessionEnqueueThrottle > TimeSpan.Zero)
|
||||
{
|
||||
try { await Task.Delay(PerSessionEnqueueThrottle, ct); }
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; }
|
||||
}
|
||||
}
|
||||
|
||||
// Enumeration finished cleanly. Persist ECBU BEFORE the upload so a subsequent
|
||||
// upload failure is recognized as upload-retry-eligible on the next pickup.
|
||||
// On retry pickups ECBU may already be set — UpsertHistoryAsync is idempotent.
|
||||
if (history.EnumerationCompletedBeforeUpload == null)
|
||||
{
|
||||
history.EnumerationCompletedBeforeUpload = DateTime.UtcNow;
|
||||
await _auditRepo.UpsertHistoryAsync(history, ct);
|
||||
}
|
||||
|
||||
// Plan Rev-7-F2: EnumerationCompleted on the payload = the in-blob flag the drain
|
||||
// probe checks. We can only set it true here because the iterator above ran to
|
||||
// completion without throwing (the throw-path skipped this whole block).
|
||||
var payload = new OffboardingExpectations
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
TenantId = tenantId,
|
||||
HistoryRowKey = history.RowKey,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
EnumerationCompleted = true,
|
||||
EnumeratedSessionCount = collected.Count,
|
||||
Expectations = collected,
|
||||
};
|
||||
|
||||
var uploaded = await _expectations.TryUploadInitialAsync(payload, ct);
|
||||
if (!uploaded)
|
||||
{
|
||||
// Race: a parallel worker beat us. Their blob is the source of truth.
|
||||
_logger.LogInformation(
|
||||
"Expectations blob race lost for tenant={Tenant} history={History} — using existing blob",
|
||||
tenantId, history.RowKey);
|
||||
}
|
||||
else
|
||||
{
|
||||
history.CascadeSessionsEnqueued = collected.Count;
|
||||
await _auditRepo.UpsertHistoryAsync(history, ct);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Phase 2.C — drain predicate ─────────────────────────────────────────
|
||||
|
||||
private async Task<(bool DrainOk, string? FailReason)> EvaluateDrainAsync(
|
||||
OffboardingHistoryEntry history, string tenantId, CancellationToken ct)
|
||||
{
|
||||
var (payload, etag) = await _expectations.TryDownloadAsync(tenantId, history.RowKey, ct);
|
||||
if (payload == null) return (false, "expectations_missing");
|
||||
if (etag == null) return (false, "expectations_missing");
|
||||
if (payload.SchemaVersion != 1) return (false, "expectations_corrupt");
|
||||
if (!payload.EnumerationCompleted) return (false, "enumeration_incomplete");
|
||||
if (payload.Expectations.Count != payload.EnumeratedSessionCount)
|
||||
return (false, "expectations_size_mismatch");
|
||||
|
||||
var allSatisfied = true;
|
||||
var dirty = false;
|
||||
var actor = new DeletionActor { Type = "system", Actor = "System.TenantOffboarding" };
|
||||
|
||||
for (int i = 0; i < payload.Expectations.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var expectation = payload.Expectations[i];
|
||||
|
||||
switch (expectation.Outcome)
|
||||
{
|
||||
case nameof(SessionDeletionEnqueueOutcome.KillSwitchActive):
|
||||
return (false, "killswitch");
|
||||
|
||||
case nameof(SessionDeletionEnqueueOutcome.Poisoned):
|
||||
return (false, "poisoned");
|
||||
|
||||
case nameof(SessionDeletionEnqueueOutcome.SessionNotFound):
|
||||
// Session was already gone when we tried to enqueue — nothing to drain.
|
||||
continue;
|
||||
|
||||
case nameof(SessionDeletionEnqueueOutcome.CasExhausted):
|
||||
if (expectation.RetryCount >= MaxCasRetriesPerSession)
|
||||
return (false, "cas_exhausted");
|
||||
|
||||
// Retry the per-session enqueue. Update outcome + counter in-place.
|
||||
var retry = await _cascadeEnqueuer.EnqueueAsync(
|
||||
tenantId, expectation.SessionId, reason: "tenant_offboard", actor: actor,
|
||||
retentionContext: null, cancellationToken: ct);
|
||||
expectation.Outcome = retry.Outcome.ToString();
|
||||
expectation.ManifestId = retry.ManifestId;
|
||||
expectation.RetryCount++;
|
||||
dirty = true;
|
||||
allSatisfied = false; // re-evaluate next drain cycle
|
||||
break;
|
||||
|
||||
case nameof(SessionDeletionEnqueueOutcome.Enqueued):
|
||||
case nameof(SessionDeletionEnqueueOutcome.AlreadyInFlight):
|
||||
if (string.IsNullOrEmpty(expectation.ManifestId))
|
||||
{
|
||||
// Rev-8-F3 — AlreadyInFlight with null ManifestId is a Preparing-without-
|
||||
// snapshot state. There's no progress blob to drain against; fail-closed.
|
||||
return (false, "alreadyinflight_no_manifest");
|
||||
}
|
||||
var done = await _drainProbe.IsCascadeCompletedAsync(
|
||||
tenantId, expectation.SessionId, expectation.ManifestId, ct);
|
||||
if (!done) allSatisfied = false;
|
||||
break;
|
||||
|
||||
default:
|
||||
_logger.LogWarning(
|
||||
"Unknown Expectation.Outcome '{Outcome}' for tenant={Tenant} session={Session}",
|
||||
expectation.Outcome, tenantId, expectation.SessionId);
|
||||
allSatisfied = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dirty)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _expectations.UpdateWithEtagCasAsync(payload, etag, ct);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412)
|
||||
{
|
||||
// A parallel worker re-evaluated and wrote first. Their version wins; the
|
||||
// next drain cycle will pick up the latest state. We still report
|
||||
// allSatisfied=false so this cycle re-enqueues.
|
||||
_logger.LogInformation(
|
||||
"Expectations CAS 412 for tenant={Tenant} — parallel worker won, will re-read next cycle",
|
||||
tenantId);
|
||||
return (false, null);
|
||||
}
|
||||
}
|
||||
|
||||
return (allSatisfied, null);
|
||||
}
|
||||
|
||||
// ── Phases 2.D / 2.E / 2.D-pass-2 / 2.F / 2.G / 2.H / 2.I ───────────────
|
||||
|
||||
private async Task RunPostDrainPhasesAsync(
|
||||
OffboardingHistoryEntry history, string tenantId, CancellationToken ct)
|
||||
{
|
||||
// 2.D-archive (PR3.B §3) — Archive then wipe GatherRules / AnalyzeRules /
|
||||
// ImeLogPatterns. Fail-loud: a row that cannot be archived must NOT be wiped.
|
||||
// After the third iteration if the source still has rows, throw with
|
||||
// FailedPhase="customs_arrival_race" — marker stays Failed for operator action.
|
||||
try
|
||||
{
|
||||
await ArchiveAndWipeCustomsRulesAsync(history, tenantId, ct);
|
||||
}
|
||||
catch (InvalidOperationException ex) when (ex.Message.StartsWith("customs_arrival_race", StringComparison.Ordinal))
|
||||
{
|
||||
await FailAsync(history, tenantId, "customs_arrival_race", ex.Message, ct);
|
||||
throw; // worker dequeues + eventually poisons
|
||||
}
|
||||
|
||||
// 2.D-pass-1 — SafeWipe all tenant-scoped tables, including AuditLogs.
|
||||
var deletedCounts = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
await SafeWipeAllTenantTablesAsync(deletedCounts, tenantId, ct);
|
||||
|
||||
// 2.E — Blob-prefix wipe (deletion-manifests only; offboarding-state stays).
|
||||
int deletedBlobs = 0;
|
||||
try
|
||||
{
|
||||
deletedBlobs = await _safeWipe.WipeBlobsByTenantPrefixAsync(
|
||||
Constants.BlobContainers.DeletionManifests, tenantId, ct);
|
||||
}
|
||||
catch (SafeWipeVerificationException)
|
||||
{
|
||||
throw; // unrecoverable — let worker poison
|
||||
}
|
||||
|
||||
// 2.D-pass-2 — Re-wipe AuditLogs as defence in depth against late audits that may
|
||||
// have landed between Phase 2.D-pass-1 and now (the cascade-handler reorder from
|
||||
// PR1.5 closes the main window, but a struggling SignalR/Audit retry could still
|
||||
// bring a row in late).
|
||||
var auditPass2 = await _safeWipe.WipeByExactPartitionAsync(Constants.TableNames.AuditLogs, tenantId, ct);
|
||||
if (auditPass2 > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Audit pass-2 swept {Count} late row(s) for tenant={Tenant}", auditPass2, tenantId);
|
||||
deletedCounts[Constants.TableNames.AuditLogs] =
|
||||
deletedCounts.TryGetValue(Constants.TableNames.AuditLogs, out var existing) ? existing + auditPass2 : auditPass2;
|
||||
}
|
||||
|
||||
// ── 2.G — Order: side-effects FIRST, then History terminal write LAST ────
|
||||
//
|
||||
// Note (PR3.B review): the legacy 2.F TenantConfiguration-delete used to sit
|
||||
// BEFORE 2.G. That left a tiny window where a /api/auth/me race could re-create
|
||||
// a fresh Default-Disabled=false TenantConfiguration before the Completed
|
||||
// state landed — letting an offboarded tenant slip back through. The delete
|
||||
// has moved to AFTER 2.I (see "2.F-final" below) so the entire Phase 2 sequence
|
||||
// runs against an existing Disabled=true row, and only once everything is
|
||||
// committed do we drop the row. Self-service re-onboarding still works because
|
||||
// the user's NEXT /api/auth/me after 2.F-final sees no row → auto-create-default.
|
||||
// Plan §3 Phase 2.G + Review-Fix Finding 3: a crash between History=Completed and
|
||||
// the marker/pointer/blob/audit/ops writes would otherwise make the next pickup
|
||||
// return early (terminal-status check) and leave the side-effects dangling. By
|
||||
// flipping History last we keep DrainCompletedAt as the resume anchor — re-pickup
|
||||
// skips drain and re-runs the (idempotent) side-effect chain.
|
||||
|
||||
// Pre-populate the in-memory history with the final values so the LAST write below
|
||||
// commits the whole snapshot atomically.
|
||||
var completedAt = DateTime.UtcNow;
|
||||
history.CompletedAt = completedAt;
|
||||
history.DeletedRowCountsJson = JsonSerializer.Serialize(deletedCounts);
|
||||
history.TotalRowsDeleted = deletedCounts.Values.Sum();
|
||||
history.DeletedBlobCount = deletedBlobs;
|
||||
|
||||
// Side-effect 1: Marker → Completed (must happen FIRST so a crash here leaves
|
||||
// History=InProgress; the next pickup re-runs the idempotent post-drain chain).
|
||||
var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct);
|
||||
if (marker != null)
|
||||
{
|
||||
marker.Status = "Completed";
|
||||
marker.CompletedAt = completedAt;
|
||||
await _auditRepo.UpsertMarkerAsync(marker, ct);
|
||||
}
|
||||
|
||||
// Side-effect 2: Pointer → Completed (ETag-CAS retry).
|
||||
await UpdatePointerWithRetryAsync(tenantId, history.RowKey, "Completed", ct);
|
||||
|
||||
// Side-effect 3: Expectations blob delete (fail-soft; MarkerCleanupFunction retries).
|
||||
try
|
||||
{
|
||||
await _expectations.DeleteAsync(tenantId, history.RowKey, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Expectations blob delete failed for tenant={Tenant} history={History} — MarkerCleanupFunction will retry",
|
||||
tenantId, history.RowKey);
|
||||
}
|
||||
|
||||
// Side-effect 4: Global-tenant audit row (persists across the wiped tenant audits).
|
||||
await _maintenance.LogAuditEntryAsync(
|
||||
Constants.AuditGlobalTenantId,
|
||||
action: "DELETE",
|
||||
entityType: "Tenant",
|
||||
entityId: tenantId,
|
||||
performedBy: history.InitiatedBy,
|
||||
details: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["Action"] = "Offboard",
|
||||
["Phase"] = "Completed",
|
||||
["DomainName"] = history.DomainName ?? string.Empty,
|
||||
["TotalRowsDeleted"] = (history.TotalRowsDeleted ?? 0).ToString(),
|
||||
["DeletedBlobCount"] = (history.DeletedBlobCount ?? 0).ToString(),
|
||||
["CascadeSessionsEnqueued"] = (history.CascadeSessionsEnqueued ?? 0).ToString(),
|
||||
["HistoryRowKey"] = history.RowKey,
|
||||
});
|
||||
|
||||
// Side-effect 5: OpsEvent. Reuses the existing TenantOffboarded helper so the
|
||||
// dashboard category cell stays "Tenant" / event "TenantOffboarded".
|
||||
await _opsEvents.RecordTenantOffboardedAsync(
|
||||
tenantId, history.InitiatedBy, deletedCounts, history.DomainName);
|
||||
|
||||
// COMMIT: History → Completed LAST. Until this write lands the next pickup will
|
||||
// see Status=InProgress + DrainCompletedAt set and re-run all of the above
|
||||
// idempotently (duplicate audit/OpsEvent emit is preferable to dangling state).
|
||||
history.Status = "Completed";
|
||||
await _auditRepo.UpsertHistoryAsync(history, ct);
|
||||
|
||||
// ── 2.F-final — TenantConfiguration delete (PR3.B Codex Finding 2 reorder) ──
|
||||
//
|
||||
// This is the LAST step in Phase 2. By now Marker / Pointer / History all say
|
||||
// "Completed" and the Disabled=true auth-gate has carried the tenant through the
|
||||
// entire async wipe. Deleting the row here means the next /api/auth/me for this
|
||||
// tenant will see no row → TenantConfigurationService.GetConfigurationAsync's
|
||||
// auto-create-default path kicks in, giving the same user a fresh, clean tenant
|
||||
// (the legitimate self-service re-onboarding case).
|
||||
//
|
||||
// Fail-soft: if the delete fails, Marker stays Completed and the
|
||||
// OffboardingMarkerCleanupFunction's TenantConfig-sweep (defense-in-depth) runs
|
||||
// every 2h and finishes the cleanup. The tenant cannot re-onboard until that
|
||||
// sweep lands — acceptable, because Disabled=true still applies meanwhile.
|
||||
try
|
||||
{
|
||||
var tenantConfigDeleted = await _safeWipe.WipeByExactPartitionAsync(
|
||||
Constants.TableNames.TenantConfiguration, tenantId, ct);
|
||||
if (tenantConfigDeleted > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Phase 2.F-final: deleted TenantConfiguration row(s)={Count} for tenant={Tenant}",
|
||||
tenantConfigDeleted, tenantId);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Phase 2.F-final TenantConfiguration delete failed for tenant={Tenant} — MarkerCleanupFunction's TenantConfig-sweep will retry",
|
||||
tenantId);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Tenant offboarding Completed: tenant={Tenant} totalRows={Rows} blobs={Blobs} sessions={Sessions} archiveGather={G} archiveAnalyze={A} archivePatterns={P}",
|
||||
tenantId,
|
||||
history.TotalRowsDeleted, history.DeletedBlobCount, history.CascadeSessionsEnqueued,
|
||||
history.CustomGatherRulesArchived, history.CustomAnalyzeRulesArchived, history.ImeLogPatternOverridesArchived);
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// PR3.B §3 — Phase 2.D-archive. For each rules table, archive every row to
|
||||
/// <see cref="ITenantCustomsArchiveRepository"/> THEN delete that exact (PK, RK)
|
||||
/// from the source. Up to <see cref="ArchiveIterationCap"/> iterations to absorb
|
||||
/// late writes that may have landed during the drain barrier; if the source still
|
||||
/// has rows after the cap, throw <see cref="InvalidOperationException"/> so the
|
||||
/// caller fail-closes the offboarding with <c>FailedPhase="customs_arrival_race"</c>.
|
||||
/// <para>
|
||||
/// Crash-safety: the archive upsert is idempotent (Replace), so a worker that
|
||||
/// crashed between archive-insert and source-delete will re-enumerate the same
|
||||
/// row on re-pickup, re-write the (identical) archive entry, and delete the
|
||||
/// source row that survived the crash. Counters are recomputed from the archive
|
||||
/// table at the end so a transient crash mid-loop does not produce a stale 0.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private async Task ArchiveAndWipeCustomsRulesAsync(
|
||||
OffboardingHistoryEntry history, string tenantId, CancellationToken ct)
|
||||
{
|
||||
foreach (var (tableName, fieldName) in ArchivedRuleTables)
|
||||
{
|
||||
await ArchiveAndWipeRulesTableAsync(tableName, tenantId, history.RowKey, ct);
|
||||
|
||||
// Recompute the counter from the archive table (per run + table) rather than
|
||||
// from in-loop accumulation. A crash between this assignment and the final
|
||||
// UpsertHistoryAsync below would otherwise leave a stale 0.
|
||||
var counted = await _customsArchive.CountByRunAndTableAsync(tenantId, history.RowKey, tableName, ct);
|
||||
AssignArchivedCounter(history, fieldName, counted);
|
||||
}
|
||||
|
||||
await _auditRepo.UpsertHistoryAsync(history, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Archive-then-wipe loop for a single rules table. Up to <see cref="ArchiveIterationCap"/>
|
||||
/// iterations; final-iteration verify-read triggers <c>customs_arrival_race</c> when a
|
||||
/// writer is still racing us. Marked <c>internal virtual</c> so test harnesses can
|
||||
/// override it to a no-op without faking the underlying TableServiceClient.
|
||||
/// </summary>
|
||||
internal virtual async Task ArchiveAndWipeRulesTableAsync(
|
||||
string tableName, string tenantId, string historyRowKey, CancellationToken ct)
|
||||
{
|
||||
var sourceClient = _storage.GetTableClient(tableName);
|
||||
|
||||
for (var attempt = 1; attempt <= ArchiveIterationCap; attempt++)
|
||||
{
|
||||
var archivedThisAttempt = 0;
|
||||
var filter = OffboardingFilters.ExactPartition(tenantId);
|
||||
await foreach (var entity in sourceClient.QueryAsync<TableEntity>(filter, cancellationToken: ct))
|
||||
{
|
||||
await _customsArchive.UpsertAsync(BuildArchiveEntry(entity, tenantId, tableName, historyRowKey), ct);
|
||||
// Only AFTER the archive entry is persisted do we delete the source.
|
||||
try
|
||||
{
|
||||
await sourceClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey, Azure.ETag.All, ct);
|
||||
}
|
||||
catch (Azure.RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
// Already gone (parallel writer / re-pickup) — archive already
|
||||
// captured the state we observed, so this is idempotent-safe.
|
||||
}
|
||||
archivedThisAttempt++;
|
||||
}
|
||||
|
||||
if (archivedThisAttempt == 0)
|
||||
{
|
||||
// Source partition empty → converged.
|
||||
return;
|
||||
}
|
||||
|
||||
if (attempt == ArchiveIterationCap)
|
||||
{
|
||||
// Final iteration still found rows. One last verify-read; if STILL
|
||||
// non-empty, a writer is racing us and we cannot guarantee complete
|
||||
// archive coverage → fail-closed.
|
||||
var leftover = 0;
|
||||
await foreach (var _ in sourceClient.QueryAsync<TableEntity>(
|
||||
filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
|
||||
{
|
||||
leftover++;
|
||||
break; // existence is enough
|
||||
}
|
||||
if (leftover > 0)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Customs archive iteration cap exhausted for {Table} tenant={Tenant} run={History}: a writer is still producing rows after {Cap} archive-then-wipe passes — fail-closing",
|
||||
tableName, tenantId, historyRowKey, ArchiveIterationCap);
|
||||
throw new InvalidOperationException(
|
||||
$"customs_arrival_race: table '{tableName}' still has rows after {ArchiveIterationCap} archive-then-wipe iterations");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal TenantOffboardingCustomsArchiveEntry BuildArchiveEntry(
|
||||
TableEntity source, string tenantId, string originalTable, string historyRowKey)
|
||||
{
|
||||
// Serialize every non-system property the source row exposes. TableEntity is
|
||||
// IDictionary<string, object> internally, so we can iterate.
|
||||
var snapshot = new Dictionary<string, object?>(StringComparer.Ordinal);
|
||||
foreach (var kv in source)
|
||||
{
|
||||
// Skip pseudo-properties Azure layers add (ETag/Timestamp/odata.*).
|
||||
if (kv.Key is "odata.etag" or "Timestamp") continue;
|
||||
snapshot[kv.Key] = kv.Value;
|
||||
}
|
||||
var entityJson = JsonSerializer.Serialize(snapshot);
|
||||
|
||||
return new TenantOffboardingCustomsArchiveEntry
|
||||
{
|
||||
PartitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey(tenantId, historyRowKey),
|
||||
RowKey = TableTenantCustomsArchiveRepository.BuildRowKey(originalTable, source.RowKey),
|
||||
TenantId = tenantId,
|
||||
OriginalTable = originalTable,
|
||||
OriginalPartitionKey = source.PartitionKey,
|
||||
OriginalRowKey = source.RowKey,
|
||||
EntityJson = entityJson,
|
||||
HistoryRowKey = historyRowKey,
|
||||
ArchivedAt = DateTime.UtcNow,
|
||||
ArchivedBy = "TenantOffboardingHandler",
|
||||
};
|
||||
}
|
||||
|
||||
private static void AssignArchivedCounter(OffboardingHistoryEntry history, string fieldName, int value)
|
||||
{
|
||||
switch (fieldName)
|
||||
{
|
||||
case nameof(OffboardingHistoryEntry.CustomGatherRulesArchived):
|
||||
history.CustomGatherRulesArchived = value;
|
||||
break;
|
||||
case nameof(OffboardingHistoryEntry.CustomAnalyzeRulesArchived):
|
||||
history.CustomAnalyzeRulesArchived = value;
|
||||
break;
|
||||
case nameof(OffboardingHistoryEntry.ImeLogPatternOverridesArchived):
|
||||
history.ImeLogPatternOverridesArchived = value;
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException($"Unknown archived counter field '{fieldName}'");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SafeWipeAllTenantTablesAsync(
|
||||
Dictionary<string, int> counts, string tenantId, CancellationToken ct)
|
||||
{
|
||||
foreach (var table in TenantPartitionTables)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
counts[table] = await _safeWipe.WipeByExactPartitionAsync(table, tenantId, ct);
|
||||
}
|
||||
foreach (var table in CompositePartitionTables)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
counts[table] = await _safeWipe.WipeByCompositePartitionRangeAsync(table, tenantId, ct);
|
||||
}
|
||||
foreach (var (table, discriminator) in DiscriminatorTables)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var key = $"{table}/{discriminator}";
|
||||
counts[key] = await _safeWipe.WipeByDiscriminatorAndTenantPropertyAsync(table, discriminator, tenantId, ct);
|
||||
}
|
||||
foreach (var table in PropertyOnlyTables)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
counts[table] = await _safeWipe.WipeByTenantIdPropertyAsync(table, tenantId, ct);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Failure path ────────────────────────────────────────────────────────
|
||||
//
|
||||
// Same Review-Fix Finding 3 invariant as the Completed path: write Marker/Pointer/Ops
|
||||
// FIRST, then History terminal LAST. A crash between the side-effects and the History
|
||||
// commit makes the next pickup re-run the side-effects idempotently — which is exactly
|
||||
// what we want for the Failed path, because the marker/pointer/ops outcome is the
|
||||
// operator-facing signal, not History.Status by itself.
|
||||
|
||||
private async Task FailAsync(
|
||||
OffboardingHistoryEntry history, string tenantId,
|
||||
string failedPhase, string errorMessage, CancellationToken ct)
|
||||
{
|
||||
// Pre-populate in-memory history with the final values so the LAST write below
|
||||
// commits the snapshot.
|
||||
history.ErrorMessage = errorMessage;
|
||||
|
||||
// Side-effect 1: Marker → Failed.
|
||||
var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct);
|
||||
if (marker != null)
|
||||
{
|
||||
marker.Status = "Failed";
|
||||
marker.FailedAt = DateTime.UtcNow;
|
||||
marker.FailedPhase = failedPhase;
|
||||
await _auditRepo.UpsertMarkerAsync(marker, ct);
|
||||
}
|
||||
|
||||
// Side-effect 2: Pointer → Failed.
|
||||
await UpdatePointerWithRetryAsync(tenantId, history.RowKey, "Failed", ct);
|
||||
|
||||
// Side-effect 3: OpsEvent (Telegram-routable).
|
||||
await _opsEvents.RecordTenantOffboardingFailedAsync(
|
||||
tenantId, history.InitiatedBy, failedPhase, errorMessage, history.RetryCount, history.DomainName);
|
||||
|
||||
// COMMIT: History → Failed LAST.
|
||||
history.Status = "Failed";
|
||||
await _auditRepo.UpsertHistoryAsync(history, ct);
|
||||
|
||||
_logger.LogError(
|
||||
"Tenant offboarding FAILED: tenant={Tenant} phase={Phase} message={Message}",
|
||||
tenantId, failedPhase, errorMessage);
|
||||
}
|
||||
|
||||
// ── Worker-side poison transition (Review-Fix Finding 2) ────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Called by <see cref="TenantOffboardingWorker"/> when an envelope hits max-dequeue and
|
||||
/// moves to the poison queue. Idempotently transitions History/Pointer/Marker to Failed
|
||||
/// so the operator dashboard reflects the dead-letter rather than leaving the tenant
|
||||
/// hanging in InProgress. No-op when the row is already Completed/Failed.
|
||||
/// </summary>
|
||||
public async Task MarkEnvelopeFailedFromPoisonAsync(
|
||||
TenantOffboardingEnvelope envelope, int dequeueCount, CancellationToken ct = default)
|
||||
{
|
||||
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
|
||||
var tenantId = envelope.TenantId.ToLowerInvariant();
|
||||
|
||||
var history = await _auditRepo.TryGetHistoryAsync(envelope.HistoryRowKey, ct);
|
||||
if (history == null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"MarkEnvelopeFailedFromPoison: history row missing for tenant={Tenant} history={History} — nothing to transition",
|
||||
tenantId, envelope.HistoryRowKey);
|
||||
return;
|
||||
}
|
||||
if (history.Status is "Completed" or "Failed")
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"MarkEnvelopeFailedFromPoison: history already terminal ({Status}) for tenant={Tenant} — skipping",
|
||||
history.Status, tenantId);
|
||||
return;
|
||||
}
|
||||
|
||||
history.RetryCount = Math.Max(history.RetryCount, dequeueCount);
|
||||
await FailAsync(history, tenantId,
|
||||
failedPhase: "max_dequeue",
|
||||
errorMessage: $"Envelope poisoned after {dequeueCount} failed handler attempts",
|
||||
ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
+345
@@ -0,0 +1,345 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Identity;
|
||||
using Azure.Storage.Queues;
|
||||
using Azure.Storage.Queues.Models;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Background poll-loop for the <c>tenant-offboarding</c> queue (Plan §7.3). Modeled on
|
||||
/// <see cref="Deletion.SessionDeletionWorker"/>:
|
||||
/// <list type="bullet">
|
||||
/// <item><see cref="BatchSize"/> = <b>1</b> — one tenant cascade per receive bounds memory
|
||||
/// (per-tenant enumerate + per-session enqueue is long-running) and limits the
|
||||
/// blast radius of a poison.</item>
|
||||
/// <item>Heartbeat extends visibility every <see cref="DefaultHeartbeatInterval"/> while
|
||||
/// the handler runs so concurrent re-delivery can't spawn a parallel offboarding
|
||||
/// for the same tenant.</item>
|
||||
/// <item>Max-dequeue <see cref="MaxDequeueCount"/> → poison queue + the handler's own
|
||||
/// Failed-state transition (the handler emits <c>TenantOffboardingFailed</c> when
|
||||
/// it fails closed; the worker emits one too on dequeue exhaustion so an envelope
|
||||
/// that never reaches the handler still surfaces).</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class TenantOffboardingWorker : BackgroundService
|
||||
{
|
||||
internal const int BatchSize = 1;
|
||||
|
||||
internal static readonly TimeSpan VisibilityTimeout = TimeSpan.FromMinutes(5);
|
||||
internal static readonly TimeSpan DefaultHeartbeatInterval = TimeSpan.FromSeconds(60);
|
||||
internal static readonly TimeSpan HeartbeatExtendBy = TimeSpan.FromMinutes(5);
|
||||
internal static readonly TimeSpan DefaultPollInterval = TimeSpan.FromSeconds(10);
|
||||
internal static readonly TimeSpan ErrorBackoff = TimeSpan.FromSeconds(30);
|
||||
|
||||
internal const int MaxDequeueCount = 5;
|
||||
private const string PoisonQueueSuffix = "-poison";
|
||||
|
||||
private readonly QueueClient _mainQueue;
|
||||
private readonly QueueClient _poisonQueue;
|
||||
private readonly TenantOffboardingHandler _handler;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ILogger<TenantOffboardingWorker> _logger;
|
||||
|
||||
private readonly TimeSpan _heartbeatInterval;
|
||||
private readonly TimeSpan _pollInterval;
|
||||
|
||||
public TenantOffboardingWorker(
|
||||
IConfiguration configuration,
|
||||
TenantOffboardingHandler handler,
|
||||
OpsEventService opsEvents,
|
||||
ILogger<TenantOffboardingWorker> logger)
|
||||
{
|
||||
_handler = handler ?? throw new ArgumentNullException(nameof(handler));
|
||||
_opsEvents = opsEvents ?? throw new ArgumentNullException(nameof(opsEvents));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_heartbeatInterval = DefaultHeartbeatInterval;
|
||||
_pollInterval = DefaultPollInterval;
|
||||
|
||||
var options = new QueueClientOptions { MessageEncoding = QueueMessageEncoding.Base64 };
|
||||
var storageAccountName = configuration["AzureStorageAccountName"];
|
||||
var connectionString = configuration["AzureTableStorageConnectionString"];
|
||||
|
||||
if (!string.IsNullOrEmpty(storageAccountName))
|
||||
{
|
||||
var mainUri = new Uri(
|
||||
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboarding}");
|
||||
var poisonUri = new Uri(
|
||||
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboardingPoison}");
|
||||
var credential = new DefaultAzureCredential();
|
||||
_mainQueue = new QueueClient(mainUri, credential, options);
|
||||
_poisonQueue = new QueueClient(poisonUri, credential, options);
|
||||
_logger.LogInformation(
|
||||
"TenantOffboardingWorker initialized with Managed Identity (account: {Account})",
|
||||
storageAccountName);
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(connectionString))
|
||||
{
|
||||
_mainQueue = new QueueClient(connectionString, Constants.QueueNames.TenantOffboarding, options);
|
||||
_poisonQueue = new QueueClient(connectionString, Constants.QueueNames.TenantOffboardingPoison, options);
|
||||
_logger.LogInformation("TenantOffboardingWorker initialized with connection string");
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Queue Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureTableStorageConnectionString'.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test seam — inject mocked queues + shorter intervals for unit tests.</summary>
|
||||
internal TenantOffboardingWorker(
|
||||
QueueClient mainQueue,
|
||||
QueueClient poisonQueue,
|
||||
TenantOffboardingHandler handler,
|
||||
OpsEventService opsEvents,
|
||||
ILogger<TenantOffboardingWorker> logger,
|
||||
TimeSpan? heartbeatInterval = null,
|
||||
TimeSpan? pollInterval = null)
|
||||
{
|
||||
_mainQueue = mainQueue ?? throw new ArgumentNullException(nameof(mainQueue));
|
||||
_poisonQueue = poisonQueue ?? throw new ArgumentNullException(nameof(poisonQueue));
|
||||
_handler = handler ?? throw new ArgumentNullException(nameof(handler));
|
||||
_opsEvents = opsEvents ?? throw new ArgumentNullException(nameof(opsEvents));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_heartbeatInterval = heartbeatInterval ?? DefaultHeartbeatInterval;
|
||||
_pollInterval = pollInterval ?? DefaultPollInterval;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await TryCreateQueueAsync(_mainQueue, "main", stoppingToken).ConfigureAwait(false);
|
||||
await TryCreateQueueAsync(_poisonQueue, "poison", stoppingToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation("TenantOffboardingWorker: poll loop started (queue {Name})", _mainQueue.Name);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
var batch = await _mainQueue
|
||||
.ReceiveMessagesAsync(BatchSize, VisibilityTimeout, stoppingToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (batch?.Value is null || batch.Value.Length == 0)
|
||||
{
|
||||
await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var msg in batch.Value)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) break;
|
||||
await ProcessOneAsync(msg, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"TenantOffboardingWorker: poll-loop error — backing off {Backoff}",
|
||||
ErrorBackoff);
|
||||
try { await Task.Delay(ErrorBackoff, stoppingToken).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("TenantOffboardingWorker: poll loop stopped");
|
||||
}
|
||||
|
||||
private async Task ProcessOneAsync(QueueMessage msg, CancellationToken ct)
|
||||
{
|
||||
if (msg.DequeueCount > MaxDequeueCount)
|
||||
{
|
||||
await MoveToPoisonAsync(msg, ct).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
TenantOffboardingEnvelope? envelope;
|
||||
try
|
||||
{
|
||||
envelope = JsonConvert.DeserializeObject<TenantOffboardingEnvelope>(msg.Body.ToString());
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"TenantOffboardingWorker: malformed envelope JSON — dropping (msg {Id})",
|
||||
msg.MessageId);
|
||||
await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (envelope is null
|
||||
|| string.IsNullOrEmpty(envelope.TenantId)
|
||||
|| string.IsNullOrEmpty(envelope.HistoryRowKey))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"TenantOffboardingWorker: envelope missing required fields — dropping (msg {Id})",
|
||||
msg.MessageId);
|
||||
await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var heartbeatState = new HeartbeatState(msg.MessageId, msg.PopReceipt);
|
||||
var heartbeatTask = Task.Run(() => HeartbeatLoopAsync(heartbeatState, heartbeatCts.Token), heartbeatCts.Token);
|
||||
|
||||
try
|
||||
{
|
||||
await _handler.HandleAsync(envelope, ct).ConfigureAwait(false);
|
||||
|
||||
heartbeatCts.Cancel();
|
||||
try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { }
|
||||
|
||||
await SafeDeleteAsync(msg.MessageId, heartbeatState.PopReceipt, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
heartbeatCts.Cancel();
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
heartbeatCts.Cancel();
|
||||
try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { }
|
||||
|
||||
_logger.LogWarning(ex,
|
||||
"TenantOffboardingWorker: handler failed (tenant={Tenant} history={History} dequeue={N}) — visibility-timeout retry",
|
||||
envelope.TenantId, envelope.HistoryRowKey, msg.DequeueCount);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HeartbeatLoopAsync(HeartbeatState state, CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await Task.Delay(_heartbeatInterval, ct).ConfigureAwait(false); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
|
||||
try
|
||||
{
|
||||
var updated = await _mainQueue.UpdateMessageAsync(
|
||||
state.MessageId, state.PopReceipt,
|
||||
visibilityTimeout: HeartbeatExtendBy,
|
||||
cancellationToken: ct).ConfigureAwait(false);
|
||||
state.PopReceipt = updated.Value.PopReceipt;
|
||||
_logger.LogDebug(
|
||||
"TenantOffboardingWorker: heartbeat extended visibility for message {Id} by {Extend}",
|
||||
state.MessageId, HeartbeatExtendBy);
|
||||
}
|
||||
catch (OperationCanceledException) { return; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"TenantOffboardingWorker: heartbeat UpdateMessageAsync failed for message {Id} — exiting heartbeat",
|
||||
state.MessageId);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MoveToPoisonAsync(QueueMessage msg, CancellationToken ct)
|
||||
{
|
||||
TenantOffboardingEnvelope? envelope = null;
|
||||
try { envelope = JsonConvert.DeserializeObject<TenantOffboardingEnvelope>(msg.Body.ToString()); }
|
||||
catch (JsonException) { /* still poison the malformed envelope below */ }
|
||||
|
||||
// Review-Fix (second-pass) Finding 1: the durable Failed-state transition MUST land
|
||||
// BEFORE the poison-queue send + main-queue delete. Otherwise a transient storage /
|
||||
// OpsEvent / pointer-CAS failure on the transition would leave the message in poison
|
||||
// but the Tenant stuck at InProgress. By doing the transition first we keep the
|
||||
// message visible on the main queue when the transition fails — the worker will
|
||||
// retry on the next dequeue. The transition itself is idempotent
|
||||
// (MarkEnvelopeFailedFromPoisonAsync no-ops on terminal History.Status) so retries
|
||||
// are safe.
|
||||
if (envelope != null
|
||||
&& !string.IsNullOrEmpty(envelope.TenantId)
|
||||
&& !string.IsNullOrEmpty(envelope.HistoryRowKey))
|
||||
{
|
||||
try
|
||||
{
|
||||
await _handler.MarkEnvelopeFailedFromPoisonAsync(
|
||||
envelope, (int)(msg.DequeueCount - 1), ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Critical: leave the message visible so the next dequeue can retry the
|
||||
// transition. We do NOT proceed to the poison send / SafeDelete here —
|
||||
// otherwise the operator would see a queue with nothing in it and a tenant
|
||||
// hanging on InProgress.
|
||||
_logger.LogError(ex,
|
||||
"TenantOffboardingWorker: MarkEnvelopeFailedFromPoisonAsync failed for tenant={Tenant} history={History} — message left visible for retry (durable Failed state has higher priority than poison move)",
|
||||
envelope.TenantId, envelope.HistoryRowKey);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Failed state is now durable (or envelope was malformed and couldn't transition).
|
||||
// Best-effort poison move + SafeDelete; transient failure here leaves the message
|
||||
// visible but the next pickup will see History.Status=Failed and SafeDelete cleanly.
|
||||
try
|
||||
{
|
||||
await _poisonQueue.SendMessageAsync(msg.Body.ToString(), ct).ConfigureAwait(false);
|
||||
await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false);
|
||||
_logger.LogWarning(
|
||||
"TenantOffboardingWorker: moved message {Id} to poison queue after {N} failed attempts",
|
||||
msg.MessageId, msg.DequeueCount - 1);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"TenantOffboardingWorker: poison move failed for message {Id} (will retry; durable Failed state already written)",
|
||||
msg.MessageId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SafeDeleteAsync(string messageId, string popReceipt, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _mainQueue.DeleteMessageAsync(messageId, popReceipt, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"TenantOffboardingWorker: DeleteMessageAsync failed for message {Id} — message will reappear after visibility expires",
|
||||
messageId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TryCreateQueueAsync(QueueClient queue, string label, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await queue.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"TenantOffboardingWorker: CreateIfNotExists failed on {Label} queue {Name}",
|
||||
label, queue.Name);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class HeartbeatState
|
||||
{
|
||||
public string MessageId { get; }
|
||||
public string PopReceipt { get; set; }
|
||||
|
||||
public HeartbeatState(string messageId, string popReceipt)
|
||||
{
|
||||
MessageId = messageId;
|
||||
PopReceipt = popReceipt;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -275,6 +275,26 @@ namespace AutopilotMonitor.Functions.Services
|
||||
tenantId, performedBy, details);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fired by the tenant-offboarding worker when the cascade fails closed (kill-switch
|
||||
/// active mid-enqueue, drain timeout, expectations blob corruption, ETag/CAS exhaustion,
|
||||
/// SafeWipe verify abort, …). Marker stays Failed until operator action; this event is
|
||||
/// the Telegram-routable signal that something needs human attention. Plan Rev-4 Q2.
|
||||
/// </summary>
|
||||
public Task RecordTenantOffboardingFailedAsync(
|
||||
string tenantId, string performedBy, string failedPhase, string errorMessage,
|
||||
int retryCount, string? domainName = null)
|
||||
{
|
||||
var tenantLabel = string.IsNullOrWhiteSpace(domainName)
|
||||
? tenantId
|
||||
: $"{domainName} ({tenantId})";
|
||||
|
||||
return WriteAsync(OpsEventCategory.Tenant, "TenantOffboardingFailed", OpsEventSeverity.Error,
|
||||
$"Tenant {tenantLabel} offboarding failed at phase '{failedPhase}': {errorMessage}",
|
||||
tenantId, performedBy,
|
||||
new { domainName, failedPhase, errorMessage, retryCount });
|
||||
}
|
||||
|
||||
// ── Agent ──────────────────────────────────────────────────────────────
|
||||
|
||||
public Task RecordSessionTimeoutsAsync(string tenantId, int sessionCount, int timeoutHours)
|
||||
|
||||
@@ -124,7 +124,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// Returns (config, exists). exists=false when no row was found — does NOT auto-create.
|
||||
/// Use for agent security gates where unknown tenants must be rejected.
|
||||
/// </summary>
|
||||
public async Task<(TenantConfiguration config, bool exists)> TryGetConfigurationAsync(string tenantId)
|
||||
public virtual async Task<(TenantConfiguration config, bool exists)> TryGetConfigurationAsync(string tenantId)
|
||||
{
|
||||
if (string.IsNullOrEmpty(tenantId))
|
||||
return (TenantConfiguration.CreateDefault("unknown"), false);
|
||||
|
||||
@@ -493,6 +493,25 @@ namespace AutopilotMonitor.Shared
|
||||
public const string ClassifierVerdictsByIdLevel = "ClassifierVerdictsByIdLevel";
|
||||
public const string SignalsByKind = "SignalsByKind";
|
||||
|
||||
// Tenant offboarding audit (cascade-worker plan, Rev 9). Houses three
|
||||
// PartitionKey patterns side-by-side:
|
||||
// - "OffboardingMarker" (1 row/tenant, RK=tenantId) active 403-gate marker
|
||||
// - "OffboardingHistory" (N rows/tenant, RK="{yyyyMMddHHmmssfff}_{tenantId}") audit trail
|
||||
// - "OffboardingByTenant" (1 row/tenant, RK=tenantId) O(1) re-onboarding pointer
|
||||
// Marker survives the Completed transition for at least 15min so that warm
|
||||
// function-instance caches of TenantConfiguration cannot leak auth after wipe.
|
||||
// See OffboardingPartitionKeys for the canonical PK string literals.
|
||||
public const string OffboardingAudit = "OffboardingAudit";
|
||||
|
||||
// Tenant offboarding customs archive (PR3.B). Phase 2.D-archive writes a
|
||||
// snapshot of every GatherRules / AnalyzeRules / ImeLogPatterns row keyed by
|
||||
// PK=tenantId before the originals are safe-wiped, so a Global Admin can
|
||||
// review them post-offboarding from the /admin/customs-archive page.
|
||||
// - PartitionKey = "{normalizedTenantId}_{historyRowKey}" (one partition per offboarding run)
|
||||
// - RowKey = "{originalTable}_{base64url(originalRowKey)}"
|
||||
// Lives forever; cleanup is operator-driven via the admin UI.
|
||||
public const string TenantOffboardingCustomsArchive = "TenantOffboardingCustomsArchive";
|
||||
|
||||
/// <summary>
|
||||
/// Returns all table names for initialization
|
||||
/// </summary>
|
||||
@@ -546,10 +565,54 @@ namespace AutopilotMonitor.Shared
|
||||
SessionsByStage,
|
||||
DeadEndsByReason,
|
||||
ClassifierVerdictsByIdLevel,
|
||||
SignalsByKind
|
||||
SignalsByKind,
|
||||
OffboardingAudit,
|
||||
TenantOffboardingCustomsArchive
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Canonical PartitionKey strings stored in <see cref="TableNames.OffboardingAudit"/>.
|
||||
/// All three patterns coexist in the same table; consumers must always read by
|
||||
/// (PartitionKey, RowKey) point-lookup, never by table scan.
|
||||
/// </summary>
|
||||
public static class OffboardingPartitionKeys
|
||||
{
|
||||
/// <summary>Active 403 sperrmarker. RowKey = normalized tenantId. 1 row per active offboarding.</summary>
|
||||
public const string Marker = "OffboardingMarker";
|
||||
|
||||
/// <summary>Chronological audit history. RowKey = "{yyyyMMddHHmmssfff}_{normalizedTenantId}". N rows per tenant.</summary>
|
||||
public const string History = "OffboardingHistory";
|
||||
|
||||
/// <summary>O(1) pointer index for re-onboarding lookup. RowKey = normalized tenantId. 1 row per tenant.</summary>
|
||||
public const string ByTenant = "OffboardingByTenant";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Azure Blob Storage container names. Single source of truth for container strings
|
||||
/// so that storage helpers, cascade workers, and offboarding workers all agree.
|
||||
/// </summary>
|
||||
public static class BlobContainers
|
||||
{
|
||||
/// <summary>
|
||||
/// Houses cascade-deletion manifests (snapshot + progress blobs) under prefix
|
||||
/// <c>{tenantId}/{sessionId}/{manifestId}.{snapshot.json.gz | progress.json}</c>.
|
||||
/// 30d lifecycle delete + 3d soft delete = ~33d effective retention.
|
||||
/// Phase 2.E of tenant offboarding wipes this container by <c>{tenantId}/</c> prefix.
|
||||
/// </summary>
|
||||
public const string DeletionManifests = "deletion-manifests";
|
||||
|
||||
/// <summary>
|
||||
/// Houses tenant-offboarding state blobs that MUST survive the deletion-manifests wipe.
|
||||
/// Currently: per-tenant Expectations-Blob written at the first handler pickup
|
||||
/// (<c>{tenantId}/{historyRowKey}.expectations.json</c>). Phase 2.E intentionally
|
||||
/// skips this container; Phase 2.G deletes the blob explicitly as its last step,
|
||||
/// with <see cref="Functions.Maintenance.OffboardingMarkerCleanupFunction"/> as a
|
||||
/// defense-in-depth second-pass.
|
||||
/// </summary>
|
||||
public const string OffboardingState = "offboarding-state";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Azure Storage Queue names. Queues feed async fan-out / reconciliation paths
|
||||
/// that would otherwise block the hot ingest loop.
|
||||
@@ -593,6 +656,20 @@ namespace AutopilotMonitor.Shared
|
||||
/// the Sessions row. Poison suffix <c>-poison</c>, max-dequeue 5.
|
||||
/// </summary>
|
||||
public const string SessionDeletion = "session-deletion";
|
||||
|
||||
/// <summary>
|
||||
/// Tenant offboarding cascade-worker queue. Producer (TenantOffboardFunction) writes
|
||||
/// one envelope per tenant after inserting the History/Pointer/Marker rows in
|
||||
/// <see cref="TableNames.OffboardingAudit"/>. Worker (TenantOffboardingWorker)
|
||||
/// enumerates sessions, enqueues per-session cascades, drains against an
|
||||
/// Expectations-Blob in <see cref="BlobContainers.OffboardingState"/>, then safe-wipes
|
||||
/// all remaining tenant-scoped rows. Max-dequeue 5, paired with
|
||||
/// <see cref="TenantOffboardingPoison"/>.
|
||||
/// </summary>
|
||||
public const string TenantOffboarding = "tenant-offboarding";
|
||||
|
||||
/// <summary>Poison sibling of <see cref="TenantOffboarding"/>.</summary>
|
||||
public const string TenantOffboardingPoison = "tenant-offboarding-poison";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,7 +69,16 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId);
|
||||
|
||||
// --- Tenant Offboarding ---
|
||||
Task<Dictionary<string, int>> DeleteAllTenantDataAsync(string tenantId);
|
||||
|
||||
/// <summary>
|
||||
/// Fail-loud iterator over every session belonging to <paramref name="tenantId"/>.
|
||||
/// Used by the tenant-offboarding worker to drive per-session cascade enqueue.
|
||||
/// Unlike <see cref="GetSessionsByDateRangeAsync"/>, this method MUST NOT swallow
|
||||
/// storage exceptions — a transient failure during enumeration must propagate so
|
||||
/// the queue worker can retry / poison, instead of silently returning zero sessions
|
||||
/// which would trigger a same-cycle wipe without cascade backup.
|
||||
/// </summary>
|
||||
IAsyncEnumerable<string> EnumerateSessionsForOffboardingAsync(string tenantId, System.Threading.CancellationToken ct = default);
|
||||
}
|
||||
|
||||
public class OrphanedEventSession
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
|
||||
namespace AutopilotMonitor.Shared.DataAccess
|
||||
{
|
||||
/// <summary>
|
||||
/// Access to the <c>OffboardingAudit</c> table. Owns three PartitionKey patterns
|
||||
/// (Marker / History / ByTenant). All write paths fail-loud — Queue retry / poison
|
||||
/// owns recovery semantics.
|
||||
/// </summary>
|
||||
public interface IOffboardingAuditRepository
|
||||
{
|
||||
// ── Marker (offboarding lifecycle anchor; idempotency + cleanup) ───────
|
||||
|
||||
/// <summary>
|
||||
/// Uncached point lookup used by <c>TenantOffboardFunction</c>'s idempotency check
|
||||
/// + <c>OffboardingMarkerCleanupFunction</c>'s timer sweep. NOT read in the agent
|
||||
/// or web auth hotpath any more — the active auth-gate is
|
||||
/// <c>TenantConfiguration.Disabled=true</c>. Returns null when no marker exists.
|
||||
/// </summary>
|
||||
Task<OffboardingMarkerEntry?> TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default);
|
||||
|
||||
/// <summary>Inserts a fresh marker. Throws if a marker for the tenant already exists.</summary>
|
||||
Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default);
|
||||
|
||||
/// <summary>Idempotent upsert; preferred path from worker phase transitions.</summary>
|
||||
Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default);
|
||||
|
||||
/// <summary>Removes a marker by tenant id. 404 silently swallowed (idempotent).</summary>
|
||||
Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default);
|
||||
|
||||
/// <summary>Enumerates ALL active markers — used by <c>OffboardingMarkerCleanupFunction</c>.</summary>
|
||||
IAsyncEnumerable<OffboardingMarkerEntry> QueryMarkersAsync(CancellationToken ct = default);
|
||||
|
||||
// ── History (audit trail, one row per offboarding attempt) ──────────────
|
||||
|
||||
Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default);
|
||||
Task<OffboardingHistoryEntry?> TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default);
|
||||
Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default);
|
||||
|
||||
// ── Pointer (O(1) re-onboarding index) ──────────────────────────────────
|
||||
//
|
||||
// Plan §4.4 update protocol is strictly Read-Modify-Write with ETag-CAS:
|
||||
// 1. TryGet → null + null ETag → Insert with OffboardCount=1
|
||||
// 2. TryGet → (pointer, etag) → modify in-memory + UpdateWithEtag(etag),
|
||||
// 412 on mismatch ⇒ re-read and retry.
|
||||
// The "blind upsert" path was removed because it cannot satisfy OffboardCount=existing+1
|
||||
// under concurrent writers — the read-modify-write window is the whole point of the
|
||||
// pointer's design.
|
||||
|
||||
/// <summary>
|
||||
/// Reads the pointer row plus its ETag for CAS. Returns <c>(null, null)</c> on 404.
|
||||
/// </summary>
|
||||
Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync(
|
||||
string normalizedTenantId, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a fresh pointer row. Throws <see cref="Azure.RequestFailedException"/> with
|
||||
/// status 409 when a pointer for the tenant already exists — caller is expected to fall
|
||||
/// back to <see cref="UpdateByTenantPointerWithEtagAsync"/> in that case.
|
||||
/// </summary>
|
||||
Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Conditional replace: succeeds only when the stored row's ETag still matches
|
||||
/// <paramref name="ifMatchEtag"/>. Throws <see cref="Azure.RequestFailedException"/> with
|
||||
/// status 412 on mismatch so the caller can re-read + retry within their bounded loop.
|
||||
/// </summary>
|
||||
Task UpdateByTenantPointerWithEtagAsync(
|
||||
OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared.Models.Offboarding;
|
||||
|
||||
namespace AutopilotMonitor.Shared.DataAccess
|
||||
{
|
||||
/// <summary>
|
||||
/// Access to <c>TenantOffboardingCustomsArchive</c>. Writes happen during Phase 2.D-archive
|
||||
/// of the offboarding worker; reads / deletes are driven by the Global Admin
|
||||
/// <c>/admin/customs-archive</c> UI.
|
||||
/// <para>
|
||||
/// All write paths fail-loud — when an Azure storage call throws, exceptions propagate
|
||||
/// to the caller (queue worker), which owns retry/poison semantics. Reads + deletes
|
||||
/// from the admin API similarly surface errors to the HTTP caller.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public interface ITenantCustomsArchiveRepository
|
||||
{
|
||||
/// <summary>
|
||||
/// Idempotent upsert (Replace). Used during Phase 2.D-archive so a worker that
|
||||
/// crashed mid-archive can re-pickup without duplicate-key errors.
|
||||
/// </summary>
|
||||
Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default);
|
||||
|
||||
/// <summary>Count entries within a single offboarding run filtered by source table.</summary>
|
||||
Task<int> CountByRunAndTableAsync(
|
||||
string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default);
|
||||
|
||||
/// <summary>List one offboarding run's archive partition (all 3 source tables interleaved).</summary>
|
||||
IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByRunAsync(
|
||||
string normalizedTenantId, string historyRowKey, CancellationToken ct = default);
|
||||
|
||||
/// <summary>List every archive partition for a tenant (one entry per row across all runs).</summary>
|
||||
IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByTenantAsync(
|
||||
string normalizedTenantId, CancellationToken ct = default);
|
||||
|
||||
/// <summary>List every archived run as a (tenantId, historyRowKey)-pair summary across all tenants.</summary>
|
||||
IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryAllAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>Fetch a single archive entry by (PK, RK).</summary>
|
||||
Task<TenantOffboardingCustomsArchiveEntry?> TryGetEntryAsync(
|
||||
string partitionKey, string rowKey, CancellationToken ct = default);
|
||||
|
||||
/// <summary>Delete a single archived entry. 404 silently swallowed (idempotent).</summary>
|
||||
Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Delete every entry in a single run's archive partition. Used by the Global Admin
|
||||
/// "Delete run" action after they have reviewed the contents.
|
||||
/// </summary>
|
||||
Task<int> DeleteRunAsync(
|
||||
string normalizedTenantId, string historyRowKey, CancellationToken ct = default);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// O(1) point-lookup pointer used by the re-onboarding hook to find the latest history
|
||||
/// row for a given tenant without scanning the <c>OffboardingHistory</c> partition.
|
||||
/// Azure Tables cannot serve <c>RowKey endswith '_{tenantId}'</c> efficiently
|
||||
/// server-side (full partition scan + client-side filter), so we maintain a sidecar
|
||||
/// row keyed by tenantId.
|
||||
/// <para>
|
||||
/// Lives in <c>OffboardingAudit</c> under
|
||||
/// <c>PartitionKey = OffboardingPartitionKeys.ByTenant</c> with
|
||||
/// <c>RowKey = normalizedTenantId</c>. Upserted with two-step writes (history-first,
|
||||
/// pointer-second) and updated as the worker progresses through Status transitions.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class OffboardingByTenantPointer
|
||||
{
|
||||
/// <summary>Always <c>OffboardingPartitionKeys.ByTenant</c>.</summary>
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Normalized (lowercase GUID) tenant id.</summary>
|
||||
public string RowKey { get; set; } = string.Empty;
|
||||
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>"{yyyyMMddHHmmssfff}_{tenantId}" of the current/latest history row.</summary>
|
||||
public string LatestHistoryRowKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Mirrors the matching history row's Status:
|
||||
/// <c>Initiated | InProgress | Completed | Failed | Reonboarded</c>.</summary>
|
||||
public string LatestStatus { get; set; } = "Initiated";
|
||||
|
||||
/// <summary>UTC of the most recent pointer update; monotonically increasing.</summary>
|
||||
public DateTime LatestUpdatedAt { get; set; }
|
||||
|
||||
/// <summary>How many distinct offboarding rounds this tenant has been through.
|
||||
/// Incremented on each new Initiate. Useful for "Tenant ging und kam zurück"-Reporting.</summary>
|
||||
public int OffboardCount { get; set; } = 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-tenant snapshot of the cascade enqueue outcome captured at the first handler pickup.
|
||||
/// Drives the drain predicate: every entry is matched against either a progress blob
|
||||
/// (Enqueued/AlreadyInFlight), a no-op (SessionNotFound), a fail-closed condition
|
||||
/// (KillSwitchActive/Poisoned), or a bounded retry loop (CasExhausted).
|
||||
/// <para>
|
||||
/// Stored as a single blob at
|
||||
/// <c>{BlobContainers.OffboardingState}/{tenantId}/{historyRowKey}.expectations.json</c>.
|
||||
/// The container is deliberately separate from <c>deletion-manifests</c> because the latter
|
||||
/// is wiped in Phase 2.E; storing this blob there would make crash-recovery between 2.E
|
||||
/// and 2.G impossible.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class OffboardingExpectations
|
||||
{
|
||||
/// <summary>Bump on breaking-schema changes. Drain-probe fail-closed if mismatch.</summary>
|
||||
public int SchemaVersion { get; set; } = 1;
|
||||
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Matching history row's RowKey, so a blob can be paired back to its run.</summary>
|
||||
public string HistoryRowKey { get; set; } = string.Empty;
|
||||
|
||||
public DateTime CreatedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// True only after the session enumerator iterated to completion without exception AND
|
||||
/// the upload is being written. Default false; drain-probe treats <c>false</c> as
|
||||
/// "this blob was uploaded with incomplete data" → fail-closed with
|
||||
/// <c>FailedPhase="enumeration_incomplete"</c>. Critical disambiguation: lets the
|
||||
/// probe tell apart "0 sessions, happy path" from "0 entries because enumerator
|
||||
/// crashed before yielding any".
|
||||
/// </summary>
|
||||
public bool EnumerationCompleted { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of sessions the enumerator yielded BEFORE per-session enqueue ran. Must equal
|
||||
/// <see cref="Expectations"/>.Count on upload; mismatch → fail-closed.
|
||||
/// </summary>
|
||||
public int EnumeratedSessionCount { get; set; }
|
||||
|
||||
public List<OffboardingExpectation> Expectations { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>One per session that the enumerator returned.</summary>
|
||||
public sealed class OffboardingExpectation
|
||||
{
|
||||
public string SessionId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Manifest id returned by <c>SessionDeletionProducer.EnqueueAsync</c>. Null for outcomes
|
||||
/// that did not produce a cascade (SessionNotFound, KillSwitchActive, CasExhausted) or
|
||||
/// when the producer returned <c>AlreadyInFlight</c> against a Preparing-without-snapshot
|
||||
/// state. The drain probe fails closed on Enqueued/AlreadyInFlight with a null/empty
|
||||
/// manifest id.
|
||||
/// </summary>
|
||||
public string? ManifestId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// String form of <c>SessionDeletionProducer.EnqueueOutcome</c>:
|
||||
/// <c>Enqueued | AlreadyInFlight | SessionNotFound | KillSwitchActive | Poisoned | CasExhausted</c>.
|
||||
/// Stored as string so the shared assembly doesn't need to reference the
|
||||
/// Functions-only enum and so future enum additions don't break already-written blobs.
|
||||
/// </summary>
|
||||
public string Outcome { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// CasExhausted retry counter. Drain-probe re-enqueues this session and increments;
|
||||
/// after 3 retries the offboarding fails closed with <c>FailedPhase="cas_exhausted"</c>.
|
||||
/// </summary>
|
||||
public int RetryCount { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Chronological audit-trail row for one tenant-offboarding attempt. Multiple rows per
|
||||
/// tenant when the tenant offboarded, re-onboarded, and offboarded again. Lives in
|
||||
/// <c>OffboardingAudit</c> under <c>PartitionKey = OffboardingPartitionKeys.History</c>
|
||||
/// with <c>RowKey = "{yyyyMMddHHmmssfff}_{normalizedTenantId}"</c> so rows sort
|
||||
/// newest-last lexicographically within the partition.
|
||||
/// </summary>
|
||||
public sealed class OffboardingHistoryEntry
|
||||
{
|
||||
/// <summary>Always <c>OffboardingPartitionKeys.History</c>.</summary>
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>"{yyyyMMddHHmmssfff}_{normalizedTenantId}". Unique per offboarding attempt.</summary>
|
||||
public string RowKey { get; set; } = string.Empty;
|
||||
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
public string DomainName { get; set; } = string.Empty;
|
||||
public string InitiatedBy { get; set; } = string.Empty;
|
||||
|
||||
public DateTime OffboardedAt { get; set; }
|
||||
public DateTime? CompletedAt { get; set; }
|
||||
|
||||
/// <summary><c>Initiated</c> | <c>InProgress</c> | <c>Completed</c> | <c>Failed</c>.</summary>
|
||||
public string Status { get; set; } = "Initiated";
|
||||
|
||||
/// <summary>
|
||||
/// Cache-Drain-Barrier (plan v2 §2). The worker MUST NOT begin destructive Phase 2
|
||||
/// before this UTC timestamp. <c>TenantOffboardFunction</c> sets it to
|
||||
/// <c>OffboardedAt + DrainBarrier</c> (default 6 min) and enqueues the envelope
|
||||
/// with the matching <c>visibilityDelay</c>, so the message is invisible to the
|
||||
/// worker until then. All warm function-host instances will have hit their 5-min
|
||||
/// <c>TenantConfigurationService</c> cache TTL by the time it expires, so the
|
||||
/// existing <c>Disabled=true</c> gate carries the 403 instead of the marker.
|
||||
/// <para>
|
||||
/// Also rendered by the Web UI as the "data deletion starts in mm ss" countdown
|
||||
/// during the drain barrier state, and used by the resume-path to compute the
|
||||
/// remaining delay when an admin re-clicks mid-barrier.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public DateTime? EarliestProcessingAt { get; set; }
|
||||
|
||||
// ── Diagnostics ────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>JSON: <c>{ tableName: deletedRowCount }</c> across SafeWipe + Cascade phases.</summary>
|
||||
public string? DeletedRowCountsJson { get; set; }
|
||||
|
||||
public int? TotalRowsDeleted { get; set; }
|
||||
public int? DeletedBlobCount { get; set; }
|
||||
public int? CascadeSessionsEnqueued { get; set; }
|
||||
public string? ErrorMessage { get; set; }
|
||||
public int RetryCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Rev-9-F1 phase marker. Set once by the handler immediately after the drain predicate
|
||||
/// returns true, BEFORE Phase 2.D begins wiping tables. Re-pickup after crash skips the
|
||||
/// drain predicate when this is non-null because Phase 2.E may have wiped the cascade
|
||||
/// progress blobs that the drain predicate reads from; the post-drain phases (2.D-G)
|
||||
/// are all idempotent so a clean resume requires no progress blobs.
|
||||
/// </summary>
|
||||
public DateTime? DrainCompletedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Stamped by the handler the first time it enters <c>EnsureExpectationsBlobAsync</c>,
|
||||
/// BEFORE the session enumerator starts iterating. On re-pickup after an enumerator
|
||||
/// crash this field is non-null while the Expectations blob is still missing — the
|
||||
/// handler then fail-closes with <c>FailedPhase="expectations_missing"</c> instead of
|
||||
/// re-enumerating against a tenant whose underlying Sessions table may already be
|
||||
/// mid-mutation. Plan §7.4 step 3 (Rev-5-F3 + Rev-7-F2 crash-after-enumerate-start).
|
||||
/// </summary>
|
||||
public DateTime? EnumerationStartedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Stamped after the session enumerator loop finishes cleanly, BEFORE the Expectations
|
||||
/// blob upload. Disambiguates two failure modes that both leave the blob missing with
|
||||
/// <see cref="EnumerationStartedAt"/> set:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Mid-enumeration crash</b>: ECBU is null → fail-closed
|
||||
/// (<c>FailedPhase="expectations_missing"</c>); the tenant may be mid-mutation
|
||||
/// and re-enumerating is unsafe.</item>
|
||||
/// <item><b>Upload failure</b>: ECBU is non-null → retry the enumerate+upload pair
|
||||
/// on the next pickup. The session-deletion producer is idempotent
|
||||
/// (<c>AlreadyInFlight</c> outcome with the existing ManifestId), so re-running
|
||||
/// the loop produces the same cascade-state set.</item>
|
||||
/// </list>
|
||||
/// Plan §7.4 step 3 + Review-Fix (second-pass) Finding 2.
|
||||
/// </summary>
|
||||
public DateTime? EnumerationCompletedBeforeUpload { get; set; }
|
||||
|
||||
// ── Archived Customs (PR3.B §3) ────────────────────────────────────────
|
||||
//
|
||||
// Counts of rows that were snapshotted into TenantOffboardingCustomsArchive
|
||||
// during Phase 2.D-archive, immediately before the originals were safe-wiped.
|
||||
// Recomputed from the archive table by post-archive query so a crash between
|
||||
// archive-insert and counter-write does not leave a stale 0.
|
||||
|
||||
public int? CustomGatherRulesArchived { get; set; }
|
||||
public int? CustomAnalyzeRulesArchived { get; set; }
|
||||
public int? ImeLogPatternOverridesArchived { get; set; }
|
||||
|
||||
// ── Re-onboarding tracking (legacy, kept for audit history continuity) ─
|
||||
|
||||
public DateTime? ReonboardedAt { get; set; }
|
||||
public string? ReonboardedBy { get; set; }
|
||||
public int? CustomsAutoWipedOnReonboard { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Lifecycle anchor + audit row for a tenant currently (or recently) being offboarded.
|
||||
/// Lives in <c>OffboardingAudit</c> table under
|
||||
/// <c>PartitionKey = OffboardingPartitionKeys.Marker</c> with <c>RowKey = normalizedTenantId</c>.
|
||||
/// <para>
|
||||
/// In the PR3-revised architecture this row is NO LONGER read in the agent or web
|
||||
/// auth hotpath. The active gate is <c>TenantConfiguration.Disabled=true</c> + the
|
||||
/// 6-minute cache-drain barrier on the worker envelope. The marker exists to:
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item>Provide an idempotency anchor for the <c>TenantOffboardFunction</c>
|
||||
/// admin endpoint (re-clicks resolve to the existing marker and resume).</item>
|
||||
/// <item>Track the offboarding lifecycle status (<see cref="Status"/>) across
|
||||
/// Phase 1 → Phase 2 → Completed/Failed for ops visibility.</item>
|
||||
/// <item>Anchor the post-completion cleanup sweeps run by
|
||||
/// <c>OffboardingMarkerCleanupFunction</c> (Expectations blob delete +
|
||||
/// defense-in-depth TenantConfiguration sweep if Phase 2.F-final failed).</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// Marker survives the Completed transition: Phase 2.G sets <see cref="Status"/> =
|
||||
/// <c>"Completed"</c> + <see cref="CompletedAt"/>. <c>OffboardingMarkerCleanupFunction</c>
|
||||
/// removes Completed markers after 15min (post-cleanup-sweep settle buffer). Failed
|
||||
/// markers are never auto-deleted — operator action required.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class OffboardingMarkerEntry
|
||||
{
|
||||
/// <summary>Always <c>OffboardingPartitionKeys.Marker</c>.</summary>
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Normalized (lowercase GUID) tenant id.</summary>
|
||||
public string RowKey { get; set; } = string.Empty;
|
||||
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Cross-reference to the matching <c>OffboardingHistory</c> row's RowKey.</summary>
|
||||
public string OffboardingHistoryRowKey { get; set; } = string.Empty;
|
||||
|
||||
public DateTime InitiatedAt { get; set; }
|
||||
|
||||
/// <summary>UPN of the admin (or "System.*") that triggered the offboarding.</summary>
|
||||
public string InitiatedBy { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Lifecycle: <c>Initiated</c> → <c>InProgress</c> → <c>Completed</c> | <c>Failed</c>.
|
||||
/// Marker remains active in all four states; only the cleanup function removes
|
||||
/// Completed markers (and never Failed).
|
||||
/// </summary>
|
||||
public string Status { get; set; } = "Initiated";
|
||||
|
||||
/// <summary>UTC timestamp set by Phase 2.G; gates 15-min cleanup window.</summary>
|
||||
public DateTime? CompletedAt { get; set; }
|
||||
|
||||
/// <summary>UTC timestamp set after max-dequeue → Failed; marker stays active until operator action.</summary>
|
||||
public DateTime? FailedAt { get; set; }
|
||||
|
||||
/// <summary>Phase that failed: <c>drain</c>, <c>wipe</c>, <c>blobs</c>, <c>tenantconfig</c>,
|
||||
/// <c>killswitch</c>, <c>poisoned</c>, <c>cas_exhausted</c>, <c>drain_timeout</c>,
|
||||
/// <c>expectations_missing</c>, <c>expectations_corrupt</c>, <c>enumeration_incomplete</c>,
|
||||
/// <c>expectations_size_mismatch</c>, <c>alreadyinflight_no_manifest</c>.</summary>
|
||||
public string? FailedPhase { get; set; }
|
||||
}
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Snapshot of a tenant's custom rule row written by the offboarding handler during
|
||||
/// Phase 2.D-archive, BEFORE the original row is safe-wiped. Survives forever so a
|
||||
/// Global Admin can review (and selectively delete) the tenant's custom rules from
|
||||
/// <c>/admin/customs-archive</c> after offboarding completes.
|
||||
/// <para>
|
||||
/// Storage layout (see PR3.B plan §3.2):
|
||||
/// <list type="bullet">
|
||||
/// <item><b>PartitionKey</b>: <c>"{normalizedTenantId}_{historyRowKey}"</c>. One
|
||||
/// partition per offboarding run — Re-Re-Offboarding produces a fresh, immutable
|
||||
/// partition each time so multiple runs co-exist without RowKey collisions.</item>
|
||||
/// <item><b>RowKey</b>: <c>"{originalTable}_{base64url(originalRowKey)}"</c>. The
|
||||
/// table-name prefix disambiguates collisions across the three rules tables; the
|
||||
/// base64url encoding of <see cref="OriginalRowKey"/> avoids the Azure-Tables
|
||||
/// RowKey-forbidden characters (<c>#</c>, <c>?</c>, <c>/</c>, <c>\</c>, control chars).</item>
|
||||
/// </list>
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class TenantOffboardingCustomsArchiveEntry
|
||||
{
|
||||
/// <summary>"{normalizedTenantId}_{historyRowKey}".</summary>
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>"{originalTable}_{base64url(originalRowKey)}".</summary>
|
||||
public string RowKey { get; set; } = string.Empty;
|
||||
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>One of <c>GatherRules</c> / <c>AnalyzeRules</c> / <c>ImeLogPatterns</c>.</summary>
|
||||
public string OriginalTable { get; set; } = string.Empty;
|
||||
|
||||
public string OriginalPartitionKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Original RowKey verbatim — base64url-encoded into <see cref="RowKey"/>'s suffix.</summary>
|
||||
public string OriginalRowKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// JSON dump of every property on the source row (excluding system properties).
|
||||
/// The full body is small enough to fit in a single Azure Table string property
|
||||
/// (64 KB limit; observed rule bodies are well under 4 KB). If a future rule type
|
||||
/// ever exceeds that, an EntityJsonBlobUrl overflow pointer can be added — not
|
||||
/// part of this PR.
|
||||
/// </summary>
|
||||
public string EntityJson { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Cross-reference to the <c>OffboardingHistory</c> row for the run.</summary>
|
||||
public string HistoryRowKey { get; set; } = string.Empty;
|
||||
|
||||
public DateTime ArchivedAt { get; set; }
|
||||
|
||||
/// <summary>Constant <c>"TenantOffboardingHandler"</c> in production. Test fakes may override.</summary>
|
||||
public string ArchivedBy { get; set; } = "TenantOffboardingHandler";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models.Offboarding
|
||||
{
|
||||
/// <summary>
|
||||
/// Queue message envelope carried on <c>tenant-offboarding</c> (and its poison sibling).
|
||||
/// Producer (<c>TenantOffboardFunction</c>) writes one envelope per tenant after the
|
||||
/// History/Pointer/Marker rows are committed. Worker (<c>TenantOffboardingWorker</c>)
|
||||
/// dequeues, runs <c>TenantOffboardingHandler.HandleAsync</c>, and either acks on
|
||||
/// success or re-enqueues itself with <see cref="DrainPollCount"/> incremented when the
|
||||
/// drain predicate is still in flight.
|
||||
/// </summary>
|
||||
public sealed class TenantOffboardingEnvelope
|
||||
{
|
||||
/// <summary>Schema version — bump on breaking envelope changes.</summary>
|
||||
public string EnvelopeVersion { get; set; } = "1";
|
||||
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Always <c>OffboardingPartitionKeys.History</c>; kept for self-documentation.</summary>
|
||||
public string HistoryPartitionKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>"{yyyyMMddHHmmssfff}_{tenantId}" pointing at the matching history row.</summary>
|
||||
public string HistoryRowKey { get; set; } = string.Empty;
|
||||
|
||||
public string InitiatedBy { get; set; } = string.Empty;
|
||||
public DateTime InitiatedAt { get; set; }
|
||||
public DateTime EnqueuedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Drain self-poll counter. Handler re-enqueues with this incremented when the
|
||||
/// cascade-progress drain predicate has not yet settled. Capped at 60 (~2h) before
|
||||
/// failing the offboarding with <c>FailedPhase="drain_timeout"</c>.
|
||||
/// </summary>
|
||||
public int DrainPollCount { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -44,7 +44,11 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
"PoisonQueueBacklogHigh",
|
||||
"PoisonQueueBacklogCritical",
|
||||
],
|
||||
Tenant: ["TenantOffboarded"],
|
||||
// TenantOffboardingFailed is dispatched by TenantOffboardingWorker when the async cascade fails
|
||||
// closed (drain timeout, KillSwitch active, Poisoned session, ETag/CAS exhaustion, SafeWipe verify
|
||||
// abort, etc.). Marker remains in Failed state until operator action — this event is the
|
||||
// Telegram-routable signal that something needs human attention. Plan Rev-4 Q2 + Rev-9 §11 PR3.
|
||||
Tenant: ["TenantOffboarded", "TenantOffboardingFailed"],
|
||||
Agent: ["BlobStorageMissing", "BlobStorageUnreachable", "NewImeVersionDetected", "ExcessiveSessionEvents"],
|
||||
};
|
||||
|
||||
|
||||
+243
@@ -0,0 +1,243 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState, use as usePromise } from "react";
|
||||
import Link from "next/link";
|
||||
import {
|
||||
api,
|
||||
type CustomsArchiveEntrySummary,
|
||||
type CustomsArchiveFullEntry,
|
||||
type CustomsArchiveListEntriesResponse,
|
||||
} from "@/lib/api";
|
||||
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
import { useAdminConfig } from "../../../AdminConfigContext";
|
||||
import { AdminNotifications } from "../../../AdminNotifications";
|
||||
import { DeleteConfirmModal } from "../../components/DeleteConfirmModal";
|
||||
|
||||
interface PageProps {
|
||||
params: Promise<{ tenantId: string; historyRowKey: string }>;
|
||||
}
|
||||
|
||||
export default function CustomsArchiveDetailPage({ params }: PageProps) {
|
||||
// Next.js 15: route params are async. Resolve once.
|
||||
const resolved = usePromise(params);
|
||||
const tenantId = decodeURIComponent(resolved.tenantId);
|
||||
const historyRowKey = decodeURIComponent(resolved.historyRowKey);
|
||||
|
||||
const { getAccessToken, setError, setSuccessMessage } = useAdminConfig();
|
||||
const [entries, setEntries] = useState<CustomsArchiveEntrySummary[]>([]);
|
||||
const [loading, setLoading] = useState<boolean>(false);
|
||||
const [expanded, setExpanded] = useState<Record<string, CustomsArchiveFullEntry | "loading" | undefined>>({});
|
||||
const [deletingRk, setDeletingRk] = useState<string | null>(null);
|
||||
const [pendingDelete, setPendingDelete] = useState<CustomsArchiveEntrySummary | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
const url = api.customsArchive.listEntries(tenantId, historyRowKey);
|
||||
const res = await authenticatedFetch(url, getAccessToken);
|
||||
if (!res.ok) {
|
||||
throw new Error(`List entries failed: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
const body = (await res.json()) as CustomsArchiveListEntriesResponse;
|
||||
setEntries(body.entries ?? []);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError(err.message);
|
||||
} else {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [tenantId, historyRowKey, getAccessToken, setError]);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
const toggleExpand = useCallback(async (entry: CustomsArchiveEntrySummary) => {
|
||||
const key = entry.rowKey;
|
||||
if (expanded[key]) {
|
||||
// Collapse
|
||||
setExpanded((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[key];
|
||||
return next;
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
setExpanded((prev) => ({ ...prev, [key]: "loading" }));
|
||||
try {
|
||||
const res = await authenticatedFetch(
|
||||
api.customsArchive.getEntry(tenantId, historyRowKey, entry.rowKey),
|
||||
getAccessToken,
|
||||
);
|
||||
if (!res.ok) throw new Error(`Get entry failed: ${res.status}`);
|
||||
const body = await res.json();
|
||||
setExpanded((prev) => ({ ...prev, [key]: body.entry as CustomsArchiveFullEntry }));
|
||||
} catch (err) {
|
||||
setExpanded((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[key];
|
||||
return next;
|
||||
});
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
}, [expanded, tenantId, historyRowKey, getAccessToken, setError]);
|
||||
|
||||
const confirmDelete = useCallback(async () => {
|
||||
const entry = pendingDelete;
|
||||
if (!entry) return;
|
||||
try {
|
||||
setDeletingRk(entry.rowKey);
|
||||
const res = await authenticatedFetch(
|
||||
api.customsArchive.deleteEntry(tenantId, historyRowKey, entry.rowKey),
|
||||
getAccessToken,
|
||||
{ method: "DELETE" },
|
||||
);
|
||||
if (!res.ok) throw new Error(`Delete failed: ${res.status}`);
|
||||
setSuccessMessage(`Deleted ${entry.originalTable} / ${entry.originalRowKey}`);
|
||||
setPendingDelete(null);
|
||||
await load();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setDeletingRk(null);
|
||||
}
|
||||
}, [pendingDelete, tenantId, historyRowKey, getAccessToken, load, setError, setSuccessMessage]);
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
|
||||
<header className="bg-white dark:bg-gray-800 shadow dark:shadow-gray-700">
|
||||
<div className="py-6 px-4 sm:px-6 lg:px-8">
|
||||
<Link
|
||||
href="/admin/customs-archive"
|
||||
className="text-sm text-blue-600 dark:text-blue-400 hover:underline"
|
||||
>
|
||||
← Back to runs
|
||||
</Link>
|
||||
<h1 className="text-2xl font-normal text-gray-900 dark:text-white mt-2">
|
||||
Customs Archive — Run detail
|
||||
</h1>
|
||||
<p className="text-xs font-mono text-gray-600 dark:text-gray-400 mt-1">
|
||||
Tenant: {tenantId} · Run: {historyRowKey}
|
||||
</p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main className="max-w-6xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-4">
|
||||
<AdminNotifications />
|
||||
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-sm text-gray-600 dark:text-gray-300">
|
||||
{entries.length} entry/entries{loading ? " — loading…" : ""}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={load}
|
||||
disabled={loading}
|
||||
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
{loading ? "Loading…" : "Refresh"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{entries.length === 0 && !loading && (
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 text-sm text-gray-600 dark:text-gray-400">
|
||||
No archived entries for this run.
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-2">
|
||||
{entries.map((entry) => {
|
||||
const detail = expanded[entry.rowKey];
|
||||
const isOpen = !!detail;
|
||||
const isLoadingFull = detail === "loading";
|
||||
const fullEntry = typeof detail === "object" ? (detail as CustomsArchiveFullEntry) : null;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={entry.rowKey}
|
||||
className="bg-white dark:bg-gray-800 rounded-lg shadow border border-gray-200 dark:border-gray-700"
|
||||
>
|
||||
<div className="p-3 flex flex-wrap items-center justify-between gap-2">
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="inline-block px-2 py-0.5 rounded text-xs font-medium bg-indigo-100 text-indigo-800 dark:bg-indigo-900/40 dark:text-indigo-200">
|
||||
{entry.originalTable}
|
||||
</span>
|
||||
<span className="text-sm font-mono text-gray-900 dark:text-white truncate">
|
||||
{entry.originalRowKey}
|
||||
</span>
|
||||
</div>
|
||||
<div className="text-xs text-gray-500 dark:text-gray-400 mt-1">
|
||||
Archived {new Date(entry.archivedAt).toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => toggleExpand(entry)}
|
||||
className="px-3 py-1.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-600 text-gray-900 dark:text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
{isOpen ? "Collapse" : "Inspect"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setPendingDelete(entry)}
|
||||
disabled={deletingRk === entry.rowKey}
|
||||
className="px-3 py-1.5 bg-red-600 hover:bg-red-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
{deletingRk === entry.rowKey ? "Deleting…" : "Delete entry"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isOpen && (
|
||||
<div className="border-t border-gray-200 dark:border-gray-700 p-3 bg-gray-50 dark:bg-gray-900/50">
|
||||
{isLoadingFull ? (
|
||||
<div className="text-sm text-gray-500">Loading…</div>
|
||||
) : fullEntry ? (
|
||||
<pre className="text-xs font-mono whitespace-pre-wrap break-all text-gray-800 dark:text-gray-200">
|
||||
{prettyJson(fullEntry.entityJson)}
|
||||
</pre>
|
||||
) : null}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<DeleteConfirmModal
|
||||
open={pendingDelete !== null}
|
||||
title="Delete archive entry"
|
||||
description={pendingDelete && (
|
||||
<>
|
||||
<p>
|
||||
Delete this archived entry from{" "}
|
||||
<span className="font-mono">{pendingDelete.originalTable}</span>?
|
||||
</p>
|
||||
<p className="mt-2 font-mono text-xs break-all">
|
||||
{pendingDelete.originalRowKey}
|
||||
</p>
|
||||
<p className="mt-2">This action is irreversible.</p>
|
||||
</>
|
||||
)}
|
||||
busy={deletingRk !== null}
|
||||
onCancel={() => setPendingDelete(null)}
|
||||
onConfirm={confirmDelete}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function prettyJson(raw: string): string {
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(raw), null, 2);
|
||||
} catch {
|
||||
return raw;
|
||||
}
|
||||
}
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
interface DeleteConfirmModalProps {
|
||||
open: boolean;
|
||||
title: string;
|
||||
description: React.ReactNode;
|
||||
busy?: boolean;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
}
|
||||
|
||||
const CONFIRM_WORD = "DELETE";
|
||||
|
||||
/**
|
||||
* Lightweight typed-confirmation modal for destructive customs-archive actions.
|
||||
* Replaces the native window.confirm() so the dialog can render rich context
|
||||
* (tenant id, table/row, count) and require a typed "DELETE" before the
|
||||
* primary button activates.
|
||||
*/
|
||||
export function DeleteConfirmModal({
|
||||
open,
|
||||
title,
|
||||
description,
|
||||
busy = false,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
}: DeleteConfirmModalProps) {
|
||||
const [text, setText] = useState("");
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) setText("");
|
||||
}, [open]);
|
||||
|
||||
if (!open) return null;
|
||||
|
||||
const canConfirm = text === CONFIRM_WORD && !busy;
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4">
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow-xl max-w-md w-full p-6">
|
||||
<h3 className="text-lg font-bold text-gray-900 dark:text-white mb-2">{title}</h3>
|
||||
<div className="text-sm text-gray-700 dark:text-gray-300 mb-4">{description}</div>
|
||||
|
||||
<div className="mb-4">
|
||||
<label className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">
|
||||
Type <span className="font-bold text-red-600 dark:text-red-400">{CONFIRM_WORD}</span> to confirm
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
value={text}
|
||||
onChange={(e) => setText(e.target.value)}
|
||||
placeholder={CONFIRM_WORD}
|
||||
disabled={busy}
|
||||
autoFocus
|
||||
autoComplete="off"
|
||||
className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-900 text-gray-900 dark:text-white focus:outline-none focus:ring-2 focus:ring-red-500 focus:border-red-500"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex space-x-3">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onCancel}
|
||||
disabled={busy}
|
||||
className="flex-1 px-4 py-2 border border-gray-300 dark:border-gray-600 rounded-md text-gray-700 dark:text-gray-200 bg-white dark:bg-gray-700 hover:bg-gray-50 dark:hover:bg-gray-600 disabled:opacity-50 transition-colors"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onConfirm}
|
||||
disabled={!canConfirm}
|
||||
className="flex-1 px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
{busy ? "Deleting…" : "Delete"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import Link from "next/link";
|
||||
import {
|
||||
api,
|
||||
type CustomsArchiveListRunsResponse,
|
||||
type CustomsArchiveRunSummary,
|
||||
} from "@/lib/api";
|
||||
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
import { useAdminConfig } from "../AdminConfigContext";
|
||||
import { AdminNotifications } from "../AdminNotifications";
|
||||
import { DeleteConfirmModal } from "./components/DeleteConfirmModal";
|
||||
|
||||
export default function CustomsArchivePage() {
|
||||
const { getAccessToken, setError, setSuccessMessage } = useAdminConfig();
|
||||
const [filter, setFilter] = useState<string>("");
|
||||
const [runs, setRuns] = useState<CustomsArchiveRunSummary[]>([]);
|
||||
const [loading, setLoading] = useState<boolean>(false);
|
||||
const [deletingPk, setDeletingPk] = useState<string | null>(null);
|
||||
const [pendingDelete, setPendingDelete] = useState<CustomsArchiveRunSummary | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
const url = api.customsArchive.listRuns({ tenantId: filter.trim() || undefined });
|
||||
const res = await authenticatedFetch(url, getAccessToken);
|
||||
if (!res.ok) {
|
||||
throw new Error(`List runs failed: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
const body = (await res.json()) as CustomsArchiveListRunsResponse;
|
||||
setRuns(body.runs ?? []);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError(err.message);
|
||||
} else {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [filter, getAccessToken, setError]);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
const confirmDelete = useCallback(async () => {
|
||||
const run = pendingDelete;
|
||||
if (!run) return;
|
||||
try {
|
||||
setDeletingPk(run.partitionKey);
|
||||
const res = await authenticatedFetch(
|
||||
api.customsArchive.deleteRun(run.tenantId, run.historyRowKey),
|
||||
getAccessToken,
|
||||
{ method: "DELETE" }
|
||||
);
|
||||
if (!res.ok) {
|
||||
throw new Error(`Delete run failed: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
const body = await res.json();
|
||||
setSuccessMessage(`Deleted ${body.deleted ?? 0} archived rules.`);
|
||||
setPendingDelete(null);
|
||||
await load();
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError(err.message);
|
||||
} else {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
} finally {
|
||||
setDeletingPk(null);
|
||||
}
|
||||
}, [pendingDelete, getAccessToken, load, setError, setSuccessMessage]);
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
|
||||
<header className="bg-white dark:bg-gray-800 shadow dark:shadow-gray-700">
|
||||
<div className="py-6 px-4 sm:px-6 lg:px-8">
|
||||
<h1 className="text-2xl font-normal text-gray-900 dark:text-white">Customs Archive</h1>
|
||||
<p className="text-sm text-gray-600 dark:text-gray-400 mt-1">
|
||||
Snapshots of each tenant's custom <strong>GatherRules</strong> / <strong>AnalyzeRules</strong> /{" "}
|
||||
<strong>ImeLogPatterns</strong> taken during offboarding (Phase 2.D-archive). Review the content,
|
||||
then delete entries selectively when no longer needed. No auto-cleanup.
|
||||
</p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
|
||||
<AdminNotifications />
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={filter}
|
||||
onChange={(e) => setFilter(e.target.value)}
|
||||
placeholder="Filter by tenant ID (optional)"
|
||||
className="px-3 py-1.5 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm text-gray-900 dark:text-white"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={load}
|
||||
disabled={loading}
|
||||
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
{loading ? "Loading…" : "Refresh"}
|
||||
</button>
|
||||
<span className="text-sm text-gray-500 dark:text-gray-400 ml-2">{runs.length} run(s)</span>
|
||||
</div>
|
||||
|
||||
{runs.length === 0 && !loading && (
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 text-sm text-gray-600 dark:text-gray-400">
|
||||
No archived offboarding runs found{filter ? ` for tenant ${filter}.` : "."}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-3">
|
||||
{runs.map((run) => (
|
||||
<div
|
||||
key={run.partitionKey}
|
||||
className="bg-white dark:bg-gray-800 rounded-lg shadow p-4 flex flex-wrap items-center justify-between gap-3"
|
||||
>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="text-sm font-mono text-gray-900 dark:text-white truncate">{run.tenantId}</div>
|
||||
<div className="text-xs text-gray-500 dark:text-gray-400 mt-0.5">
|
||||
Run: <span className="font-mono">{run.historyRowKey}</span> · Archived{" "}
|
||||
{new Date(run.archivedAt).toLocaleString()}
|
||||
</div>
|
||||
<div className="text-xs text-gray-600 dark:text-gray-300 mt-1 space-x-3">
|
||||
<span>Gather: <strong>{run.gatherRulesCount}</strong></span>
|
||||
<span>Analyze: <strong>{run.analyzeRulesCount}</strong></span>
|
||||
<span>ImePatterns: <strong>{run.imeLogPatternsCount}</strong></span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Link
|
||||
href={`/admin/customs-archive/${encodeURIComponent(run.tenantId)}/${encodeURIComponent(run.historyRowKey)}`}
|
||||
className="px-3 py-1.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-600 text-gray-900 dark:text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
Inspect
|
||||
</Link>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setPendingDelete(run)}
|
||||
disabled={deletingPk === run.partitionKey}
|
||||
className="px-3 py-1.5 bg-red-600 hover:bg-red-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
{deletingPk === run.partitionKey ? "Deleting…" : "Delete run"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<DeleteConfirmModal
|
||||
open={pendingDelete !== null}
|
||||
title="Delete archive run"
|
||||
description={pendingDelete && (
|
||||
<>
|
||||
<p>
|
||||
Delete the entire archive run for tenant{" "}
|
||||
<span className="font-mono">{pendingDelete.tenantId}</span>?
|
||||
</p>
|
||||
<p className="mt-2">
|
||||
This permanently removes{" "}
|
||||
<strong>
|
||||
{pendingDelete.gatherRulesCount +
|
||||
pendingDelete.analyzeRulesCount +
|
||||
pendingDelete.imeLogPatternsCount}
|
||||
</strong>{" "}
|
||||
archived rules. This action is irreversible.
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
busy={deletingPk !== null}
|
||||
onCancel={() => setPendingDelete(null)}
|
||||
onConfirm={confirmDelete}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -15,6 +15,18 @@ import { type BootstrapSessionItem } from "./components/BootstrapSessionsSection
|
||||
// ---------------------------------------------------------------------------
|
||||
// Context value interface
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* State surfaced by handleOffboard once the DELETE returns 202. Drives the post-confirm
|
||||
* banner in OffboardingSection. EarliestProcessingAt is rendered as a live countdown;
|
||||
* when it elapses, handleDrainBarrierElapsed() runs and logs the user out.
|
||||
*/
|
||||
export interface OffboardingInProgressInfo {
|
||||
status: string;
|
||||
historyRowKey: string;
|
||||
earliestProcessingAt?: string | null;
|
||||
message: string;
|
||||
}
|
||||
|
||||
interface TenantConfigContextValue {
|
||||
// Core
|
||||
config: TenantConfiguration | null;
|
||||
@@ -200,6 +212,12 @@ interface TenantConfigContextValue {
|
||||
setOffboardError: (v: string | null) => void;
|
||||
handleOffboard: () => Promise<void>;
|
||||
|
||||
/** Set after the DELETE returns 202; drives the post-confirm drain-barrier banner. */
|
||||
offboardingInProgress: OffboardingInProgressInfo | null;
|
||||
|
||||
/** Called by the banner countdown when the cache-drain barrier elapses → triggers logout. */
|
||||
handleDrainBarrierElapsed: () => void;
|
||||
|
||||
// Auth helpers
|
||||
user: ReturnType<typeof useAuth>["user"];
|
||||
getAccessToken: () => Promise<string | null>;
|
||||
@@ -243,6 +261,7 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode }
|
||||
const [offboardConfirmText, setOffboardConfirmText] = useState("");
|
||||
const [offboarding, setOffboarding] = useState(false);
|
||||
const [offboardError, setOffboardError] = useState<string | null>(null);
|
||||
const [offboardingInProgress, setOffboardingInProgress] = useState<OffboardingInProgressInfo | null>(null);
|
||||
|
||||
// Bootstrap sessions
|
||||
const [bootstrapSessions, setBootstrapSessions] = useState<BootstrapSessionItem[]>([]);
|
||||
@@ -1163,18 +1182,40 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode }
|
||||
throw new Error(data?.error || `Offboard failed: ${response.statusText}`);
|
||||
}
|
||||
|
||||
// Offboard successful – sign out the user as their admin access is gone
|
||||
// Backend returns 202 (or 200 for idempotent re-clicks) with the History row pointer
|
||||
// and EarliestProcessingAt (cache-drain barrier deadline). Switch the UI into the
|
||||
// drain-barrier banner state; the banner's countdown will auto-logout once the
|
||||
// barrier elapses (by then the worker has started Phase 2 and the auth pipeline
|
||||
// returns 403 via the existing Disabled-flag gate).
|
||||
const body = await response.json().catch(() => ({}));
|
||||
trackEvent("tenant_offboarded");
|
||||
logout();
|
||||
|
||||
setOffboardingInProgress({
|
||||
status: body?.status ?? "Queued",
|
||||
historyRowKey: body?.historyRowKey ?? "",
|
||||
earliestProcessingAt: body?.earliestProcessingAt ?? null,
|
||||
message: body?.message ?? "Tenant offboarding queued.",
|
||||
});
|
||||
// Dismiss the confirmation dialog now that the banner has taken over.
|
||||
setShowOffboardDialog(false);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
addNotification('error', 'Session Expired', err.message, 'session-expired-error');
|
||||
} else {
|
||||
setOffboardError(err instanceof Error ? err.message : 'Offboard failed');
|
||||
}
|
||||
} finally {
|
||||
setOffboarding(false);
|
||||
}
|
||||
}, [tenantId, getAccessToken, addNotification, logout]);
|
||||
}, [tenantId, getAccessToken, addNotification]);
|
||||
|
||||
const handleDrainBarrierElapsed = useCallback(() => {
|
||||
// The cache-drain barrier has expired. The worker is starting Phase 2 right now and
|
||||
// all function-host instances have refreshed their TenantConfiguration cache to see
|
||||
// Disabled=true. Sign the user out — any further authenticated call will fail with
|
||||
// 403 anyway.
|
||||
logout();
|
||||
}, [logout]);
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Provider value
|
||||
@@ -1281,6 +1322,7 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode }
|
||||
offboardConfirmText, setOffboardConfirmText,
|
||||
offboarding, offboardError, setOffboardError,
|
||||
handleOffboard,
|
||||
offboardingInProgress, handleDrainBarrierElapsed,
|
||||
|
||||
// Auth
|
||||
user, getAccessToken,
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
interface OffboardingInProgressInfo {
|
||||
status: string; // "Queued" | "Initiated" | "InProgress" | "Completed" | "Failed"
|
||||
historyRowKey: string;
|
||||
earliestProcessingAt?: string | null; // ISO UTC; drives countdown in drain-barrier state
|
||||
message: string;
|
||||
}
|
||||
|
||||
interface OffboardingSectionProps {
|
||||
showOffboardDialog: boolean;
|
||||
setShowOffboardDialog: (value: boolean) => void;
|
||||
@@ -9,6 +18,58 @@ interface OffboardingSectionProps {
|
||||
offboardError: string | null;
|
||||
setOffboardError: (value: string | null) => void;
|
||||
onOffboard: () => void;
|
||||
|
||||
/** Set after the DELETE returns 202; switches the section into the drain-barrier banner state. */
|
||||
offboardingInProgress: OffboardingInProgressInfo | null;
|
||||
|
||||
/** Invoked by the banner's auto-logout timer once the drain barrier has fully elapsed. */
|
||||
onDrainBarrierElapsed: () => void;
|
||||
}
|
||||
|
||||
function formatRemaining(totalSeconds: number): string {
|
||||
if (totalSeconds <= 0) return "0 min";
|
||||
return `~${Math.ceil(totalSeconds / 60)} min`;
|
||||
}
|
||||
|
||||
/** Live countdown — re-renders every second until EarliestProcessingAt passes, then calls onElapsed once. */
|
||||
function useCountdown(targetIso: string | null | undefined, onElapsed: () => void): number {
|
||||
const [secondsLeft, setSecondsLeft] = useState<number>(() => {
|
||||
if (!targetIso) return 0;
|
||||
return Math.max(0, Math.floor((new Date(targetIso).getTime() - Date.now()) / 1000));
|
||||
});
|
||||
const firedRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!targetIso) return;
|
||||
firedRef.current = false;
|
||||
|
||||
// Anchor the remaining-time once at mount, then drive subsequent ticks off
|
||||
// performance.now() — a monotonic clock that is NOT affected by NTP-syncs
|
||||
// or manual clock changes during the ~6 min drain barrier. The previous
|
||||
// implementation re-read Date.now() on every tick, so a wall-clock jump
|
||||
// forward would fire onElapsed (auto-logout) prematurely.
|
||||
const initialRemainingMs = Math.max(
|
||||
0,
|
||||
new Date(targetIso).getTime() - Date.now(),
|
||||
);
|
||||
const monoAtMount = performance.now();
|
||||
|
||||
const tick = () => {
|
||||
const elapsedMs = performance.now() - monoAtMount;
|
||||
const remaining = Math.max(0, Math.floor((initialRemainingMs - elapsedMs) / 1000));
|
||||
setSecondsLeft(remaining);
|
||||
if (remaining === 0 && !firedRef.current) {
|
||||
firedRef.current = true;
|
||||
onElapsed();
|
||||
}
|
||||
};
|
||||
|
||||
tick(); // immediate sync
|
||||
const interval = setInterval(tick, 1000);
|
||||
return () => clearInterval(interval);
|
||||
}, [targetIso, onElapsed]);
|
||||
|
||||
return secondsLeft;
|
||||
}
|
||||
|
||||
export default function OffboardingSection({
|
||||
@@ -20,7 +81,20 @@ export default function OffboardingSection({
|
||||
offboardError,
|
||||
setOffboardError,
|
||||
onOffboard,
|
||||
offboardingInProgress,
|
||||
onDrainBarrierElapsed,
|
||||
}: OffboardingSectionProps) {
|
||||
// When the offboarding has been queued, replace the section entirely with the
|
||||
// drain-barrier banner. The dialog is dismissed by handleOffboard on success.
|
||||
if (offboardingInProgress) {
|
||||
return (
|
||||
<OffboardingInProgressBanner
|
||||
info={offboardingInProgress}
|
||||
onDrainBarrierElapsed={onDrainBarrierElapsed}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
{/* Danger Zone: Tenant Offboarding */}
|
||||
@@ -79,7 +153,10 @@ export default function OffboardingSection({
|
||||
<li>Tenant configuration</li>
|
||||
<li>All admin accounts (including yours)</li>
|
||||
</ul>
|
||||
<p className="mt-3 font-semibold">After offboarding you will be signed out and lose all access.</p>
|
||||
<p className="mt-3 font-semibold">
|
||||
Data deletion will start after a ~6 minute preparation window and then run in the background.
|
||||
You can close this tab once the request has been sent.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="mb-4">
|
||||
@@ -118,10 +195,10 @@ export default function OffboardingSection({
|
||||
{offboarding ? (
|
||||
<>
|
||||
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-white"></div>
|
||||
<span>Offboarding...</span>
|
||||
<span>Queuing...</span>
|
||||
</>
|
||||
) : (
|
||||
<span>Permanently Delete All Data</span>
|
||||
<span>Queue Permanent Deletion</span>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
@@ -131,3 +208,95 @@ export default function OffboardingSection({
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Banner shown after the DELETE returns 202. Replaces the entire Danger Zone section.
|
||||
* Renders a live countdown until `earliestProcessingAt`, then auto-logs the user out
|
||||
* (the worker has by then started Phase 2 and the auth pipeline will return 403
|
||||
* anyway via the Disabled-flag gate).
|
||||
*/
|
||||
function OffboardingInProgressBanner({
|
||||
info,
|
||||
onDrainBarrierElapsed,
|
||||
}: {
|
||||
info: OffboardingInProgressInfo;
|
||||
onDrainBarrierElapsed: () => void;
|
||||
}) {
|
||||
const secondsLeft = useCountdown(info.earliestProcessingAt ?? null, onDrainBarrierElapsed);
|
||||
const inDrainBarrier = (info.earliestProcessingAt && secondsLeft > 0) === true;
|
||||
|
||||
// Single 4-state visual; status text drives the badge color + label.
|
||||
// "Preparing" is derived from earliestProcessingAt + countdown rather than from status,
|
||||
// because the backend status stays "Queued" during the cache-drain window.
|
||||
const stateLabel = inDrainBarrier
|
||||
? "Preparing"
|
||||
: info.status === "Failed"
|
||||
? "Failed"
|
||||
: info.status === "Completed"
|
||||
? "Completed"
|
||||
: "In Progress";
|
||||
|
||||
const stateColors: Record<string, string> = {
|
||||
"Preparing": "bg-amber-100 text-amber-800 border-amber-300",
|
||||
"In Progress": "bg-blue-100 text-blue-800 border-blue-300",
|
||||
"Completed": "bg-green-100 text-green-800 border-green-300",
|
||||
"Failed": "bg-red-100 text-red-800 border-red-300",
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="bg-white rounded-lg shadow border-2 border-red-200 p-6 space-y-4">
|
||||
<div className="flex items-center space-x-3">
|
||||
<div className="w-12 h-12 bg-red-100 rounded-full flex items-center justify-center flex-shrink-0">
|
||||
<svg className="w-6 h-6 text-red-600" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
|
||||
</svg>
|
||||
</div>
|
||||
<div>
|
||||
<h2 className="text-xl font-semibold text-red-900">Tenant Offboarding queued</h2>
|
||||
<p className="text-sm text-gray-600 mt-1">
|
||||
This tenant will be permanently deleted. You do not need to keep this page open.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center space-x-3">
|
||||
<span className={`inline-flex items-center px-3 py-1 rounded-full text-sm font-medium border ${stateColors[stateLabel] ?? stateColors["In Progress"]}`}>
|
||||
{stateLabel}
|
||||
</span>
|
||||
{inDrainBarrier && (
|
||||
<span className="text-sm text-gray-700">
|
||||
Data deletion starts in <span className="font-mono font-semibold">{formatRemaining(secondsLeft)}</span>
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{inDrainBarrier ? (
|
||||
<div className="bg-amber-50 border border-amber-200 rounded-lg p-4 text-sm text-amber-900 space-y-1">
|
||||
<p className="font-semibold">Preparing for deletion</p>
|
||||
<p>
|
||||
Your tenant has been disabled and is being prepared for permanent deletion. The wipe will start
|
||||
automatically once the countdown reaches zero.
|
||||
</p>
|
||||
<p className="mt-2">
|
||||
Everything needed for the deletion is already recorded — you can sign out and close this tab.
|
||||
The deletion continues in the background.
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="bg-blue-50 border border-blue-200 rounded-lg p-4 text-sm text-blue-900 space-y-1">
|
||||
<p className="font-semibold">Deletion in progress</p>
|
||||
<p>
|
||||
Your tenant is being permanently deleted. You can close this tab safely.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="text-xs text-gray-500 space-y-0.5">
|
||||
<p>History row: <span className="font-mono">{info.historyRowKey}</span></p>
|
||||
{info.earliestProcessingAt && (
|
||||
<p>Earliest processing at: <span className="font-mono">{info.earliestProcessingAt}</span></p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ export function SectionOffboarding() {
|
||||
offboardConfirmText, setOffboardConfirmText,
|
||||
offboarding, offboardError, setOffboardError,
|
||||
handleOffboard,
|
||||
offboardingInProgress, handleDrainBarrierElapsed,
|
||||
} = useTenantConfig();
|
||||
|
||||
return (
|
||||
@@ -24,6 +25,8 @@ export function SectionOffboarding() {
|
||||
offboardError={offboardError}
|
||||
setOffboardError={setOffboardError}
|
||||
onOffboard={handleOffboard}
|
||||
offboardingInProgress={offboardingInProgress}
|
||||
onDrainBarrierElapsed={handleDrainBarrierElapsed}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
|
||||
@@ -402,6 +402,21 @@ export const api = {
|
||||
})}`,
|
||||
},
|
||||
|
||||
// ── Customs Archive (PR3.B Tenant offboarding archive) ────────────────────
|
||||
// Response types defined below (CustomsArchive*).
|
||||
customsArchive: {
|
||||
listRuns: (opts?: { tenantId?: string }) =>
|
||||
`${API_BASE_URL}/api/global/customs-archive${qs({ tenantId: opts?.tenantId })}`,
|
||||
listEntries: (tenantId: string, historyRowKey: string) =>
|
||||
`${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}`,
|
||||
getEntry: (tenantId: string, historyRowKey: string, archiveRowKey: string) =>
|
||||
`${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}/${encodeURIComponent(archiveRowKey)}`,
|
||||
deleteEntry: (tenantId: string, historyRowKey: string, archiveRowKey: string) =>
|
||||
`${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}/${encodeURIComponent(archiveRowKey)}`,
|
||||
deleteRun: (tenantId: string, historyRowKey: string) =>
|
||||
`${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}`,
|
||||
},
|
||||
|
||||
// ── Feedback ──────────────────────────────────────────────────────────────
|
||||
feedback: {
|
||||
status: () => `${API_BASE_URL}/api/feedback/status`,
|
||||
@@ -484,3 +499,45 @@ export const api = {
|
||||
leaveGroup: () => `${API_BASE_URL}/api/realtime/groups/leave`,
|
||||
},
|
||||
};
|
||||
|
||||
// ── Customs Archive response types ────────────────────────────────────────
|
||||
// Shared by /admin/customs-archive (list runs) and
|
||||
// /admin/customs-archive/[tenantId]/[historyRowKey] (list/inspect entries).
|
||||
export interface CustomsArchiveRunSummary {
|
||||
partitionKey: string;
|
||||
tenantId: string;
|
||||
historyRowKey: string;
|
||||
archivedAt: string;
|
||||
gatherRulesCount: number;
|
||||
analyzeRulesCount: number;
|
||||
imeLogPatternsCount: number;
|
||||
}
|
||||
|
||||
export interface CustomsArchiveListRunsResponse {
|
||||
success: boolean;
|
||||
count: number;
|
||||
runs: CustomsArchiveRunSummary[];
|
||||
}
|
||||
|
||||
export interface CustomsArchiveEntrySummary {
|
||||
partitionKey: string;
|
||||
rowKey: string;
|
||||
originalTable: string;
|
||||
originalRowKey: string;
|
||||
archivedAt: string;
|
||||
entityJsonPreview: string;
|
||||
}
|
||||
|
||||
export interface CustomsArchiveListEntriesResponse {
|
||||
success: boolean;
|
||||
count: number;
|
||||
entries: CustomsArchiveEntrySummary[];
|
||||
}
|
||||
|
||||
export interface CustomsArchiveFullEntry extends CustomsArchiveEntrySummary {
|
||||
tenantId: string;
|
||||
originalPartitionKey: string;
|
||||
entityJson: string;
|
||||
historyRowKey: string;
|
||||
archivedBy: string;
|
||||
}
|
||||
|
||||
@@ -263,6 +263,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [
|
||||
items: [
|
||||
{ id: "ga-maintenance", label: "Maintenance", href: "/admin/ops" },
|
||||
{ id: "ga-session-cleanup", label: "Session Cleanup", href: "/admin/ops/session-cleanup" },
|
||||
{ id: "ga-customs-archive", label: "Customs Archive", href: "/admin/customs-archive" },
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user