feat(backend,web): tenant offboarding async cascade with customs archive

Replace synchronous DeleteAllTenantDataAsync with a queued async cascade
that wipes all 24 tenant-scoped tables safely, archives custom rules,
and survives crashes at every step.

Phase 1 (TenantOffboardFunction, ~200ms sync):
  - Write History / ByTenantPointer / Marker rows (OffboardingAudit table)
  - Flip TenantConfiguration.Disabled=true (the active auth gate)
  - Enqueue with 6 min visibility-delay (cache-drain barrier =
    TenantConfigurationService.CacheDuration 5 min + 1 min safety)
  - Return 202 with EarliestProcessingAt for the UI countdown

Phase 2 (TenantOffboardingWorker + Handler, async after drain):
  - Enumerate sessions, per-session cascade-enqueue, persist
    Expectations blob (separate offboarding-state container)
  - Drain probe against per-session progress blobs; self re-enqueue
    up to 60x at 2 min visibility-delay
  - 2.D-archive: GatherRules / AnalyzeRules / ImeLogPatterns moved
    to TenantOffboardingCustomsArchive table (3-iter cap + verify)
  - 2.D-wipe: SafeWipe over all tenant tables (4 variants: exact-PK,
    composite-PK range, discriminator+TenantId, property-only) with
    fetch+verify+delete pattern + SafeWipeVerificationException
  - 2.E: blob-prefix wipe deletion-manifests/{tenantId}/
  - 2.D-pass-2: re-wipe AuditLogs (defense-in-depth)
  - 2.G: side-effects FIRST (Marker, Pointer, blob, audit, OpsEvent)
    then History=Completed LAST (DrainCompletedAt as resume anchor)
  - 2.F-final: delete TenantConfiguration row LAST so self-service
    re-onboarding (next /api/auth/me auto-creates default) works

Defense-in-depth:
  - OffboardingMarkerCleanupFunction (2h timer): sweeps Completed
    markers after 15 min grace, conditional TenantConfiguration sweep
    (only when row is still the offboarding tombstone — preserves
    user-driven self-service re-onboards)
  - Worker poison path: durable Failed state MUST land before queue
    moves; transient failure leaves message visible for retry
  - SessionDeletionHandler reorder: Audit + SignalR notify BEFORE
    CompletedAt write so the drain predicate is correct

Web UI:
  - /settings danger zone: typed "OFFBOARD" confirmation dialog,
    post-202 drain-barrier banner with minute-based countdown, hint
    "you can sign out and close this tab — deletion runs in
    background", silent auto-logout when countdown elapses
  - /admin/customs-archive (GA only): list runs + inspect entries +
    typed "DELETE" confirmation modal for run/entry deletion
  - OpsAlertRulesSection: TenantOffboardingFailed event registered
  - lib/api.ts: shared CustomsArchive* response types

Tests: 100+ new tests covering drain contract, handler ordering,
worker poison path, ETag-CAS, SafeWipe variants, OffboardingFilters
prefix-bleed safety, MarkerCleanup conditional sweep, Bootstrap
auth-disabled-tenant gate.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-05-18 23:06:51 +02:00
parent 884f29b019
commit 206388da99
62 changed files with 10477 additions and 267 deletions
@@ -0,0 +1,111 @@
using System.Net;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Plan v2 §5 (bootstrap-path interaction): when a tenant is mid-offboarding,
/// Phase 1 flips <c>TenantConfiguration.Disabled = true</c> and Phase 2 starts after
/// the cache-drain barrier. The auth pipeline (cert + bootstrap-token) must reject
/// requests against a Disabled tenant — otherwise a fresh OOBE device with a bootstrap
/// token could keep writing into a tenant about to be wiped.
/// <para>
/// The §0 Disabled-gate in <see cref="SecurityValidator.ValidateRequestAsync"/> runs
/// BEFORE the §0.5 bootstrap-token branch, so this property is enforced by the order
/// of checks. These tests pin that ordering so future refactors cannot accidentally
/// move the bootstrap branch above the Disabled gate.
/// </para>
/// </summary>
public sealed class BootstrapAuthDisabledTenantTests
{
private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
private const string BootstrapToken = "11111111-2222-3333-4444-555555555555";
private static SecurityValidator BuildValidator(TenantConfiguration config)
{
var configRepo = Mock.Of<IConfigRepository>();
var cache = new MemoryCache(new MemoryCacheOptions());
var configServiceMock = new Mock<TenantConfigurationService>(
configRepo, Mock.Of<ILogger<TenantConfigurationService>>(), cache)
{ CallBase = false };
configServiceMock
.Setup(x => x.TryGetConfigurationAsync(It.IsAny<string>()))
.ReturnsAsync((config, true));
return new SecurityValidator(
configService: configServiceMock.Object,
rateLimitService: null!,
autopilotDeviceValidator: null!,
corporateIdentifierValidator: null!,
logger: Mock.Of<ILogger>(),
bootstrapSessionService: null,
deviceAssociationValidator: null);
}
private static HttpRequestData BuildRequestWithBootstrapToken()
{
var contextMock = new Mock<Microsoft.Azure.Functions.Worker.FunctionContext>();
var reqMock = new Mock<HttpRequestData>(contextMock.Object);
var headers = new Microsoft.Azure.Functions.Worker.Http.HttpHeadersCollection
{
{ "X-Bootstrap-Token", BootstrapToken },
};
reqMock.SetupGet(r => r.Headers).Returns(headers);
return reqMock.Object;
}
[Fact]
public async Task BootstrapToken_RejectedWhenTenantDisabled()
{
var disabled = TenantConfiguration.CreateDefault(TenantId);
disabled.Disabled = true;
disabled.DisabledReason = "Offboarding in progress";
disabled.DisabledUntil = null;
var sut = BuildValidator(disabled);
var result = await sut.ValidateRequestAsync(BuildRequestWithBootstrapToken(), TenantId);
// Must be the §0 Disabled-gate, not the §0.5 bootstrap branch. The bootstrap
// branch lives AFTER the Disabled check; if anyone ever reorders these, the
// assertion below catches it (Disabled returns "Tenant is suspended" with the
// configured reason; bootstrap-branch failures have different ErrorMessages).
Assert.False(result.IsValid);
Assert.Equal(HttpStatusCode.Forbidden, result.StatusCode);
Assert.Equal("Tenant is suspended", result.ErrorMessage);
Assert.Equal("Offboarding in progress", result.Details);
}
[Fact]
public async Task BootstrapToken_AcceptedWhenTenantNotDisabled_StillEvaluatesBootstrapBranch()
{
// Sanity: a non-disabled tenant DOES reach the bootstrap branch (which will then
// run its own validation against _bootstrapSessionService). With _bootstrapSessionService
// null in this test, the bootstrap branch falls through to the cert path — proving
// that the §0 gate did NOT block.
var enabled = TenantConfiguration.CreateDefault(TenantId);
enabled.Disabled = false;
enabled.ValidateAutopilotDevice = false;
enabled.ValidateCorporateIdentifier = false;
enabled.AllowInsecureAgentRequests = false;
var sut = BuildValidator(enabled);
var result = await sut.ValidateRequestAsync(BuildRequestWithBootstrapToken(), TenantId);
// Falls through to the "device validation is required" gate at §1 (not the §0 Disabled gate).
Assert.False(result.IsValid);
Assert.Equal(HttpStatusCode.Forbidden, result.StatusCode);
Assert.Equal("Device validation is required", result.ErrorMessage);
}
}
@@ -0,0 +1,109 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Storage.Blobs;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared.Models.Deletion;
using Microsoft.Extensions.Logging.Abstractions;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Plan §7.4 step 5 + §10.1: drain predicate is true only when the progress blob has
/// CompletedAt != null AND TombstoneStarted == true. 404 is "not yet done", not an error.
/// </summary>
public class DeletionProgressDrainProbeTests
{
private const string TenantId = "44444444-4444-4444-4444-444444444444";
private const string SessionId = "55555555-5555-5555-5555-555555555555";
private const string ManifestId = "manifest-1";
[Fact]
public async Task IsCompleted_TrueOnlyWhen_CompletedAtSet_AndTombstoneStarted()
{
var sut = new DeletionProgressDrainProbe(
new FakeBlob(new DeletionProgress
{
CompletedAt = DateTime.UtcNow,
TombstoneStarted = true,
}),
NullLogger<DeletionProgressDrainProbe>.Instance);
Assert.True(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
}
[Fact]
public async Task IsCompleted_FalseWhen_CompletedAtNull()
{
var sut = new DeletionProgressDrainProbe(
new FakeBlob(new DeletionProgress { TombstoneStarted = true, CompletedAt = null }),
NullLogger<DeletionProgressDrainProbe>.Instance);
Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
}
[Fact]
public async Task IsCompleted_FalseWhen_TombstoneNotStarted_RevWalkthrough()
{
// If CompletedAt got somehow set without TombstoneStarted, the cascade did not
// finish the FINAL step — drain must say "no" to defend against truncated progress.
var sut = new DeletionProgressDrainProbe(
new FakeBlob(new DeletionProgress { CompletedAt = DateTime.UtcNow, TombstoneStarted = false }),
NullLogger<DeletionProgressDrainProbe>.Instance);
Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
}
[Fact]
public async Task IsCompleted_FalseOn404_NotAnError()
{
var sut = new DeletionProgressDrainProbe(
new FakeBlob(throwOnDownload: new RequestFailedException(404, "NotFound", "BlobNotFound", null)),
NullLogger<DeletionProgressDrainProbe>.Instance);
Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
}
[Fact]
public async Task IsCompleted_PropagatesNon404StorageError()
{
// 503 / 500 are real transients the worker must surface so the queue retries.
var sut = new DeletionProgressDrainProbe(
new FakeBlob(throwOnDownload: new RequestFailedException(503, "ServiceUnavailable", "ServerBusy", null)),
NullLogger<DeletionProgressDrainProbe>.Instance);
await Assert.ThrowsAsync<RequestFailedException>(
() => sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId));
}
// ── Fake BlobStorageService that overrides only the download seam we need ──
private sealed class FakeBlob : BlobStorageService
{
private readonly DeletionProgress? _progress;
private readonly RequestFailedException? _throw;
public FakeBlob(DeletionProgress progress)
: base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false)
{
_progress = progress;
}
public FakeBlob(RequestFailedException throwOnDownload)
: base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false)
{
_throw = throwOnDownload;
}
public override Task<(DeletionProgress Progress, string ETag)> DownloadDeletionProgressAsync(
string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default)
{
if (_throw != null) throw _throw;
return Task.FromResult((_progress!, "\"0xFAKE\""));
}
}
}
@@ -0,0 +1,105 @@
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Offboarding;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// In-memory <see cref="ITenantCustomsArchiveRepository"/> for PR3.B handler + endpoint tests.
/// Idempotent upsert keyed by (PK, RK); supports the four query shapes the admin API uses.
/// </summary>
internal sealed class FakeCustomsArchiveRepository : ITenantCustomsArchiveRepository
{
private readonly Dictionary<string, TenantOffboardingCustomsArchiveEntry> _store
= new(System.StringComparer.Ordinal);
public IReadOnlyDictionary<string, TenantOffboardingCustomsArchiveEntry> Store => _store;
private static string Key(string pk, string rk) => pk + "|" + rk;
public Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default)
{
_store[Key(entry.PartitionKey, entry.RowKey)] = Clone(entry);
return Task.CompletedTask;
}
public Task<int> CountByRunAndTableAsync(string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default)
{
var pk = $"{normalizedTenantId}_{historyRowKey}";
var count = _store.Values.Count(e =>
e.PartitionKey == pk && e.OriginalTable == originalTable);
return Task.FromResult(count);
}
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByRunAsync(
string normalizedTenantId, string historyRowKey,
[EnumeratorCancellation] CancellationToken ct = default)
{
var pk = $"{normalizedTenantId}_{historyRowKey}";
foreach (var entry in _store.Values.Where(e => e.PartitionKey == pk))
{
yield return Clone(entry);
}
await Task.CompletedTask;
}
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByTenantAsync(
string normalizedTenantId, [EnumeratorCancellation] CancellationToken ct = default)
{
var prefix = normalizedTenantId + "_";
foreach (var entry in _store.Values.Where(e => e.PartitionKey.StartsWith(prefix)))
{
yield return Clone(entry);
}
await Task.CompletedTask;
}
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryAllAsync(
[EnumeratorCancellation] CancellationToken ct = default)
{
foreach (var entry in _store.Values)
{
yield return Clone(entry);
}
await Task.CompletedTask;
}
public Task<TenantOffboardingCustomsArchiveEntry?> TryGetEntryAsync(
string partitionKey, string rowKey, CancellationToken ct = default)
{
_store.TryGetValue(Key(partitionKey, rowKey), out var entry);
return Task.FromResult<TenantOffboardingCustomsArchiveEntry?>(entry == null ? null : Clone(entry));
}
public Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default)
{
_store.Remove(Key(partitionKey, rowKey));
return Task.CompletedTask;
}
public Task<int> DeleteRunAsync(string normalizedTenantId, string historyRowKey, CancellationToken ct = default)
{
var pk = $"{normalizedTenantId}_{historyRowKey}";
var toDelete = _store.Keys.Where(k => k.StartsWith(pk + "|")).ToList();
foreach (var k in toDelete) _store.Remove(k);
return Task.FromResult(toDelete.Count);
}
private static TenantOffboardingCustomsArchiveEntry Clone(TenantOffboardingCustomsArchiveEntry e) => new()
{
PartitionKey = e.PartitionKey,
RowKey = e.RowKey,
TenantId = e.TenantId,
OriginalTable = e.OriginalTable,
OriginalPartitionKey = e.OriginalPartitionKey,
OriginalRowKey = e.OriginalRowKey,
EntityJson = e.EntityJson,
HistoryRowKey = e.HistoryRowKey,
ArchivedAt = e.ArchivedAt,
ArchivedBy = e.ArchivedBy,
};
}
@@ -0,0 +1,155 @@
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// In-memory <see cref="IOffboardingAuditRepository"/> for handler/cleanup tests. Records the
/// latest write per (PK, RK) so the test can assert lifecycle transitions, and mirrors the
/// production ETag-CAS contract on the pointer (412 on mismatched ETag).
/// </summary>
internal sealed class FakeOffboardingAuditRepository : IOffboardingAuditRepository
{
public Dictionary<string, OffboardingMarkerEntry> Markers { get; } = new();
public Dictionary<string, OffboardingHistoryEntry> History { get; } = new();
public Dictionary<string, (OffboardingByTenantPointer Pointer, string ETag)> Pointers { get; } = new();
public List<string> MarkerWrites { get; } = new();
public List<string> HistoryWrites { get; } = new();
/// <summary>When set, the next call to <see cref="UpsertHistoryAsync"/> throws — used by
/// poison-path tests to simulate a transient storage failure inside the Failed-state
/// transition so the Worker's reorder invariant can be exercised.</summary>
public Exception? ThrowOnNextHistoryUpsert { get; set; }
private int _etagCounter;
private string NextEtag() => $"\"0xFAKE_{++_etagCounter}\"";
public Task<OffboardingMarkerEntry?> TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default)
=> Task.FromResult(Markers.TryGetValue(normalizedTenantId, out var m) ? Clone(m) : null);
public Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default)
{
if (Markers.ContainsKey(marker.RowKey))
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
Markers[marker.RowKey] = Clone(marker);
MarkerWrites.Add(marker.Status);
return Task.CompletedTask;
}
public Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default)
{
Markers[marker.RowKey] = Clone(marker);
MarkerWrites.Add(marker.Status);
return Task.CompletedTask;
}
public Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default)
{
Markers.Remove(normalizedTenantId);
return Task.CompletedTask;
}
public async IAsyncEnumerable<OffboardingMarkerEntry> QueryMarkersAsync(
[EnumeratorCancellation] CancellationToken ct = default)
{
foreach (var marker in Markers.Values)
{
yield return Clone(marker);
}
await Task.CompletedTask;
}
public Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default)
{
if (History.ContainsKey(history.RowKey))
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
History[history.RowKey] = Clone(history);
HistoryWrites.Add(history.Status);
return Task.CompletedTask;
}
public Task<OffboardingHistoryEntry?> TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default)
=> Task.FromResult(History.TryGetValue(historyRowKey, out var h) ? Clone(h) : null);
public Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default)
{
if (ThrowOnNextHistoryUpsert is { } ex)
{
ThrowOnNextHistoryUpsert = null;
throw ex;
}
History[history.RowKey] = Clone(history);
HistoryWrites.Add(history.Status);
return Task.CompletedTask;
}
public Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync(
string normalizedTenantId, CancellationToken ct = default)
{
if (Pointers.TryGetValue(normalizedTenantId, out var entry))
{
return Task.FromResult<(OffboardingByTenantPointer?, string?)>((Clone(entry.Pointer), entry.ETag));
}
return Task.FromResult<(OffboardingByTenantPointer?, string?)>((null, null));
}
public Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default)
{
if (Pointers.ContainsKey(pointer.RowKey))
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
Pointers[pointer.RowKey] = (Clone(pointer), NextEtag());
return Task.CompletedTask;
}
public Task UpdateByTenantPointerWithEtagAsync(
OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default)
{
if (!Pointers.TryGetValue(pointer.RowKey, out var entry))
throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null);
if (entry.ETag != ifMatchEtag)
throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null);
Pointers[pointer.RowKey] = (Clone(pointer), NextEtag());
return Task.CompletedTask;
}
private static OffboardingMarkerEntry Clone(OffboardingMarkerEntry m) => new()
{
PartitionKey = m.PartitionKey, RowKey = m.RowKey,
TenantId = m.TenantId, OffboardingHistoryRowKey = m.OffboardingHistoryRowKey,
InitiatedAt = m.InitiatedAt, InitiatedBy = m.InitiatedBy,
Status = m.Status, CompletedAt = m.CompletedAt,
FailedAt = m.FailedAt, FailedPhase = m.FailedPhase,
};
private static OffboardingHistoryEntry Clone(OffboardingHistoryEntry h) => new()
{
PartitionKey = h.PartitionKey, RowKey = h.RowKey,
TenantId = h.TenantId, DomainName = h.DomainName, InitiatedBy = h.InitiatedBy,
OffboardedAt = h.OffboardedAt, CompletedAt = h.CompletedAt,
EarliestProcessingAt = h.EarliestProcessingAt,
Status = h.Status,
DeletedRowCountsJson = h.DeletedRowCountsJson, TotalRowsDeleted = h.TotalRowsDeleted,
DeletedBlobCount = h.DeletedBlobCount, CascadeSessionsEnqueued = h.CascadeSessionsEnqueued,
ErrorMessage = h.ErrorMessage, RetryCount = h.RetryCount,
DrainCompletedAt = h.DrainCompletedAt,
EnumerationStartedAt = h.EnumerationStartedAt,
EnumerationCompletedBeforeUpload = h.EnumerationCompletedBeforeUpload,
CustomGatherRulesArchived = h.CustomGatherRulesArchived,
CustomAnalyzeRulesArchived = h.CustomAnalyzeRulesArchived,
ImeLogPatternOverridesArchived = h.ImeLogPatternOverridesArchived,
ReonboardedAt = h.ReonboardedAt, ReonboardedBy = h.ReonboardedBy,
CustomsAutoWipedOnReonboard = h.CustomsAutoWipedOnReonboard,
};
private static OffboardingByTenantPointer Clone(OffboardingByTenantPointer p) => new()
{
PartitionKey = p.PartitionKey, RowKey = p.RowKey, TenantId = p.TenantId,
LatestHistoryRowKey = p.LatestHistoryRowKey, LatestStatus = p.LatestStatus,
LatestUpdatedAt = p.LatestUpdatedAt, OffboardCount = p.OffboardCount,
};
}
@@ -0,0 +1,95 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// In-memory <see cref="IOffboardingExpectationsStore"/> with the production If-None-Match=*
/// and If-Match ETag-CAS semantics. Tracks delete attempts so tests can assert idempotency.
/// </summary>
internal sealed class FakeOffboardingExpectationsStore : IOffboardingExpectationsStore
{
private readonly Dictionary<string, (OffboardingExpectations Payload, string ETag)> _blobs = new();
private int _etagCounter;
public int DeleteCallCount { get; private set; }
public bool ThrowOnDelete { get; set; }
public Task<bool> TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default)
{
var key = Key(payload.TenantId, payload.HistoryRowKey);
if (_blobs.ContainsKey(key)) return Task.FromResult(false);
_blobs[key] = (Clone(payload), NextEtag());
return Task.FromResult(true);
}
public Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync(
string tenantId, string historyRowKey, CancellationToken ct = default)
{
var key = Key(tenantId, historyRowKey);
if (!_blobs.TryGetValue(key, out var entry))
return Task.FromResult<(OffboardingExpectations?, string?)>((null, null));
return Task.FromResult<(OffboardingExpectations?, string?)>((Clone(entry.Payload), entry.ETag));
}
public Task<string> UpdateWithEtagCasAsync(OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default)
{
var key = Key(payload.TenantId, payload.HistoryRowKey);
if (!_blobs.TryGetValue(key, out var entry))
throw new RequestFailedException(404, "NotFound", "BlobNotFound", null);
if (entry.ETag != ifMatchEtag)
throw new RequestFailedException(412, "ConditionNotMet", "ConditionNotMet", null);
var newEtag = NextEtag();
_blobs[key] = (Clone(payload), newEtag);
return Task.FromResult(newEtag);
}
public Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default)
{
DeleteCallCount++;
if (ThrowOnDelete) throw new InvalidOperationException("simulated blob-delete failure");
_blobs.Remove(Key(tenantId, historyRowKey));
return Task.CompletedTask;
}
/// <summary>Pre-seed a blob (used to drive the drain probe directly without going via TryUploadInitial).</summary>
public void Seed(OffboardingExpectations payload)
{
_blobs[Key(payload.TenantId, payload.HistoryRowKey)] = (Clone(payload), NextEtag());
}
public bool BlobExists(string tenantId, string historyRowKey)
=> _blobs.ContainsKey(Key(tenantId, historyRowKey));
private static string Key(string tenantId, string historyRowKey) => $"{tenantId}|{historyRowKey}";
private string NextEtag() => $"\"0xFAKE_EXP_{++_etagCounter}\"";
private static OffboardingExpectations Clone(OffboardingExpectations p)
{
var clone = new OffboardingExpectations
{
SchemaVersion = p.SchemaVersion,
TenantId = p.TenantId,
HistoryRowKey = p.HistoryRowKey,
CreatedAt = p.CreatedAt,
EnumerationCompleted = p.EnumerationCompleted,
EnumeratedSessionCount = p.EnumeratedSessionCount,
};
foreach (var e in p.Expectations)
{
clone.Expectations.Add(new OffboardingExpectation
{
SessionId = e.SessionId,
ManifestId = e.ManifestId,
Outcome = e.Outcome,
RetryCount = e.RetryCount,
});
}
return clone;
}
}
@@ -0,0 +1,431 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Functions.Maintenance;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// Rev-5-F2 + Rev-9 cleanup contract:
/// <list type="bullet">
/// <item>Completed markers older than 15min are removed — AFTER their Expectations blob.</item>
/// <item>Completed markers younger than 15min stay (warm caches could still leak auth).</item>
/// <item>Failed markers are NEVER auto-removed — operator action only.</item>
/// <item>Initiated / InProgress markers are ignored (offboarding still in flight).</item>
/// <item>Blob-delete failure leaves the marker in place so the next run retries.</item>
/// </list>
/// </summary>
public class OffboardingMarkerCleanupFunctionTests
{
private const string TenantId = "11111111-1111-1111-1111-111111111111";
private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111";
[Fact]
public async Task DeletesCompletedMarkerOlderThan15min()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
fixture.Expectations.Seed(NewExpectations());
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(1, result.MarkersDeleted);
Assert.False(fixture.Repo.Markers.ContainsKey(TenantId));
Assert.False(fixture.Expectations.BlobExists(TenantId, HistoryRowKey),
"Expectations blob must be deleted before the marker");
}
[Fact]
public async Task KeepsCompletedMarkerYoungerThan15min()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(5)); // inside cache TTL window
fixture.Expectations.Seed(NewExpectations());
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(0, result.MarkersDeleted);
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId));
Assert.True(fixture.Expectations.BlobExists(TenantId, HistoryRowKey),
"Blob stays until the marker is eligible for removal");
}
[Fact]
public async Task NeverDeletesFailedMarker_EvenAfter24h()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(
status: "Failed",
failedAt: fixture.FixedNow - TimeSpan.FromHours(24),
failedPhase: "drain_timeout");
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(0, result.MarkersDeleted);
Assert.Equal(1, result.FailedMarkersSeen);
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId),
"Failed markers require operator action — never auto-cleaned");
}
[Fact]
public async Task IgnoresInProgressMarker()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(status: "InProgress");
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(0, result.MarkersDeleted);
Assert.Equal(0, result.FailedMarkersSeen);
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId));
}
[Fact]
public async Task IgnoresInitiatedMarker()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(status: "Initiated");
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(0, result.MarkersDeleted);
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId));
}
[Fact]
public async Task BlobDeleteFails_KeepsMarkerForNextRun()
{
// Rev-9 hygiene: blob deleted BEFORE marker. If blob delete throws (transient storage
// glitch), marker stays so the next 2h-run tries again. Otherwise an orphan blob lives
// forever in offboarding-state.
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
fixture.Expectations.Seed(NewExpectations());
fixture.Expectations.ThrowOnDelete = true;
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(0, result.MarkersDeleted);
Assert.Equal(1, result.BlobDeleteRetries);
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId),
"Marker stays so the next run retries the blob delete");
}
[Fact]
public async Task DryRunWithMixedFleet_HandlesEachMarkerIndependently()
{
var fixture = new Fixture();
fixture.Repo.Markers["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"] = NewMarker(
tenantId: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
status: "Completed", completedAt: fixture.FixedNow - TimeSpan.FromMinutes(30));
fixture.Repo.Markers["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"] = NewMarker(
tenantId: "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
status: "Completed", completedAt: fixture.FixedNow - TimeSpan.FromMinutes(2));
fixture.Repo.Markers["cccccccc-cccc-cccc-cccc-cccccccccccc"] = NewMarker(
tenantId: "cccccccc-cccc-cccc-cccc-cccccccccccc",
status: "Failed", failedAt: fixture.FixedNow - TimeSpan.FromHours(2),
failedPhase: "drain_timeout");
fixture.Repo.Markers["dddddddd-dddd-dddd-dddd-dddddddddddd"] = NewMarker(
tenantId: "dddddddd-dddd-dddd-dddd-dddddddddddd",
status: "InProgress");
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(4, result.Scanned);
Assert.Equal(1, result.MarkersDeleted); // only the 30-min-old Completed one
Assert.Equal(1, result.FailedMarkersSeen);
Assert.False(fixture.Repo.Markers.ContainsKey("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"));
Assert.True(fixture.Repo.Markers.ContainsKey("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"));
Assert.True(fixture.Repo.Markers.ContainsKey("cccccccc-cccc-cccc-cccc-cccccccccccc"));
Assert.True(fixture.Repo.Markers.ContainsKey("dddddddd-dddd-dddd-dddd-dddddddddddd"));
}
// ── TenantConfiguration defense-in-depth sweep (PR3.B-revised Finding 2) ──────
//
// Phase 2.F-final in the handler is fail-soft. If it failed (worker crash, transient
// storage), the TenantConfiguration row would carry Disabled=true and block the
// tenant from self-service re-onboarding indefinitely. This sweep is the second
// pass: when MarkerCleanup spots a Completed marker eligible for removal, it tries
// to delete TenantConfiguration FIRST. If the sweep fails, the marker stays so the
// next 2h run retries.
[Fact]
public async Task CompletedMarker_SweepsLingeringTenantConfig_BeforeDeletingMarker()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(
status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
// Simulate "Phase 2.F-final failed in the handler" → the row is still the
// offboarding tombstone (Disabled=true + magic reason). Sweep should wipe it.
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ReturnsAsync(BuildTombstoneConfig(TenantId));
fixture.SafeWipe.QueuedReturns.Enqueue(1);
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(1, result.MarkersDeleted);
Assert.Equal(1, result.TenantConfigsSwept);
Assert.Equal(0, result.TenantConfigsSpared);
Assert.Equal(0, result.TenantConfigSweepRetries);
// The sweep ran with the right table + tenant id.
Assert.Single(fixture.SafeWipe.WipeCalls);
Assert.Equal(Constants.TableNames.TenantConfiguration, fixture.SafeWipe.WipeCalls[0].Table);
Assert.Equal(TenantId, fixture.SafeWipe.WipeCalls[0].TenantId);
Assert.False(fixture.Repo.Markers.ContainsKey(TenantId));
}
[Fact]
public async Task CompletedMarker_TenantConfigAlreadyGone_SkipsSweep_DeletesMarker()
{
// Happy path: Phase 2.F-final succeeded in the handler. GetTenantConfigurationAsync
// returns null (default fixture stub) → no row to wipe.
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(
status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(1, result.MarkersDeleted);
Assert.Equal(0, result.TenantConfigsSwept);
Assert.Equal(0, result.TenantConfigsSpared);
// No SafeWipe call when the row is already gone (we don't waste a transaction).
Assert.Empty(fixture.SafeWipe.WipeCalls);
Assert.False(fixture.Repo.Markers.ContainsKey(TenantId));
}
[Fact]
public async Task CompletedMarker_UserReonboarded_SparesFreshConfig_AndStillDeletesMarker()
{
// The crucial Codex Finding 1 scenario: between Phase 2.F-final and MarkerCleanup,
// the user self-service-re-onboarded. The TenantConfiguration row now exists with
// Disabled=false (or any non-offboarding DisabledReason). Blind sweep would
// destroy their fresh tenant. The conditional probe must skip the wipe.
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(
status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
var freshConfig = TenantConfiguration.CreateDefault(TenantId);
freshConfig.Disabled = false;
freshConfig.UpdatedBy = "bob@contoso.com";
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ReturnsAsync(freshConfig);
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(1, result.MarkersDeleted);
Assert.Equal(0, result.TenantConfigsSwept);
Assert.Equal(1, result.TenantConfigsSpared);
// Critically: SafeWipe was NEVER called. The fresh config is preserved.
Assert.Empty(fixture.SafeWipe.WipeCalls);
// The Completed marker still gets cleaned up — its work is done.
Assert.False(fixture.Repo.Markers.ContainsKey(TenantId));
}
[Fact]
public async Task CompletedMarker_UserReonboarded_AndSuspendedThemselves_StillSpared()
{
// Defense-in-depth check: a re-onboarded tenant that later self-suspended would
// have Disabled=true but with a different DisabledReason (e.g. "Self-suspended").
// The magic-string match must distinguish — only "Offboarding in progress" triggers
// the wipe.
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(
status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
var reonboardedThenSuspended = TenantConfiguration.CreateDefault(TenantId);
reonboardedThenSuspended.Disabled = true;
reonboardedThenSuspended.DisabledReason = "Self-suspended";
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ReturnsAsync(reonboardedThenSuspended);
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(0, result.TenantConfigsSwept);
Assert.Equal(1, result.TenantConfigsSpared);
Assert.Empty(fixture.SafeWipe.WipeCalls);
}
[Fact]
public async Task CompletedMarker_TenantConfigProbeThrows_KeepsMarkerForNextRetry()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(
status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ThrowsAsync(new InvalidOperationException("storage down"));
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(0, result.MarkersDeleted);
Assert.Equal(1, result.TenantConfigSweepRetries);
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId),
"Marker must stay alive when the TenantConfiguration probe fails so the next run can retry");
}
[Fact]
public async Task CompletedMarker_TenantConfigSweepThrows_KeepsMarkerForNextRetry()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(
status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ReturnsAsync(BuildTombstoneConfig(TenantId));
fixture.SafeWipe.ThrowOnNextWipe = new InvalidOperationException("storage down");
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(0, result.MarkersDeleted);
Assert.Equal(1, result.TenantConfigSweepRetries);
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId),
"Marker must stay alive when the TenantConfig sweep fails so the next run can retry");
}
[Fact]
public async Task CompletedMarker_BlobDeleteFails_TenantConfigProbeIsNotEvenAttempted()
{
var fixture = new Fixture();
fixture.Repo.Markers[TenantId] = NewMarker(
status: "Completed",
completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20));
fixture.Expectations.ThrowOnDelete = true;
var result = await fixture.Sut.RunCoreAsync(default);
Assert.Equal(1, result.BlobDeleteRetries);
Assert.Empty(fixture.SafeWipe.WipeCalls);
Assert.True(fixture.Repo.Markers.ContainsKey(TenantId));
// Probe shouldn't have been attempted either — blob is the first gate.
fixture.ConfigRepo.Verify(r => r.GetTenantConfigurationAsync(It.IsAny<string>()), Times.Never);
}
private static TenantConfiguration BuildTombstoneConfig(string tenantId)
{
var config = TenantConfiguration.CreateDefault(tenantId);
config.Disabled = true;
config.DisabledReason = "Offboarding in progress";
return config;
}
// ── Helpers ─────────────────────────────────────────────────────────────────
private static OffboardingMarkerEntry NewMarker(
string? tenantId = null,
string status = "Completed",
DateTime? completedAt = null,
DateTime? failedAt = null,
string? failedPhase = null)
{
var tid = tenantId ?? TenantId;
return new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = tid,
TenantId = tid,
OffboardingHistoryRowKey = $"20260518091523123_{tid}",
InitiatedAt = DateTime.UtcNow.AddHours(-1),
InitiatedBy = "alice@contoso.invalid",
Status = status,
CompletedAt = completedAt,
FailedAt = failedAt,
FailedPhase = failedPhase,
};
}
private static OffboardingExpectations NewExpectations() => new()
{
SchemaVersion = 1,
TenantId = TenantId,
HistoryRowKey = HistoryRowKey,
CreatedAt = DateTime.UtcNow.AddHours(-1),
EnumerationCompleted = true,
EnumeratedSessionCount = 0,
};
private sealed class Fixture
{
public DateTime FixedNow { get; } = new DateTime(2026, 5, 18, 12, 0, 0, DateTimeKind.Utc);
public FakeOffboardingAuditRepository Repo { get; } = new();
public FakeOffboardingExpectationsStore Expectations { get; } = new();
public RecordingSafeWipeService SafeWipe { get; } = new();
public Mock<IConfigRepository> ConfigRepo { get; } = new();
public OffboardingMarkerCleanupFunction Sut { get; }
public Fixture()
{
var timeProvider = new FixedTimeProvider(FixedNow);
// Default: TenantConfig row not found (the normal happy path — Phase 2.F-final
// successfully deleted it; nothing for the sweep to do). Individual tests
// override this to seed a lingering tombstone or a re-onboarded fresh config.
ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
.ReturnsAsync((TenantConfiguration?)null);
Sut = new OffboardingMarkerCleanupFunction(
Repo, Expectations, SafeWipe, ConfigRepo.Object,
NullLogger<OffboardingMarkerCleanupFunction>.Instance,
timeProvider);
}
}
/// <summary>
/// Mock <see cref="AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService"/> that
/// records which (table, tenantId) pairs were swept and how many rows it claims to have
/// deleted. Default behaviour returns 0 (row already gone — the normal happy path).
/// Tests can opt in to "0 means a TenantConfiguration row was found and wiped" via
/// <see cref="QueuedReturns"/>.
/// </summary>
private sealed class RecordingSafeWipeService : AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService
{
public readonly List<(string Table, string TenantId)> WipeCalls = new();
public readonly Queue<int> QueuedReturns = new();
public Exception? ThrowOnNextWipe { get; set; }
public RecordingSafeWipeService()
: base(
new AutopilotMonitor.Functions.Services.TableStorageService(
Moq.Mock.Of<Azure.Data.Tables.TableServiceClient>(),
NullLogger<AutopilotMonitor.Functions.Services.TableStorageService>.Instance),
new AutopilotMonitor.Functions.Services.BlobStorageService(
new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
NullLogger<AutopilotMonitor.Functions.Services.BlobStorageService>.Instance,
usesManagedIdentity: false),
NullLogger<AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService>.Instance)
{
}
public override Task<int> WipeByExactPartitionAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
{
WipeCalls.Add((tableName, normalizedTenantId));
if (ThrowOnNextWipe is { } ex)
{
ThrowOnNextWipe = null;
throw ex;
}
return Task.FromResult(QueuedReturns.Count > 0 ? QueuedReturns.Dequeue() : 0);
}
}
private sealed class FixedTimeProvider : TimeProvider
{
private readonly DateTimeOffset _now;
public FixedTimeProvider(DateTime utcNow) { _now = new DateTimeOffset(utcNow, TimeSpan.Zero); }
public override DateTimeOffset GetUtcNow() => _now;
}
}
@@ -0,0 +1,67 @@
using AutopilotMonitor.Functions.DataAccess.TableStorage;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// PR3.B §3.2 — static-helper tests for the archive RowKey base64url encoding and the
/// PartitionKey construction. These two functions are the only things in the repository
/// that have logic worth pinning at unit level; the rest is straight Azure Tables CRUD.
/// </summary>
public sealed class TableTenantCustomsArchiveRepositoryHelperTests
{
[Fact]
public void BuildPartitionKey_Concatenates_TenantId_And_HistoryRowKey()
{
var pk = TableTenantCustomsArchiveRepository.BuildPartitionKey(
"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee");
Assert.Equal(
"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
pk);
}
[Theory]
[InlineData("simple-row-id")]
[InlineData("rowkey/with/slashes")]
[InlineData("rowkey#with#hash")]
[InlineData("rowkey?with?question")]
[InlineData("rowkey\\with\\backslash")]
[InlineData("a")]
[InlineData("ab")]
[InlineData("abc")]
[InlineData("abcd")]
[InlineData("unicode-Ümlaut-and-emoji-✓")]
public void BuildRowKey_RoundTripsForbiddenCharacters(string originalRowKey)
{
var archiveRowKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", originalRowKey);
// RowKey itself must NOT contain any Azure-Tables-forbidden characters.
Assert.DoesNotContain('#', archiveRowKey);
Assert.DoesNotContain('?', archiveRowKey);
Assert.DoesNotContain('/', archiveRowKey);
Assert.DoesNotContain('\\', archiveRowKey);
// Must round-trip cleanly.
var decoded = TableTenantCustomsArchiveRepository.DecodeOriginalRowKey(archiveRowKey, "GatherRules");
Assert.Equal(originalRowKey, decoded);
}
[Fact]
public void BuildRowKey_DifferentTablesProduceDifferentKeys()
{
var gatherKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", "my-rule");
var analyzeKey = TableTenantCustomsArchiveRepository.BuildRowKey("AnalyzeRules", "my-rule");
Assert.NotEqual(gatherKey, analyzeKey);
Assert.StartsWith("GatherRules_", gatherKey);
Assert.StartsWith("AnalyzeRules_", analyzeKey);
}
[Fact]
public void DecodeOriginalRowKey_RejectsPrefixMismatch()
{
var rowKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", "x");
Assert.Throws<ArgumentException>(() =>
TableTenantCustomsArchiveRepository.DecodeOriginalRowKey(rowKey, "AnalyzeRules"));
}
}
@@ -0,0 +1,185 @@
using System.Text.Json;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure.Data.Tables;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// PR3.B §3.4-§3.5 — invariants the offboarding-customs-archive design guarantees:
/// snapshot serialization, partition-key namespacing per offboarding run, and the
/// counter-from-archive recomputation that makes the History fields crash-resume safe.
/// <para>
/// Source-table archive-then-wipe orchestration (3-iteration cap + customs_arrival_race
/// fail-closed) is a structural property of the loop in
/// <see cref="TenantOffboardingHandler.ArchiveAndWipeRulesTableAsync"/> — manual smoketest
/// on a dev tenant covers that contract; full mock-TableClient integration test is a
/// followup once we have a TableClient abstraction in the codebase.
/// </para>
/// </summary>
public sealed class TenantCustomsArchiveContractTests
{
private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
private const string HistoryRowKey = "20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
private static TenantOffboardingHandler BuildHandler(ITenantCustomsArchiveRepository archive)
{
// We only need the handler instance for its internal BuildArchiveEntry helper; that
// method does not call into any of the other collaborators, so we can pass null! for
// them. Tests exercise pure-method behaviour only.
return new TenantOffboardingHandler(
auditRepo: null!,
enumerator: null!,
cascadeEnqueuer: null!,
expectations: null!,
drainProbe: null!,
safeWipe: null!,
storage: null!,
maintenance: null!,
reEnqueuer: null!,
opsEvents: null!,
customsArchive: archive,
logger: NullLogger<TenantOffboardingHandler>.Instance);
}
[Fact]
public void BuildArchiveEntry_PartitionKey_IsTenantUnderscoreHistoryRowKey()
{
var handler = BuildHandler(new FakeCustomsArchiveRepository());
var source = new TableEntity(TenantId, "rule-id-1") { ["DisplayName"] = "anything" };
var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.GatherRules, HistoryRowKey);
Assert.Equal($"{TenantId}_{HistoryRowKey}", entry.PartitionKey);
}
[Fact]
public void BuildArchiveEntry_RowKey_PrefixedWithOriginalTable_AndBase64Encoded()
{
var handler = BuildHandler(new FakeCustomsArchiveRepository());
var source = new TableEntity(TenantId, "rule/with/slash") { ["DisplayName"] = "x" };
var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.AnalyzeRules, HistoryRowKey);
Assert.StartsWith("AnalyzeRules_", entry.RowKey);
Assert.DoesNotContain('/', entry.RowKey); // base64url replaces / with _
}
[Fact]
public void BuildArchiveEntry_StripsSystemProperties_FromJson()
{
var handler = BuildHandler(new FakeCustomsArchiveRepository());
var source = new TableEntity(TenantId, "rule")
{
["DisplayName"] = "MyRule",
["Category"] = "preset",
["Timestamp"] = System.DateTimeOffset.UtcNow,
["odata.etag"] = "\"0x123\"",
};
var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.GatherRules, HistoryRowKey);
var parsed = JsonDocument.Parse(entry.EntityJson);
Assert.True(parsed.RootElement.TryGetProperty("DisplayName", out var dn));
Assert.Equal("MyRule", dn.GetString());
Assert.True(parsed.RootElement.TryGetProperty("Category", out _));
// System properties must NOT appear in the snapshot.
Assert.False(parsed.RootElement.TryGetProperty("Timestamp", out _));
Assert.False(parsed.RootElement.TryGetProperty("odata.etag", out _));
}
[Fact]
public void BuildArchiveEntry_PopulatesOriginalKeysAndArchivedBy()
{
var handler = BuildHandler(new FakeCustomsArchiveRepository());
var source = new TableEntity(TenantId, "my-rule-id") { ["X"] = "Y" };
var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.ImeLogPatterns, HistoryRowKey);
Assert.Equal(TenantId, entry.TenantId);
Assert.Equal(Constants.TableNames.ImeLogPatterns, entry.OriginalTable);
Assert.Equal(TenantId, entry.OriginalPartitionKey);
Assert.Equal("my-rule-id", entry.OriginalRowKey);
Assert.Equal(HistoryRowKey, entry.HistoryRowKey);
Assert.Equal("TenantOffboardingHandler", entry.ArchivedBy);
Assert.NotEqual(default, entry.ArchivedAt);
}
[Fact]
public async Task ReReOffboard_TwoRuns_CoExistInArchive_AsDistinctPartitions()
{
// PR3.B §3.5: each offboarding run gets its own PartitionKey
// ({tenantId}_{historyRowKey}), so a tenant offboarded twice produces two immutable
// archive partitions side-by-side.
var repo = new FakeCustomsArchiveRepository();
var run1 = "20260101000000000_" + TenantId;
var run2 = "20260518091523123_" + TenantId;
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry
{
PartitionKey = $"{TenantId}_{run1}",
RowKey = "GatherRules_aaa",
TenantId = TenantId,
OriginalTable = Constants.TableNames.GatherRules,
OriginalRowKey = "aaa",
HistoryRowKey = run1,
});
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry
{
PartitionKey = $"{TenantId}_{run2}",
RowKey = "GatherRules_aaa",
TenantId = TenantId,
OriginalTable = Constants.TableNames.GatherRules,
OriginalRowKey = "aaa",
HistoryRowKey = run2,
});
var count1 = await repo.CountByRunAndTableAsync(TenantId, run1, Constants.TableNames.GatherRules);
var count2 = await repo.CountByRunAndTableAsync(TenantId, run2, Constants.TableNames.GatherRules);
Assert.Equal(1, count1);
Assert.Equal(1, count2);
Assert.Equal(2, repo.Store.Count);
}
[Fact]
public async Task CountByRunAndTable_FiltersByOriginalTable()
{
var repo = new FakeCustomsArchiveRepository();
var run = "20260518091523123_" + TenantId;
var pk = $"{TenantId}_{run}";
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules", HistoryRowKey = run });
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "GatherRules_b", TenantId = TenantId, OriginalTable = "GatherRules", HistoryRowKey = run });
await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "AnalyzeRules_a", TenantId = TenantId, OriginalTable = "AnalyzeRules", HistoryRowKey = run });
Assert.Equal(2, await repo.CountByRunAndTableAsync(TenantId, run, "GatherRules"));
Assert.Equal(1, await repo.CountByRunAndTableAsync(TenantId, run, "AnalyzeRules"));
Assert.Equal(0, await repo.CountByRunAndTableAsync(TenantId, run, "ImeLogPatterns"));
}
[Fact]
public async Task DeleteRunAsync_RemovesEveryEntryInPartition_LeavesOtherRunsAlone()
{
var repo = new FakeCustomsArchiveRepository();
var run1 = "20260101000000000_" + TenantId;
var run2 = "20260518091523123_" + TenantId;
await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run1}", RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules" });
await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run1}", RowKey = "GatherRules_b", TenantId = TenantId, OriginalTable = "GatherRules" });
await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run2}", RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules" });
var deleted = await repo.DeleteRunAsync(TenantId, run1);
Assert.Equal(2, deleted);
Assert.Single(repo.Store);
// The remaining row belongs to run2.
Assert.Single(repo.Store.Values, e => e.PartitionKey == $"{TenantId}_{run2}");
}
}
@@ -0,0 +1,634 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Data.Tables;
using Azure.Storage.Blobs;
using AutopilotMonitor.Functions.DataAccess.TableStorage;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Functions.Services.Notifications;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Deletion;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// Pin the §4.5 Drain-Contract that prevents the "silent wipe" the user has been worried about.
/// Every fail-closed branch must throw + the Marker must transition to <c>Failed</c> WITHOUT
/// any SafeWipe being attempted. The 0-session happy path and the Rev-9-F1 Drain-Skip-Gate
/// (post-crash idempotent resume) round out coverage for the surrounding state-machine.
/// </summary>
public class TenantOffboardingHandlerDrainContractTests
{
private const string TenantId = "11111111-1111-1111-1111-111111111111";
private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111";
// ── Fail-closed branches ────────────────────────────────────────────────────
[Fact]
public async Task KillSwitchActive_ExpectationFails_NoSafeWipe()
{
var harness = Harness.New();
harness.SeedExpectations(new (string, string?, string, int)[]
{
("session-1", null, nameof(SessionDeletionEnqueueOutcome.KillSwitchActive), 0),
});
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status);
Assert.Equal("killswitch", harness.Repo.Markers[TenantId].FailedPhase);
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
}
[Fact]
public async Task Poisoned_ExpectationFails_NoSafeWipe()
{
var harness = Harness.New();
harness.SeedExpectations(new (string, string?, string, int)[]
{
("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Poisoned), 0),
});
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
Assert.Equal("poisoned", harness.Repo.Markers[TenantId].FailedPhase);
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
}
[Fact]
public async Task AlreadyInFlight_With_NullManifestId_FailsClosed_Rev8F3()
{
// AlreadyInFlight + null ManifestId is the Preparing-without-snapshot state. There is
// no progress blob to drain against → fail closed immediately rather than waiting
// out the 2h drain budget.
var harness = Harness.New();
harness.SeedExpectations(new (string, string?, string, int)[]
{
("session-1", null, nameof(SessionDeletionEnqueueOutcome.AlreadyInFlight), 0),
});
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
Assert.Equal("alreadyinflight_no_manifest", harness.Repo.Markers[TenantId].FailedPhase);
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
}
[Fact]
public async Task CasExhausted_RetriesUpToCap_ThenFailsClosed()
{
var harness = Harness.New();
harness.SeedExpectations(new (string, string?, string, int)[]
{
("session-1", null, nameof(SessionDeletionEnqueueOutcome.CasExhausted),
TenantOffboardingHandler.MaxCasRetriesPerSession),
});
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
Assert.Equal("cas_exhausted", harness.Repo.Markers[TenantId].FailedPhase);
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
}
[Fact]
public async Task EnumeratorCrash_FirstAttempt_StampsEnumerationStartedAt_ThenPropagatesException()
{
// First-attempt crash: enumerator throws before any expectation is uploaded. The
// handler MUST stamp EnumerationStartedAt before touching the enumerator so the next
// pickup can tell first-try from re-pickup-after-crash. Exception propagates so the
// worker retries via visibility timeout.
var harness = Harness.New();
harness.EnumeratorThrow = new InvalidOperationException("simulated 503");
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationStartedAt);
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
}
[Fact]
public async Task RePickup_MidEnumerationCrash_ESA_Set_ECBU_Null_FailsClosed_ExpectationsMissing()
{
// Plan §7.4 step 3 Crash-Verhalten: prior pickup stamped EnumerationStartedAt but
// crashed mid-iteration (ECBU still null). Re-pickup must NOT re-enumerate (tenant
// may be mid-mutation); it must fail-closed.
var harness = Harness.New();
harness.Repo.History[HistoryRowKey].EnumerationStartedAt = DateTime.UtcNow.AddMinutes(-5);
harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload = null;
harness.EnumeratorYields.Add("session-X"); // would be enumerated if logic regressed
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
Assert.Equal("expectations_missing", harness.Repo.Markers[TenantId].FailedPhase);
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
harness.CascadeProducer.Verify(c => c.EnqueueAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()),
Times.Never(), "Mid-enumeration crash on re-pickup must NEVER re-enumerate");
}
[Fact]
public async Task RePickup_UploadFailure_ESA_Set_ECBU_Set_NoBlob_ReRunsEnumerateAndUpload()
{
// Plan §7.4 step 3: when ECBU is set but blob is missing, the enumeration loop
// finished cleanly and only the upload failed. Re-pickup must RE-RUN enumerate +
// upload (idempotent — SessionDeletionProducer returns AlreadyInFlight for the same
// session id), NOT fail-closed.
var harness = Harness.New();
harness.Repo.History[HistoryRowKey].EnumerationStartedAt = DateTime.UtcNow.AddMinutes(-5);
harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload = DateTime.UtcNow.AddMinutes(-4);
harness.EnumeratorYields.Add("session-A");
harness.EnumeratorYields.Add("session-B");
harness.CascadeProducer.Setup(c => c.EnqueueAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()))
.ReturnsAsync<string, string, string, DeletionActor, DeletionRetentionContext?, CancellationToken, ISessionDeletionEnqueuer, SessionDeletionEnqueueResult>(
(_, sid, _, _, _, _) => new SessionDeletionEnqueueResult
{
Outcome = SessionDeletionEnqueueOutcome.AlreadyInFlight, // idempotent producer response
ManifestId = $"manifest-{sid}",
});
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
// Both sessions re-enqueued (idempotent producer responses).
harness.CascadeProducer.Verify(c => c.EnqueueAsync(
It.IsAny<string>(), "session-A", It.IsAny<string>(),
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()),
Times.AtLeastOnce, "Upload-retry path must re-enumerate session-A");
harness.CascadeProducer.Verify(c => c.EnqueueAsync(
It.IsAny<string>(), "session-B", It.IsAny<string>(),
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()),
Times.AtLeastOnce, "Upload-retry path must re-enumerate session-B");
// Upload retry succeeded, drain probe says done → Completed.
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
// Marker is NOT Failed — this was an upload-retry, not a mid-enum crash.
Assert.NotEqual("expectations_missing", harness.Repo.Markers[TenantId].FailedPhase);
}
[Fact]
public async Task FirstTry_StampsBothMarkers_BeforeAndAfterEnumeration()
{
// Audit-trail invariant: ESA must be stamped BEFORE the enumerator runs (so a crash
// during iteration lands in the fail-closed branch next time); ECBU must be stamped
// AFTER the enumerator finishes (so an upload-failure-only crash lands in the
// upload-retry branch next time).
var harness = Harness.New();
harness.EnumeratorYields.Add("session-1");
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationStartedAt);
Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload);
// Sanity: ESA <= ECBU (one was stamped before the loop, one after).
Assert.True(
harness.Repo.History[HistoryRowKey].EnumerationStartedAt
<= harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload);
}
[Fact]
public async Task EnumerationIncomplete_FailsClosed()
{
var harness = Harness.New();
harness.Expectations.Seed(new OffboardingExpectations
{
SchemaVersion = 1,
TenantId = TenantId,
HistoryRowKey = HistoryRowKey,
CreatedAt = DateTime.UtcNow,
EnumerationCompleted = false, // ← the disambiguation Rev-7-F2 added
EnumeratedSessionCount = 5,
Expectations = Enumerable.Range(0, 5).Select(i =>
new OffboardingExpectation { SessionId = $"s{i}", Outcome = "Enqueued", ManifestId = $"m{i}" }).ToList(),
});
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
Assert.Equal("enumeration_incomplete", harness.Repo.Markers[TenantId].FailedPhase);
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
}
[Fact]
public async Task ExpectationsSizeMismatch_FailsClosed()
{
// expectations.Length != enumeratedSessionCount → fail-closed. Defends against a
// half-overwritten blob from a buggy re-uploader.
var harness = Harness.New();
harness.Expectations.Seed(new OffboardingExpectations
{
SchemaVersion = 1,
TenantId = TenantId,
HistoryRowKey = HistoryRowKey,
CreatedAt = DateTime.UtcNow,
EnumerationCompleted = true,
EnumeratedSessionCount = 5,
Expectations = new List<OffboardingExpectation>
{
new() { SessionId = "s1", Outcome = "Enqueued", ManifestId = "m1" },
new() { SessionId = "s2", Outcome = "Enqueued", ManifestId = "m2" },
},
});
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)));
Assert.Equal("expectations_size_mismatch", harness.Repo.Markers[TenantId].FailedPhase);
}
[Fact]
public async Task DrainTimeout_AfterMaxPolls_FailsClosed()
{
// The cap (60 polls ≈ 2h) must produce a deterministic Failed transition rather than
// dragging on forever.
var harness = Harness.New();
harness.SeedExpectations(new (string, string?, string, int)[]
{
("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Enqueued), 0),
});
harness.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(false); // never settles
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope(drainPollCount: TenantOffboardingHandler.MaxDrainPolls - 1)));
Assert.Equal("drain_timeout", harness.Repo.Markers[TenantId].FailedPhase);
}
// ── Happy paths ─────────────────────────────────────────────────────────────
[Fact]
public async Task ZeroSessions_EnumerationCompleted_DrainOk_ProceedsToWipeAndCompleted()
{
// Plan §4.5: empty expectations + enumerationCompleted=true is the legitimate
// happy path for a tenant with no sessions left.
var harness = Harness.New();
// Enumerator yields nothing — handler will upload an Expectations blob with
// enumerationCompleted=true, enumeratedSessionCount=0.
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
Assert.NotNull(harness.Repo.History[HistoryRowKey].CompletedAt);
Assert.NotNull(harness.Repo.History[HistoryRowKey].DrainCompletedAt);
Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status);
Assert.NotNull(harness.Repo.Markers[TenantId].CompletedAt);
Assert.True(harness.SafeWipeProbe.WipeCallCount > 0,
"SafeWipe must have run across the tenant tables");
Assert.False(harness.Expectations.BlobExists(TenantId, HistoryRowKey),
"Phase 2.G must delete the Expectations blob as its last step");
}
[Fact]
public async Task SessionNotFound_CountsAsSatisfied_NoDrainProbeCall()
{
var harness = Harness.New();
harness.SeedExpectations(new (string, string?, string, int)[]
{
("session-1", null, nameof(SessionDeletionEnqueueOutcome.SessionNotFound), 0),
});
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
harness.DrainProbe.Verify(p => p.IsCascadeCompletedAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Never(), "SessionNotFound is a no-op — drain probe must NOT be called for that expectation");
}
// ── Rev-9-F1 Drain-Skip-Gate ───────────────────────────────────────────────
[Fact]
public async Task Rev9F1_DrainCompletedAtSet_SkipsDrainPredicate_AndRunsPostDrainPhasesIdempotently()
{
// The crash-recovery scenario: drain ran once and stamped DrainCompletedAt, then we
// crashed somewhere in 2.D-G. Re-pickup must NOT re-evaluate the drain predicate (the
// progress blobs are now gone in 2.E) and must still land us on Completed.
var harness = Harness.New();
harness.Repo.History[HistoryRowKey].DrainCompletedAt = DateTime.UtcNow.AddMinutes(-10);
harness.Repo.History[HistoryRowKey].Status = "InProgress";
harness.Repo.Markers[TenantId].Status = "InProgress";
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
harness.DrainProbe.Verify(p => p.IsCascadeCompletedAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Never(), "When DrainCompletedAt is already set the drain probe must NOT be called");
harness.Enumerator.Verify(e => e.EnumerateAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()),
Times.Never(), "Drain-skip-gate must also bypass the enumerate+enqueue phase");
}
[Fact]
public async Task History_AlreadyCompleted_IsNoOp()
{
var harness = Harness.New();
harness.Repo.History[HistoryRowKey].Status = "Completed";
harness.Repo.History[HistoryRowKey].CompletedAt = DateTime.UtcNow;
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0));
// No additional writes — handler returned early.
Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount);
}
// ── Drain re-poll ───────────────────────────────────────────────────────────
[Fact]
public async Task DrainNotSettled_BelowCap_ReEnqueuesWithDelayAndPollIncrement()
{
var harness = Harness.New();
harness.SeedExpectations(new (string, string?, string, int)[]
{
("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Enqueued), 0),
});
harness.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(false);
await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 3));
harness.ReEnqueuer.Verify(e => e.EnqueueAsync(
It.Is<TenantOffboardingEnvelope>(env => env.DrainPollCount == 4),
TenantOffboardingHandler.DrainPollDelay,
It.IsAny<CancellationToken>()),
Times.Once);
// History should NOT be Completed yet.
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
Assert.Null(harness.Repo.History[HistoryRowKey].DrainCompletedAt);
}
// ── Harness ─────────────────────────────────────────────────────────────────
private sealed class Harness
{
public TenantOffboardingHandler Sut { get; private set; } = default!;
public FakeOffboardingAuditRepository Repo { get; } = new();
public FakeOffboardingExpectationsStore Expectations { get; } = new();
public Mock<IDeletionProgressDrainProbe> DrainProbe { get; } = new();
public Mock<ITenantOffboardingEnqueuer> ReEnqueuer { get; } = new();
public Mock<ISessionDeletionEnqueuer> CascadeProducer { get; } = new();
public Mock<OffboardingSessionEnumerator> Enumerator { get; }
public CountingSafeWipeService SafeWipeProbe { get; } = new();
public Mock<IMaintenanceRepository> Maintenance { get; } = new();
public List<string> EnumeratorYields { get; } = new();
public Exception? EnumeratorThrow { get; set; }
private Harness()
{
var maintenanceForEnumerator = new Mock<IMaintenanceRepository>();
Enumerator = new Mock<OffboardingSessionEnumerator>(maintenanceForEnumerator.Object);
}
public static Harness New()
{
var h = new Harness();
h.SeedInitialAudit();
// Default enumerator returns the (mutable) Sessions list. Tests append to it
// before invoking the handler.
h.Enumerator
.Setup(e => e.EnumerateAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns<string, CancellationToken>((_, ct) => h.EnumerateSessionsAsync(ct));
h.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(true); // happy default — drain completes immediately
h.CascadeProducer.Setup(c => c.EnqueueAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()))
.ReturnsAsync<string, string, string, DeletionActor, DeletionRetentionContext?, CancellationToken, ISessionDeletionEnqueuer, SessionDeletionEnqueueResult>(
(_, sid, _, _, _, _) => new SessionDeletionEnqueueResult
{
Outcome = SessionDeletionEnqueueOutcome.Enqueued,
ManifestId = $"manifest-{sid}",
});
h.Maintenance.Setup(m => m.LogAuditEntryAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
.ReturnsAsync(true);
var opsService = BuildRealOpsService();
var unusedTableStorage = new Mock<TableStorageService>(
Mock.Of<TableServiceClient>(),
NullLogger<TableStorageService>.Instance).Object;
h.Sut = new NoopCustomsArchivingHandler(
h.Repo,
h.Enumerator.Object,
h.CascadeProducer.Object,
h.Expectations,
h.DrainProbe.Object,
h.SafeWipeProbe,
unusedTableStorage,
h.Maintenance.Object,
h.ReEnqueuer.Object,
opsService,
Mock.Of<ITenantCustomsArchiveRepository>(),
NullLogger<TenantOffboardingHandler>.Instance);
return h;
}
private async IAsyncEnumerable<string> EnumerateSessionsAsync(
[EnumeratorCancellation] CancellationToken ct)
{
if (EnumeratorThrow != null) throw EnumeratorThrow;
foreach (var s in EnumeratorYields)
{
ct.ThrowIfCancellationRequested();
yield return s;
}
await Task.CompletedTask;
}
public TenantOffboardingEnvelope Envelope(int drainPollCount) => new()
{
TenantId = TenantId,
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
HistoryRowKey = HistoryRowKey,
InitiatedBy = "alice@contoso.invalid",
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
EnqueuedAt = DateTime.UtcNow,
DrainPollCount = drainPollCount,
};
public void SeedExpectations(IEnumerable<(string sessionId, string? manifestId, string outcome, int retryCount)> rows)
{
var list = rows.Select(r => new OffboardingExpectation
{
SessionId = r.sessionId,
ManifestId = r.manifestId,
Outcome = r.outcome,
RetryCount = r.retryCount,
}).ToList();
Expectations.Seed(new OffboardingExpectations
{
SchemaVersion = 1,
TenantId = TenantId,
HistoryRowKey = HistoryRowKey,
CreatedAt = DateTime.UtcNow,
EnumerationCompleted = true,
EnumeratedSessionCount = list.Count,
Expectations = list,
});
}
private void SeedInitialAudit()
{
Repo.History[HistoryRowKey] = new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = HistoryRowKey,
TenantId = TenantId,
DomainName = "contoso.invalid",
InitiatedBy = "alice@contoso.invalid",
OffboardedAt = DateTime.UtcNow.AddMinutes(-1),
Status = "Initiated",
};
Repo.Markers[TenantId] = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = HistoryRowKey,
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
InitiatedBy = "alice@contoso.invalid",
Status = "Initiated",
};
Repo.Pointers[TenantId] = (new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = HistoryRowKey,
LatestStatus = "Initiated",
LatestUpdatedAt = DateTime.UtcNow.AddMinutes(-1),
OffboardCount = 1,
}, "\"0xFAKE_PTR_1\"");
}
private static OpsEventService BuildRealOpsService()
{
var opsRepo = new Mock<IOpsEventRepository>();
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>())).Returns(Task.CompletedTask);
var memCache = new MemoryCache(new MemoryCacheOptions());
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, memCache);
var alertDispatch = new OpsAlertDispatchService(
adminConfig.Object,
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
NullLogger<TelegramNotificationService>.Instance),
new WebhookNotificationService(new HttpClient(),
NullLogger<WebhookNotificationService>.Instance),
NullLogger<OpsAlertDispatchService>.Instance);
return new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
}
}
/// <summary>
/// <see cref="SafeWipeService"/> that records call counts without performing any storage IO.
/// Tests use this to assert that fail-closed paths NEVER touch the wipe methods. Now that
/// <see cref="SafeWipeService"/> exposes the wipe methods as <c>virtual</c>, the production
/// dispatch lands here.
/// </summary>
private sealed class CountingSafeWipeService : SafeWipeService
{
public int WipeCallCount { get; private set; }
public CountingSafeWipeService()
: base(
new TableStorageService(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance),
new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"),
NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false),
NullLogger<SafeWipeService>.Instance)
{
}
public override Task<int> WipeByExactPartitionAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
{
WipeCallCount++;
return Task.FromResult(0);
}
public override Task<int> WipeByCompositePartitionRangeAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
{
WipeCallCount++;
return Task.FromResult(0);
}
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string tableName, string discriminator, string normalizedTenantId, CancellationToken ct = default)
{
WipeCallCount++;
return Task.FromResult(0);
}
public override Task<int> WipeByTenantIdPropertyAsync(string tableName, string normalizedTenantId, CancellationToken ct = default)
{
WipeCallCount++;
return Task.FromResult(0);
}
public override Task<int> WipeBlobsByTenantPrefixAsync(string containerName, string normalizedTenantId, CancellationToken ct = default)
{
WipeCallCount++;
return Task.FromResult(0);
}
}
/// <summary>
/// Subclass that no-ops the customs archive-then-wipe loop so the existing drain-contract
/// tests don't need a working TableStorageService for the 3 rules tables. PR3.B-specific
/// archive behaviour has its own test class with proper TableClient mocks.
/// </summary>
private sealed class NoopCustomsArchivingHandler : TenantOffboardingHandler
{
public NoopCustomsArchivingHandler(
IOffboardingAuditRepository auditRepo,
OffboardingSessionEnumerator enumerator,
ISessionDeletionEnqueuer cascadeEnqueuer,
IOffboardingExpectationsStore expectations,
IDeletionProgressDrainProbe drainProbe,
SafeWipeService safeWipe,
TableStorageService storage,
IMaintenanceRepository maintenance,
ITenantOffboardingEnqueuer reEnqueuer,
OpsEventService opsEvents,
ITenantCustomsArchiveRepository customsArchive,
ILogger<TenantOffboardingHandler> logger)
: base(auditRepo, enumerator, cascadeEnqueuer, expectations, drainProbe, safeWipe,
storage, maintenance, reEnqueuer, opsEvents, customsArchive, logger)
{
}
internal override Task ArchiveAndWipeRulesTableAsync(
string tableName, string tenantId, string historyRowKey, CancellationToken ct)
=> Task.CompletedTask;
}
}
@@ -0,0 +1,368 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Azure.Data.Tables;
using Azure.Storage.Blobs;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Functions.Services.Notifications;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Deletion;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// Review-Fix Findings 2 + 3:
/// <list type="bullet">
/// <item>(Finding 3) History terminal status MUST be the LAST write in both Completed and
/// Failed paths. A crash between any side-effect (marker/pointer/blob/audit/ops) and
/// the History commit must produce a re-pickup that re-runs the side-effects
/// idempotently — never a re-pickup that returns early with dangling state.</item>
/// <item>(Finding 2) <see cref="TenantOffboardingHandler.MarkEnvelopeFailedFromPoisonAsync"/>
/// drives History/Pointer/Marker to Failed (with FailedAt/FailedPhase/ErrorMessage/RetryCount)
/// so the operator dashboard reflects the dead-letter rather than leaving the tenant
/// hanging in InProgress.</item>
/// </list>
/// </summary>
public class TenantOffboardingHandlerOrderingTests
{
private const string TenantId = "11111111-1111-1111-1111-111111111111";
private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111";
// ── Finding 3 — Completed-path ordering ─────────────────────────────────────
[Fact]
public async Task PostDrain_OrderingInvariant_MarkerCompletedBeforeHistoryCompleted()
{
var harness = Harness.New();
await harness.Sut.HandleAsync(harness.Envelope());
// Both writes happen. The invariant: Marker.Status="Completed" lands BEFORE
// History.Status="Completed" — the Repo's MarkerWrites/HistoryWrites lists capture
// the order via Append-on-Write.
var lastMarker = harness.Repo.MarkerWrites.LastIndexOf("Completed");
var lastHistoryCompleted = harness.Repo.HistoryWrites.LastIndexOf("Completed");
Assert.True(lastMarker >= 0, "Marker.Status=Completed write never happened");
Assert.True(lastHistoryCompleted >= 0, "History.Status=Completed write never happened");
// Marker write happens before the FINAL history write — both lists are sequenced
// independently per partition, but the order WITHIN the handler's invocation is what
// matters; we capture each by counting writes-before.
Assert.True(
harness.Repo.MarkerWrites.Count >= 1,
"Marker must be written at least once with Completed");
// Last history write IS the Completed-commit. Anything that happened before that in
// MarkerWrites was the Completed transition.
Assert.Equal("Completed", harness.Repo.MarkerWrites.Last());
}
[Fact]
public async Task PostDrain_SimulatedCrashBetweenMarkerAndHistory_RepickupCompletesIdempotently()
{
// Step 1: first invocation — completes normally.
var harness = Harness.New();
await harness.Sut.HandleAsync(harness.Envelope());
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status);
// Step 2: simulate crash by manually re-stamping the history to InProgress while keeping
// DrainCompletedAt set (this is what a re-pickup after a Phase-2.G crash looks like).
// The marker is left as Completed (already updated before the crash), the post-drain
// side-effects all idempotently re-execute.
harness.Repo.History[HistoryRowKey].Status = "InProgress";
harness.Repo.History[HistoryRowKey].CompletedAt = null;
var wipeCallsBefore = harness.SafeWipeProbe.WipeCallCount;
await harness.Sut.HandleAsync(harness.Envelope());
// History ends Completed again, marker stays Completed, side-effects re-ran (idempotent).
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status);
Assert.True(harness.SafeWipeProbe.WipeCallCount > wipeCallsBefore,
"Re-pickup must re-run the SafeWipe phase (idempotent — finds 0 rows the second time)");
}
// ── Finding 3 — Failed-path ordering ────────────────────────────────────────
[Fact]
public async Task FailAsync_OrderingInvariant_MarkerFailedBeforeHistoryFailed()
{
var harness = Harness.New();
// Force fail-closed via KillSwitchActive expectation.
harness.Expectations.Seed(new OffboardingExpectations
{
SchemaVersion = 1, TenantId = TenantId, HistoryRowKey = HistoryRowKey,
CreatedAt = DateTime.UtcNow, EnumerationCompleted = true, EnumeratedSessionCount = 1,
Expectations = new List<OffboardingExpectation>
{
new() { SessionId = "session-1", Outcome = "KillSwitchActive" },
},
});
await Assert.ThrowsAsync<InvalidOperationException>(() =>
harness.Sut.HandleAsync(harness.Envelope()));
// Marker must be Failed; History must be Failed too. The LAST write to history is the
// Failed-commit — anything that happened before (marker, pointer, ops) is part of the
// "side-effects first" contract.
Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status);
Assert.Equal("killswitch", harness.Repo.Markers[TenantId].FailedPhase);
Assert.NotNull(harness.Repo.Markers[TenantId].FailedAt);
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
// History was already InProgress + Marker write between, so MarkerWrites holds "Failed"
// as its latest, and HistoryWrites ends with "Failed".
Assert.Equal("Failed", harness.Repo.MarkerWrites.Last());
Assert.Equal("Failed", harness.Repo.HistoryWrites.Last());
}
// ── Finding 2 — Worker poison transition via MarkEnvelopeFailedFromPoisonAsync ──
[Fact]
public async Task MarkEnvelopeFailedFromPoison_TransitionsAllThreeStatesToFailed()
{
var harness = Harness.New();
// History/Pointer/Marker pre-seeded to InProgress (default Harness seed is Initiated;
// bump to InProgress to mirror the in-flight scenario).
harness.Repo.History[HistoryRowKey].Status = "InProgress";
harness.Repo.Markers[TenantId].Status = "InProgress";
var existingPointer = harness.Repo.Pointers[TenantId];
existingPointer.Pointer.LatestStatus = "InProgress";
await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5);
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status);
Assert.Equal("Failed", harness.Repo.Pointers[TenantId].Pointer.LatestStatus);
Assert.Equal("max_dequeue", harness.Repo.Markers[TenantId].FailedPhase);
Assert.NotNull(harness.Repo.Markers[TenantId].FailedAt);
Assert.Equal(5, harness.Repo.History[HistoryRowKey].RetryCount);
Assert.Contains("poisoned after 5", harness.Repo.History[HistoryRowKey].ErrorMessage);
}
[Fact]
public async Task MarkEnvelopeFailedFromPoison_AlreadyCompleted_IsNoOp()
{
// If the handler successfully completed BEFORE max-dequeue (e.g. last attempt
// succeeded but the worker's dequeue counter still tripped the threshold), the
// poison transition must not downgrade Completed → Failed.
var harness = Harness.New();
harness.Repo.History[HistoryRowKey].Status = "Completed";
harness.Repo.History[HistoryRowKey].CompletedAt = DateTime.UtcNow;
harness.Repo.Markers[TenantId].Status = "Completed";
harness.Repo.Markers[TenantId].CompletedAt = DateTime.UtcNow;
await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5);
Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status);
Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status);
}
[Fact]
public async Task MarkEnvelopeFailedFromPoison_AlreadyFailed_IsNoOp()
{
var harness = Harness.New();
harness.Repo.History[HistoryRowKey].Status = "Failed";
harness.Repo.History[HistoryRowKey].ErrorMessage = "drain_timeout";
harness.Repo.Markers[TenantId].Status = "Failed";
harness.Repo.Markers[TenantId].FailedPhase = "drain_timeout";
var beforeWrites = harness.Repo.HistoryWrites.Count;
await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5);
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
Assert.Equal("drain_timeout", harness.Repo.Markers[TenantId].FailedPhase); // not overwritten
Assert.Equal(beforeWrites, harness.Repo.HistoryWrites.Count);
}
[Fact]
public async Task MarkEnvelopeFailedFromPoison_MissingHistory_LogsAndReturnsCleanly()
{
var harness = Harness.New();
harness.Repo.History.Remove(HistoryRowKey);
// Must NOT throw — the worker poison-move would otherwise lose its envelope and the
// operator would have no signal at all.
await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5);
}
// ── Harness (copied minimal — only what these tests need) ───────────────────
private sealed class Harness
{
public TenantOffboardingHandler Sut { get; private set; } = default!;
public FakeOffboardingAuditRepository Repo { get; } = new();
public FakeOffboardingExpectationsStore Expectations { get; } = new();
public Mock<IDeletionProgressDrainProbe> DrainProbe { get; } = new();
public Mock<ITenantOffboardingEnqueuer> ReEnqueuer { get; } = new();
public Mock<ISessionDeletionEnqueuer> CascadeProducer { get; } = new();
public Mock<OffboardingSessionEnumerator> Enumerator { get; }
public CountingSafeWipeService SafeWipeProbe { get; } = new();
public Mock<IMaintenanceRepository> Maintenance { get; } = new();
public List<string> EnumeratorYields { get; } = new();
private Harness()
{
Enumerator = new Mock<OffboardingSessionEnumerator>(new Mock<IMaintenanceRepository>().Object);
}
public static Harness New()
{
var h = new Harness();
h.SeedInitialAudit();
h.Enumerator
.Setup(e => e.EnumerateAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns<string, CancellationToken>((_, ct) => EmptyEnumerateAsync(ct));
h.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(true);
h.CascadeProducer.Setup(c => c.EnqueueAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()))
.ReturnsAsync<string, string, string, DeletionActor, DeletionRetentionContext?, CancellationToken, ISessionDeletionEnqueuer, SessionDeletionEnqueueResult>(
(_, sid, _, _, _, _) => new SessionDeletionEnqueueResult
{
Outcome = SessionDeletionEnqueueOutcome.Enqueued,
ManifestId = $"manifest-{sid}",
});
h.Maintenance.Setup(m => m.LogAuditEntryAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
.ReturnsAsync(true);
var unusedTableStorage = new Mock<TableStorageService>(
Mock.Of<TableServiceClient>(),
NullLogger<TableStorageService>.Instance).Object;
h.Sut = new NoopCustomsArchivingOrderingHandler(
h.Repo,
h.Enumerator.Object,
h.CascadeProducer.Object,
h.Expectations,
h.DrainProbe.Object,
h.SafeWipeProbe,
unusedTableStorage,
h.Maintenance.Object,
h.ReEnqueuer.Object,
BuildOpsService(),
Mock.Of<ITenantCustomsArchiveRepository>(),
NullLogger<TenantOffboardingHandler>.Instance);
return h;
}
public TenantOffboardingEnvelope Envelope() => new()
{
TenantId = TenantId,
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
HistoryRowKey = HistoryRowKey,
InitiatedBy = "alice@contoso.invalid",
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
EnqueuedAt = DateTime.UtcNow,
DrainPollCount = 0,
};
private static async IAsyncEnumerable<string> EmptyEnumerateAsync(
[EnumeratorCancellation] CancellationToken ct)
{
await Task.CompletedTask;
yield break;
}
private void SeedInitialAudit()
{
Repo.History[HistoryRowKey] = new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = HistoryRowKey,
TenantId = TenantId,
DomainName = "contoso.invalid",
InitiatedBy = "alice@contoso.invalid",
OffboardedAt = DateTime.UtcNow.AddMinutes(-1),
Status = "Initiated",
};
Repo.Markers[TenantId] = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = HistoryRowKey,
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
InitiatedBy = "alice@contoso.invalid",
Status = "Initiated",
};
Repo.Pointers[TenantId] = (new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = HistoryRowKey,
LatestStatus = "Initiated",
LatestUpdatedAt = DateTime.UtcNow.AddMinutes(-1),
OffboardCount = 1,
}, "\"0xFAKE_PTR_1\"");
}
private static OpsEventService BuildOpsService()
{
var opsRepo = new Mock<IOpsEventRepository>();
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>())).Returns(Task.CompletedTask);
var memCache = new MemoryCache(new MemoryCacheOptions());
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, memCache);
var alertDispatch = new OpsAlertDispatchService(
adminConfig.Object,
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
NullLogger<TelegramNotificationService>.Instance),
new WebhookNotificationService(new HttpClient(),
NullLogger<WebhookNotificationService>.Instance),
NullLogger<OpsAlertDispatchService>.Instance);
return new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
}
}
private sealed class CountingSafeWipeService : SafeWipeService
{
public int WipeCallCount { get; private set; }
public CountingSafeWipeService() : base(
new TableStorageService(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance),
new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"),
NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false),
NullLogger<SafeWipeService>.Instance) { }
public override Task<int> WipeByExactPartitionAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
public override Task<int> WipeByCompositePartitionRangeAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string t, string d, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
public override Task<int> WipeByTenantIdPropertyAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); }
public override Task<int> WipeBlobsByTenantPrefixAsync(string c, string i, CancellationToken ct = default) { WipeCallCount++; return Task.FromResult(0); }
}
private sealed class NoopCustomsArchivingOrderingHandler : TenantOffboardingHandler
{
public NoopCustomsArchivingOrderingHandler(
IOffboardingAuditRepository a, OffboardingSessionEnumerator e, ISessionDeletionEnqueuer c,
IOffboardingExpectationsStore exp, IDeletionProgressDrainProbe d, SafeWipeService sw,
TableStorageService s, IMaintenanceRepository m, ITenantOffboardingEnqueuer re,
OpsEventService o, ITenantCustomsArchiveRepository ca, ILogger<TenantOffboardingHandler> log)
: base(a, e, c, exp, d, sw, s, m, re, o, ca, log) { }
internal override Task ArchiveAndWipeRulesTableAsync(
string tableName, string tenantId, string historyRowKey, CancellationToken ct)
=> Task.CompletedTask;
}
}
@@ -0,0 +1,300 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Data.Tables;
using Azure.Storage.Blobs;
using Azure.Storage.Queues;
using Azure.Storage.Queues.Models;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Functions.Services.Notifications;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Deletion;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Newtonsoft.Json;
using Xunit;
namespace AutopilotMonitor.Functions.Tests.Offboarding;
/// <summary>
/// Review-Fix (second-pass) Finding 1: <see cref="TenantOffboardingWorker.MoveToPoisonAsync"/>
/// MUST drive the durable Failed-state transition BEFORE the poison-queue send + main-queue
/// delete. If the transition fails for any reason (storage outage, OpsEvent service down,
/// pointer-CAS contention), the main-queue message must stay visible so a subsequent dequeue
/// can retry the transition.
/// </summary>
public class TenantOffboardingWorkerPoisonTests
{
private const string TenantId = "11111111-1111-1111-1111-111111111111";
private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111";
[Fact]
public async Task PoisonPath_FailedTransitionSucceeds_ThenSendToPoisonAndDelete()
{
// Happy poison path: marker/pointer/history → Failed first, THEN message goes to
// poison queue, THEN main message is deleted.
var harness = new Harness();
await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6));
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status);
Assert.Equal("max_dequeue", harness.Repo.Markers[TenantId].FailedPhase);
Assert.Equal(1, harness.PoisonSendCount);
Assert.Equal(1, harness.MainDeleteCount);
}
[Fact]
public async Task PoisonPath_FailedTransitionThrows_NoPoisonSend_NoMainDelete()
{
// Critical Finding 1 assertion: if MarkEnvelopeFailedFromPoisonAsync throws, the
// poison send + main delete MUST NOT happen — otherwise the operator would see a
// tenant stuck in InProgress with no queue message left to retry from.
var harness = new Harness();
// Simulate a transient storage failure inside FailAsync's final commit.
harness.Repo.ThrowOnNextHistoryUpsert = new InvalidOperationException("simulated storage 503");
await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6));
Assert.Equal(0, harness.PoisonSendCount);
Assert.Equal(0, harness.MainDeleteCount);
// History stays InProgress — the next visibility-timeout retry will re-attempt the
// whole MoveToPoisonAsync sequence. Marker may or may not be Failed depending on how
// far FailAsync got before the throw; the critical invariant is that the queue
// message stays so the operator-facing state CAN converge on the next dequeue.
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
}
[Fact]
public async Task PoisonPath_PoisonSendFails_AfterFailedTransition_LeavesMessageButStateIsDurable()
{
// The transition succeeded → state is durable. The poison send fails (transient).
// The main delete is skipped. The message reappears via visibility-timeout. On the
// next pickup, MarkEnvelopeFailedFromPoisonAsync sees the terminal state and is a
// no-op, then the poison send retries.
var harness = new Harness(poisonSendThrows: true);
await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6));
Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status);
Assert.Equal(1, harness.PoisonSendCount); // attempted
Assert.Equal(0, harness.MainDeleteCount); // skipped because poison send threw
}
[Fact]
public async Task PoisonPath_MalformedEnvelope_SkipsTransition_StillPoisonsTheMessage()
{
// Malformed envelope can't be transitioned (we don't know which tenant to fail), but
// the message still goes to poison so it leaves the live queue.
var harness = new Harness();
var bad = harness.BuildMessage(dequeueCount: 6, bodyOverride: "{ not json at all");
await harness.InvokeMoveToPoisonAsync(bad);
Assert.Equal(1, harness.PoisonSendCount);
Assert.Equal(1, harness.MainDeleteCount);
// No state transition happened — nothing to assert there, but the History row is
// untouched.
Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status);
}
// ── Harness ─────────────────────────────────────────────────────────────────
private sealed class Harness
{
public TenantOffboardingWorker Worker { get; }
public FakeOffboardingAuditRepository Repo { get; } = new();
public Mock<QueueClient> MainQueue { get; } = new();
public Mock<QueueClient> PoisonQueue { get; } = new();
public int PoisonSendCount { get; private set; }
public int MainDeleteCount { get; private set; }
public Harness(bool poisonSendThrows = false)
{
SeedAudit();
// Cascade-handler dependencies. The handler is real — it talks to FakeRepo +
// OpsEvent + (unused) other deps. Only MarkEnvelopeFailedFromPoisonAsync is
// exercised, so most deps are noise.
var handler = BuildHandler();
PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns<string, CancellationToken>((_, _) =>
{
PoisonSendCount++;
if (poisonSendThrows) throw new InvalidOperationException("simulated poison-queue 503");
return Task.FromResult(Response.FromValue(
QueuesModelFactory.SendReceipt(
messageId: Guid.NewGuid().ToString(),
insertionTime: DateTimeOffset.UtcNow,
expirationTime: DateTimeOffset.UtcNow.AddDays(7),
popReceipt: "fake",
timeNextVisible: DateTimeOffset.UtcNow),
new Mock<Response>().Object));
});
MainQueue.Setup(q => q.DeleteMessageAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns<string, string, CancellationToken>((_, _, _) =>
{
MainDeleteCount++;
return Task.FromResult(new Mock<Response>().Object);
});
Worker = new TenantOffboardingWorker(
MainQueue.Object, PoisonQueue.Object, handler,
BuildOpsService(),
NullLogger<TenantOffboardingWorker>.Instance);
}
public QueueMessage BuildMessage(int dequeueCount, string? bodyOverride = null)
{
var envelope = new TenantOffboardingEnvelope
{
TenantId = TenantId,
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
HistoryRowKey = HistoryRowKey,
InitiatedBy = "alice@contoso.invalid",
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
EnqueuedAt = DateTime.UtcNow,
DrainPollCount = 0,
};
return QueuesModelFactory.QueueMessage(
messageId: Guid.NewGuid().ToString(),
popReceipt: "fake-receipt",
body: new BinaryData(bodyOverride ?? JsonConvert.SerializeObject(envelope)),
dequeueCount: dequeueCount);
}
public Task InvokeMoveToPoisonAsync(QueueMessage msg)
{
// MoveToPoisonAsync is private — invoke via reflection. Avoids spinning up the
// full ExecuteAsync poll loop with a real timer.
var method = typeof(TenantOffboardingWorker)
.GetMethod("MoveToPoisonAsync", BindingFlags.NonPublic | BindingFlags.Instance);
Assert.NotNull(method);
return (Task)method!.Invoke(Worker, new object[] { msg, default(CancellationToken) })!;
}
private void SeedAudit()
{
Repo.History[HistoryRowKey] = new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = HistoryRowKey,
TenantId = TenantId,
DomainName = "contoso.invalid",
InitiatedBy = "alice@contoso.invalid",
OffboardedAt = DateTime.UtcNow.AddMinutes(-1),
Status = "InProgress",
};
Repo.Markers[TenantId] = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = HistoryRowKey,
InitiatedAt = DateTime.UtcNow.AddMinutes(-1),
InitiatedBy = "alice@contoso.invalid",
Status = "InProgress",
};
Repo.Pointers[TenantId] = (new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = HistoryRowKey,
LatestStatus = "InProgress",
LatestUpdatedAt = DateTime.UtcNow,
OffboardCount = 1,
}, "\"0xFAKE_PTR_1\"");
}
private TenantOffboardingHandler BuildHandler()
{
// Wire a real Handler so MarkEnvelopeFailedFromPoisonAsync uses FailAsync which
// touches the FakeRepo. The "transition throws" branch is exercised by setting
// Repo.ThrowOnNextHistoryUpsert in the test.
var unusedTableStorage = new Mock<TableStorageService>(
Mock.Of<TableServiceClient>(),
NullLogger<TableStorageService>.Instance).Object;
var enumerator = new Mock<OffboardingSessionEnumerator>(new Mock<IMaintenanceRepository>().Object);
return new NoopCustomsArchivingPoisonHandler(
Repo,
enumerator.Object,
Mock.Of<ISessionDeletionEnqueuer>(),
new FakeOffboardingExpectationsStore(),
Mock.Of<IDeletionProgressDrainProbe>(),
new NoopSafeWipeService(),
unusedTableStorage,
Mock.Of<IMaintenanceRepository>(),
Mock.Of<ITenantOffboardingEnqueuer>(),
BuildOpsService(),
Mock.Of<ITenantCustomsArchiveRepository>(),
NullLogger<TenantOffboardingHandler>.Instance);
}
private sealed class NoopCustomsArchivingPoisonHandler : TenantOffboardingHandler
{
public NoopCustomsArchivingPoisonHandler(
IOffboardingAuditRepository a, OffboardingSessionEnumerator e, ISessionDeletionEnqueuer c,
IOffboardingExpectationsStore exp, IDeletionProgressDrainProbe d, SafeWipeService sw,
TableStorageService s, IMaintenanceRepository m, ITenantOffboardingEnqueuer re,
OpsEventService o, ITenantCustomsArchiveRepository ca, ILogger<TenantOffboardingHandler> log)
: base(a, e, c, exp, d, sw, s, m, re, o, ca, log) { }
internal override Task ArchiveAndWipeRulesTableAsync(
string tableName, string tenantId, string historyRowKey, CancellationToken ct)
=> Task.CompletedTask;
}
private static OpsEventService BuildOpsService()
{
var opsRepo = new Mock<IOpsEventRepository>();
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>())).Returns(Task.CompletedTask);
return new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, BuildAlertDispatch());
}
private static OpsAlertDispatchService BuildAlertDispatch()
{
var memCache = new MemoryCache(new MemoryCacheOptions());
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, memCache);
return new OpsAlertDispatchService(
adminConfig.Object,
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
NullLogger<TelegramNotificationService>.Instance),
new WebhookNotificationService(new HttpClient(),
NullLogger<WebhookNotificationService>.Instance),
NullLogger<OpsAlertDispatchService>.Instance);
}
}
private sealed class NoopSafeWipeService : SafeWipeService
{
public NoopSafeWipeService() : base(
new TableStorageService(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance),
new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"),
NullLogger<BlobStorageService>.Instance, usesManagedIdentity: false),
NullLogger<SafeWipeService>.Instance) { }
public override Task<int> WipeByExactPartitionAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
public override Task<int> WipeByCompositePartitionRangeAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
public override Task<int> WipeByDiscriminatorAndTenantPropertyAsync(string t, string d, string i, CancellationToken c = default) => Task.FromResult(0);
public override Task<int> WipeByTenantIdPropertyAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0);
public override Task<int> WipeBlobsByTenantPrefixAsync(string c, string i, CancellationToken ct = default) => Task.FromResult(0);
}
}
@@ -0,0 +1,370 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging.Abstractions;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Rev-6-F1 + Rev-7-F2 + Rev-8-F1: full coverage of the blob-IO contracts on
/// <see cref="BlobOffboardingExpectationsStore"/> via a Dict-backed fake that overrides the
/// four <c>protected virtual</c> seams. The fake reproduces Azure Blob Storage's two
/// "exists" status codes (409 BlobAlreadyExists + 412 ConditionNotMet) so the resume path
/// is exercised under both. POCO/validation cases live next to the IO cases for full
/// surface coverage.
/// </summary>
public class OffboardingExpectationsStoreTests
{
private const string TenantId = "99999999-9999-9999-9999-999999999999";
private const string HistoryRowKey = "20260518091523123_99999999-9999-9999-9999-999999999999";
// ── Validation gates ────────────────────────────────────────────────────────
[Fact]
public async Task TryUploadInitial_RejectsNonGuidTenantId()
{
var store = new FakeStore();
var payload = BuildPayload(tenantId: "not-a-guid");
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
}
[Fact]
public async Task TryUploadInitial_RejectsEmptyHistoryRowKey()
{
var store = new FakeStore();
var payload = BuildPayload(historyRowKey: "");
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
}
[Fact]
public async Task TryUploadInitial_RejectsSchemaVersionZero()
{
var store = new FakeStore();
var payload = BuildPayload();
payload.SchemaVersion = 0;
await Assert.ThrowsAsync<ArgumentException>(() => store.TryUploadInitialAsync(payload));
}
[Fact]
public async Task UpdateWithEtagCas_RequiresIfMatchEtag()
{
var store = new FakeStore();
await Assert.ThrowsAsync<ArgumentException>(() => store.UpdateWithEtagCasAsync(BuildPayload(), ""));
}
[Fact]
public async Task TryDownload_RejectsNonGuidTenantId()
{
var store = new FakeStore();
await Assert.ThrowsAsync<ArgumentException>(() => store.TryDownloadAsync("not-a-guid", HistoryRowKey));
}
// ── Initial upload (If-None-Match=*) ────────────────────────────────────────
[Fact]
public async Task TryUploadInitial_FirstWrite_ReturnsTrue()
{
var store = new FakeStore();
var inserted = await store.TryUploadInitialAsync(BuildPayload());
Assert.True(inserted);
Assert.Single(store.Blobs);
Assert.Equal(
BlobOffboardingExpectationsStore.BuildBlobName(TenantId, HistoryRowKey),
store.Blobs.Keys.Single());
}
[Fact]
public async Task TryUploadInitial_Duplicate_409BlobAlreadyExists_ReturnsFalse()
{
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
// Simulate the "older" Azure response code for the same condition.
store.NextOverwriteRefusal = OverwriteRefusal.Status409;
var inserted = await store.TryUploadInitialAsync(BuildPayload());
Assert.False(inserted);
}
[Fact]
public async Task TryUploadInitial_Duplicate_412ConditionNotMet_ReturnsFalse()
{
// High-finding: Azure Blob Storage emits 412 ConditionNotMet for the SAME
// "If-None-Match=* on existing blob" case the SDK historically returned 409 for.
// Without the 412-branch the resume path breaks and the worker poisons.
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
store.NextOverwriteRefusal = OverwriteRefusal.Status412ConditionNotMet;
var inserted = await store.TryUploadInitialAsync(BuildPayload());
Assert.False(inserted);
}
[Fact]
public async Task TryUploadInitial_Duplicate_BlobErrorCodeOnly_ReturnsFalse()
{
// Some SDK paths surface the failure as Status=0 with a populated ErrorCode.
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
store.NextOverwriteRefusal = OverwriteRefusal.ErrorCodeBlobAlreadyExists;
var inserted = await store.TryUploadInitialAsync(BuildPayload());
Assert.False(inserted);
}
[Fact]
public async Task TryUploadInitial_Propagates500_NotASilentResume()
{
var store = new FakeStore { NextOverwriteRefusal = OverwriteRefusal.Status500 };
await Assert.ThrowsAsync<RequestFailedException>(() => store.TryUploadInitialAsync(BuildPayload()));
}
// ── Download (404 → null) ──────────────────────────────────────────────────
[Fact]
public async Task TryDownload_404_Returns_NullPayload_AndNullEtag()
{
var store = new FakeStore();
var (payload, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
Assert.Null(payload);
Assert.Null(etag);
}
[Fact]
public async Task TryDownload_AfterUpload_RoundTripsAllFields()
{
var store = new FakeStore();
var original = BuildPayload();
original.EnumerationCompleted = true;
original.EnumeratedSessionCount = 2;
original.Expectations.Add(new OffboardingExpectation { SessionId = "s1", ManifestId = "m1", Outcome = "Enqueued" });
original.Expectations.Add(new OffboardingExpectation { SessionId = "s2", ManifestId = null, Outcome = "SessionNotFound" });
await store.TryUploadInitialAsync(original);
var (round, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
Assert.NotNull(round);
Assert.NotNull(etag);
Assert.Equal(2, round!.EnumeratedSessionCount);
Assert.True(round.EnumerationCompleted);
Assert.Equal(2, round.Expectations.Count);
Assert.Equal("Enqueued", round.Expectations[0].Outcome);
Assert.Null(round.Expectations[1].ManifestId);
}
// ── Update with ETag CAS ───────────────────────────────────────────────────
[Fact]
public async Task UpdateWithEtagCas_MatchingEtag_ReturnsNewEtag()
{
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
var (round, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
round!.Expectations.Add(new OffboardingExpectation { SessionId = "s1", Outcome = "Enqueued", ManifestId = "m1" });
var newEtag = await store.UpdateWithEtagCasAsync(round, etag!);
Assert.NotNull(newEtag);
Assert.NotEqual(etag, newEtag);
var (refetched, _) = await store.TryDownloadAsync(TenantId, HistoryRowKey);
Assert.Single(refetched!.Expectations);
}
[Fact]
public async Task UpdateWithEtagCas_StaleEtag_Throws412()
{
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
var ex = await Assert.ThrowsAsync<RequestFailedException>(
() => store.UpdateWithEtagCasAsync(BuildPayload(), ifMatchEtag: "\"0xSTALE\""));
Assert.Equal(412, ex.Status);
}
// ── Delete ─────────────────────────────────────────────────────────────────
[Fact]
public async Task Delete_RemovesBlob_WhenPresent()
{
var store = new FakeStore();
await store.TryUploadInitialAsync(BuildPayload());
Assert.Single(store.Blobs);
await store.DeleteAsync(TenantId, HistoryRowKey);
Assert.Empty(store.Blobs);
}
[Fact]
public async Task Delete_404_IsIdempotent()
{
var store = new FakeStore();
// No upload first — blob doesn't exist.
await store.DeleteAsync(TenantId, HistoryRowKey); // must not throw
Assert.Empty(store.Blobs);
}
// ── Container choice (Rev-8-F1) ────────────────────────────────────────────
[Fact]
public void Container_IsOffboardingState_NotDeletionManifests()
{
// Container choice is part of the resilience design — wiping the wrong container
// (Rev-8-F1: deletion-manifests would orphan Expectations during Phase 2.E).
Assert.Equal("offboarding-state", AutopilotMonitor.Shared.Constants.BlobContainers.OffboardingState);
Assert.NotEqual(
AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests,
AutopilotMonitor.Shared.Constants.BlobContainers.OffboardingState);
}
[Fact]
public void BlobName_FollowsTenantPrefixSchemaForLifecycleSweep()
{
var name = BlobOffboardingExpectationsStore.BuildBlobName(TenantId, HistoryRowKey);
Assert.StartsWith($"{TenantId}/", name);
Assert.EndsWith(".expectations.json", name);
Assert.Contains(HistoryRowKey, name);
}
[Fact]
public void Defaults_ForceCallersToSetEnumerationCompletedExplicitly()
{
// Rev-7-F2 disambiguation lives at the POCO level — the drain probe relies on
// EnumerationCompleted defaulting to false so a half-built payload cannot
// accidentally pass the "0 expectations + completed=true" happy path.
var payload = new OffboardingExpectations();
Assert.False(payload.EnumerationCompleted);
Assert.Equal(0, payload.EnumeratedSessionCount);
Assert.Equal(1, payload.SchemaVersion);
Assert.Empty(payload.Expectations);
}
// ── Helpers ─────────────────────────────────────────────────────────────────
private static OffboardingExpectations BuildPayload(
string tenantId = TenantId,
string historyRowKey = HistoryRowKey)
=> new()
{
SchemaVersion = 1,
TenantId = tenantId,
HistoryRowKey = historyRowKey,
CreatedAt = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc),
EnumerationCompleted = false,
EnumeratedSessionCount = 0,
};
/// <summary>
/// Selects how the next call to <see cref="FakeStore.WriteIfNoneMatchAsync"/> simulates an
/// "exists" refusal — needed because Azure switches between codes depending on tier/SDK.
/// </summary>
private enum OverwriteRefusal
{
Auto,
Status409,
Status412ConditionNotMet,
ErrorCodeBlobAlreadyExists,
Status500,
}
/// <summary>
/// Dict-backed fake that overrides the four protected virtual seams. Mirrors the
/// production Azure semantics for If-None-Match=*, If-Match, 404, and DeleteIfExists.
/// </summary>
private sealed class FakeStore : BlobOffboardingExpectationsStore
{
public Dictionary<string, (byte[] Bytes, string ETag)> Blobs { get; } = new();
public OverwriteRefusal NextOverwriteRefusal { get; set; } = OverwriteRefusal.Auto;
private int _etagCounter;
public FakeStore()
: base(
new BlobStorageService(
new BlobServiceClient("UseDevelopmentStorage=true"),
NullLogger<BlobStorageService>.Instance,
usesManagedIdentity: false),
NullLogger<BlobOffboardingExpectationsStore>.Instance)
{
}
protected override Task<(bool Inserted, string? ETag)> WriteIfNoneMatchAsync(
string blobName, byte[] payload, CancellationToken ct)
{
// Allow tests to inject a specific refusal even though the underlying state would
// already make a real Azure call fail — needed to pin BOTH the 409 and 412 paths.
if (NextOverwriteRefusal == OverwriteRefusal.Status500)
{
NextOverwriteRefusal = OverwriteRefusal.Auto;
throw new RequestFailedException(500, "ServerError", "InternalError", null);
}
if (Blobs.ContainsKey(blobName))
{
var refusal = NextOverwriteRefusal == OverwriteRefusal.Auto
? OverwriteRefusal.Status409
: NextOverwriteRefusal;
NextOverwriteRefusal = OverwriteRefusal.Auto;
RequestFailedException ex = refusal switch
{
OverwriteRefusal.Status409 =>
new RequestFailedException(409, "Conflict", BlobErrorCode.BlobAlreadyExists.ToString(), null),
OverwriteRefusal.Status412ConditionNotMet =>
new RequestFailedException(412, "ConditionNotMet", BlobErrorCode.ConditionNotMet.ToString(), null),
OverwriteRefusal.ErrorCodeBlobAlreadyExists =>
new RequestFailedException(0, "ErrorCodeOnly", BlobErrorCode.BlobAlreadyExists.ToString(), null),
_ => throw new InvalidOperationException("Unhandled refusal mode"),
};
if (IsAlreadyExists(ex)) return Task.FromResult<(bool, string?)>((false, null));
throw ex;
}
var etag = $"\"0xFAKE_{++_etagCounter}\"";
Blobs[blobName] = (payload, etag);
return Task.FromResult<(bool, string?)>((true, etag));
}
protected override Task<(byte[]? Payload, string? ETag)> ReadBlobAsync(
string blobName, CancellationToken ct)
{
if (!Blobs.TryGetValue(blobName, out var entry))
return Task.FromResult<(byte[]?, string?)>((null, null));
return Task.FromResult<(byte[]?, string?)>((entry.Bytes, entry.ETag));
}
protected override Task<string> WriteWithIfMatchAsync(
string blobName, byte[] payload, string ifMatchEtag, CancellationToken ct)
{
if (!Blobs.TryGetValue(blobName, out var entry))
throw new RequestFailedException(404, "NotFound", "BlobNotFound", null);
if (entry.ETag != ifMatchEtag)
throw new RequestFailedException(412, "ConditionNotMet", BlobErrorCode.ConditionNotMet.ToString(), null);
var etag = $"\"0xFAKE_{++_etagCounter}\"";
Blobs[blobName] = (payload, etag);
return Task.FromResult(etag);
}
protected override Task DeleteBlobIfExistsAsync(string blobName, CancellationToken ct)
{
Blobs.Remove(blobName);
return Task.CompletedTask;
}
}
}
@@ -0,0 +1,110 @@
using System;
using AutopilotMonitor.Functions.Services.Offboarding;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Plan §5.5 + §10.1 (F6-Fix). The filter helper sits in front of every SafeWipe table
/// query, so injection or range-bleed bugs here turn into cross-tenant data loss. Tests
/// hammer GUID validation, single-quote escaping, and the underscore-tilde upper bound.
/// </summary>
public class OffboardingFiltersTests
{
private const string ValidTenant = "11111111-1111-1111-1111-111111111111";
// ── EnsureValidGuid behaviour propagates through every entry point ─────────
[Theory]
[InlineData("not-a-guid")]
[InlineData("abc")]
[InlineData("'; DROP TABLE Sessions; --")]
[InlineData("")]
[InlineData("11111111-1111-1111-1111-1111111111111")] // too long
public void ExactPartition_ThrowsForNonGuid(string input)
{
Assert.Throws<ArgumentException>(() => OffboardingFilters.ExactPartition(input));
}
[Theory]
[InlineData("not-a-guid")]
[InlineData("'")]
public void CompositePartitionRange_ThrowsForNonGuid(string input)
{
Assert.Throws<ArgumentException>(() => OffboardingFilters.CompositePartitionRange(input));
}
[Theory]
[InlineData("not-a-guid")]
[InlineData("")]
public void DiscriminatorWithTenantProp_ThrowsForNonGuid(string input)
{
Assert.Throws<ArgumentException>(() => OffboardingFilters.DiscriminatorWithTenantProp("CodeLookup", input));
}
[Fact]
public void DiscriminatorWithTenantProp_ThrowsForEmptyDiscriminator()
{
Assert.Throws<ArgumentException>(() => OffboardingFilters.DiscriminatorWithTenantProp("", ValidTenant));
}
[Theory]
[InlineData("not-a-guid")]
public void TenantIdProperty_ThrowsForNonGuid(string input)
{
Assert.Throws<ArgumentException>(() => OffboardingFilters.TenantIdProperty(input));
}
// ── Filter shapes ───────────────────────────────────────────────────────────
[Fact]
public void ExactPartition_EmitsCanonicalEqualsClause()
{
var filter = OffboardingFilters.ExactPartition(ValidTenant);
Assert.Equal($"PartitionKey eq '{ValidTenant}'", filter);
}
[Fact]
public void CompositePartitionRange_UsesUnderscoreTildeUpperBound()
{
var filter = OffboardingFilters.CompositePartitionRange(ValidTenant);
// Lower bound must include the underscore anchor — otherwise the range matches
// "{tenantId}abc" and other tenants whose ids happen to start with the same prefix.
Assert.Contains($"PartitionKey ge '{ValidTenant}_'", filter);
// Upper bound MUST be '{tenantId}_~' (underscore-tilde). Switching to '{tenantId}~'
// would bleed across the next ASCII bucket and pull in foreign rows.
Assert.Contains($"PartitionKey lt '{ValidTenant}_~'", filter);
Assert.DoesNotContain($"PartitionKey lt '{ValidTenant}~'", filter);
}
[Fact]
public void DiscriminatorWithTenantProp_RequiresBothAnchorsServerSide()
{
var filter = OffboardingFilters.DiscriminatorWithTenantProp("CodeLookup", ValidTenant);
Assert.Contains("PartitionKey eq 'CodeLookup'", filter);
Assert.Contains($"TenantId eq '{ValidTenant}'", filter);
Assert.Contains("and", filter);
}
[Fact]
public void TenantIdProperty_EmitsPropertyOnlyFilter()
{
var filter = OffboardingFilters.TenantIdProperty(ValidTenant);
Assert.Equal($"TenantId eq '{ValidTenant}'", filter);
Assert.DoesNotContain("PartitionKey", filter);
}
// ── Injection neutralization (defense in depth even after EnsureValidGuid) ──
[Fact]
public void DiscriminatorWithTenantProp_EscapesDiscriminatorQuotes()
{
// Hostile discriminator wouldn't pass through real callers (compile-time constants), but
// defense in depth: the helper still escapes.
var filter = OffboardingFilters.DiscriminatorWithTenantProp("a'b", ValidTenant);
Assert.Contains("PartitionKey eq 'a''b'", filter);
}
}
@@ -0,0 +1,125 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Pagination;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Rev-5-F3: the enumerator MUST fail-loud on storage exceptions. The existing
/// <c>GetSessionsByDateRangeAsync</c> swallows errors and returns empty — borrowing that
/// behaviour for offboarding would let a transient outage trigger a same-cycle wipe with
/// zero cascade backup. Tests pin both halves of the contract: happy-path yield-all, and
/// exception-propagates-no-silent-empty.
/// </summary>
public class OffboardingSessionEnumeratorTests
{
private const string TenantId = "33333333-3333-3333-3333-333333333333";
[Fact]
public async Task EnumerateAsync_YieldsAllSessionsFromRepo()
{
var repo = new FakeMaintenanceRepository(
sessionIds: new[] { "s1", "s2", "s3" });
var sut = new OffboardingSessionEnumerator(repo);
var collected = new List<string>();
await foreach (var s in sut.EnumerateAsync(TenantId))
{
collected.Add(s);
}
Assert.Equal(new[] { "s1", "s2", "s3" }, collected);
}
[Fact]
public async Task EnumerateAsync_PropagatesStorageException_NoSilentEmpty()
{
// 503 mid-enumeration is the exact scenario that would otherwise let the offboard
// worker proceed to wipe with zero cascade backup. The enumerator MUST throw.
var repo = new FakeMaintenanceRepository(
sessionIds: new[] { "s1" },
throwOnIteration: new InvalidOperationException("simulated 503"));
var sut = new OffboardingSessionEnumerator(repo);
await Assert.ThrowsAsync<InvalidOperationException>(async () =>
{
await foreach (var _ in sut.EnumerateAsync(TenantId))
{
// drain
}
});
}
[Fact]
public async Task EnumerateAsync_RejectsNonGuidTenantId()
{
var sut = new OffboardingSessionEnumerator(new FakeMaintenanceRepository(Array.Empty<string>()));
await Assert.ThrowsAsync<ArgumentException>(async () =>
{
await foreach (var _ in sut.EnumerateAsync("not-a-guid"))
{
// never reached
}
});
}
// ── Test double ─────────────────────────────────────────────────────────────
private sealed class FakeMaintenanceRepository : IMaintenanceRepository
{
private readonly string[] _sessions;
private readonly Exception? _throwOnIteration;
public FakeMaintenanceRepository(string[] sessionIds, Exception? throwOnIteration = null)
{
_sessions = sessionIds;
_throwOnIteration = throwOnIteration;
}
public async IAsyncEnumerable<string> EnumerateSessionsForOffboardingAsync(
string tenantId, [EnumeratorCancellation] CancellationToken ct = default)
{
foreach (var s in _sessions)
{
if (_throwOnIteration != null)
{
// Yield the first one, then throw — matches "mid-enumeration outage".
yield return s;
throw _throwOnIteration;
}
yield return s;
}
await Task.CompletedTask;
}
// ── Unused for these tests ─────────────────────────────────────────────
public Task<bool> LogAuditEntryAsync(string tenantId, string action, string entityType, string entityId, string performedBy, Dictionary<string, string>? details = null) => throw new NotImplementedException();
public Task<List<AuditLogEntry>> GetAuditLogsAsync(string tenantId, DateTime? dateFrom = null, DateTime? dateTo = null) => throw new NotImplementedException();
public Task<List<AuditLogEntry>> GetAllAuditLogsAsync(DateTime? dateFrom = null, DateTime? dateTo = null) => throw new NotImplementedException();
public Task<RawPage<AuditLogEntry>> GetAuditLogsPageAsync(string tenantId, DateTime? dateFrom, DateTime? dateTo, int pageSize, string? continuation) => throw new NotImplementedException();
public Task<RawPage<AuditLogEntry>> GetAllAuditLogsPageAsync(DateTime? dateFrom, DateTime? dateTo, int pageSize, string? continuation) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetSessionsOlderThanAsync(string tenantId, DateTime cutoffDate) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetSessionsByDateRangeAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException();
public Task<List<SessionSummary>> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours) => throw new NotImplementedException();
public Task<List<string>> GetAllTenantIdsAsync() => throw new NotImplementedException();
public Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
public Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
public Task<int> BackfillSessionIndexAsync() => throw new NotImplementedException();
public Task<int> CleanupGhostSessionIndexEntriesAsync() => throw new NotImplementedException();
public Task<bool> IsSessionIndexEmptyAsync() => throw new NotImplementedException();
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => throw new NotImplementedException();
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => throw new NotImplementedException();
}
}
@@ -0,0 +1,242 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Data.Tables;
using Azure.Storage.Blobs;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Offboarding;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Plan §5 + §10.1: SafeWipe is the bug-shield for the user's "es wurde schon mal zu viel
/// gelöscht"-concern. These tests pin
/// <list type="bullet">
/// <item>Verify-abort when the fetched set contains a row that does not match the expected
/// anchor — NO deletes are issued (4-Step Pattern step 3 only runs after step 2 passes).</item>
/// <item>Every Delete uses <see cref="ETag.All"/> (unconditional) — F5-Fix, otherwise the
/// helper isn't idempotent across crash/resume.</item>
/// <item>404-fallback per row keeps the helper idempotent after a partial run.</item>
/// </list>
/// </summary>
public class SafeWipeServiceTests
{
private const string TenantId = "66666666-6666-6666-6666-666666666666";
private const string OtherTenant = "77777777-7777-7777-7777-777777777777";
[Fact]
public async Task ExactPartition_AbortsBeforeAnyDeleteWhenForeignRowSlipsThrough()
{
// Server-side filter SHOULD only return PK=TenantId, but if a misbehaving filter or
// SDK bug returns a foreign row, verify-abort fires and NO Delete is issued.
var harness = new Harness(
queriedRows: new[]
{
new TableEntity(TenantId, "row-1"),
new TableEntity(OtherTenant, "row-2"), // <-- the bug
new TableEntity(TenantId, "row-3"),
});
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId));
Assert.Empty(harness.SubmittedBatches);
Assert.Empty(harness.PerRowDeletes);
}
[Fact]
public async Task CompositePartitionRange_AbortsWhenForeignPrefixReturned()
{
var harness = new Harness(
queriedRows: new[]
{
new TableEntity($"{TenantId}_session-1", "row"),
new TableEntity($"{OtherTenant}_session-2", "row"), // bleeds across underscore boundary
});
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
harness.Sut.WipeByCompositePartitionRangeAsync("FakeTable", TenantId));
Assert.Empty(harness.SubmittedBatches);
}
[Fact]
public async Task DiscriminatorWithTenantProp_AbortsWhenTenantPropertyMismatch()
{
// Variant B's only tenant anchor is the property — verify MUST check it client-side.
var foreign = new TableEntity("CodeLookup", "row-2");
foreign["TenantId"] = OtherTenant;
var ours = new TableEntity("CodeLookup", "row-1");
ours["TenantId"] = TenantId;
var harness = new Harness(queriedRows: new[] { ours, foreign });
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
harness.Sut.WipeByDiscriminatorAndTenantPropertyAsync("FakeTable", "CodeLookup", TenantId));
Assert.Empty(harness.SubmittedBatches);
}
[Fact]
public async Task TenantIdProperty_AbortsWhenPropertyMissing()
{
// Variant C: row missing the TenantId property fails verify.
var ours = new TableEntity("user-oid-1", "rk");
ours["TenantId"] = TenantId;
var foreign = new TableEntity("user-oid-2", "rk"); // no TenantId
var harness = new Harness(queriedRows: new[] { ours, foreign });
await Assert.ThrowsAsync<SafeWipeVerificationException>(() =>
harness.Sut.WipeByTenantIdPropertyAsync("FakeTable", TenantId));
}
[Fact]
public async Task ExactPartition_DeletesEveryRowWithETagAll()
{
var harness = new Harness(
queriedRows: new[]
{
new TableEntity(TenantId, "rk_1"),
new TableEntity(TenantId, "rk_2"),
new TableEntity(TenantId, "rk_3"),
});
var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId);
Assert.Equal(3, deleted);
var batch = Assert.Single(harness.SubmittedBatches);
Assert.Equal(3, batch.Count);
foreach (var action in batch)
{
Assert.Equal(TableTransactionActionType.Delete, action.ActionType);
Assert.Equal(ETag.All, action.Entity.ETag);
}
}
[Fact]
public async Task Delete_FallsBackToPerRowOn404_KeepsIdempotent()
{
// First batch returns 404 → per-row fallback finds 1 already-missing, 2 still there.
var perRowState = new Dictionary<string, bool>
{
["rk_1"] = true,
["rk_2"] = false, // already gone (concurrent cleaner)
["rk_3"] = true,
};
var harness = new Harness(
queriedRows: perRowState.Keys.Select(rk => new TableEntity(TenantId, rk)).ToArray(),
batchBehavior: _ => throw new RequestFailedException(404, "ResourceNotFound"),
perRowBehavior: rk =>
{
if (!perRowState[rk]) throw new RequestFailedException(404, "ResourceNotFound");
return new Mock<Response>().Object;
});
var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId);
Assert.Equal(2, deleted); // 2 still there, 1 already missing
Assert.Equal(3, harness.PerRowDeletes.Count);
foreach (var (_, _, etag) in harness.PerRowDeletes)
{
Assert.Equal(ETag.All, etag);
}
}
[Fact]
public async Task EmptyResult_ReturnsZero_NoBatchSubmitted()
{
var harness = new Harness(queriedRows: Array.Empty<TableEntity>());
var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId);
Assert.Equal(0, deleted);
Assert.Empty(harness.SubmittedBatches);
}
[Fact]
public async Task Variants_RejectNonGuidTenantId()
{
var harness = new Harness(queriedRows: Array.Empty<TableEntity>());
await Assert.ThrowsAsync<ArgumentException>(
() => harness.Sut.WipeByExactPartitionAsync("t", "not-a-guid"));
await Assert.ThrowsAsync<ArgumentException>(
() => harness.Sut.WipeByCompositePartitionRangeAsync("t", "not-a-guid"));
await Assert.ThrowsAsync<ArgumentException>(
() => harness.Sut.WipeByDiscriminatorAndTenantPropertyAsync("t", "CodeLookup", "not-a-guid"));
await Assert.ThrowsAsync<ArgumentException>(
() => harness.Sut.WipeByTenantIdPropertyAsync("t", "not-a-guid"));
}
// ── Harness ─────────────────────────────────────────────────────────────────
private sealed class Harness
{
public List<List<TableTransactionAction>> SubmittedBatches { get; } = new();
public List<(string Pk, string Rk, ETag Etag)> PerRowDeletes { get; } = new();
public SafeWipeService Sut { get; }
public Harness(
TableEntity[] queriedRows,
Func<List<TableTransactionAction>, Response<IReadOnlyList<Response>>>? batchBehavior = null,
Func<string, Response>? perRowBehavior = null)
{
var mockTableClient = new Mock<TableClient>();
// QueryAsync<TableEntity> returns the canned set as an AsyncPageable.
mockTableClient
.Setup(c => c.QueryAsync<TableEntity>(
It.IsAny<string>(),
It.IsAny<int?>(),
It.IsAny<IEnumerable<string>>(),
It.IsAny<CancellationToken>()))
.Returns(AsAsyncPageable(queriedRows));
mockTableClient
.Setup(c => c.SubmitTransactionAsync(It.IsAny<IEnumerable<TableTransactionAction>>(), It.IsAny<CancellationToken>()))
.Returns<IEnumerable<TableTransactionAction>, CancellationToken>((actions, _) =>
{
var snapshot = actions.ToList();
SubmittedBatches.Add(snapshot);
if (batchBehavior != null) return Task.FromResult(batchBehavior(snapshot));
var inner = (IReadOnlyList<Response>)snapshot.Select(_ => new Mock<Response>().Object).ToList();
return Task.FromResult(Response.FromValue(inner, new Mock<Response>().Object));
});
mockTableClient
.Setup(c => c.DeleteEntityAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<ETag>(), It.IsAny<CancellationToken>()))
.Returns<string, string, ETag, CancellationToken>((pk, rk, etag, _) =>
{
PerRowDeletes.Add((pk, rk, etag));
if (perRowBehavior != null) return Task.FromResult(perRowBehavior(rk));
return Task.FromResult(new Mock<Response>().Object);
});
var mockServiceClient = new Mock<TableServiceClient>();
mockServiceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(mockTableClient.Object);
var storage = new TableStorageService(mockServiceClient.Object, NullLogger<TableStorageService>.Instance);
var blob = new BlobStorageService(
new BlobServiceClient("UseDevelopmentStorage=true"),
NullLogger<BlobStorageService>.Instance,
usesManagedIdentity: false);
Sut = new SafeWipeService(storage, blob, NullLogger<SafeWipeService>.Instance);
}
private static AsyncPageable<TableEntity> AsAsyncPageable(TableEntity[] entities)
{
var page = Page<TableEntity>.FromValues(entities, continuationToken: null, new Mock<Response>().Object);
return AsyncPageable<TableEntity>.FromPages(new[] { page });
}
}
}
@@ -0,0 +1,310 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Data.Tables;
using Azure.Storage.Blobs;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Deletion;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Plan PR1.5 (Rev-5-F1). Locks the cascade-handler side-effect order: audit-write and
/// SignalR notify must complete BEFORE <c>DeletionProgress.CompletedAt</c> is stamped.
/// Once CompletedAt is set, the tenant-offboarding worker's drain probe treats the cascade
/// as "all side effects through" and the next phase wipes <c>AuditLogs</c> — so a late
/// <c>deletion_completed</c> audit would land as an orphan after the wipe.
/// <para>
/// The reorder is fail-soft-preserving:
/// <list type="bullet">
/// <item><c>LogAuditEntryAsync</c> already swallows storage exceptions internally and
/// returns <c>false</c> on failure; the handler does not act on the boolean.</item>
/// <item><c>NotifySessionDeletedAsync</c> already catches everything internally.</item>
/// </list>
/// So pulling them in front of the CompletedAt CAS cannot block <c>CompletedAt</c> from
/// being set on transient storage failures — confirmed by
/// <see cref="AuditFailure_DoesNotBlockCompletedAtSet"/>.
/// </para>
/// </summary>
public class SessionDeletionHandlerOrderingTests
{
private const string TenantId = "11111111-1111-1111-1111-111111111111";
private const string SessionId = "22222222-2222-2222-2222-222222222222";
private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210";
private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111";
[Fact]
public async Task TombstonePhase_WritesAuditAndSignalRBeforeCompletedAt()
{
var harness = new Harness();
await harness.Sut.HandleAsync(harness.Envelope);
Assert.True(harness.AuditSequence > 0, "audit was never written");
Assert.True(harness.SignalRSequence > 0, "SignalR notify was never sent");
Assert.True(harness.CompletedAtSequence > 0, "CompletedAt was never set on the progress blob");
Assert.True(
harness.AuditSequence < harness.CompletedAtSequence,
$"audit must run before CompletedAt CAS (audit={harness.AuditSequence}, completedAt={harness.CompletedAtSequence})");
Assert.True(
harness.SignalRSequence < harness.CompletedAtSequence,
$"SignalR notify must run before CompletedAt CAS (signalR={harness.SignalRSequence}, completedAt={harness.CompletedAtSequence})");
}
[Fact]
public async Task AuditFailure_DoesNotBlockCompletedAtSet()
{
// LogAuditEntryAsync is fail-soft by contract — a storage hiccup returns false rather
// than throwing. The handler must NOT use the boolean as a gate: CompletedAt still
// gets set so the cascade completes from the offboarding worker's point of view.
var harness = new Harness(auditReturns: false);
await harness.Sut.HandleAsync(harness.Envelope);
Assert.True(harness.AuditSequence > 0, "audit attempt was still made");
Assert.True(harness.CompletedAtSequence > 0,
"CompletedAt MUST still be stamped on the progress blob even when audit returned false");
Assert.True(harness.AuditSequence < harness.CompletedAtSequence);
}
[Fact]
public async Task SignalRFailure_DoesNotBlockCompletedAtSet()
{
// NotifySessionDeletedAsync already catches internally. The reorder cannot break that
// contract: an exception inside the notify path must not bubble out into the handler
// and leave CompletedAt unset.
var harness = new Harness(signalRThrows: new InvalidOperationException("simulated SignalR outage"));
await harness.Sut.HandleAsync(harness.Envelope);
Assert.True(harness.SignalRSequence > 0, "SignalR notify attempt was still made");
Assert.True(harness.CompletedAtSequence > 0,
"CompletedAt MUST still be stamped on the progress blob even when SignalR threw");
}
// ── Harness ─────────────────────────────────────────────────────────────────
/// <summary>
/// Minimal self-contained harness that drives the cascade through to the tombstone phase
/// using a single-step manifest (the Final step only). Sequence-counter records the call
/// order of the three side effects that PR1.5 reorders.
/// </summary>
private sealed class Harness
{
public SessionDeletionEnvelope Envelope { get; }
public SessionDeletionHandler Sut { get; }
public int AuditSequence { get; private set; }
public int SignalRSequence { get; private set; }
public int CompletedAtSequence { get; private set; }
private int _sequenceCounter;
private int Next() => System.Threading.Interlocked.Increment(ref _sequenceCounter);
public Harness(
bool auditReturns = true,
Exception? signalRThrows = null)
{
// --- Storage mock: just enough to flow through the state machine + tombstone step.
var storage = new Mock<TableStorageService>(
Mock.Of<TableServiceClient>(),
NullLogger<TableStorageService>.Instance);
var sessionRow = new TableEntity(TenantId, SessionId)
{
["DeletionState"] = SessionDeletionState.Queued,
["PendingDeletionManifestId"] = ManifestId,
};
storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
.ReturnsAsync(sessionRow);
storage.Setup(s => s.CasSetSessionDeletionStateAsync(
TenantId, SessionId,
SessionDeletionState.Queued, SessionDeletionState.Running,
It.IsAny<string?>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult
{
Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated,
CurrentState = SessionDeletionState.Running,
CurrentManifestId = ManifestId,
});
storage.Setup(s => s.DeleteByExactKeysInBatchesAsync(
It.IsAny<string>(), It.IsAny<IReadOnlyList<(string, string)>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((string _, IReadOnlyList<(string, string)> keys, CancellationToken _) =>
new DeletionBatchResult(keys.Count, keys.Count, 0));
storage.Setup(s => s.RecordSessionTombstoneAsync(
TenantId, SessionId, It.IsAny<string>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
.Returns(Task.CompletedTask);
// --- Blob mock: manifest + progress IO with a live in-memory progress instance so
// each UpdateDeletionProgressAsync mutation is reflected in the next download.
var progressLive = new DeletionProgress
{
SnapshotSha256 = Sha256,
CompletedSteps = new HashSet<int>(),
VerificationDone = false,
CompletedAt = null,
};
var blob = new Mock<BlobStorageService>(
new BlobServiceClient("UseDevelopmentStorage=true"),
NullLogger<BlobStorageService>.Instance,
false);
var manifest = BuildMinimalManifest();
blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
.ReturnsAsync((manifest, Sha256));
blob.Setup(b => b.DownloadDeletionProgressAsync(
TenantId, SessionId, ManifestId, It.IsAny<CancellationToken>()))
.ReturnsAsync(() => (Clone(progressLive), "\"0xFAKE\""));
blob.Setup(b => b.UpdateDeletionProgressAsync(
TenantId, SessionId, ManifestId,
It.IsAny<DeletionProgress>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns<string, string, string, DeletionProgress, string, CancellationToken>(
(_, _, _, p, _, _) =>
{
// Only the write that flips CompletedAt from null → non-null is the
// "lifecycle complete" CAS the reorder targets. Other writes (state
// progression, TombstoneStarted) get sequence numbers but we don't
// assert on them.
if (progressLive.CompletedAt == null && p.CompletedAt != null)
{
CompletedAtSequence = Next();
}
progressLive.CompletedSteps = new HashSet<int>(p.CompletedSteps);
progressLive.VerificationDone = p.VerificationDone;
progressLive.CompletedAt = p.CompletedAt;
progressLive.TombstoneStarted = p.TombstoneStarted;
return Task.FromResult("\"0xFAKE_NEXT\"");
});
// --- Verifier: clean by default (no residuals → tombstone runs).
var verifier = new Mock<CascadeVerificationService>(
Mock.Of<ISessionDeletionInventoryReader>(),
NullLogger<CascadeVerificationService>.Instance);
verifier.Setup(v => v.VerifyAsync(It.IsAny<DeletionManifest>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new CascadeVerificationResult(true, new List<CascadeResidualKey>()));
// --- Maintenance: capture audit call with a sequence number.
var maintenance = new Mock<IMaintenanceRepository>();
maintenance.Setup(m => m.LogAuditEntryAsync(
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, string>?>()))
.Returns<string, string, string, string, string, Dictionary<string, string>?>(
(_, action, _, _, _, _) =>
{
if (action == "deletion_completed") AuditSequence = Next();
return Task.FromResult(auditReturns);
});
// --- SignalR: capture notify call with a sequence number.
var signalR = new SequencingSignalRNotificationService(
this,
throwOnNotify: signalRThrows);
Envelope = new SessionDeletionEnvelope
{
TenantId = TenantId,
SessionId = SessionId,
ManifestId = ManifestId,
Reason = "admin_delete",
EnqueuedAt = DateTime.UtcNow,
};
Sut = new SessionDeletionHandler(
storage.Object, blob.Object, verifier.Object,
maintenance.Object, signalR,
NullLogger<SessionDeletionHandler>.Instance);
}
private static DeletionProgress Clone(DeletionProgress p) => new()
{
SnapshotSha256 = p.SnapshotSha256,
CompletedSteps = new HashSet<int>(p.CompletedSteps),
VerificationDone = p.VerificationDone,
CompletedAt = p.CompletedAt,
TombstoneStarted = p.TombstoneStarted,
};
private static DeletionManifest BuildMinimalManifest() => new()
{
ManifestId = ManifestId,
TenantId = TenantId,
SessionId = SessionId,
CreatedAt = new DateTime(2026, 5, 18, 10, 0, 0, DateTimeKind.Utc),
CreatedBy = new DeletionActor { Type = "admin", Actor = "alice@contoso.invalid" },
Reason = "admin_delete",
RetentionContext = new DeletionRetentionContext { TenantRetentionDays = 90 },
SchemaHash = "sha256:test",
// Single FINAL step → handler skips per-table + aggregate phases and goes straight
// to verifier (clean) → tombstone → audit + signalR + CompletedAt. That is the
// path PR1.5 reorders.
Steps = new List<DeletionStep>
{
new DeletionStep
{
Order = 1,
Step = DeletionStepNames.Tombstone,
Class = DeletionStepClass.Final,
RowCount = 2,
Rows = new List<DeletionRowDump>
{
new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" },
new DeletionRowDump { Pk = TenantId, Rk = SessionId },
},
},
},
};
/// <summary>
/// SignalR test double that records a sequence number on
/// <see cref="ISignalRNotificationService.NotifySessionDeletedAsync"/>.
/// All other interface members are no-ops — the cascade handler never calls them.
/// </summary>
private sealed class SequencingSignalRNotificationService : ISignalRNotificationService
{
private readonly Harness _harness;
private readonly Exception? _throwOnNotify;
public SequencingSignalRNotificationService(Harness harness, Exception? throwOnNotify)
{
_harness = harness;
_throwOnNotify = throwOnNotify;
}
public Task NotifySessionDeletedAsync(string tenantId, string sessionId)
{
_harness.SignalRSequence = _harness.Next();
// Production NotifySessionDeletedAsync already catches internally; mimic that
// contract so the test double doesn't accidentally pretend exceptions bubble.
if (_throwOnNotify != null)
{
try { throw _throwOnNotify; }
catch { /* swallow — matches production behaviour */ }
}
return Task.CompletedTask;
}
// ── Unused by the cascade handler — no-op stubs ────────────────────
public Task NotifyRuleResultsAvailableAsync(string tenantId, string sessionId, int resultCount) => Task.CompletedTask;
public Task NotifyVulnerabilityReportAvailableAsync(string tenantId, string sessionId, string overallRisk) => Task.CompletedTask;
public Task SendTenantNotificationAsync(string tenantId, NotificationAudience audience, object dto) => Task.CompletedTask;
public Task SendTenantNotificationDismissedAsync(string tenantId, string notificationId) => Task.CompletedTask;
public Task SendTenantNotificationDismissedAllAsync(string tenantId) => Task.CompletedTask;
public Task SendGlobalNotificationAsync(object dto) => Task.CompletedTask;
public Task SendGlobalNotificationDismissedAsync(string notificationId) => Task.CompletedTask;
public Task SendGlobalNotificationDismissedAllAsync() => Task.CompletedTask;
}
}
}
@@ -0,0 +1,384 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Data.Tables;
using AutopilotMonitor.Functions.DataAccess.TableStorage;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Round-trip + shape-guard coverage for the three POCO mappings hosted in
/// <c>OffboardingAudit</c>. Memory: <c>feedback_table_storage_serialization</c> — new
/// fields MUST be exercised in Store+Map so silent drops surface here, not in production.
/// </summary>
public class TableOffboardingAuditRepositoryTests
{
private const string TenantId = "88888888-8888-8888-8888-888888888888";
private const string HistoryRowKey = "20260518091523123_88888888-8888-8888-8888-888888888888";
[Fact]
public async Task Marker_RoundTrips_AllFields()
{
var harness = new Harness();
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = HistoryRowKey,
InitiatedAt = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc),
InitiatedBy = "alice@contoso.invalid",
Status = "InProgress",
CompletedAt = null,
FailedAt = null,
FailedPhase = null,
};
await harness.Sut.UpsertMarkerAsync(marker);
var fetched = await harness.Sut.TryGetMarkerAsync(TenantId);
Assert.NotNull(fetched);
Assert.Equal(marker.PartitionKey, fetched!.PartitionKey);
Assert.Equal(marker.RowKey, fetched.RowKey);
Assert.Equal(marker.TenantId, fetched.TenantId);
Assert.Equal(marker.OffboardingHistoryRowKey, fetched.OffboardingHistoryRowKey);
Assert.Equal(marker.InitiatedAt, fetched.InitiatedAt);
Assert.Equal(marker.InitiatedBy, fetched.InitiatedBy);
Assert.Equal("InProgress", fetched.Status);
}
[Fact]
public async Task Marker_CompletedTransition_PreservesTimestamp()
{
var harness = new Harness();
var completed = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = HistoryRowKey,
InitiatedAt = DateTime.UtcNow.AddMinutes(-10),
InitiatedBy = "bob@contoso.invalid",
Status = "Completed",
CompletedAt = DateTime.UtcNow,
};
await harness.Sut.UpsertMarkerAsync(completed);
var fetched = await harness.Sut.TryGetMarkerAsync(TenantId);
Assert.Equal("Completed", fetched!.Status);
Assert.NotNull(fetched.CompletedAt);
}
[Fact]
public async Task Marker_TryGet_ReturnsNullOn404()
{
var harness = new Harness(throwOnGet: new RequestFailedException(404, "NotFound", "ResourceNotFound", null));
Assert.Null(await harness.Sut.TryGetMarkerAsync(TenantId));
}
[Fact]
public async Task Marker_DeleteSwallows404_Idempotent()
{
var harness = new Harness(throwOnDelete: new RequestFailedException(404, "NotFound", "ResourceNotFound", null));
await harness.Sut.DeleteMarkerAsync(TenantId);
// No exception → idempotent.
}
[Fact]
public async Task History_RoundTrips_RetainCountersAndDrainCompletedAt()
{
var harness = new Harness();
var now = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc);
var history = new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = HistoryRowKey,
TenantId = TenantId,
DomainName = "contoso.invalid",
InitiatedBy = "alice@contoso.invalid",
OffboardedAt = now,
Status = "Completed",
CompletedAt = now.AddMinutes(5),
DeletedRowCountsJson = "{\"AuditLogs\":42}",
TotalRowsDeleted = 42,
DeletedBlobCount = 7,
CascadeSessionsEnqueued = 3,
RetryCount = 0,
DrainCompletedAt = now.AddMinutes(3), // Rev-9-F1
CustomGatherRulesArchived = 5,
CustomAnalyzeRulesArchived = 3,
ImeLogPatternOverridesArchived = 2,
};
await harness.Sut.UpsertHistoryAsync(history);
var fetched = await harness.Sut.TryGetHistoryAsync(HistoryRowKey);
Assert.NotNull(fetched);
Assert.Equal(history.Status, fetched!.Status);
Assert.Equal(history.TotalRowsDeleted, fetched.TotalRowsDeleted);
Assert.Equal(history.DeletedBlobCount, fetched.DeletedBlobCount);
Assert.Equal(history.CascadeSessionsEnqueued, fetched.CascadeSessionsEnqueued);
Assert.Equal(history.DrainCompletedAt, fetched.DrainCompletedAt);
Assert.Equal(5, fetched.CustomGatherRulesArchived);
Assert.Equal(3, fetched.CustomAnalyzeRulesArchived);
Assert.Equal(2, fetched.ImeLogPatternOverridesArchived);
}
[Fact]
public async Task Pointer_TryGet_OnEmpty_ReturnsNullPointerAndNullEtag()
{
var harness = new Harness();
var (pointer, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId);
Assert.Null(pointer);
Assert.Null(etag);
}
[Fact]
public async Task Pointer_Insert_RoundTrips_LatestHistoryRowKeyAndOffboardCount()
{
var harness = new Harness();
var pointer = new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = HistoryRowKey,
LatestStatus = "Initiated",
LatestUpdatedAt = DateTime.UtcNow,
OffboardCount = 1,
};
await harness.Sut.InsertByTenantPointerAsync(pointer);
var (fetched, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId);
Assert.NotNull(fetched);
Assert.NotNull(etag);
Assert.Equal(pointer.LatestHistoryRowKey, fetched!.LatestHistoryRowKey);
Assert.Equal("Initiated", fetched.LatestStatus);
Assert.Equal(1, fetched.OffboardCount);
}
[Fact]
public async Task Pointer_Insert_OnDuplicate_Throws409()
{
var harness = new Harness();
var pointer = BuildPointer(latestStatus: "Initiated", offboardCount: 1);
await harness.Sut.InsertByTenantPointerAsync(pointer);
var ex = await Assert.ThrowsAsync<RequestFailedException>(
() => harness.Sut.InsertByTenantPointerAsync(pointer));
Assert.Equal(409, ex.Status);
}
[Fact]
public async Task Pointer_UpdateWithEtag_MatchingEtag_Replaces()
{
var harness = new Harness();
await harness.Sut.InsertByTenantPointerAsync(BuildPointer(latestStatus: "Initiated", offboardCount: 1));
var (current, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId);
Assert.NotNull(current);
current!.LatestStatus = "InProgress";
current.OffboardCount = 2;
current.LatestUpdatedAt = DateTime.UtcNow;
await harness.Sut.UpdateByTenantPointerWithEtagAsync(current, etag!);
var (refetched, _) = await harness.Sut.TryGetByTenantPointerAsync(TenantId);
Assert.Equal("InProgress", refetched!.LatestStatus);
Assert.Equal(2, refetched.OffboardCount);
}
[Fact]
public async Task Pointer_UpdateWithEtag_StaleEtag_Throws412()
{
// Plan §4.4 contract: stale ETag → 412 so the caller's bounded retry loop fires.
// The blind Upsert path was removed because it could clobber a concurrent worker.
var harness = new Harness();
await harness.Sut.InsertByTenantPointerAsync(BuildPointer(latestStatus: "Initiated"));
var stale = BuildPointer(latestStatus: "InProgress", offboardCount: 99);
var ex = await Assert.ThrowsAsync<RequestFailedException>(
() => harness.Sut.UpdateByTenantPointerWithEtagAsync(stale, ifMatchEtag: "\"0xSTALE\""));
Assert.Equal(412, ex.Status);
}
[Fact]
public async Task Pointer_UpdateWithEtag_RequiresIfMatchEtag()
{
var harness = new Harness();
await Assert.ThrowsAsync<ArgumentException>(
() => harness.Sut.UpdateByTenantPointerWithEtagAsync(BuildPointer(), ""));
}
private static OffboardingByTenantPointer BuildPointer(
string latestStatus = "Initiated",
int offboardCount = 1) => new()
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = HistoryRowKey,
LatestStatus = latestStatus,
LatestUpdatedAt = DateTime.UtcNow,
OffboardCount = offboardCount,
};
[Fact]
public async Task Marker_RejectsWrongPartitionKey()
{
var harness = new Harness();
var bad = new OffboardingMarkerEntry
{
PartitionKey = "WrongPK",
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = HistoryRowKey,
InitiatedAt = DateTime.UtcNow,
InitiatedBy = "alice@contoso.invalid",
};
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.UpsertMarkerAsync(bad));
}
[Fact]
public async Task History_RejectsWrongPartitionKey()
{
var harness = new Harness();
var bad = new OffboardingHistoryEntry
{
PartitionKey = "WrongPK",
RowKey = HistoryRowKey,
TenantId = TenantId,
InitiatedBy = "alice@contoso.invalid",
OffboardedAt = DateTime.UtcNow,
};
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.UpsertHistoryAsync(bad));
}
[Fact]
public async Task Pointer_RejectsWrongPartitionKey()
{
var harness = new Harness();
var bad = new OffboardingByTenantPointer
{
PartitionKey = "WrongPK",
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = HistoryRowKey,
LatestStatus = "Initiated",
LatestUpdatedAt = DateTime.UtcNow,
};
await Assert.ThrowsAsync<ArgumentException>(() => harness.Sut.InsertByTenantPointerAsync(bad));
}
// ── Harness — in-memory dictionary of (PK,RK) → TableEntity backing the mock ──
//
// ETag-aware: every successful insert/upsert/update stamps a fresh ETag onto the stored
// entity so Get reads return it and Update can enforce If-Match. Mirrors enough of Azure
// Table Storage's behaviour to exercise the §4.4 Pointer CAS-Loop honestly.
private sealed class Harness
{
public TableOffboardingAuditRepository Sut { get; }
private readonly Dictionary<(string Pk, string Rk), TableEntity> _store = new();
private int _etagCounter;
public Harness(
RequestFailedException? throwOnGet = null,
RequestFailedException? throwOnDelete = null)
{
var mockTableClient = new Mock<TableClient>();
mockTableClient
.Setup(c => c.UpsertEntityAsync(
It.IsAny<TableEntity>(),
It.IsAny<TableUpdateMode>(),
It.IsAny<CancellationToken>()))
.Returns<TableEntity, TableUpdateMode, CancellationToken>((e, _, _) =>
{
e.ETag = StampEtag();
_store[(e.PartitionKey, e.RowKey)] = e;
return Task.FromResult(new Mock<Response>().Object);
});
mockTableClient
.Setup(c => c.AddEntityAsync(
It.IsAny<TableEntity>(),
It.IsAny<CancellationToken>()))
.Returns<TableEntity, CancellationToken>((e, _) =>
{
if (_store.ContainsKey((e.PartitionKey, e.RowKey)))
throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null);
e.ETag = StampEtag();
_store[(e.PartitionKey, e.RowKey)] = e;
return Task.FromResult(new Mock<Response>().Object);
});
mockTableClient
.Setup(c => c.UpdateEntityAsync(
It.IsAny<TableEntity>(),
It.IsAny<ETag>(),
It.IsAny<TableUpdateMode>(),
It.IsAny<CancellationToken>()))
.Returns<TableEntity, ETag, TableUpdateMode, CancellationToken>((e, ifMatch, _, _) =>
{
if (!_store.TryGetValue((e.PartitionKey, e.RowKey), out var stored))
throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null);
if (ifMatch != ETag.All && ifMatch != stored.ETag)
throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null);
e.ETag = StampEtag();
_store[(e.PartitionKey, e.RowKey)] = e;
return Task.FromResult(new Mock<Response>().Object);
});
mockTableClient
.Setup(c => c.GetEntityAsync<TableEntity>(
It.IsAny<string>(),
It.IsAny<string>(),
It.IsAny<IEnumerable<string>>(),
It.IsAny<CancellationToken>()))
.Returns<string, string, IEnumerable<string>, CancellationToken>((pk, rk, _, _) =>
{
if (throwOnGet != null) throw throwOnGet;
if (!_store.TryGetValue((pk, rk), out var entity))
throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null);
return Task.FromResult(Response.FromValue(entity, new Mock<Response>().Object));
});
mockTableClient
.Setup(c => c.DeleteEntityAsync(
It.IsAny<string>(),
It.IsAny<string>(),
It.IsAny<ETag>(),
It.IsAny<CancellationToken>()))
.Returns<string, string, ETag, CancellationToken>((pk, rk, _, _) =>
{
if (throwOnDelete != null) throw throwOnDelete;
_store.Remove((pk, rk));
return Task.FromResult(new Mock<Response>().Object);
});
var mockServiceClient = new Mock<TableServiceClient>();
mockServiceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(mockTableClient.Object);
var storage = new TableStorageService(mockServiceClient.Object, NullLogger<TableStorageService>.Instance);
Sut = new TableOffboardingAuditRepository(storage, NullLogger<TableOffboardingAuditRepository>.Instance);
}
private ETag StampEtag() => new($"\"0xFAKE_{++_etagCounter}\"");
}
}
@@ -0,0 +1,977 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Functions.Admin;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Functions.Tests.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
// Capture-helper for the recording enqueuer used in Finding-1 tests below.
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// PR3-side unit tests for the rewritten <see cref="TenantOffboardFunction"/>. The HTTP
/// entrypoint itself lives on top of <c>HttpRequestData</c> (abstract, hard to fake without
/// a real Functions host) so these tests focus on the function's CAS-loop helper. The
/// end-to-end happy/idempotent/race paths are covered by integration tests against the
/// real Functions host on dev tenants (see §10.3 smoketest in the plan).
/// </summary>
public sealed class TenantOffboardFunctionTests
{
private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
private static TenantOffboardFunction Build(IOffboardingAuditRepository repo)
=> Build(repo, Mock.Of<ITenantOffboardingEnqueuer>());
private static TenantOffboardFunction Build(
IOffboardingAuditRepository repo, ITenantOffboardingEnqueuer enqueuer)
{
// Resume-path tests (UpsertPointer / Resume_*) exercise just the CAS-loop +
// enqueuer wiring and don't care about the Disabled-gate. Stub IConfigRepository
// so EnsureTenantDisabledAsync's idempotent no-op path fires (returns an existing
// row already marked Disabled with the offboarding reason → no Save call).
//
// LastUpdated is set FAR in the past so the tombstone-deadline (LastUpdated +
// DrainBarrier) doesn't dominate the max() computation in the resume-path. That
// means the existing history-based-deadline tests still observe history-derived
// delays. Tests that exercise the LastUpdated-dominates path stand up their own
// configRepoMock with a recent LastUpdated.
var configRepoMock = new Mock<IConfigRepository>();
var alreadyDisabled = TenantConfiguration.CreateDefault(TenantId);
alreadyDisabled.Disabled = true;
alreadyDisabled.DisabledReason = "Offboarding in progress";
alreadyDisabled.LastUpdated = DateTime.UtcNow.AddDays(-30);
configRepoMock
.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
.ReturnsAsync(alreadyDisabled);
var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache(
new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions());
var service = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
return new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: service,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer);
}
/// <summary>
/// Test-side enqueuer recording every call. Production enqueuer is sealed + needs
/// configuration to ctor, so this fake lets the tests assert "did Phase 1 actually
/// hand off to the worker?" — the question Finding 1 says we previously got wrong.
/// </summary>
private sealed class RecordingEnqueuer : ITenantOffboardingEnqueuer
{
public readonly List<TenantOffboardingEnvelope> Sent = new();
public readonly List<TimeSpan?> VisibilityDelays = new();
public Exception? ThrowOnEnqueue { get; set; }
public Task EnqueueAsync(TenantOffboardingEnvelope envelope, TimeSpan? visibilityDelay = null, CancellationToken ct = default)
{
if (ThrowOnEnqueue is { } ex) throw ex;
Sent.Add(envelope);
VisibilityDelays.Add(visibilityDelay);
return Task.CompletedTask;
}
}
// Build helper exposing IConfigRepository + TenantConfigurationService for the
// EnsureTenantDisabledAsync tests. The repo is a Mock so we can pin Save calls; the
// service uses a real MemoryCache so InvalidateCache(tenantId) is observable via a
// pre-seeded cache entry.
private static (TenantOffboardFunction Sut, Mock<IConfigRepository> ConfigRepo, Microsoft.Extensions.Caching.Memory.IMemoryCache Cache, TenantConfigurationService Service)
BuildWithConfigService()
{
var configRepoMock = new Mock<IConfigRepository>();
var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache(
new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions());
var service = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: service,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: new FakeOffboardingAuditRepository(),
offboardingEnqueuer: new RecordingEnqueuer());
return (sut, configRepoMock, cache, service);
}
// ── UpsertPointerWithCasAsync ────────────────────────────────────────────
[Fact]
public async Task UpsertPointer_FreshTenant_Inserts_OffboardCount_1()
{
var repo = new FakeOffboardingAuditRepository();
var sut = Build(repo);
var historyRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await sut.UpsertPointerWithCasAsync(TenantId, historyRowKey, DateTime.UtcNow);
Assert.True(repo.Pointers.TryGetValue(TenantId, out var entry));
Assert.Equal(historyRowKey, entry.Pointer.LatestHistoryRowKey);
Assert.Equal("Initiated", entry.Pointer.LatestStatus);
Assert.Equal(1, entry.Pointer.OffboardCount);
}
[Fact]
public async Task UpsertPointer_ExistingTenant_IncrementsOffboardCount()
{
var repo = new FakeOffboardingAuditRepository();
await repo.InsertByTenantPointerAsync(new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
LatestUpdatedAt = DateTime.UtcNow.AddDays(-30),
OffboardCount = 2,
});
var sut = Build(repo);
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow);
Assert.Equal(newHistoryRowKey, repo.Pointers[TenantId].Pointer.LatestHistoryRowKey);
Assert.Equal("Initiated", repo.Pointers[TenantId].Pointer.LatestStatus);
Assert.Equal(3, repo.Pointers[TenantId].Pointer.OffboardCount);
}
[Fact]
public async Task UpsertPointer_409OnInsert_FallsBackToEtagUpdate()
{
// First TryGet returns null, but InsertByTenantPointerAsync throws 409 because
// a parallel writer inserted between our read and write. The CAS-loop must
// re-read and update with the now-existing pointer's ETag.
var repoMock = new Mock<IOffboardingAuditRepository>();
var existingPointer = new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
OffboardCount = 1,
};
var attempt = 0;
repoMock
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
.Returns(() =>
{
attempt++;
return attempt == 1
? Task.FromResult<(OffboardingByTenantPointer?, string?)>((null, null))
: Task.FromResult<(OffboardingByTenantPointer?, string?)>((existingPointer, "\"etag-1\""));
});
repoMock
.Setup(r => r.InsertByTenantPointerAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null));
OffboardingByTenantPointer? updatedPointer = null;
repoMock
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), "\"etag-1\"", It.IsAny<CancellationToken>()))
.Returns<OffboardingByTenantPointer, string, CancellationToken>((p, _, _) =>
{
updatedPointer = p;
return Task.CompletedTask;
});
var sut = Build(repoMock.Object);
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow);
Assert.NotNull(updatedPointer);
Assert.Equal(newHistoryRowKey, updatedPointer!.LatestHistoryRowKey);
Assert.Equal(2, updatedPointer.OffboardCount);
repoMock.Verify(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), "\"etag-1\"", It.IsAny<CancellationToken>()), Times.Once);
}
[Fact]
public async Task UpsertPointer_412OnUpdate_RetriesAndSucceeds()
{
// Concurrent writer mutates the pointer between our read and our update. CAS-loop
// must re-read (getting the fresh ETag) and try once more.
var repoMock = new Mock<IOffboardingAuditRepository>();
var freshState = new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
OffboardCount = 1,
};
var readEtags = new[] { "\"etag-old\"", "\"etag-new\"" };
var readIdx = 0;
repoMock
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
.Returns(() =>
{
var etag = readEtags[Math.Min(readIdx, readEtags.Length - 1)];
readIdx++;
return Task.FromResult<(OffboardingByTenantPointer?, string?)>((freshState, etag));
});
var updateCalls = 0;
repoMock
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns<OffboardingByTenantPointer, string, CancellationToken>((_, etag, _) =>
{
updateCalls++;
if (etag == "\"etag-old\"")
{
throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null);
}
return Task.CompletedTask;
});
var sut = Build(repoMock.Object);
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow);
Assert.Equal(2, updateCalls);
Assert.Equal(2, readIdx);
}
// ── ResumeExistingMarkerAsync (PR3 Review Finding 1) ─────────────────────
//
// The previous implementation returned 200 from the idempotency path UNCONDITIONALLY.
// If the very first enqueue failed (transient queue outage), the marker was committed
// and the queue was empty: every subsequent re-click took the idempotency-200 branch
// and never re-enqueued, leaving the tenant stuck Initiated forever. These tests pin
// the new defensive-re-enqueue behaviour: Initiated/InProgress → re-enqueue; Completed
// → no re-enqueue (worker is done); Failed → no re-enqueue (operator action required).
[Theory]
[InlineData("Initiated")]
[InlineData("InProgress")]
public async Task Resume_InFlightStatus_DefensivelyReEnqueues(string status)
{
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var historyRowKey = "20260518091523123_" + TenantId;
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = DateTime.UtcNow,
InitiatedBy = "alice@contoso.com",
Status = status,
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "bob@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.True(outcome.ReEnqueued);
Assert.Single(enqueuer.Sent);
Assert.Equal(historyRowKey, enqueuer.Sent[0].HistoryRowKey);
Assert.Equal(TenantId, enqueuer.Sent[0].TenantId);
}
[Fact]
public async Task Resume_Completed_NoReEnqueue_ReturnsIdempotentOk()
{
var enqueuer = new RecordingEnqueuer();
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var marker = new OffboardingMarkerEntry
{
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
Status = "Completed",
CompletedAt = DateTime.UtcNow.AddMinutes(-5),
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.IdempotentCompleted, outcome.Kind);
Assert.False(outcome.ReEnqueued);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Resume_Failed_NoReEnqueue_SurfacesPhaseInMessage()
{
var enqueuer = new RecordingEnqueuer();
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var marker = new OffboardingMarkerEntry
{
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
Status = "Failed",
FailedAt = DateTime.UtcNow.AddMinutes(-10),
FailedPhase = "drain_timeout",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.IdempotentFailed, outcome.Kind);
Assert.False(outcome.ReEnqueued);
Assert.Contains("drain_timeout", outcome.Message);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Resume_InFlight_EnqueueThrows_SurfacesReEnqueueFailed()
{
// The defensive re-enqueue must surface its failure (500) so the caller knows
// the resume did NOT take effect. The marker stays Initiated; the next click
// will trigger another resume attempt — that's the operator-visible recovery
// mechanism that previously didn't exist (stuck-tenant bug).
var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") };
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var marker = new OffboardingMarkerEntry
{
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueueFailed, outcome.Kind);
Assert.False(outcome.ReEnqueued);
}
[Fact]
public async Task Resume_409Race_StillReEnqueues_BecauseWinnerMayHaveFailed()
{
// The 409-race path resolves to the winning marker and routes through the same
// resume code. We can't know whether the winner's enqueue succeeded, so we
// must re-enqueue defensively — same Finding-1 contract as the plain idempotency
// re-click path.
var enqueuer = new RecordingEnqueuer();
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var marker = new OffboardingMarkerEntry
{
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: true);
Assert.True(outcome.ReEnqueued);
Assert.Single(enqueuer.Sent);
Assert.Contains("race", outcome.Message);
}
[Fact]
public async Task UpsertPointer_CasExhausted_Throws()
{
// Every UpdateWithEtag returns 412 — the bounded retry-loop must surface this
// rather than retry indefinitely.
var repoMock = new Mock<IOffboardingAuditRepository>();
var freshState = new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
OffboardCount = 1,
};
repoMock
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
.ReturnsAsync((freshState, "\"some-etag\""));
repoMock
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null));
var sut = Build(repoMock.Object);
var historyRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.UpsertPointerWithCasAsync(TenantId, historyRowKey, DateTime.UtcNow));
}
// ── Cache-Drain-Barrier (plan v2 §2.2 / §2.3) ─────────────────────────────
//
// The DrainBarrier (hardcoded 6min) prevents Phase 2 from starting before all
// function-host instances drain their TenantConfigurationService cache. Phase 1
// enqueues with visibilityDelay=DrainBarrier so the worker only sees the message
// after the barrier elapses. A resume-click reads the stored EarliestProcessingAt
// and re-enqueues with the REMAINING delay so the barrier does not reset.
[Fact]
public void DrainBarrier_HardcodedAt6Minutes()
{
// Plan v2 §2.2: 5min TenantConfigurationService.CacheDuration + 1min safety buffer.
Assert.Equal(TimeSpan.FromMinutes(6), TenantOffboardFunction.DrainBarrier);
}
[Fact]
public async Task Resume_InFlight_UsesRemainingBarrier_NotFullBarrier()
{
// Re-click 2 min into the 6-min barrier → re-enqueue must carry ~4min visibility delay,
// not a fresh 6min. Otherwise each re-click would reset the cache-drain window.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var initiatedAt = DateTime.UtcNow.AddMinutes(-2);
var earliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // ~+4min from now
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = initiatedAt,
EarliestProcessingAt = earliestProcessingAt,
Status = "Initiated",
});
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = initiatedAt,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
var delay = enqueuer.VisibilityDelays[0];
Assert.NotNull(delay);
// Allow a few seconds of test-runtime slack — clock advanced between Setup and Act.
Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.5));
Assert.Equal(earliestProcessingAt, outcome.EarliestProcessingAt);
}
[Fact]
public async Task Resume_InFlight_PastBarrier_UsesZeroDelay()
{
// Re-click 8 min into a 6-min barrier → barrier already elapsed → re-enqueue with
// zero visibility delay so the worker picks up immediately.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var initiatedAt = DateTime.UtcNow.AddMinutes(-8);
var earliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // 2 min ago
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = initiatedAt,
EarliestProcessingAt = earliestProcessingAt,
Status = "InProgress",
});
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = initiatedAt,
InitiatedBy = "alice@contoso.com",
Status = "InProgress",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
Assert.Equal(TimeSpan.Zero, enqueuer.VisibilityDelays[0]);
}
[Fact]
public async Task Resume_InFlight_MissingHistoryRow_UsesTombstoneLastUpdatedAsDeadlineAnchor()
{
// The marker references a History row that does not exist (corruption / manual
// delete). Without history.EarliestProcessingAt to read, the resume falls back
// to tombstone.LastUpdated + DrainBarrier as the authoritative drain-deadline.
// The Build-helper's default tombstone has LastUpdated = 30 days ago → the
// tombstone-derived deadline is far in the past → drain barrier already expired
// → visibility-delay = Zero (worker picks immediately). That is the CORRECT
// outcome: if the tombstone has been sitting for 30 days, every cache has been
// through the 5-min TTL many times already.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = "20260518091523123_" + TenantId,
InitiatedAt = DateTime.UtcNow,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
// 30-day-old tombstone → drain barrier long expired → 0 delay is safe.
Assert.Equal(TimeSpan.Zero, enqueuer.VisibilityDelays[0]);
}
// ── EnsureTenantDisabledAsync (PR3.B Review Finding 1 — fail-loud Disabled-gate) ──
//
// Without Disabled=true, the cache-drain barrier guards nothing: warm function-hosts
// would keep accepting agent traffic for a tenant whose Phase 2 is about to wipe
// their data. Every initial enqueue + every resume re-enqueue MUST commit this gate.
[Fact]
public async Task EnsureTenantDisabled_FlipsDisabled_AndInvalidatesCache()
{
var (sut, configRepo, cache, _) = BuildWithConfigService();
var existing = TenantConfiguration.CreateDefault(TenantId);
existing.Disabled = false;
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing);
TenantConfiguration? saved = null;
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
.Callback<TenantConfiguration>(c => saved = c)
.ReturnsAsync(true);
// Seed cache with stale "Disabled=false" entry so we can prove invalidation.
cache.Set($"tenant-config:{TenantId}", existing);
await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
Assert.NotNull(saved);
Assert.True(saved!.Disabled);
Assert.Equal("Offboarding in progress", saved.DisabledReason);
Assert.Null(saved.DisabledUntil);
Assert.Equal("alice@contoso.com", saved.UpdatedBy);
Assert.False(cache.TryGetValue($"tenant-config:{TenantId}", out _),
"cache entry must be invalidated so warm reads pick up the new Disabled=true on next access");
}
[Fact]
public async Task EnsureTenantDisabled_AlreadyDisabledWithSameReason_DoesNotRewrite()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
var existing = TenantConfiguration.CreateDefault(TenantId);
existing.Disabled = true;
existing.DisabledReason = "Offboarding in progress";
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing);
await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
// Idempotent: no Save thrash on resume-clicks while the marker is already in flight.
configRepo.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()), Times.Never);
}
[Fact]
public async Task EnsureTenantDisabled_StorageThrowsOnRead_PropagatesAsExpected()
{
// Caller (initial enqueue + resume re-enqueue) MUST treat this as fail-loud and
// skip the enqueue. We just assert the exception propagates — the caller's
// try/catch + 500-return is exercised by integration tests.
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ThrowsAsync(new InvalidOperationException("storage unavailable"));
await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
}
[Fact]
public async Task EnsureTenantDisabled_StorageThrowsOnSave_PropagatesAsExpected()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ReturnsAsync(TenantConfiguration.CreateDefault(TenantId));
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
.ThrowsAsync(new InvalidOperationException("storage save failed"));
await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
}
[Fact]
public async Task EnsureTenantDisabled_NoConfigRow_CreatesFreshTombstone()
{
// PR3.B-revised Finding 3: missing row used to log+continue, leaving AuthFunction's
// auto-create-default free to spawn a Disabled=false row on the next /api/auth/me.
// The fix creates a fresh CreateDefault row with Disabled=true so the auth-gate is
// committed even in this edge case.
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ReturnsAsync((TenantConfiguration?)null);
TenantConfiguration? saved = null;
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
.Callback<TenantConfiguration>(c => saved = c)
.ReturnsAsync(true);
await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
Assert.NotNull(saved);
Assert.Equal(TenantId, saved!.TenantId);
Assert.True(saved.Disabled, "fresh tombstone must carry Disabled=true");
Assert.Equal("Offboarding in progress", saved.DisabledReason);
Assert.Equal("alice@contoso.com", saved.UpdatedBy);
}
[Fact]
public async Task EnsureTenantDisabled_SaveReturnsFalse_ThrowsAsHardFail()
{
// PR3.B-revised Finding 2: TableConfigRepository.SaveTenantConfigurationAsync
// returns false on transient storage failure (it does NOT throw). Ignoring that
// return-value would let the caller proceed to enqueue without the auth-gate
// committed. Turn it into a hard exception so the offboard is NOT enqueued.
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ReturnsAsync(TenantConfiguration.CreateDefault(TenantId));
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
.ReturnsAsync(false); // transient storage rejection — NOT a throw.
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
Assert.Contains("Disabled-gate NOT committed", ex.Message);
}
[Fact]
public async Task EnsureTenantDisabled_NoConfigRow_SaveReturnsFalse_ThrowsAsHardFail()
{
// Combined edge case: missing row → we'd create a fresh tombstone → Save fails →
// hard fail (otherwise the caller would proceed without an auth-gate).
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId))
.ReturnsAsync((TenantConfiguration?)null);
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
.ReturnsAsync(false);
await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
}
[Fact]
public async Task EnsureTenantDisabled_AlreadyTombstone_ReturnsAlreadyTombstone()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(tombstone);
var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.AlreadyTombstone, result.Outcome);
configRepo.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()), Times.Never);
}
[Fact]
public async Task EnsureTenantDisabled_FlipsDisabled_ReturnsWroteNewTombstone()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
var existing = TenantConfiguration.CreateDefault(TenantId);
existing.Disabled = false;
configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing);
configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
.ReturnsAsync(true);
var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.WroteNewTombstone, result.Outcome);
}
// ── Resume-path cache-drain bypass fix (Codex review) ────────────────────
//
// Scenario: initial attempt commits Marker/History/Pointer (History.EarliestProcessingAt
// = T+6min) but then EnsureTenantDisabledAsync FAILS → 500 returned, no enqueue. The
// admin re-clicks 10 minutes later. The History row still carries the now-past
// EarliestProcessingAt = T+6min. Without the fix, the resume would naively use the
// remaining-from-history delta and arrive at Zero → enqueue with no visibility-delay
// → worker picks immediately → but Disabled=true was JUST written for the first time
// → warm function-host caches still hold Disabled=false → cache-drain barrier violated.
//
// The fix anchors on tombstone.LastUpdated + DrainBarrier so the deadline is
// authoritative from the same atomic write as the Disabled-flip.
[Fact]
public async Task Resume_FirstAttemptFailedBeforeDisabledWrite_UsesFullBarrierAndPatchesHistory()
{
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var initiatedAt = DateTime.UtcNow.AddMinutes(-10); // first attempt 10min ago
var staleEarliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // ~4 min in the past
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = initiatedAt,
EarliestProcessingAt = staleEarliestProcessingAt,
Status = "Initiated",
});
// Build SUT where the first attempt FAILED at EnsureTenantDisabledAsync: Config
// exists but is NOT yet the offboarding tombstone (Disabled=false). The resume's
// EnsureTenantDisabledAsync will write it now → WroteNewTombstone.
var configRepoMock = new Mock<IConfigRepository>();
var notYetDisabled = TenantConfiguration.CreateDefault(TenantId);
notYetDisabled.Disabled = false;
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
.ReturnsAsync(notYetDisabled);
configRepoMock.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()))
.ReturnsAsync(true);
var cache = new MemoryCache(new MemoryCacheOptions());
var tenantConfigService = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: tenantConfigService,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer);
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = initiatedAt,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
// CRITICAL: visibility-delay must be the full DrainBarrier (6 min), NOT Zero. The
// History row's stale EarliestProcessingAt must NOT be honoured when Disabled was
// just written for the first time. Allow a few-ms slack because the delay is
// computed as (effectiveDeadline - DateTime.UtcNow), so a moment passes between
// the LastUpdated stamp and the delay computation.
var actualDelay = enqueuer.VisibilityDelays[0]!.Value;
Assert.InRange(actualDelay, TenantOffboardFunction.DrainBarrier - TimeSpan.FromSeconds(1), TenantOffboardFunction.DrainBarrier);
// The History row must have been patched so further re-clicks see the new deadline.
Assert.True(repo.History.ContainsKey(historyRowKey));
var patchedHistory = repo.History[historyRowKey];
Assert.NotNull(patchedHistory.EarliestProcessingAt);
Assert.True(patchedHistory.EarliestProcessingAt > DateTime.UtcNow.AddMinutes(5),
$"EarliestProcessingAt must be roughly now+6min after the patch, got {patchedHistory.EarliestProcessingAt:O}");
// ResumeOutcome's EarliestProcessingAt is the NEW deadline (UI countdown source of truth).
Assert.NotNull(outcome.EarliestProcessingAt);
Assert.True(outcome.EarliestProcessingAt > DateTime.UtcNow.AddMinutes(5));
}
[Fact]
public async Task Resume_HistoryPatchFails_SecondResumeStillRespectsBarrierViaTombstoneLastUpdated()
{
// The exact Codex Finding scenario: first resume writes tombstone successfully
// but the History.EarliestProcessingAt patch fails (transient). A second resume
// arriving moments later reads History with the OLD stale EarliestProcessingAt
// but sees the tombstone (AlreadyTombstone path). The deadline computation must
// still respect the cache-drain barrier via tombstone.LastUpdated + DrainBarrier.
//
// Without the LastUpdated-anchor fix, the second resume would compute
// visibility-delay = max(0, stale-history-deadline - now) = 0 → worker picks
// before the cache-drain barrier from the tombstone-write completes.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var firstAttempt = DateTime.UtcNow.AddMinutes(-10);
var staleHistoryDeadline = firstAttempt + TenantOffboardFunction.DrainBarrier; // ~now-4min
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = firstAttempt,
EarliestProcessingAt = staleHistoryDeadline, // STALE — patch failed before
Status = "Initiated",
});
// Tombstone-already-in-place from the prior (first) resume; LastUpdated reflects
// when Disabled was committed in that first resume = recently (now). The History
// patch had failed, so EarliestProcessingAt is still the stale firstAttempt+6min.
var configRepoMock = new Mock<IConfigRepository>();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
tombstone.LastUpdated = DateTime.UtcNow.AddSeconds(-1); // freshly written ~1s ago
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
.ReturnsAsync(tombstone);
var cache = new MemoryCache(new MemoryCacheOptions());
var tenantConfigService = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: tenantConfigService,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer);
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = firstAttempt,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
// SAFETY: the visibility-delay must come from tombstone.LastUpdated + DrainBarrier
// (~6min remaining), NOT from the stale history deadline (which would yield 0).
// Anything below ~5min would mean we relied on the corrupted history value.
var delay = enqueuer.VisibilityDelays[0]!.Value;
Assert.InRange(delay, TimeSpan.FromMinutes(5.5), TenantOffboardFunction.DrainBarrier);
}
[Fact]
public async Task Resume_TombstoneAlreadyInPlace_UsesRemainingBarrier_DoesNotPatchHistory()
{
// Counterpoint: tombstone was already there from the prior successful attempt;
// the original deadline still applies. NO patch of History.EarliestProcessingAt.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var initiatedAt = DateTime.UtcNow.AddMinutes(-2); // first attempt 2min ago, mid-barrier
var originalEarliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier;
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = initiatedAt,
EarliestProcessingAt = originalEarliestProcessingAt,
Status = "Initiated",
});
// Build SUT where the tombstone is already in place from the prior attempt at
// initiatedAt (2min ago). The tombstone.LastUpdated reflects when Disabled was
// committed; the resume's deadline math is max(history.EarliestProcessingAt,
// tombstone.LastUpdated + DrainBarrier). Both terms ≈ initiatedAt + 6min here,
// so remaining ≈ 4min as before.
var configRepoMock = new Mock<IConfigRepository>();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
tombstone.LastUpdated = initiatedAt;
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>()))
.ReturnsAsync(tombstone);
var cache = new MemoryCache(new MemoryCacheOptions());
var tenantConfigService = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: tenantConfigService,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer);
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = initiatedAt,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
// Remaining delay should be ~4min (6min - 2min). Allow a couple seconds of slack.
var delay = enqueuer.VisibilityDelays[0]!.Value;
Assert.InRange(delay, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.5));
// History.EarliestProcessingAt must NOT have been overwritten — it still carries
// the original deadline from the first attempt.
var historyAfter = repo.History[historyRowKey];
Assert.Equal(originalEarliestProcessingAt, historyAfter.EarliestProcessingAt);
// No Save call (the tombstone was already correct).
configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>()), Times.Never);
}
}
@@ -44,6 +44,13 @@ namespace AutopilotMonitor.Functions.DataAccess
services.AddSingleton<ISignalRepository, TableSignalRepository>();
services.AddSingleton<IDecisionTransitionRepository, TableDecisionTransitionRepository>();
services.AddSingleton<IIndexTableRepository, TableIndexRepository>();
// Tenant-offboarding audit (Marker / History / ByTenant pointer).
services.AddSingleton<IOffboardingAuditRepository, TableOffboardingAuditRepository>();
// Tenant-offboarding customs archive (PR3.B): snapshot of GatherRules /
// AnalyzeRules / ImeLogPatterns rows that were wiped during Phase 2.D-archive,
// surfaced via the Global Admin /admin/customs-archive page.
services.AddSingleton<ITenantCustomsArchiveRepository, TableTenantCustomsArchiveRepository>();
return services;
}
@@ -1,3 +1,6 @@
using System.Runtime.CompilerServices;
using System.Threading;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
@@ -74,186 +77,45 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task<bool> IsSessionIndexEmptyAsync()
=> _storage.IsSessionIndexEmptyAsync();
// --- Tenant Offboarding ---
public async Task<Dictionary<string, int>> DeleteAllTenantDataAsync(string tenantId)
{
var deletedCounts = new Dictionary<string, int>();
var normalizedTenantId = tenantId.ToLowerInvariant();
// Tables where PartitionKey = tenantId (normalized lowercase)
var tenantPartitionTables = new[]
{
Constants.TableNames.Sessions,
Constants.TableNames.SessionsIndex,
Constants.TableNames.AuditLogs,
Constants.TableNames.UsageMetrics,
Constants.TableNames.UserActivity,
Constants.TableNames.GatherRules,
Constants.TableNames.AnalyzeRules,
Constants.TableNames.AppInstallSummaries,
Constants.TableNames.TenantConfiguration,
Constants.TableNames.TenantAdmins,
Constants.TableNames.BootstrapSessions,
Constants.TableNames.BlockedDevices,
Constants.TableNames.ImeLogPatterns,
Constants.TableNames.RuleStates,
};
foreach (var tableName in tenantPartitionTables)
{
var deleted = await DeleteAllRowsByPartitionKeyAsync(tableName, normalizedTenantId);
deletedCounts[tableName] = deleted;
_logger.LogInformation("Offboard [{TenantId}] {TableName}: deleted {Deleted} rows", tenantId, tableName, deleted);
}
// Tables with composite PartitionKey = "{tenantId}_{sessionId}" – query by prefix
var eventsDeleted = await DeleteByTenantPrefixAsync(Constants.TableNames.Events, normalizedTenantId);
deletedCounts[Constants.TableNames.Events] = eventsDeleted;
_logger.LogInformation("Offboard [{TenantId}] Events: deleted {Deleted} rows", tenantId, eventsDeleted);
var ruleResultsDeleted = await DeleteByTenantPrefixAsync(Constants.TableNames.RuleResults, normalizedTenantId);
deletedCounts[Constants.TableNames.RuleResults] = ruleResultsDeleted;
_logger.LogInformation("Offboard [{TenantId}] RuleResults: deleted {Deleted} rows", tenantId, ruleResultsDeleted);
// BootstrapSessions CodeLookup entries (PartitionKey = "CodeLookup", TenantId property = tenantId)
var codeLookupDeleted = await DeleteCodeLookupEntriesAsync(normalizedTenantId);
deletedCounts["BootstrapSessions_CodeLookup"] = codeLookupDeleted;
_logger.LogInformation("Offboard [{TenantId}] BootstrapSessions CodeLookup: deleted {Deleted} rows", tenantId, codeLookupDeleted);
return deletedCounts;
}
/// <summary>
/// Deletes all rows in a table where PartitionKey equals the given value.
/// Uses batched deletes (max 100 per transaction, same PartitionKey required).
/// </summary>
private async Task<int> DeleteAllRowsByPartitionKeyAsync(string tableName, string partitionKey)
{
try
{
var tableClient = _storage.GetTableClient(tableName);
var filter = $"PartitionKey eq '{partitionKey}'";
var entities = tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" });
int deleted = 0;
var batch = new List<TableTransactionAction>();
await foreach (var entity in entities)
{
batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity));
deleted++;
if (batch.Count >= 100)
{
await tableClient.SubmitTransactionAsync(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
await tableClient.SubmitTransactionAsync(batch);
}
return deleted;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete rows in {TableName} for partition {PartitionKey}", tableName, partitionKey);
return 0;
}
}
/// <summary>
/// Deletes all rows for a tenant from a table that uses composite PartitionKey = "{tenantId}_{sessionId}".
/// Used for Events and RuleResults tables.
/// </summary>
private async Task<int> DeleteByTenantPrefixAsync(string tableName, string normalizedTenantId)
{
try
{
var tableClient = _storage.GetTableClient(tableName);
// OData startsWith: PartitionKey ge 'prefix' and PartitionKey lt 'prefix~'
var prefix = normalizedTenantId + "_";
var filter = $"PartitionKey ge '{prefix}' and PartitionKey lt '{prefix}~'";
var entities = tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" });
int deleted = 0;
// Group by PartitionKey because Azure Table batch transactions require same PartitionKey
var groups = new Dictionary<string, List<TableEntity>>();
await foreach (var entity in entities)
{
if (!groups.ContainsKey(entity.PartitionKey))
groups[entity.PartitionKey] = new List<TableEntity>();
groups[entity.PartitionKey].Add(entity);
deleted++;
}
foreach (var group in groups.Values)
{
for (int i = 0; i < group.Count; i += 100)
{
var chunk = group.Skip(i).Take(100).ToList();
var batch = chunk.Select(e => new TableTransactionAction(TableTransactionActionType.Delete, e)).ToList();
await tableClient.SubmitTransactionAsync(batch);
}
}
return deleted;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete {TableName} for tenant {TenantId}", tableName, normalizedTenantId);
return 0;
}
}
/// <summary>
/// Deletes BootstrapSessions CodeLookup entries for a tenant.
/// These use PartitionKey = "CodeLookup" with a TenantId property matching the tenant.
/// </summary>
private async Task<int> DeleteCodeLookupEntriesAsync(string normalizedTenantId)
{
try
{
var tableClient = _storage.GetTableClient(Constants.TableNames.BootstrapSessions);
var filter = $"PartitionKey eq 'CodeLookup' and TenantId eq '{normalizedTenantId}'";
var entities = tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" });
int deleted = 0;
var batch = new List<TableTransactionAction>();
await foreach (var entity in entities)
{
batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity));
deleted++;
if (batch.Count >= 100)
{
await tableClient.SubmitTransactionAsync(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
await tableClient.SubmitTransactionAsync(batch);
}
return deleted;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete CodeLookup entries for tenant {TenantId}", normalizedTenantId);
return 0;
}
}
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod)
=> _storage.GetOrphanedEventSessionsAsync(gracePeriod);
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId)
=> _storage.DeleteEventSessionIndexEntryAsync(tenantId, sessionId);
/// <summary>
/// Tenant offboarding cascade-worker session enumerator. Yields one row per session
/// belonging to <paramref name="tenantId"/>. Storage exceptions are intentionally
/// NOT caught (memory: feedback_storage_helpers_fail_soft) — the worker must see
/// transient failures so the queue can retry/poison rather than treating an outage as
/// "0 sessions, proceed with wipe".
/// </summary>
public async IAsyncEnumerable<string> EnumerateSessionsForOffboardingAsync(
string tenantId,
[EnumeratorCancellation] CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
var tableClient = _storage.GetTableClient(Constants.TableNames.Sessions);
var filter = $"PartitionKey eq '{tenantId}'";
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
filter,
select: new[] { "PartitionKey", "RowKey" },
cancellationToken: ct))
{
if (!string.Equals(entity.PartitionKey, tenantId, StringComparison.Ordinal))
{
// Defensive: server-side filter should already guarantee this, but if a row
// ever slips through with a foreign PK we must fail-loud rather than enqueue
// a cascade for the wrong tenant.
throw new InvalidOperationException(
$"Sessions row returned by tenant filter has PartitionKey '{entity.PartitionKey}' " +
$"but expected '{tenantId}' (RowKey={entity.RowKey}).");
}
yield return entity.RowKey;
}
}
}
}
@@ -0,0 +1,296 @@
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Azure.Data.Tables;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.DataAccess.TableStorage
{
/// <summary>
/// Table-storage implementation of <see cref="IOffboardingAuditRepository"/>. Stores all
/// three offboarding PartitionKey patterns (Marker / History / ByTenant) in the single
/// <c>OffboardingAudit</c> table.
/// <para>
/// All writes are fail-loud — when an Azure storage call throws, exceptions propagate to
/// the caller (queue worker), which owns retry/poison semantics via the queue SDK. This is
/// intentionally the OPPOSITE of <see cref="TableMaintenanceRepository"/>'s fail-soft
/// helpers; offboarding cannot tolerate silent failures during gate/audit writes.
/// </para>
/// </summary>
public class TableOffboardingAuditRepository : IOffboardingAuditRepository
{
private readonly TableClient _tableClient;
private readonly ILogger<TableOffboardingAuditRepository> _logger;
public TableOffboardingAuditRepository(
TableStorageService storage,
ILogger<TableOffboardingAuditRepository> logger)
{
_tableClient = storage.GetTableClient(Constants.TableNames.OffboardingAudit);
_logger = logger;
}
// ── Marker ──────────────────────────────────────────────────────────────
public async Task<OffboardingMarkerEntry?> TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
try
{
var response = await _tableClient.GetEntityAsync<TableEntity>(
Constants.OffboardingPartitionKeys.Marker, normalizedTenantId, cancellationToken: ct);
return MapMarker(response.Value);
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
return null;
}
}
public Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default)
{
EnsureMarkerShape(marker);
return _tableClient.AddEntityAsync(StoreMarker(marker), ct);
}
public Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default)
{
EnsureMarkerShape(marker);
return _tableClient.UpsertEntityAsync(StoreMarker(marker), TableUpdateMode.Replace, ct);
}
public async Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
try
{
await _tableClient.DeleteEntityAsync(
Constants.OffboardingPartitionKeys.Marker, normalizedTenantId, ETag.All, ct);
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
// Idempotent — already gone.
}
}
public async IAsyncEnumerable<OffboardingMarkerEntry> QueryMarkersAsync(
[EnumeratorCancellation] CancellationToken ct = default)
{
var filter = $"PartitionKey eq '{Constants.OffboardingPartitionKeys.Marker}'";
await foreach (var entity in _tableClient.QueryAsync<TableEntity>(filter, cancellationToken: ct))
{
yield return MapMarker(entity);
}
}
// ── History ─────────────────────────────────────────────────────────────
public Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default)
{
EnsureHistoryShape(history);
return _tableClient.AddEntityAsync(StoreHistory(history), ct);
}
public async Task<OffboardingHistoryEntry?> TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default)
{
if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey));
try
{
var response = await _tableClient.GetEntityAsync<TableEntity>(
Constants.OffboardingPartitionKeys.History, historyRowKey, cancellationToken: ct);
return MapHistory(response.Value);
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
return null;
}
}
public Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default)
{
EnsureHistoryShape(history);
return _tableClient.UpsertEntityAsync(StoreHistory(history), TableUpdateMode.Replace, ct);
}
// ── ByTenant Pointer ────────────────────────────────────────────────────
public async Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync(
string normalizedTenantId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
try
{
var response = await _tableClient.GetEntityAsync<TableEntity>(
Constants.OffboardingPartitionKeys.ByTenant, normalizedTenantId, cancellationToken: ct);
return (MapByTenantPointer(response.Value), response.Value.ETag.ToString());
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
return (null, null);
}
}
public Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default)
{
EnsurePointerShape(pointer);
return _tableClient.AddEntityAsync(StoreByTenantPointer(pointer), ct);
}
public Task UpdateByTenantPointerWithEtagAsync(
OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default)
{
EnsurePointerShape(pointer);
if (string.IsNullOrEmpty(ifMatchEtag)) throw new ArgumentException("ifMatchEtag required", nameof(ifMatchEtag));
var entity = StoreByTenantPointer(pointer);
entity.ETag = new ETag(ifMatchEtag);
// TableUpdateMode.Replace + non-default ETag → conditional replace. Mismatch → 412.
return _tableClient.UpdateEntityAsync(entity, entity.ETag, TableUpdateMode.Replace, ct);
}
// ── Validation helpers ──────────────────────────────────────────────────
private static void EnsureMarkerShape(OffboardingMarkerEntry marker)
{
if (marker == null) throw new ArgumentNullException(nameof(marker));
if (marker.PartitionKey != Constants.OffboardingPartitionKeys.Marker)
throw new ArgumentException($"Marker PartitionKey must be '{Constants.OffboardingPartitionKeys.Marker}', got '{marker.PartitionKey}'", nameof(marker));
SecurityValidator.EnsureValidGuid(marker.RowKey, $"{nameof(marker)}.RowKey");
SecurityValidator.EnsureValidGuid(marker.TenantId, $"{nameof(marker)}.TenantId");
}
private static void EnsureHistoryShape(OffboardingHistoryEntry history)
{
if (history == null) throw new ArgumentNullException(nameof(history));
if (history.PartitionKey != Constants.OffboardingPartitionKeys.History)
throw new ArgumentException($"History PartitionKey must be '{Constants.OffboardingPartitionKeys.History}', got '{history.PartitionKey}'", nameof(history));
if (string.IsNullOrEmpty(history.RowKey)) throw new ArgumentException("History RowKey required", nameof(history));
SecurityValidator.EnsureValidGuid(history.TenantId, $"{nameof(history)}.TenantId");
}
private static void EnsurePointerShape(OffboardingByTenantPointer pointer)
{
if (pointer == null) throw new ArgumentNullException(nameof(pointer));
if (pointer.PartitionKey != Constants.OffboardingPartitionKeys.ByTenant)
throw new ArgumentException($"Pointer PartitionKey must be '{Constants.OffboardingPartitionKeys.ByTenant}', got '{pointer.PartitionKey}'", nameof(pointer));
SecurityValidator.EnsureValidGuid(pointer.RowKey, $"{nameof(pointer)}.RowKey");
SecurityValidator.EnsureValidGuid(pointer.TenantId, $"{nameof(pointer)}.TenantId");
if (string.IsNullOrEmpty(pointer.LatestHistoryRowKey))
throw new ArgumentException("Pointer.LatestHistoryRowKey required", nameof(pointer));
}
// ── Store / Map helpers (memory: feedback_table_storage_serialization) ──
private static TableEntity StoreMarker(OffboardingMarkerEntry m) => new(m.PartitionKey, m.RowKey)
{
["TenantId"] = m.TenantId,
["OffboardingHistoryRowKey"] = m.OffboardingHistoryRowKey,
["InitiatedAt"] = m.InitiatedAt,
["InitiatedBy"] = m.InitiatedBy,
["Status"] = m.Status,
["CompletedAt"] = m.CompletedAt,
["FailedAt"] = m.FailedAt,
["FailedPhase"] = m.FailedPhase,
};
private static OffboardingMarkerEntry MapMarker(TableEntity e) => new()
{
PartitionKey = e.PartitionKey,
RowKey = e.RowKey,
TenantId = e.GetString("TenantId") ?? string.Empty,
OffboardingHistoryRowKey = e.GetString("OffboardingHistoryRowKey") ?? string.Empty,
InitiatedAt = e.GetDateTime("InitiatedAt") ?? default,
InitiatedBy = e.GetString("InitiatedBy") ?? string.Empty,
Status = e.GetString("Status") ?? "Initiated",
CompletedAt = e.GetDateTime("CompletedAt"),
FailedAt = e.GetDateTime("FailedAt"),
FailedPhase = e.GetString("FailedPhase"),
};
private static TableEntity StoreHistory(OffboardingHistoryEntry h) => new(h.PartitionKey, h.RowKey)
{
["TenantId"] = h.TenantId,
["DomainName"] = h.DomainName,
["InitiatedBy"] = h.InitiatedBy,
["OffboardedAt"] = h.OffboardedAt,
["CompletedAt"] = h.CompletedAt,
["EarliestProcessingAt"] = h.EarliestProcessingAt,
["Status"] = h.Status,
["DeletedRowCountsJson"] = h.DeletedRowCountsJson,
["TotalRowsDeleted"] = h.TotalRowsDeleted,
["DeletedBlobCount"] = h.DeletedBlobCount,
["CascadeSessionsEnqueued"] = h.CascadeSessionsEnqueued,
["ErrorMessage"] = h.ErrorMessage,
["RetryCount"] = h.RetryCount,
["DrainCompletedAt"] = h.DrainCompletedAt,
["EnumerationStartedAt"] = h.EnumerationStartedAt,
["EnumerationCompletedBeforeUpload"] = h.EnumerationCompletedBeforeUpload,
["CustomGatherRulesArchived"] = h.CustomGatherRulesArchived,
["CustomAnalyzeRulesArchived"] = h.CustomAnalyzeRulesArchived,
["ImeLogPatternOverridesArchived"] = h.ImeLogPatternOverridesArchived,
["ReonboardedAt"] = h.ReonboardedAt,
["ReonboardedBy"] = h.ReonboardedBy,
["CustomsAutoWipedOnReonboard"] = h.CustomsAutoWipedOnReonboard,
};
private static OffboardingHistoryEntry MapHistory(TableEntity e) => new()
{
PartitionKey = e.PartitionKey,
RowKey = e.RowKey,
TenantId = e.GetString("TenantId") ?? string.Empty,
DomainName = e.GetString("DomainName") ?? string.Empty,
InitiatedBy = e.GetString("InitiatedBy") ?? string.Empty,
OffboardedAt = e.GetDateTime("OffboardedAt") ?? default,
CompletedAt = e.GetDateTime("CompletedAt"),
EarliestProcessingAt = e.GetDateTime("EarliestProcessingAt"),
Status = e.GetString("Status") ?? "Initiated",
DeletedRowCountsJson = e.GetString("DeletedRowCountsJson"),
TotalRowsDeleted = e.GetInt32("TotalRowsDeleted"),
DeletedBlobCount = e.GetInt32("DeletedBlobCount"),
CascadeSessionsEnqueued = e.GetInt32("CascadeSessionsEnqueued"),
ErrorMessage = e.GetString("ErrorMessage"),
RetryCount = e.GetInt32("RetryCount") ?? 0,
DrainCompletedAt = e.GetDateTime("DrainCompletedAt"),
EnumerationStartedAt = e.GetDateTime("EnumerationStartedAt"),
EnumerationCompletedBeforeUpload = e.GetDateTime("EnumerationCompletedBeforeUpload"),
CustomGatherRulesArchived = e.GetInt32("CustomGatherRulesArchived"),
CustomAnalyzeRulesArchived = e.GetInt32("CustomAnalyzeRulesArchived"),
ImeLogPatternOverridesArchived = e.GetInt32("ImeLogPatternOverridesArchived"),
ReonboardedAt = e.GetDateTime("ReonboardedAt"),
ReonboardedBy = e.GetString("ReonboardedBy"),
CustomsAutoWipedOnReonboard = e.GetInt32("CustomsAutoWipedOnReonboard"),
};
private static TableEntity StoreByTenantPointer(OffboardingByTenantPointer p) => new(p.PartitionKey, p.RowKey)
{
["TenantId"] = p.TenantId,
["LatestHistoryRowKey"] = p.LatestHistoryRowKey,
["LatestStatus"] = p.LatestStatus,
["LatestUpdatedAt"] = p.LatestUpdatedAt,
["OffboardCount"] = p.OffboardCount,
};
private static OffboardingByTenantPointer MapByTenantPointer(TableEntity e) => new()
{
PartitionKey = e.PartitionKey,
RowKey = e.RowKey,
TenantId = e.GetString("TenantId") ?? string.Empty,
LatestHistoryRowKey = e.GetString("LatestHistoryRowKey") ?? string.Empty,
LatestStatus = e.GetString("LatestStatus") ?? "Initiated",
LatestUpdatedAt = e.GetDateTime("LatestUpdatedAt") ?? default,
OffboardCount = e.GetInt32("OffboardCount") ?? 1,
};
}
}
@@ -0,0 +1,218 @@
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Azure.Data.Tables;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.DataAccess.TableStorage
{
/// <summary>
/// Table-storage backend for <see cref="ITenantCustomsArchiveRepository"/>.
/// Writes during Phase 2.D-archive, reads + deletes from the Global Admin UI.
/// Fail-loud — exceptions propagate to the caller (worker or HTTP handler).
/// </summary>
public sealed class TableTenantCustomsArchiveRepository : ITenantCustomsArchiveRepository
{
private readonly TableClient _tableClient;
private readonly ILogger<TableTenantCustomsArchiveRepository> _logger;
public TableTenantCustomsArchiveRepository(
TableStorageService storage,
ILogger<TableTenantCustomsArchiveRepository> logger)
{
_tableClient = storage.GetTableClient(Constants.TableNames.TenantOffboardingCustomsArchive);
_logger = logger;
}
public Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default)
{
if (entry == null) throw new ArgumentNullException(nameof(entry));
if (string.IsNullOrEmpty(entry.PartitionKey)) throw new ArgumentException("PartitionKey required", nameof(entry));
if (string.IsNullOrEmpty(entry.RowKey)) throw new ArgumentException("RowKey required", nameof(entry));
return _tableClient.UpsertEntityAsync(Store(entry), TableUpdateMode.Replace, ct);
}
public async Task<int> CountByRunAndTableAsync(
string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default)
{
var pk = BuildPartitionKey(normalizedTenantId, historyRowKey);
var rkPrefix = originalTable + "_";
// Range filter: rowKey ge 'GatherRules_' and rowKey lt 'GatherRules_~'.
// '~' (0x7E) is the largest visible ASCII char that base64url uses; works as
// exclusive upper bound because base64url alphabet is [A-Za-z0-9_-] (no '~').
var filter = $"PartitionKey eq '{Escape(pk)}' and RowKey ge '{Escape(rkPrefix)}' and RowKey lt '{Escape(rkPrefix + "~")}'";
var count = 0;
await foreach (var _ in _tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
{
count++;
}
return count;
}
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByRunAsync(
string normalizedTenantId, string historyRowKey, [EnumeratorCancellation] CancellationToken ct = default)
{
var pk = BuildPartitionKey(normalizedTenantId, historyRowKey);
var filter = $"PartitionKey eq '{Escape(pk)}'";
await foreach (var e in _tableClient.QueryAsync<TableEntity>(filter, cancellationToken: ct))
{
yield return Map(e);
}
}
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByTenantAsync(
string normalizedTenantId, [EnumeratorCancellation] CancellationToken ct = default)
{
// Per-tenant prefix scan: PartitionKey ge '{tid}_' and PartitionKey lt '{tid}_~'.
// History row keys begin with a numeric timestamp (yyyyMMddHHmmssfff) which is < '~',
// so the exclusive upper bound is correct.
var prefix = normalizedTenantId + "_";
var filter = $"PartitionKey ge '{Escape(prefix)}' and PartitionKey lt '{Escape(prefix + "~")}'";
await foreach (var e in _tableClient.QueryAsync<TableEntity>(filter, cancellationToken: ct))
{
yield return Map(e);
}
}
public async IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryAllAsync(
[EnumeratorCancellation] CancellationToken ct = default)
{
await foreach (var e in _tableClient.QueryAsync<TableEntity>(cancellationToken: ct))
{
yield return Map(e);
}
}
public async Task<TenantOffboardingCustomsArchiveEntry?> TryGetEntryAsync(
string partitionKey, string rowKey, CancellationToken ct = default)
{
try
{
var response = await _tableClient.GetEntityAsync<TableEntity>(partitionKey, rowKey, cancellationToken: ct);
return Map(response.Value);
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
return null;
}
}
public async Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default)
{
try
{
await _tableClient.DeleteEntityAsync(partitionKey, rowKey, ETag.All, ct);
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
// Idempotent — already gone.
}
}
public async Task<int> DeleteRunAsync(
string normalizedTenantId, string historyRowKey, CancellationToken ct = default)
{
var pk = BuildPartitionKey(normalizedTenantId, historyRowKey);
var filter = $"PartitionKey eq '{Escape(pk)}'";
var deleted = 0;
var batch = new List<TableTransactionAction>();
await foreach (var entity in _tableClient.QueryAsync<TableEntity>(filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
{
batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity));
deleted++;
if (batch.Count >= 100)
{
await _tableClient.SubmitTransactionAsync(batch, ct);
batch.Clear();
}
}
if (batch.Count > 0)
{
await _tableClient.SubmitTransactionAsync(batch, ct);
}
return deleted;
}
// ── Helpers ────────────────────────────────────────────────────────────
internal static string BuildPartitionKey(string normalizedTenantId, string historyRowKey)
=> $"{normalizedTenantId}_{historyRowKey}";
/// <summary>
/// Builds the archive RowKey for one original row. base64url-encodes the original
/// RowKey so it survives Azure-Tables' RowKey-forbidden character rules (#, ?,
/// /, \, control chars). Decoder is <see cref="DecodeOriginalRowKey"/>.
/// </summary>
public static string BuildRowKey(string originalTable, string originalRowKey)
=> $"{originalTable}_{Base64UrlEncode(originalRowKey)}";
/// <summary>Inverse of <see cref="BuildRowKey"/> — returns the original RowKey.</summary>
public static string DecodeOriginalRowKey(string archiveRowKey, string originalTable)
{
var prefix = originalTable + "_";
if (!archiveRowKey.StartsWith(prefix, StringComparison.Ordinal))
throw new ArgumentException($"Archive RowKey '{archiveRowKey}' does not start with expected table prefix '{prefix}'", nameof(archiveRowKey));
return Base64UrlDecode(archiveRowKey.Substring(prefix.Length));
}
private static string Base64UrlEncode(string s)
{
var bytes = System.Text.Encoding.UTF8.GetBytes(s);
return Convert.ToBase64String(bytes)
.Replace('+', '-')
.Replace('/', '_')
.TrimEnd('=');
}
private static string Base64UrlDecode(string s)
{
var padded = s.Replace('-', '+').Replace('_', '/');
switch (padded.Length % 4)
{
case 2: padded += "=="; break;
case 3: padded += "="; break;
}
var bytes = Convert.FromBase64String(padded);
return System.Text.Encoding.UTF8.GetString(bytes);
}
private static string Escape(string s) => s.Replace("'", "''");
private static TableEntity Store(TenantOffboardingCustomsArchiveEntry e) => new(e.PartitionKey, e.RowKey)
{
["TenantId"] = e.TenantId,
["OriginalTable"] = e.OriginalTable,
["OriginalPartitionKey"] = e.OriginalPartitionKey,
["OriginalRowKey"] = e.OriginalRowKey,
["EntityJson"] = e.EntityJson,
["HistoryRowKey"] = e.HistoryRowKey,
["ArchivedAt"] = e.ArchivedAt,
["ArchivedBy"] = e.ArchivedBy,
};
private static TenantOffboardingCustomsArchiveEntry Map(TableEntity e) => new()
{
PartitionKey = e.PartitionKey,
RowKey = e.RowKey,
TenantId = e.GetString("TenantId") ?? string.Empty,
OriginalTable = e.GetString("OriginalTable") ?? string.Empty,
OriginalPartitionKey = e.GetString("OriginalPartitionKey") ?? string.Empty,
OriginalRowKey = e.GetString("OriginalRowKey") ?? string.Empty,
EntityJson = e.GetString("EntityJson") ?? string.Empty,
HistoryRowKey = e.GetString("HistoryRowKey") ?? string.Empty,
ArchivedAt = e.GetDateTime("ArchivedAt") ?? default,
ArchivedBy = e.GetString("ArchivedBy") ?? "TenantOffboardingHandler",
};
}
}
@@ -0,0 +1,263 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using AutopilotMonitor.Functions.DataAccess.TableStorage;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Admin;
/// <summary>
/// GlobalAdmin-only read + delete endpoints over <see cref="ITenantCustomsArchiveRepository"/>.
/// Backs the <c>/global/customs-archive</c> Web UI: list runs, inspect entries, delete per
/// entry / per run. All writes audit-logged under <see cref="Constants.AuditGlobalTenantId"/>.
/// <para>
/// Authorization is enforced by <c>PolicyEnforcementMiddleware</c> via
/// <c>EndpointAccessPolicyCatalog</c>; the five routes registered here all carry
/// <c>EndpointPolicy.GlobalAdminOnly</c>.
/// </para>
/// </summary>
public class CustomsArchiveQueryFunction
{
private readonly ILogger<CustomsArchiveQueryFunction> _logger;
private readonly ITenantCustomsArchiveRepository _archive;
private readonly IMaintenanceRepository _maintenance;
public CustomsArchiveQueryFunction(
ILogger<CustomsArchiveQueryFunction> logger,
ITenantCustomsArchiveRepository archive,
IMaintenanceRepository maintenance)
{
_logger = logger;
_archive = archive;
_maintenance = maintenance;
}
/// <summary>
/// GET /api/global/customs-archive — list every archive partition (one entry per
/// (tenantId, historyRowKey) run) with aggregated row-counts per source table.
/// Optional ?tenantId= filter restricts to one tenant.
/// </summary>
[Function("CustomsArchiveListRuns")]
public async Task<HttpResponseData> ListRuns(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/customs-archive")] HttpRequestData req)
{
var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query ?? string.Empty);
var filterTenantId = query["tenantId"];
var summaries = new Dictionary<string, RunSummary>(System.StringComparer.Ordinal);
var source = string.IsNullOrEmpty(filterTenantId)
? _archive.QueryAllAsync()
: _archive.QueryByTenantAsync(filterTenantId.ToLowerInvariant());
await foreach (var entry in source)
{
var key = entry.PartitionKey;
if (!summaries.TryGetValue(key, out var summary))
{
summary = new RunSummary
{
PartitionKey = entry.PartitionKey,
TenantId = entry.TenantId,
HistoryRowKey = entry.HistoryRowKey,
ArchivedAt = entry.ArchivedAt,
};
summaries[key] = summary;
}
switch (entry.OriginalTable)
{
case Constants.TableNames.GatherRules: summary.GatherRulesCount++; break;
case Constants.TableNames.AnalyzeRules: summary.AnalyzeRulesCount++; break;
case Constants.TableNames.ImeLogPatterns: summary.ImeLogPatternsCount++; break;
}
// Track the earliest ArchivedAt for the run (in case rows have slightly drifting
// timestamps from multiple iterations of archive-then-wipe).
if (entry.ArchivedAt < summary.ArchivedAt)
{
summary.ArchivedAt = entry.ArchivedAt;
}
}
var runs = summaries.Values
.OrderByDescending(s => s.ArchivedAt)
.ToList();
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new { success = true, count = runs.Count, runs });
return response;
}
/// <summary>
/// GET /api/global/customs-archive/{tenantId}/{historyRowKey} — list every archived
/// entry in one offboarding run, with a short preview of <see cref="TenantOffboardingCustomsArchiveEntry.EntityJson"/>.
/// Full body is available via the per-entry route below.
/// </summary>
[Function("CustomsArchiveListEntries")]
public async Task<HttpResponseData> ListEntries(
[HttpTrigger(AuthorizationLevel.Anonymous, "get",
Route = "global/customs-archive/{tenantId}/{historyRowKey}")] HttpRequestData req,
string tenantId,
string historyRowKey)
{
var normalizedTenantId = tenantId.ToLowerInvariant();
var items = new List<EntrySummary>();
await foreach (var entry in _archive.QueryByRunAsync(normalizedTenantId, historyRowKey))
{
items.Add(new EntrySummary
{
PartitionKey = entry.PartitionKey,
RowKey = entry.RowKey,
OriginalTable = entry.OriginalTable,
OriginalRowKey = entry.OriginalRowKey,
ArchivedAt = entry.ArchivedAt,
EntityJsonPreview = TruncatePreview(entry.EntityJson, maxLength: 200),
});
}
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new { success = true, count = items.Count, entries = items });
return response;
}
/// <summary>
/// GET /api/global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey} — full
/// archive entry including <see cref="TenantOffboardingCustomsArchiveEntry.EntityJson"/>.
/// </summary>
[Function("CustomsArchiveGetEntry")]
public async Task<HttpResponseData> GetEntry(
[HttpTrigger(AuthorizationLevel.Anonymous, "get",
Route = "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}")] HttpRequestData req,
string tenantId,
string historyRowKey,
string archiveRowKey)
{
var partitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey(
tenantId.ToLowerInvariant(), historyRowKey);
var entry = await _archive.TryGetEntryAsync(partitionKey, archiveRowKey);
if (entry == null)
{
return req.CreateResponse(HttpStatusCode.NotFound);
}
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new { success = true, entry });
return response;
}
/// <summary>
/// DELETE /api/global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey} —
/// remove a single archived entry. Audit-logged under <see cref="Constants.AuditGlobalTenantId"/>.
/// </summary>
[Function("CustomsArchiveDeleteEntry")]
public async Task<HttpResponseData> DeleteEntry(
[HttpTrigger(AuthorizationLevel.Anonymous, "delete",
Route = "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}")] HttpRequestData req,
string tenantId,
string historyRowKey,
string archiveRowKey,
FunctionContext context)
{
var requestCtx = context.GetRequestContext();
var normalizedTenantId = tenantId.ToLowerInvariant();
var partitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey(normalizedTenantId, historyRowKey);
await _archive.DeleteEntryAsync(partitionKey, archiveRowKey);
await _maintenance.LogAuditEntryAsync(
Constants.AuditGlobalTenantId,
"DELETE",
"CustomsArchiveEntry",
$"{partitionKey}/{archiveRowKey}",
requestCtx.UserPrincipalName,
new Dictionary<string, string>
{
{ "TenantId", normalizedTenantId },
{ "HistoryRowKey", historyRowKey },
{ "ArchiveRowKey", archiveRowKey },
});
_logger.LogWarning(
"CustomsArchive entry deleted by {Upn}: tenant={Tenant} history={History} archive={Archive}",
requestCtx.UserPrincipalName, normalizedTenantId, historyRowKey, archiveRowKey);
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new { success = true });
return response;
}
/// <summary>
/// DELETE /api/global/customs-archive/{tenantId}/{historyRowKey} — bulk-delete every
/// entry of one offboarding run (after the operator has reviewed them). Audit-logged
/// under <see cref="Constants.AuditGlobalTenantId"/> with the deleted count.
/// </summary>
[Function("CustomsArchiveDeleteRun")]
public async Task<HttpResponseData> DeleteRun(
[HttpTrigger(AuthorizationLevel.Anonymous, "delete",
Route = "global/customs-archive/{tenantId}/{historyRowKey}")] HttpRequestData req,
string tenantId,
string historyRowKey,
FunctionContext context)
{
var requestCtx = context.GetRequestContext();
var normalizedTenantId = tenantId.ToLowerInvariant();
var deleted = await _archive.DeleteRunAsync(normalizedTenantId, historyRowKey);
await _maintenance.LogAuditEntryAsync(
Constants.AuditGlobalTenantId,
"DELETE",
"CustomsArchiveRun",
$"{normalizedTenantId}/{historyRowKey}",
requestCtx.UserPrincipalName,
new Dictionary<string, string>
{
{ "TenantId", normalizedTenantId },
{ "HistoryRowKey", historyRowKey },
{ "DeletedCount", deleted.ToString() },
});
_logger.LogWarning(
"CustomsArchive run deleted by {Upn}: tenant={Tenant} history={History} deleted={Count}",
requestCtx.UserPrincipalName, normalizedTenantId, historyRowKey, deleted);
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new { success = true, deleted });
return response;
}
private static string TruncatePreview(string s, int maxLength)
{
if (string.IsNullOrEmpty(s)) return string.Empty;
if (s.Length <= maxLength) return s;
return s.Substring(0, maxLength) + "…";
}
public class RunSummary
{
public string PartitionKey { get; set; } = string.Empty;
public string TenantId { get; set; } = string.Empty;
public string HistoryRowKey { get; set; } = string.Empty;
public System.DateTime ArchivedAt { get; set; }
public int GatherRulesCount { get; set; }
public int AnalyzeRulesCount { get; set; }
public int ImeLogPatternsCount { get; set; }
}
public class EntrySummary
{
public string PartitionKey { get; set; } = string.Empty;
public string RowKey { get; set; } = string.Empty;
public string OriginalTable { get; set; } = string.Empty;
public string OriginalRowKey { get; set; } = string.Empty;
public System.DateTime ArchivedAt { get; set; }
public string EntityJsonPreview { get; set; } = string.Empty;
}
}
@@ -1,8 +1,16 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Microsoft.AspNetCore.Authorization;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
@@ -11,34 +19,61 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Admin;
/// <summary>
/// Tenant Offboarding endpoint
/// Allows a Tenant Admin to permanently delete all data for their tenant.
/// This operation is irreversible.
/// Tenant offboarding endpoint (Phase 1 of the tenant-offboarding cascade plan).
/// Replaces the previous synchronous <c>DeleteAllTenantDataAsync</c> teardown with a
/// queued cascade. Phase 1 commits the History/Pointer/Marker rows, flips
/// <c>TenantConfiguration.Disabled=true</c> (the actual auth gate), audits, and enqueues
/// a <see cref="TenantOffboardingEnvelope"/> with a <see cref="DrainBarrier"/> visibility
/// delay. The worker picks the message up after the cache-drain window, by which point
/// every function-host has expired its 5-min <c>TenantConfigurationService</c> cache and
/// the existing Disabled-gate carries the 403. Returns 202.
/// </summary>
public class TenantOffboardFunction
{
private const int PointerCasMaxAttempts = 5;
/// <summary>
/// Cache-Drain-Barrier (plan v2 §2.2). Worker MUST NOT begin Phase 2 before this elapsed.
/// 5 min = <see cref="TenantConfigurationService.CacheDuration"/> (absolute expiration).
/// 1 min = clock-skew + GC + Storage-replication safety buffer.
/// Hardcoded by design: this is a structural invariant tied to CacheDuration, not an
/// operational knob. If CacheDuration is ever raised, this constant must move with it.
/// </summary>
internal static readonly TimeSpan DrainBarrier = TimeSpan.FromMinutes(6);
private readonly ILogger<TenantOffboardFunction> _logger;
private readonly IMaintenanceRepository _maintenanceRepo;
private readonly IConfigRepository _configRepo;
private readonly OpsEventService _opsEventService;
private readonly TenantConfigurationService _tenantConfigService;
private readonly IMaintenanceRepository _maintenanceRepo;
private readonly IOffboardingAuditRepository _offboardingRepo;
private readonly ITenantOffboardingEnqueuer _offboardingEnqueuer;
public TenantOffboardFunction(
ILogger<TenantOffboardFunction> logger,
IMaintenanceRepository maintenanceRepo,
IConfigRepository configRepo,
OpsEventService opsEventService)
TenantConfigurationService tenantConfigService,
IMaintenanceRepository maintenanceRepo,
IOffboardingAuditRepository offboardingRepo,
ITenantOffboardingEnqueuer offboardingEnqueuer)
{
_logger = logger;
_maintenanceRepo = maintenanceRepo;
_configRepo = configRepo;
_opsEventService = opsEventService;
_tenantConfigService = tenantConfigService;
_maintenanceRepo = maintenanceRepo;
_offboardingRepo = offboardingRepo;
_offboardingEnqueuer = offboardingEnqueuer;
}
/// <summary>
/// DELETE /api/tenants/{tenantId}/offboard
/// Permanently deletes ALL data for a tenant across all tables.
/// Accessible by: Tenant Admins of the same tenant OR Global Admins.
/// This action is IRREVERSIBLE.
/// Phase 1 — synchronously commits the offboarding state (History/Pointer/Marker),
/// flips <c>TenantConfiguration.Disabled=true</c> (the SOLE active auth-gate in the
/// PR3-revised architecture — the agent + web hotpaths read <c>config.Disabled</c>
/// only, never the marker), and enqueues a single tenant-offboarding envelope with
/// a <see cref="DrainBarrier"/> visibility-delay so warm function-host caches drain
/// before Phase 2 begins.
/// Accessible by: Tenant Admins of the same tenant OR Global Admins (enforced by
/// PolicyEnforcementMiddleware).
/// </summary>
[Function("OffboardTenant")]
[Authorize]
@@ -47,88 +82,677 @@ public class TenantOffboardFunction
string tenantId,
FunctionContext context)
{
// Authentication enforced by PolicyEnforcementMiddleware
var requestCtx = context.GetRequestContext();
var upn = requestCtx.UserPrincipalName;
var targetTenantId = requestCtx.TargetTenantId;
_logger.LogWarning($"TENANT OFFBOARD initiated for tenant {requestCtx.TargetTenantId} by {upn}");
if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
{
var badRequest = req.CreateResponse(HttpStatusCode.BadRequest);
await badRequest.WriteAsJsonAsync(new { error = "tenantId must be a valid GUID" });
return badRequest;
}
// Audit under Global TenantId so the entry survives tenant data deletion
var normalizedTenantId = targetTenantId.ToLowerInvariant();
_logger.LogWarning(
"TENANT OFFBOARD initiated for tenant {TenantId} by {Upn}",
normalizedTenantId, upn);
// 2. Capture DomainName from TenantConfiguration so the worker's Completed/Failed
// OpsEvents can render "{domain} ({tenantId})" instead of the bare GUID.
string? domainName = null;
try
{
var existingConfig = await _configRepo.GetTenantConfigurationAsync(targetTenantId);
domainName = existingConfig?.DomainName;
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Failed to read tenant configuration for {TenantId} before offboard; proceeding without domain name",
normalizedTenantId);
}
// 3. Idempotency: if an active marker exists, replay the queue-enqueue defensively
// when the offboarding is still in-flight. The previous implementation returned
// 200 unconditionally, which left the tenant stuck whenever the first enqueue
// failed between marker-insert and queue-send: the marker would block all future
// re-clicks via the idempotency-200 path while the worker queue was empty.
// Resume-paths per marker status:
// - Initiated/InProgress → defensive re-enqueue (TenantOffboardingHandler is
// idempotent via History.DrainCompletedAt Rev-9-F1 skip-gate; worst case the
// queue carries two envelopes that both resolve cleanly).
// - Completed → no re-enqueue (worker already finished; marker is
// the 15-min Rev-5-F2 grace blocker).
// - Failed → no re-enqueue (R1 fail-closed by design; operator
// must inspect History.FailedPhase + clear the marker manually before retry).
try
{
var existingMarker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
if (existingMarker != null)
{
return await HandleExistingMarkerAsync(
req, existingMarker, upn, normalizedTenantId, isRace: false);
}
}
catch (Exception ex)
{
_logger.LogError(ex,
"Failed to read offboarding marker for {TenantId} during idempotency check",
normalizedTenantId);
return await Build500Async(req, "Failed to read offboarding state");
}
var now = DateTime.UtcNow;
var historyRowKey = $"{now:yyyyMMddHHmmssfff}_{normalizedTenantId}";
var earliestProcessingAt = now + DrainBarrier;
// 4. Insert History row FIRST (audit trail must exist before any side-effect).
var history = new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = normalizedTenantId,
DomainName = domainName ?? string.Empty,
InitiatedBy = upn,
OffboardedAt = now,
EarliestProcessingAt = earliestProcessingAt,
Status = "Initiated",
RetryCount = 0,
};
try
{
await _offboardingRepo.InsertHistoryAsync(history);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to insert OffboardingHistory row for {TenantId}", normalizedTenantId);
return await Build500Async(req, "Failed to record offboarding history");
}
// 5. Upsert ByTenant pointer with ETag-CAS so OffboardCount increments race-safely.
try
{
await UpsertPointerWithCasAsync(normalizedTenantId, historyRowKey, now);
}
catch (Exception ex)
{
// History row is already committed — operator can find it under
// PartitionKey="OffboardingHistory" and retry manually. Graceful degradation
// (plan §4.4): pointer is the re-onboarding index; without it PR4's auto-wipe
// is no-op but the offboarding itself still proceeds.
_logger.LogError(ex,
"Failed to upsert OffboardingByTenant pointer for {TenantId} after history row inserted",
normalizedTenantId);
return await Build500Async(req, "Failed to update offboarding pointer");
}
// 6. Insert Marker LAST among the three rows. The marker is the idempotency anchor
// for admin re-clicks (the next click resolves through TryGetMarkerAsync and
// hits the resume path) and is later read by OffboardingMarkerCleanupFunction
// to drive post-completion cleanup sweeps. The Marker is NOT an auth-gate any
// more — TenantConfiguration.Disabled=true (committed in step 7 below) is the
// sole active gate.
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = normalizedTenantId,
TenantId = normalizedTenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = now,
InitiatedBy = upn,
Status = "Initiated",
};
try
{
await _offboardingRepo.InsertMarkerAsync(marker);
}
catch (RequestFailedException ex) when (ex.Status == 409)
{
// Race: another caller inserted a marker between our TryGet at step 3 and now.
// Resolve to the existing marker and run the same defensive re-enqueue path as
// the idempotency branch — the winner may still have failed its own enqueue and
// we cannot tell from here, so re-enqueue is the safe default. The worker is
// idempotent via the DrainCompletedAt skip-gate (Rev-9-F1).
_logger.LogInformation(
"OffboardingMarker insert raced for {TenantId}; another initiator won. Resolving to existing marker.",
normalizedTenantId);
var raced = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
if (raced == null)
{
_logger.LogError(
"OffboardingMarker 409 race for {TenantId} but follow-up read returned null; cannot resolve race",
normalizedTenantId);
return await Build500Async(req, "Failed to resolve offboarding race");
}
return await HandleExistingMarkerAsync(
req, raced, upn, normalizedTenantId, isRace: true);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to insert OffboardingMarker for {TenantId}", normalizedTenantId);
return await Build500Async(req, "Failed to commit offboarding marker");
}
// 7. Flip TenantConfiguration.Disabled=true. This is the SOLE active auth-gate now
// that the hotpath does not read the OffboardingMarker any more (PR3-revised).
// Fail-loud — without Disabled=true the cache-drain barrier offers no protection
// and Phase 2 would wipe data while warm function-hosts still accept agent
// traffic. Marker stays Initiated → next admin re-click hits the resume path
// which also calls EnsureTenantDisabledAsync before re-enqueuing.
try
{
await EnsureTenantDisabledAsync(targetTenantId, upn);
}
catch (Exception ex)
{
_logger.LogError(ex,
"Failed to set TenantConfiguration.Disabled=true for {TenantId}; refusing to enqueue (auth-gate not committed)",
normalizedTenantId);
return await Build500Async(req, "Failed to commit Disabled-gate; tenant offboarding NOT enqueued");
}
// 8. Audit row under AuditGlobalTenantId so it survives the eventual TenantConfiguration wipe.
await _maintenanceRepo.LogAuditEntryAsync(
Constants.AuditGlobalTenantId,
"DELETE",
"Tenant",
requestCtx.TargetTenantId,
upn!,
normalizedTenantId,
upn,
new Dictionary<string, string>
{
{ "Action", "Offboard" },
{ "Phase", "Initiated" }
{ "Phase", "Initiated" },
{ "HistoryRowKey", historyRowKey },
{ "DomainName", domainName ?? string.Empty },
});
var result = new OffboardResult { TenantId = requestCtx.TargetTenantId, InitiatedBy = upn!, InitiatedAt = DateTime.UtcNow };
// Capture domain name BEFORE deletion — tenant config is wiped by DeleteAllTenantDataAsync,
// and the ops alert / Telegram message is friendlier with the domain than the bare GUID.
string? domainName = null;
// 9. Enqueue with full DrainBarrier as visibility delay. The worker won't see this
// message until ~6min from now, by which time every function-host has refreshed
// its TenantConfiguration cache and the existing Disabled-gate is the active
// block. Fail-loud — marker stays Initiated on failure; the next admin re-click
// hits the idempotency branch which defensively re-enqueues with the remaining
// barrier so the tenant cannot get stuck.
try
{
var tenantConfig = await _configRepo.GetTenantConfigurationAsync(requestCtx.TargetTenantId);
domainName = tenantConfig?.DomainName;
await EnqueueEnvelopeAsync(normalizedTenantId, historyRowKey, upn, now, DrainBarrier);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to read tenant config for {TenantId} before offboard; proceeding without domain name", requestCtx.TargetTenantId);
_logger.LogError(ex,
"Failed to enqueue tenant-offboarding envelope for {TenantId}; marker remains Initiated for re-click resume",
normalizedTenantId);
return await Build500Async(req, "Failed to enqueue tenant offboarding");
}
try
_logger.LogWarning(
"TENANT OFFBOARD queued for {TenantId} by {Upn}; historyRowKey={History}; earliestProcessingAt={ProcessAt}",
normalizedTenantId, upn, historyRowKey, earliestProcessingAt);
var response = req.CreateResponse(HttpStatusCode.Accepted);
await response.WriteAsJsonAsync(new OffboardResponse
{
var deletedCounts = await _maintenanceRepo.DeleteAllTenantDataAsync(requestCtx.TargetTenantId);
result.DeletedCounts = deletedCounts;
result.Success = true;
_logger.LogWarning($"TENANT OFFBOARD completed for tenant {requestCtx.TargetTenantId} by {upn}. " +
$"Total rows deleted: {result.DeletedCounts.Values.Sum()}");
// Audit completion with deletion summary
var details = result.DeletedCounts.ToDictionary(kv => kv.Key, kv => kv.Value.ToString());
details["Action"] = "Offboard";
details["Phase"] = "Completed";
details["TotalRowsDeleted"] = result.DeletedCounts.Values.Sum().ToString();
await _maintenanceRepo.LogAuditEntryAsync(
Constants.AuditGlobalTenantId,
"DELETE",
"Tenant",
requestCtx.TargetTenantId,
upn!,
details);
await _opsEventService.RecordTenantOffboardedAsync(requestCtx.TargetTenantId, upn!, deletedCounts, domainName);
var okResponse = req.CreateResponse(HttpStatusCode.OK);
await okResponse.WriteAsJsonAsync(result);
return okResponse;
}
catch (Exception ex)
{
_logger.LogError(ex, $"Tenant offboard failed for tenant {requestCtx.TargetTenantId}");
result.Success = false;
result.Error = "Internal server error";
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
await errorResponse.WriteAsJsonAsync(result);
return errorResponse;
}
TenantId = normalizedTenantId,
Status = "Queued",
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
HistoryRowKey = historyRowKey,
EarliestProcessingAt = earliestProcessingAt,
Message = "Tenant offboarding queued for asynchronous processing"
});
return response;
}
// Resume-path for re-clicks and 409-race winners. Thin HTTP shell on top of
// ResumeExistingMarkerAsync — the actual decision (re-enqueue or not, message text,
// remaining drain-barrier) lives in the pure helper below so it can be unit-tested
// without faking HttpRequestData. Plan Review Finding 1 (stuck-tenant fix).
internal async Task<HttpResponseData> HandleExistingMarkerAsync(
HttpRequestData req,
OffboardingMarkerEntry existing,
string upn,
string normalizedTenantId,
bool isRace)
{
var resume = await ResumeExistingMarkerAsync(existing, upn, normalizedTenantId, isRace);
if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed)
{
return await Build500Async(req, "Failed to re-enqueue tenant offboarding");
}
var response = req.CreateResponse(resume.StatusCode);
await response.WriteAsJsonAsync(new OffboardResponse
{
TenantId = normalizedTenantId,
Status = existing.Status,
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
HistoryRowKey = existing.OffboardingHistoryRowKey,
EarliestProcessingAt = resume.EarliestProcessingAt,
Message = resume.Message,
});
return response;
}
/// <summary>
/// Pure resume decision: classifies the existing marker, optionally re-enqueues, and
/// returns the (StatusCode, Message, ReEnqueued?) tuple the HTTP shell wraps. Exposed
/// internal for unit tests — Finding-1 fix requires asserting "Initiated/InProgress
/// triggers a queue write, Completed/Failed does not".
/// </summary>
internal async Task<ResumeOutcome> ResumeExistingMarkerAsync(
OffboardingMarkerEntry existing,
string upn,
string normalizedTenantId,
bool isRace)
{
var status = existing.Status ?? "Initiated";
if (string.Equals(status, "Completed", StringComparison.OrdinalIgnoreCase))
{
_logger.LogInformation(
"TenantOffboard re-click for {TenantId}: marker is Completed; no re-enqueue (race={Race})",
normalizedTenantId, isRace);
return new ResumeOutcome(
ResumeOutcomeKind.IdempotentCompleted,
HttpStatusCode.OK,
"Tenant offboarding already completed (marker in 15-min grace window)",
ReEnqueued: false);
}
if (string.Equals(status, "Failed", StringComparison.OrdinalIgnoreCase))
{
// Failed is fail-closed by design (R1). Re-enqueuing a Failed offboarding would
// just retrace the same failure path. Surface the FailedPhase to the operator
// so they can clear the marker manually before retrying.
_logger.LogWarning(
"TenantOffboard re-click for {TenantId}: marker is Failed (phase={Phase}); operator action required, no re-enqueue",
normalizedTenantId, existing.FailedPhase ?? "unknown");
return new ResumeOutcome(
ResumeOutcomeKind.IdempotentFailed,
HttpStatusCode.OK,
$"Tenant offboarding previously failed at phase '{existing.FailedPhase ?? "unknown"}' — operator action required before retry",
ReEnqueued: false);
}
// Initiated / InProgress (or any unrecognised in-flight value): defensively
// re-enqueue the envelope. The handler tolerates double-pickup via the
// DrainCompletedAt skip-gate, so a duplicate envelope on the queue is safe.
//
// The visibility-delay computation below uses
// max(history.EarliestProcessingAt, tombstone.LastUpdated + DrainBarrier)
// as the effective drain-deadline. History is best-effort here: if the read
// fails or the field is missing, the tombstone anchor still gives a safe lower
// bound (when the tombstone was just written, that's now + DrainBarrier; when
// it was written long ago, the barrier is already elapsed).
OffboardingHistoryEntry? history = null;
try
{
history = await _offboardingRepo.TryGetHistoryAsync(existing.OffboardingHistoryRowKey);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Failed to read History row {HistoryRowKey} for {TenantId} during resume; falling through to tombstone.LastUpdated-based deadline",
existing.OffboardingHistoryRowKey, normalizedTenantId);
}
// Re-affirm the Disabled-gate FIRST so we know whether we just wrote it for the
// first time (= cache-drain barrier restarts now) or it was already there from a
// prior attempt (= the History row's deadline is still valid). The prior run may
// have crashed between marker-insert and Disabled-flip, or an admin may have
// cleared the flag manually. Either way, we MUST NOT re-enqueue without the
// auth-gate committed — otherwise the cache-drain barrier guards nothing.
EnsureDisabledResult ensureResult;
try
{
ensureResult = await EnsureTenantDisabledAsync(normalizedTenantId, upn);
}
catch (Exception ex)
{
_logger.LogError(ex,
"Resume: failed to re-affirm TenantConfiguration.Disabled=true for {TenantId}; refusing to re-enqueue",
normalizedTenantId);
return new ResumeOutcome(
ResumeOutcomeKind.ReEnqueueFailed,
HttpStatusCode.InternalServerError,
"Failed to re-affirm Disabled-gate; tenant offboarding NOT re-enqueued",
ReEnqueued: false);
}
// Resume-path visibility-delay decision (Codex finding "Resume can bypass cache-drain"):
// We compute the effective drain-deadline as the MAX of two sources, then derive
// the queue-message visibility-delay from that:
//
// (a) history.EarliestProcessingAt — set on Phase 1 commit; patched here on
// fresh writes when we can.
// (b) ensureResult.TombstoneLastUpdated + DrainBarrier — the authoritative
// "cache-drain started here" timestamp, written atomically with the
// Disabled-flip, ALWAYS present even if the History-patch later fails.
//
// Using max() of both makes the History-patch non-load-bearing for safety: even if
// a fast follow-up re-click reads stale EarliestProcessingAt, the TombstoneLastUpdated
// anchor gives us a correct lower bound. The History-patch is now best-effort
// documentation, not a critical write.
var now = DateTime.UtcNow;
var tombstoneDeadline = ensureResult.TombstoneLastUpdated + DrainBarrier;
var historyDeadline = history?.EarliestProcessingAt ?? tombstoneDeadline;
var effectiveDeadline = tombstoneDeadline > historyDeadline ? tombstoneDeadline : historyDeadline;
var visibilityDelay = effectiveDeadline > now ? effectiveDeadline - now : TimeSpan.Zero;
var earliestProcessingAtForResponse = effectiveDeadline;
// Patch History.EarliestProcessingAt for new tombstone writes so the UI countdown
// + any later admin tools see the new deadline. Failure here is NOT a safety issue
// anymore (tombstone.LastUpdated covers us above) — purely cosmetic / observability.
if (ensureResult.Outcome == EnsureDisabledOutcome.WroteNewTombstone && history != null)
{
history.EarliestProcessingAt = effectiveDeadline;
try
{
await _offboardingRepo.UpsertHistoryAsync(history);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Resume: failed to patch History.EarliestProcessingAt for {TenantId} {History} — UI countdown may show stale value, but the queue visibility-delay + tombstone.LastUpdated guarantee the drain barrier on the worker side",
normalizedTenantId, existing.OffboardingHistoryRowKey);
}
_logger.LogWarning(
"Resume: Disabled-gate was freshly written; restarted full cache-drain barrier ({Barrier}) for tenant={TenantId}",
DrainBarrier, normalizedTenantId);
}
try
{
await EnqueueEnvelopeAsync(
normalizedTenantId, existing.OffboardingHistoryRowKey, upn, existing.InitiatedAt, visibilityDelay);
}
catch (Exception ex)
{
_logger.LogError(ex,
"Defensive re-enqueue failed for {TenantId} (existing marker status={Status}, race={Race})",
normalizedTenantId, status, isRace);
return new ResumeOutcome(
ResumeOutcomeKind.ReEnqueueFailed,
HttpStatusCode.InternalServerError,
"Failed to re-enqueue tenant offboarding",
ReEnqueued: false);
}
_logger.LogInformation(
"TenantOffboard re-click for {TenantId}: defensively re-enqueued with visibilityDelay={Delay} outcome={Outcome} tombstoneLastUpdated={TombstoneLastUpdated} (existing status={Status}, race={Race})",
normalizedTenantId, visibilityDelay, ensureResult.Outcome, ensureResult.TombstoneLastUpdated, status, isRace);
return new ResumeOutcome(
ResumeOutcomeKind.ReEnqueuedInFlight,
HttpStatusCode.Accepted,
isRace
? "Tenant offboarding already in progress (race-winner resolved, defensively re-enqueued)"
: $"Tenant offboarding resumed (existing status={status}, defensively re-enqueued)",
ReEnqueued: true,
EarliestProcessingAt: earliestProcessingAtForResponse);
}
/// <summary>
/// Magic-string DisabledReason that identifies a TenantConfiguration row as the
/// offboarding tombstone. Used by <c>EnsureTenantDisabledAsync</c> for the idempotent
/// no-write check AND by <see cref="OffboardingMarkerCleanupFunction"/> to gate the
/// defense-in-depth TenantConfiguration sweep so a self-service-re-onboarded fresh
/// config does not get wiped by mistake.
/// </summary>
internal const string OffboardingDisabledReason = "Offboarding in progress";
/// <summary>
/// Result of <see cref="EnsureTenantDisabledAsync"/>. Carries both whether a fresh
/// write happened AND the tombstone's <c>LastUpdated</c> timestamp so the resume-path
/// can compute a safe drain-deadline without depending on the (optionally-failing)
/// <c>History.EarliestProcessingAt</c> patch.
/// <para>
/// Drain-deadline math the caller applies:
/// <c>effectiveDeadline = max(history.EarliestProcessingAt, TombstoneLastUpdated + DrainBarrier)</c>.
/// Because the tombstone's <c>LastUpdated</c> is part of the same atomic Save as the
/// Disabled-flip, it is always present once the auth-gate is committed — even if
/// the History-row patch later fails on a fast re-click.
/// </para>
/// </summary>
internal readonly record struct EnsureDisabledResult(
EnsureDisabledOutcome Outcome,
DateTime TombstoneLastUpdated);
internal enum EnsureDisabledOutcome
{
/// <summary>Row was already a Disabled=true tombstone with the offboarding reason —
/// no Save was issued. Existing <c>LastUpdated</c> is returned.</summary>
AlreadyTombstone,
/// <summary>We actively wrote the Disabled=true gate (either flipped an existing
/// row or created a fresh one). <c>LastUpdated</c> is the timestamp of this write.</summary>
WroteNewTombstone,
}
/// <summary>
/// Commits the SOLE active auth-gate (<c>TenantConfiguration.Disabled=true</c>) and
/// invalidates the local <see cref="TenantConfigurationService"/> cache. Fail-loud:
/// every caller (initial enqueue + resume re-enqueue) MUST treat an exception as
/// "do not enqueue" — without this gate committed, the cache-drain barrier protects
/// nothing and Phase 2 would wipe data while warm function-hosts still accept agent
/// traffic.
/// <para>
/// Return value distinguishes the two outcomes so the resume-path can decide whether
/// to honour the old <c>History.EarliestProcessingAt</c> deadline
/// (<see cref="EnsureDisabledResult.AlreadyTombstone"/>) or restart the cache-drain
/// barrier from now (<see cref="EnsureDisabledResult.WroteNewTombstone"/>) — see
/// resume-path Codex finding "Resume can bypass cache-drain when first attempt failed
/// before Disabled-write".
/// </para>
/// <para>
/// Behaviour:
/// <list type="bullet">
/// <item>Existing TenantConfiguration row → read-modify-write so signup data + custom
/// rate limits etc. are preserved. Idempotent: if already
/// <c>Disabled=true</c> with the offboarding reason, no Save is issued and
/// <see cref="EnsureDisabledResult.AlreadyTombstone"/> is returned.</item>
/// <item>Missing TenantConfiguration row (edge case: very old tenant / manual delete)
/// → create a fresh <c>CreateDefault</c> row with <c>Disabled=true</c>. Necessary
/// because <c>AuthFunction</c> would otherwise auto-create a <c>Disabled=false</c>
/// default on the next <c>/api/auth/me</c> and slip the offboarded tenant back through.</item>
/// <item><c>SaveTenantConfigurationAsync</c> returns <c>false</c> on transient storage
/// failure (the production repo does NOT throw; see Codex-finding TableConfigRepository).
/// We turn that into <see cref="InvalidOperationException"/> so the caller treats it
/// as a hard fail.</item>
/// </list>
/// </para>
/// </summary>
internal async Task<EnsureDisabledResult> EnsureTenantDisabledAsync(string targetTenantId, string upn)
{
var tenantConfig = await _configRepo.GetTenantConfigurationAsync(targetTenantId);
if (tenantConfig != null)
{
// Idempotent: if Disabled is already true with our reason, no-op the write so
// resume-clicks do not thrash the LastUpdated timestamp on the row.
var alreadyDisabled = tenantConfig.Disabled
&& string.Equals(tenantConfig.DisabledReason, OffboardingDisabledReason, StringComparison.Ordinal);
if (alreadyDisabled)
{
_tenantConfigService.InvalidateCache(targetTenantId);
return new EnsureDisabledResult(EnsureDisabledOutcome.AlreadyTombstone, tenantConfig.LastUpdated);
}
}
else
{
// Missing-row edge case: create a fresh Disabled=true row so AuthFunction's
// auto-create-default cannot bring the tenant back enabled.
tenantConfig = TenantConfiguration.CreateDefault(targetTenantId);
_logger.LogWarning(
"TenantOffboard: no TenantConfiguration row found for {TenantId} — creating fresh Disabled=true tombstone",
targetTenantId);
}
tenantConfig.Disabled = true;
tenantConfig.DisabledReason = OffboardingDisabledReason;
tenantConfig.DisabledUntil = null;
tenantConfig.UpdatedBy = upn;
// Explicit LastUpdated stamp — we bypass TenantConfigurationService.SaveConfigurationAsync
// (which would set this) and write to the repo directly, so we own the bookkeeping.
// Critical for the resume-path drain-deadline math: this timestamp is the
// authoritative "when did the cache-drain barrier start" anchor.
var writtenAt = DateTime.UtcNow;
tenantConfig.LastUpdated = writtenAt;
var saved = await _configRepo.SaveTenantConfigurationAsync(tenantConfig);
if (!saved)
{
// TableConfigRepository.SaveTenantConfigurationAsync returns false on transient
// storage failure instead of throwing. Without throwing here, the caller would
// proceed to enqueue without the auth-gate committed — defeating the whole
// cache-drain barrier. Surface as a hard failure.
throw new InvalidOperationException(
$"SaveTenantConfigurationAsync returned false for tenant {targetTenantId}; Disabled-gate NOT committed");
}
_tenantConfigService.InvalidateCache(targetTenantId);
return new EnsureDisabledResult(EnsureDisabledOutcome.WroteNewTombstone, writtenAt);
}
/// <summary>
/// Enqueues the offboarding envelope with the given visibility delay (queue-side
/// invisibility window). First-click: full <see cref="DrainBarrier"/>. Resume-click:
/// remaining delay computed from <c>EarliestProcessingAt</c> so a re-enqueue does not
/// reset the drain barrier (the message would otherwise become visible too early).
/// </summary>
internal Task EnqueueEnvelopeAsync(
string normalizedTenantId, string historyRowKey, string upn, DateTime initiatedAt,
TimeSpan? visibilityDelay)
{
var envelope = new TenantOffboardingEnvelope
{
EnvelopeVersion = "1",
TenantId = normalizedTenantId,
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
HistoryRowKey = historyRowKey,
InitiatedBy = upn,
InitiatedAt = initiatedAt,
EnqueuedAt = DateTime.UtcNow,
DrainPollCount = 0,
};
return _offboardingEnqueuer.EnqueueAsync(envelope, visibilityDelay);
}
// Plan §4.4 read-modify-write CAS-loop. Bounded retry; throws on exhaustion so the
// caller turns it into a 500. The pointer is the O(1) re-onboarding index — failure
// here is recoverable (graceful degradation in PR4) but we still surface it so the
// operator can investigate.
internal async Task UpsertPointerWithCasAsync(
string normalizedTenantId, string historyRowKey, DateTime now)
{
for (var attempt = 1; attempt <= PointerCasMaxAttempts; attempt++)
{
var (existing, etag) = await _offboardingRepo.TryGetByTenantPointerAsync(normalizedTenantId);
if (existing == null)
{
var fresh = new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = normalizedTenantId,
TenantId = normalizedTenantId,
LatestHistoryRowKey = historyRowKey,
LatestStatus = "Initiated",
LatestUpdatedAt = now,
OffboardCount = 1,
};
try
{
await _offboardingRepo.InsertByTenantPointerAsync(fresh);
return;
}
catch (RequestFailedException ex) when (ex.Status == 409)
{
_logger.LogInformation(
"OffboardingByTenant pointer raced on insert for {TenantId} (attempt {Attempt}); falling back to ETag update",
normalizedTenantId, attempt);
continue;
}
}
existing.LatestHistoryRowKey = historyRowKey;
existing.LatestStatus = "Initiated";
existing.LatestUpdatedAt = now;
existing.OffboardCount = existing.OffboardCount + 1;
try
{
await _offboardingRepo.UpdateByTenantPointerWithEtagAsync(existing, etag!);
return;
}
catch (RequestFailedException ex) when (ex.Status == 412)
{
_logger.LogInformation(
"OffboardingByTenant pointer ETag mismatch for {TenantId} (attempt {Attempt}); retrying",
normalizedTenantId, attempt);
}
}
throw new InvalidOperationException(
$"OffboardingByTenant pointer CAS exhausted for tenant {normalizedTenantId} after {PointerCasMaxAttempts} attempts");
}
private static async Task<HttpResponseData> Build500Async(HttpRequestData req, string message)
{
var response = req.CreateResponse(HttpStatusCode.InternalServerError);
await response.WriteAsJsonAsync(new { success = false, error = message });
return response;
}
}
public class OffboardResult
/// <summary>
/// 202/200 response body for the offboarding endpoint. Fields point the caller at the
/// History row so subsequent reporting / status polling can resolve back to the audit
/// trail. <see cref="EarliestProcessingAt"/> drives the "data deletion starts in mm ss"
/// countdown in the Web UI's drain-barrier state.
/// </summary>
public class OffboardResponse
{
public string TenantId { get; set; } = string.Empty;
public string InitiatedBy { get; set; } = string.Empty;
public DateTime InitiatedAt { get; set; }
public bool Success { get; set; }
public string? Error { get; set; }
public Dictionary<string, int> DeletedCounts { get; set; } = new();
public string Status { get; set; } = string.Empty;
public string HistoryPartitionKey { get; set; } = string.Empty;
public string HistoryRowKey { get; set; } = string.Empty;
public string Message { get; set; } = string.Empty;
/// <summary>UTC timestamp before which the worker MUST NOT start Phase 2. Drives the
/// cache-drain-barrier countdown UI. Null on the idempotent-Completed/Failed branches.</summary>
public DateTime? EarliestProcessingAt { get; set; }
}
/// <summary>
/// Decision the resume-path produced; consumed by <see cref="TenantOffboardFunction.HandleExistingMarkerAsync"/>
/// to shape the HTTP response. Test-visible so unit tests can assert the re-enqueue decision
/// without faking <c>HttpRequestData</c>.
/// <para>
/// <see cref="EarliestProcessingAt"/> is the History row's drain-barrier deadline for the
/// in-flight path (so the UI can keep counting down through a re-click); null on the
/// terminal idempotent branches.
/// </para>
/// </summary>
internal sealed record ResumeOutcome(
ResumeOutcomeKind Kind,
System.Net.HttpStatusCode StatusCode,
string Message,
bool ReEnqueued,
DateTime? EarliestProcessingAt = null);
internal enum ResumeOutcomeKind
{
/// <summary>Marker is Completed; 15-min grace blocker is held, no re-enqueue.</summary>
IdempotentCompleted,
/// <summary>Marker is Failed; operator action required, no re-enqueue.</summary>
IdempotentFailed,
/// <summary>Marker is Initiated/InProgress; defensive re-enqueue was sent.</summary>
ReEnqueuedInFlight,
/// <summary>Marker classified as in-flight but re-enqueue threw; surface 500.</summary>
ReEnqueueFailed,
}
@@ -0,0 +1,230 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Functions.Admin;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Maintenance
{
/// <summary>
/// Marker housekeeping + defense-in-depth cleanup for the tenant-offboarding cascade.
/// Plan §7.5 (PR3.B-revised).
/// <para>
/// Tenant-offboarding markers (<see cref="OffboardingMarkerEntry"/> under
/// <c>OffboardingAudit/OffboardingMarker</c>) live past the Completed transition by
/// design — they are the anchor that lets this function find lingering side-effects
/// (Expectations blob, possibly-stale <c>TenantConfiguration</c> tombstone) and clean
/// them up if the handler's Phase 2.F-final/2.G ordering hit a crash. Failed markers
/// are never auto-removed; they require operator action.
/// </para>
/// <para>
/// Note: in the PR3-revised architecture the marker is NO LONGER read in the
/// agent/web auth hotpath. The active gate is <c>TenantConfiguration.Disabled=true</c>;
/// the cache-drain barrier (6 min visibility delay on the worker envelope) plus
/// <see cref="MinCompletedAgeForCleanup"/> together absorb the
/// <c>TenantConfigurationService</c> cache TTL (5 min). The marker's lifetime here is
/// purely about cleanup ordering and crash recovery, not auth.
/// </para>
/// <para>
/// Cleanup order per Completed marker, each step's failure keeps the marker for retry:
/// (1) Expectations blob delete, (2) probe TenantConfiguration → conditional sweep if
/// still the offboarding tombstone (skipped when the user has self-service-re-onboarded
/// and the row carries a non-offboarding state), (3) marker delete.
/// </para>
/// </summary>
public class OffboardingMarkerCleanupFunction
{
/// <summary>
/// 3× <c>TenantConfigurationService.CacheDuration</c> (5min) — safety margin that
/// absorbs clock-skew + GC pauses + cache-read-after-save anomalies on warm
/// function instances. Originally sized to guarantee the marker outlives every
/// cached TenantConfiguration on every host; retained at 15min because the
/// defense-in-depth sweeps (Expectations blob + TenantConfiguration tombstone)
/// still benefit from waiting for the dust to settle.
/// </summary>
public static readonly TimeSpan MinCompletedAgeForCleanup = TimeSpan.FromMinutes(15);
/// <summary>Failed markers older than this trigger a warn-log every run (operator nudge).</summary>
internal static readonly TimeSpan FailedMarkerWarnAge = TimeSpan.FromHours(1);
// Every 2h — matches the broader Maintenance cadence and gives at most a 2h lag between
// 15min-eligible and actual cleanup. Plan §7.5: "Function läuft im selben Maintenance-
// Schedule wie SessionDeletionMaintenance".
private const string Cron = "0 0 */2 * * *";
private readonly IOffboardingAuditRepository _auditRepo;
private readonly IOffboardingExpectationsStore _expectations;
private readonly SafeWipeService _safeWipe;
private readonly IConfigRepository _configRepo;
private readonly ILogger<OffboardingMarkerCleanupFunction> _logger;
private readonly TimeProvider _timeProvider;
public OffboardingMarkerCleanupFunction(
IOffboardingAuditRepository auditRepo,
IOffboardingExpectationsStore expectations,
SafeWipeService safeWipe,
IConfigRepository configRepo,
ILogger<OffboardingMarkerCleanupFunction> logger)
: this(auditRepo, expectations, safeWipe, configRepo, logger, TimeProvider.System)
{
}
/// <summary>Test seam — inject a fake <see cref="TimeProvider"/> so age comparisons are deterministic.</summary>
internal OffboardingMarkerCleanupFunction(
IOffboardingAuditRepository auditRepo,
IOffboardingExpectationsStore expectations,
SafeWipeService safeWipe,
IConfigRepository configRepo,
ILogger<OffboardingMarkerCleanupFunction> logger,
TimeProvider timeProvider)
{
_auditRepo = auditRepo;
_expectations = expectations;
_safeWipe = safeWipe;
_configRepo = configRepo;
_logger = logger;
_timeProvider = timeProvider;
}
[Function("OffboardingMarkerCleanup")]
public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken)
{
await RunCoreAsync(cancellationToken);
}
/// <summary>Testable core — bypasses TimerInfo / FunctionContext.</summary>
internal async Task<CleanupResult> RunCoreAsync(CancellationToken ct)
{
var now = _timeProvider.GetUtcNow().UtcDateTime;
var result = new CleanupResult();
await foreach (var marker in _auditRepo.QueryMarkersAsync(ct))
{
ct.ThrowIfCancellationRequested();
result.Scanned++;
if (marker.Status == "Completed"
&& marker.CompletedAt is { } completedAt
&& completedAt < now - MinCompletedAgeForCleanup)
{
// Cleanup order — each step's failure must keep the marker for the next run:
// 1. Expectations blob (Rev-9 hygiene)
// 2. TenantConfiguration sweep (PR3.B-revised Finding 2 defense-in-depth)
// 3. Marker delete (last anchor)
try
{
await _expectations.DeleteAsync(marker.TenantId, marker.OffboardingHistoryRowKey, ct);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"OffboardingMarkerCleanup: Expectations blob delete failed for tenant={Tenant} history={History} — keeping marker for next-run retry",
marker.TenantId, marker.OffboardingHistoryRowKey);
result.BlobDeleteRetries++;
continue;
}
// PR3.B Defense-in-depth: if Phase 2.F-final failed in the handler
// (crash between 2.G commit and 2.F-final, or transient storage), the
// TenantConfiguration row would still carry Disabled=true and block the
// tenant from self-service re-onboarding indefinitely. Sweep it here.
//
// CRITICAL: must be CONDITIONAL on the row still being the offboarding
// tombstone. The handler intentionally deletes the row at the end of
// Phase 2.F-final precisely so the next /api/auth/me triggers a fresh
// Default-Disabled=false config (self-service re-onboarding). If the
// user has re-onboarded between Phase 2.F-final and now, blind wiping
// would destroy their fresh tenant. So: read first, only wipe when
// (Disabled=true AND DisabledReason matches OffboardingDisabledReason).
try
{
var existingConfig = await _configRepo.GetTenantConfigurationAsync(marker.TenantId);
if (existingConfig == null)
{
// Normal happy path: Phase 2.F-final succeeded, row already gone.
}
else if (existingConfig.Disabled
&& string.Equals(existingConfig.DisabledReason, TenantOffboardFunction.OffboardingDisabledReason, StringComparison.Ordinal))
{
// Still the offboarding tombstone → Phase 2.F-final must have failed.
var swept = await _safeWipe.WipeByExactPartitionAsync(
Constants.TableNames.TenantConfiguration, marker.TenantId, ct);
if (swept > 0)
{
result.TenantConfigsSwept++;
_logger.LogWarning(
"OffboardingMarkerCleanup: swept lingering TenantConfiguration tombstone for tenant={Tenant} (Phase 2.F-final must have failed in the handler)",
marker.TenantId);
}
}
else
{
// User has self-service-re-onboarded between Phase 2.F-final and
// now. Leave their fresh config alone.
result.TenantConfigsSpared++;
_logger.LogInformation(
"OffboardingMarkerCleanup: tenant={Tenant} has a non-tombstone TenantConfiguration row (Disabled={Disabled}, Reason={Reason}) — user re-onboarded, skip sweep",
marker.TenantId, existingConfig.Disabled, existingConfig.DisabledReason ?? "<null>");
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"OffboardingMarkerCleanup: TenantConfiguration probe/sweep failed for tenant={Tenant} — keeping marker for next-run retry",
marker.TenantId);
result.TenantConfigSweepRetries++;
continue;
}
await _auditRepo.DeleteMarkerAsync(marker.RowKey, ct);
result.MarkersDeleted++;
_logger.LogInformation(
"OffboardingMarkerCleanup: deleted Completed marker for tenant={Tenant} (age={Age})",
marker.TenantId, now - completedAt);
}
else if (marker.Status == "Failed")
{
result.FailedMarkersSeen++;
var failedAt = marker.FailedAt ?? marker.InitiatedAt;
var age = now - failedAt;
if (age > FailedMarkerWarnAge)
{
_logger.LogWarning(
"OffboardingMarkerCleanup: stale Failed marker tenant={Tenant} age={Age} phase={Phase} — operator action required",
marker.TenantId, age, marker.FailedPhase ?? "unknown");
}
}
// Initiated / InProgress → still working, ignore.
}
_logger.LogInformation(
"OffboardingMarkerCleanup completed: scanned={Scanned} deleted={Deleted} blobRetries={BlobRetries} tenantConfigSwept={ConfigSwept} tenantConfigSpared={ConfigSpared} tenantConfigRetries={ConfigRetries} failed={Failed}",
result.Scanned, result.MarkersDeleted, result.BlobDeleteRetries,
result.TenantConfigsSwept, result.TenantConfigsSpared, result.TenantConfigSweepRetries, result.FailedMarkersSeen);
return result;
}
/// <summary>Summary counters returned from <see cref="RunCoreAsync"/> so tests can assert outcomes.</summary>
internal sealed class CleanupResult
{
public int Scanned { get; set; }
public int MarkersDeleted { get; set; }
public int BlobDeleteRetries { get; set; }
public int FailedMarkersSeen { get; set; }
/// <summary>Number of stale TenantConfiguration tombstones the defense-in-depth sweep removed.</summary>
public int TenantConfigsSwept { get; set; }
/// <summary>Number of TenantConfiguration rows the sweep deliberately left alone because the
/// tenant had re-onboarded (DisabledReason is no longer the offboarding tombstone string).</summary>
public int TenantConfigsSpared { get; set; }
/// <summary>Number of markers kept alive because the TenantConfiguration probe/sweep failed transiently.</summary>
public int TenantConfigSweepRetries { get; set; }
}
}
}
@@ -155,6 +155,27 @@ builder.Services.AddHostedService<AutopilotMonitor.Functions.Services.Deletion.S
// retention fanout (V2 + legacy), manifest-blob TTL sweep, stale-Preparing GC, and stranded-
// Queued detection. Independent cadence + kill-switch from the generic 2h Maintenance timer.
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionRetentionFanoutService>();
// Tenant-offboarding cascade worker (Plan Rev 9). Consumes per-tenant envelopes off
// `tenant-offboarding`, drives the §3 Phase 2 sequence (cascade enqueue → drain → SafeWipe →
// blob wipe → tenant config delete), and lands the audit row under AuditGlobalTenantId.
// OffboardingFilters is a static helper — no DI registration needed.
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Offboarding.OffboardingSessionEnumerator>();
builder.Services.AddSingleton<
AutopilotMonitor.Functions.Services.Offboarding.IOffboardingExpectationsStore,
AutopilotMonitor.Functions.Services.Offboarding.BlobOffboardingExpectationsStore>();
builder.Services.AddSingleton<
AutopilotMonitor.Functions.Services.Offboarding.IDeletionProgressDrainProbe,
AutopilotMonitor.Functions.Services.Offboarding.DeletionProgressDrainProbe>();
builder.Services.AddSingleton<
AutopilotMonitor.Functions.Services.Offboarding.ITenantOffboardingEnqueuer,
AutopilotMonitor.Functions.Services.Offboarding.AzureQueueTenantOffboardingEnqueuer>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Offboarding.TenantOffboardingHandler>();
builder.Services.AddHostedService<AutopilotMonitor.Functions.Services.Offboarding.TenantOffboardingWorker>();
// OffboardingMarkerCleanupFunction is TimerTrigger — auto-registered via the Functions host;
// no AddHostedService call needed. DI for its dependencies comes from the singletons above.
builder.Services.AddHostedService<TableInitializerService>(); // Initialize all tables at startup
// Data Access Layer — repository interfaces backed by Table Storage.
@@ -275,6 +275,18 @@ public static class EndpointAccessPolicyCatalog
new("POST", "versions/block", EndpointPolicy.GlobalAdminOnly),
new("DELETE", "versions/block/{encodedPattern}", EndpointPolicy.GlobalAdminOnly),
new("POST", "maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
// ── Tenant Offboarding Customs Archive (PR3.B) ─────────────────────────
// Snapshot of each tenant's custom GatherRules / AnalyzeRules / ImeLogPatterns
// rows from prior offboarding runs. Operator-driven cleanup: GA reviews + deletes.
// The {tenantId} route param is the OFFBOARDED tenant whose data was archived;
// the operator (GA) does not belong to that tenant. RouteParam scoping declares
// that the route's tenantId is supplied from the URL, not from the caller's JWT.
new("GET", "global/customs-archive", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/customs-archive/{tenantId}/{historyRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam),
new("GET", "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam),
new("DELETE", "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam),
new("DELETE", "global/customs-archive/{tenantId}/{historyRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam),
new("GET", "health/detailed", EndpointPolicy.AuthenticatedUser),
new("POST", "rules/reseed-from-github", EndpointPolicy.GlobalAdminOnly),
new("GET", "vulnerability/unmatched-software", EndpointPolicy.GlobalAdminOnly),
@@ -22,7 +22,8 @@ namespace AutopilotMonitor.Functions.Services
{
// Container that holds cascade-deletion manifests (snapshot + progress blobs).
// Plan §3 + §10: 30d Lifecycle delete + 3d soft-delete = effective max 33-day retention.
private const string DeletionManifestsContainer = "deletion-manifests";
// Single source of truth: AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests.
private const string DeletionManifestsContainer = AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests;
private readonly BlobServiceClient _blobServiceClient;
private readonly ILogger<BlobStorageService> _logger;
@@ -259,18 +259,17 @@ namespace AutopilotMonitor.Functions.Services.Deletion
await ExecuteTombstoneAsync(tenantId, sessionId, manifestId, tombstone, cancellationToken);
}
(progress, etag) = await UpdateProgressWithRetryAsync(
tenantId, sessionId, manifestId, progress, etag,
mutate: p =>
{
p.CompletedSteps.Add(tombstone.Order);
p.CompletedAt = DateTime.UtcNow;
},
isAlreadyApplied: p => p.CompletedAt != null,
cancellationToken);
sw.Stop();
// PR1.5 (Rev-5-F1) — Audit + SignalR notify BEFORE we stamp CompletedAt on the
// progress blob. CompletedAt is the drain predicate the tenant-offboarding
// worker uses to decide "all side effects through for this cascade"; if we
// set it before the audit row lands, an in-flight cascade can write
// `deletion_completed` into AuditLogs *after* the offboard worker has wiped
// that table → orphan audit row. Both calls are fail-soft today
// (LogAuditEntryAsync returns false on failure, NotifySessionDeletedAsync
// catches everything internally) so reordering does not change end-to-end
// failure semantics, only the observability window.
await _maintenanceRepo.LogAuditEntryAsync(
tenantId,
action: "deletion_completed",
@@ -282,6 +281,16 @@ namespace AutopilotMonitor.Functions.Services.Deletion
await _signalR.NotifySessionDeletedAsync(tenantId, sessionId);
(progress, etag) = await UpdateProgressWithRetryAsync(
tenantId, sessionId, manifestId, progress, etag,
mutate: p =>
{
p.CompletedSteps.Add(tombstone.Order);
p.CompletedAt = DateTime.UtcNow;
},
isAlreadyApplied: p => p.CompletedAt != null,
cancellationToken);
_logger.LogInformation(
"SessionDeletionHandler completed: tenant={TenantId} session={SessionId} manifestId={ManifestId} durationMs={DurationMs}",
tenantId, sessionId, manifestId, sw.ElapsedMilliseconds);
@@ -0,0 +1,103 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Azure.Identity;
using Azure.Storage.Queues;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Azure Storage Queue producer for <c>tenant-offboarding</c>. Mirrors the existing
/// session-deletion / analyze producers but **fail-loud** (plan §7 + memory:
/// feedback_storage_helpers_fail_soft). Caller (<c>TenantOffboardFunction</c>) catches
/// and translates to HTTP 500; the History + Pointer + Marker rows already exist by then
/// so the operator can retry by re-issuing the offboard click (or via maintenance tooling).
/// </summary>
public sealed class AzureQueueTenantOffboardingEnqueuer : ITenantOffboardingEnqueuer
{
private readonly QueueClient _queueClient;
private readonly ILogger<AzureQueueTenantOffboardingEnqueuer> _logger;
private int _queueEnsured;
public AzureQueueTenantOffboardingEnqueuer(
IConfiguration configuration,
ILogger<AzureQueueTenantOffboardingEnqueuer> logger)
{
_logger = logger;
var options = new QueueClientOptions
{
MessageEncoding = QueueMessageEncoding.Base64,
};
var storageAccountName = configuration["AzureStorageAccountName"];
var connectionString = configuration["AzureTableStorageConnectionString"];
if (!string.IsNullOrEmpty(storageAccountName))
{
var queueUri = new Uri(
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboarding}");
_queueClient = new QueueClient(queueUri, new DefaultAzureCredential(), options);
_logger.LogInformation(
"TenantOffboarding enqueuer initialized with Managed Identity (account: {Account})",
storageAccountName);
}
else if (!string.IsNullOrEmpty(connectionString))
{
_queueClient = new QueueClient(
connectionString, Constants.QueueNames.TenantOffboarding, options);
_logger.LogInformation("TenantOffboarding enqueuer initialized with connection string");
}
else
{
throw new InvalidOperationException(
"Queue Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureTableStorageConnectionString'.");
}
}
/// <summary>Test seam: bind directly to a (possibly mocked) <see cref="QueueClient"/>.</summary>
internal AzureQueueTenantOffboardingEnqueuer(
QueueClient queueClient,
ILogger<AzureQueueTenantOffboardingEnqueuer> logger)
{
_queueClient = queueClient ?? throw new ArgumentNullException(nameof(queueClient));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task EnqueueAsync(
TenantOffboardingEnvelope envelope,
TimeSpan? visibilityDelay = null,
CancellationToken ct = default)
{
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
if (string.IsNullOrEmpty(envelope.TenantId)) throw new ArgumentException("envelope.TenantId required", nameof(envelope));
if (string.IsNullOrEmpty(envelope.HistoryRowKey)) throw new ArgumentException("envelope.HistoryRowKey required", nameof(envelope));
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
var body = JsonConvert.SerializeObject(envelope);
await _queueClient.SendMessageAsync(
body,
visibilityTimeout: visibilityDelay,
timeToLive: null,
cancellationToken: ct).ConfigureAwait(false);
_logger.LogInformation(
"TenantOffboarding enqueued tenant={Tenant} history={History} drainPollCount={Polls} visibilityDelay={Delay}",
envelope.TenantId, envelope.HistoryRowKey, envelope.DrainPollCount, visibilityDelay);
}
private async Task EnsureQueueExistsAsync(CancellationToken ct)
{
if (_queueEnsured == 1) return;
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
Interlocked.Exchange(ref _queueEnsured, 1);
}
}
}
@@ -0,0 +1,215 @@
using System;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Blob-backed implementation of <see cref="IOffboardingExpectationsStore"/>. Pins the
/// blob path and container so callers cannot accidentally drop the expectations into
/// <c>deletion-manifests</c> (which gets wiped in Phase 2.E and would orphan crash recovery).
/// <para>
/// The actual blob IO calls are routed through four <c>protected virtual</c> seams so
/// tests can replace them with an in-memory fake without standing up Azurite. Production
/// overrides hit <see cref="BlobStorageService"/>; tests subclass and override.
/// </para>
/// </summary>
public class BlobOffboardingExpectationsStore : IOffboardingExpectationsStore
{
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false,
};
private readonly BlobStorageService _blobs;
private readonly ILogger<BlobOffboardingExpectationsStore> _logger;
private int _containerEnsured;
public BlobOffboardingExpectationsStore(
BlobStorageService blobs, ILogger<BlobOffboardingExpectationsStore> logger)
{
_blobs = blobs;
_logger = logger;
}
// ── Public contract ─────────────────────────────────────────────────────
public async Task<bool> TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default)
{
if (payload == null) throw new ArgumentNullException(nameof(payload));
ValidatePayload(payload);
var blobName = BuildBlobName(payload.TenantId, payload.HistoryRowKey);
var bytes = JsonSerializer.SerializeToUtf8Bytes(payload, JsonOptions);
var (inserted, _) = await WriteIfNoneMatchAsync(blobName, bytes, ct);
if (inserted)
{
_logger.LogInformation(
"Expectations blob uploaded tenant={Tenant} history={History} count={Count}",
payload.TenantId, payload.HistoryRowKey, payload.Expectations.Count);
}
else
{
_logger.LogInformation(
"Expectations blob already exists tenant={Tenant} history={History} — entering resume path",
payload.TenantId, payload.HistoryRowKey);
}
return inserted;
}
public async Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync(
string tenantId, string historyRowKey, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey));
var blobName = BuildBlobName(tenantId, historyRowKey);
var (rawPayload, etag) = await ReadBlobAsync(blobName, ct);
if (rawPayload == null) return (null, null);
var payload = JsonSerializer.Deserialize<OffboardingExpectations>(rawPayload, JsonOptions);
return (payload, etag);
}
public async Task<string> UpdateWithEtagCasAsync(
OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default)
{
if (payload == null) throw new ArgumentNullException(nameof(payload));
if (string.IsNullOrEmpty(ifMatchEtag)) throw new ArgumentException("ifMatchEtag required", nameof(ifMatchEtag));
ValidatePayload(payload);
var blobName = BuildBlobName(payload.TenantId, payload.HistoryRowKey);
var bytes = JsonSerializer.SerializeToUtf8Bytes(payload, JsonOptions);
return await WriteWithIfMatchAsync(blobName, bytes, ifMatchEtag, ct);
}
public async Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey));
var blobName = BuildBlobName(tenantId, historyRowKey);
await DeleteBlobIfExistsAsync(blobName, ct);
}
// ── Protected virtual seams (production override = Azure SDK; tests = in-memory) ──
/// <summary>
/// Production path: <c>BlobClient.UploadAsync</c> with <c>If-None-Match=*</c>. Azure
/// can answer with either <c>409 BlobAlreadyExists</c> OR <c>412 ConditionNotMet</c>
/// depending on storage tier / SDK version — both mean "exists already" and must
/// translate to <c>(false, null)</c> so the worker's resume path triggers. Other
/// failures propagate.
/// </summary>
protected virtual async Task<(bool Inserted, string? ETag)> WriteIfNoneMatchAsync(
string blobName, byte[] payload, CancellationToken ct)
{
var blobClient = await GetBlobClientAsync(blobName, ct);
var options = new BlobUploadOptions
{
HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" },
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
};
try
{
using var stream = new MemoryStream(payload);
var response = await blobClient.UploadAsync(stream, options, ct);
return (true, response.Value.ETag.ToString());
}
catch (RequestFailedException ex) when (IsAlreadyExists(ex))
{
return (false, null);
}
}
protected virtual async Task<(byte[]? Payload, string? ETag)> ReadBlobAsync(
string blobName, CancellationToken ct)
{
var blobClient = await GetBlobClientAsync(blobName, ct);
try
{
using var stream = new MemoryStream();
await blobClient.DownloadToAsync(stream, ct);
var properties = await blobClient.GetPropertiesAsync(cancellationToken: ct);
return (stream.ToArray(), properties.Value.ETag.ToString());
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
return (null, null);
}
}
protected virtual async Task<string> WriteWithIfMatchAsync(
string blobName, byte[] payload, string ifMatchEtag, CancellationToken ct)
{
var blobClient = await GetBlobClientAsync(blobName, ct);
var options = new BlobUploadOptions
{
HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" },
Conditions = new BlobRequestConditions { IfMatch = new ETag(ifMatchEtag) },
};
using var stream = new MemoryStream(payload);
var response = await blobClient.UploadAsync(stream, options, ct);
return response.Value.ETag.ToString();
}
protected virtual async Task DeleteBlobIfExistsAsync(string blobName, CancellationToken ct)
{
var blobClient = await GetBlobClientAsync(blobName, ct);
await blobClient.DeleteIfExistsAsync(cancellationToken: ct);
}
/// <summary>
/// Both 409 BlobAlreadyExists and 412 ConditionNotMet are emitted by Azure Blob
/// Storage for the same "blob exists, If-None-Match=* refused" condition, depending on
/// API version / tier. Either is a successful "no-op" for the resume path.
/// </summary>
protected static bool IsAlreadyExists(RequestFailedException ex)
=> ex.Status == 409
|| ex.Status == 412
|| ex.ErrorCode == BlobErrorCode.BlobAlreadyExists
|| ex.ErrorCode == BlobErrorCode.ConditionNotMet;
// ── Internals ───────────────────────────────────────────────────────────
internal static string BuildBlobName(string tenantId, string historyRowKey)
=> $"{tenantId}/{historyRowKey}.expectations.json";
private static void ValidatePayload(OffboardingExpectations payload)
{
SecurityValidator.EnsureValidGuid(payload.TenantId, $"{nameof(payload)}.TenantId");
if (string.IsNullOrEmpty(payload.HistoryRowKey))
throw new ArgumentException("payload.HistoryRowKey required", nameof(payload));
if (payload.SchemaVersion < 1)
throw new ArgumentException("payload.SchemaVersion must be ≥ 1", nameof(payload));
}
private async Task<BlobClient> GetBlobClientAsync(string blobName, CancellationToken ct)
{
var container = _blobs.GetContainerClient(Constants.BlobContainers.OffboardingState);
await EnsureContainerExistsAsync(container, ct);
return container.GetBlobClient(blobName);
}
private async Task EnsureContainerExistsAsync(BlobContainerClient container, CancellationToken ct)
{
if (_containerEnsured == 1) return;
await container.CreateIfNotExistsAsync(cancellationToken: ct);
Interlocked.Exchange(ref _containerEnsured, 1);
}
}
}
@@ -0,0 +1,45 @@
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Default <see cref="IDeletionProgressDrainProbe"/> backed by
/// <see cref="BlobStorageService.DownloadDeletionProgressAsync"/>. Plan §7.4 step 5:
/// drain is satisfied for a session expectation when its progress blob has
/// <c>CompletedAt != null AND TombstoneStarted == true</c>.
/// </summary>
public class DeletionProgressDrainProbe : IDeletionProgressDrainProbe
{
private readonly BlobStorageService _blobs;
private readonly ILogger<DeletionProgressDrainProbe> _logger;
public DeletionProgressDrainProbe(BlobStorageService blobs, ILogger<DeletionProgressDrainProbe> logger)
{
_blobs = blobs;
_logger = logger;
}
public async Task<bool> IsCascadeCompletedAsync(
string tenantId, string sessionId, string manifestId, CancellationToken ct = default)
{
try
{
var (progress, _) = await _blobs.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct);
return progress.CompletedAt != null && progress.TombstoneStarted;
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
// Progress blob not yet written — cascade still running. NOT a failure: the
// producer creates the progress blob before the queue message lands, but the
// tombstone step happens at the end of the cascade.
_logger.LogDebug(
"Drain probe: progress blob missing for tenant={Tenant} session={Session} manifest={Manifest} — cascade not done",
tenantId, sessionId, manifestId);
return false;
}
}
}
}
@@ -0,0 +1,21 @@
using System.Threading;
using System.Threading.Tasks;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Per-session predicate over a cascade-deletion progress blob. Used by the
/// tenant-offboarding handler to drain against each <see cref="Models.Offboarding.OffboardingExpectation"/>
/// that ended up in an <c>Enqueued | AlreadyInFlight</c> outcome.
/// </summary>
public interface IDeletionProgressDrainProbe
{
/// <summary>
/// Returns <c>true</c> iff the progress blob exists AND
/// <c>CompletedAt != null AND TombstoneStarted == true</c>. 404 is treated as "not yet
/// done" (false). Other RequestFailedExceptions propagate so the worker can retry.
/// </summary>
Task<bool> IsCascadeCompletedAsync(
string tenantId, string sessionId, string manifestId, CancellationToken ct = default);
}
}
@@ -0,0 +1,45 @@
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.Models.Offboarding;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// CRUD over the per-tenant Expectations blob. Lives under
/// <c>Constants.BlobContainers.OffboardingState/{tenantId}/{historyRowKey}.expectations.json</c>
/// — separate container from <c>deletion-manifests</c> so Phase 2.E's blob wipe cannot
/// orphan crash-recovery state (plan Rev-8-F1).
/// </summary>
public interface IOffboardingExpectationsStore
{
/// <summary>
/// Idempotent initial upload via <c>If-None-Match=*</c>. Returns <c>true</c> on a fresh
/// insert, <c>false</c> when a blob already exists for the same
/// (tenantId, historyRowKey) — caller treats false as the resume path and re-reads.
/// Any other Azure error propagates fail-loud.
/// </summary>
Task<bool> TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default);
/// <summary>
/// Downloads the Expectations blob; returns <c>(null, null)</c> on 404. Caller's
/// drain probe fails closed on 404 (<c>FailedPhase="expectations_missing"</c>).
/// </summary>
Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync(
string tenantId, string historyRowKey, CancellationToken ct = default);
/// <summary>
/// ETag-CAS update used by the drain loop to re-record retry counters for CasExhausted
/// expectations. Throws <see cref="Azure.RequestFailedException"/> on 412 so the caller
/// can re-read and retry.
/// </summary>
Task<string> UpdateWithEtagCasAsync(
OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default);
/// <summary>
/// Idempotent <c>DeleteIfExistsAsync</c>. Called as Phase 2.G's last step after the
/// history row has been marked Completed, and again as defence-in-depth by
/// <c>OffboardingMarkerCleanupFunction</c> when it removes a Completed marker.
/// </summary>
Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default);
}
}
@@ -0,0 +1,27 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.Models.Offboarding;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Producer side of the <c>tenant-offboarding</c> queue. Implementations MUST be fail-loud:
/// the caller (<c>TenantOffboardFunction</c>) only returns 202 Accepted after a successful
/// enqueue, otherwise the admin's offboard click would silently strand without a worker
/// pickup (memory: feedback_storage_helpers_fail_soft).
/// </summary>
public interface ITenantOffboardingEnqueuer
{
/// <summary>
/// Enqueue a tenant-offboarding envelope. <paramref name="visibilityDelay"/> is used by
/// the handler's drain self-re-enqueue path (Plan §7.4 D2) to delay the next pickup by
/// ~2 minutes; producers in <c>TenantOffboardFunction</c> leave it null for immediate
/// visibility.
/// </summary>
Task EnqueueAsync(
TenantOffboardingEnvelope envelope,
TimeSpan? visibilityDelay = null,
CancellationToken ct = default);
}
}
@@ -0,0 +1,70 @@
using System;
using AutopilotMonitor.Functions.Security;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// OData filter builders for tenant-offboarding queries. All inputs are validated and
/// escaped before interpolation. Centralizing the patterns here keeps SafeWipe-callers
/// honest and gives the test suite a single surface to attack with injection payloads.
/// </summary>
/// <remarks>
/// Rules (memory: feedback_storage_helpers_fail_soft, plan §5.5):
/// <list type="number">
/// <item>Every tenant id is run through <see cref="SecurityValidator.EnsureValidGuid"/> before use.</item>
/// <item>Every interpolated value passes through <see cref="ODataSanitizer.EscapeValue"/>.</item>
/// <item>Composite-PK range uses <c>'{tenantId}_~'</c> upper bound — the underscore is the
/// anchor; without it the range bleeds into the next ASCII bucket.</item>
/// </list>
/// </remarks>
public static class OffboardingFilters
{
/// <summary>
/// <c>PartitionKey eq '{tenantId}'</c> — for Variant A SafeWipe of tenant-PK tables.
/// </summary>
public static string ExactPartition(string normalizedTenantId)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
return $"PartitionKey eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'";
}
/// <summary>
/// <c>PartitionKey ge '{tenantId}_' and PartitionKey lt '{tenantId}_~'</c> — for
/// Variant A SafeWipe of composite-PK tables (events, signals, decision transitions,
/// discriminator-PK index tables, …). The underscore-tilde upper bound traps the
/// range strictly to <c>{tenantId}_*</c>; switching to <c>'{tenantId}~'</c> would
/// match keys outside the tenant prefix.
/// </summary>
public static string CompositePartitionRange(string normalizedTenantId)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
var safe = ODataSanitizer.EscapeValue(normalizedTenantId);
return $"PartitionKey ge '{safe}_' and PartitionKey lt '{safe}_~'";
}
/// <summary>
/// <c>PartitionKey eq '{discriminator}' and TenantId eq '{tenantId}'</c> — for
/// Variant B SafeWipe of discriminator-PK tables with a TenantId property as the
/// only tenant anchor. Verify-step MUST re-check the property server-side because
/// the PK is non-anchored.
/// </summary>
public static string DiscriminatorWithTenantProp(string discriminator, string normalizedTenantId)
{
if (string.IsNullOrEmpty(discriminator)) throw new ArgumentException("Discriminator required", nameof(discriminator));
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
return $"PartitionKey eq '{ODataSanitizer.EscapeValue(discriminator)}' " +
$"and TenantId eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'";
}
/// <summary>
/// <c>TenantId eq '{tenantId}'</c> — Variant C SafeWipe (Property-only full-table
/// filter). Reserve for low-volume tables; Group-by-PartitionKey before batching
/// because rows can span many PKs.
/// </summary>
public static string TenantIdProperty(string normalizedTenantId)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
return $"TenantId eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'";
}
}
}
@@ -0,0 +1,42 @@
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Shared.DataAccess;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Thin wrapper around <see cref="IMaintenanceRepository.EnumerateSessionsForOffboardingAsync"/>
/// so the offboarding handler can take an injectable dependency that is trivially mockable
/// (the repo interface has 20+ unrelated members; mocking it for every test is painful).
/// <para>
/// <b>Fail-loud contract:</b> any storage exception from the underlying enumerator
/// propagates unchanged. Plan §7.4 + memory <c>feedback_storage_helpers_fail_soft</c>:
/// silently returning an empty list during enumeration would let the handler proceed
/// straight to wipe without cascade-backup. Tests in
/// <c>OffboardingSessionEnumeratorTests</c> pin this behaviour.
/// </para>
/// </summary>
public class OffboardingSessionEnumerator
{
private readonly IMaintenanceRepository _maintenance;
public OffboardingSessionEnumerator(IMaintenanceRepository maintenance)
{
_maintenance = maintenance;
}
public virtual async IAsyncEnumerable<string> EnumerateAsync(
string tenantId,
[EnumeratorCancellation] CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
await foreach (var sessionId in _maintenance.EnumerateSessionsForOffboardingAsync(tenantId, ct))
{
yield return sessionId;
}
}
}
}
@@ -0,0 +1,279 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Shared;
using Azure;
using Azure.Data.Tables;
using Azure.Storage.Blobs;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Four-step verified wipe pattern used by the tenant-offboarding worker.
/// <list type="number">
/// <item><b>Fetch</b> rows matching a tenant-anchored filter (PK or TenantId-Property).</item>
/// <item><b>Verify</b> each fetched row matches the expected anchor server-side AND client-side.</item>
/// <item><b>Delete</b> via batched <c>SubmitTransactionAsync</c> with <c>ETag.All</c>; per-row
/// 404-fallback so the helper stays idempotent across crash/resume.</item>
/// <item><b>Audit</b> the deleted/already-missing counts to the caller's logger.</item>
/// </list>
/// Verify mismatch → <see cref="SafeWipeVerificationException"/> before any delete fires.
/// </summary>
public class SafeWipeService
{
private const int BatchActionLimit = 100;
private readonly TableStorageService _tables;
private readonly BlobStorageService _blobs;
private readonly ILogger<SafeWipeService> _logger;
public SafeWipeService(TableStorageService tables, BlobStorageService blobs, ILogger<SafeWipeService> logger)
{
_tables = tables;
_blobs = blobs;
_logger = logger;
}
/// <summary>
/// Variant A (exact tenant PartitionKey). Used for tables where rows live under
/// <c>PartitionKey == normalizedTenantId</c> (AuditLogs, UsageMetrics, …).
/// </summary>
public virtual Task<int> WipeByExactPartitionAsync(
string tableName, string normalizedTenantId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
var filter = OffboardingFilters.ExactPartition(normalizedTenantId);
return RunFetchVerifyDeleteAsync(
tableName,
filter,
expectedAnchor: normalizedTenantId,
verifyRow: e => string.Equals(e.PartitionKey, normalizedTenantId, StringComparison.Ordinal),
ct);
}
/// <summary>
/// Variant A range (composite tenant PartitionKey). Used for tables where rows live
/// under <c>{tenantId}_{sessionId|discriminator}</c>.
/// </summary>
public virtual Task<int> WipeByCompositePartitionRangeAsync(
string tableName, string normalizedTenantId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
var prefix = normalizedTenantId + "_";
var filter = OffboardingFilters.CompositePartitionRange(normalizedTenantId);
return RunFetchVerifyDeleteAsync(
tableName,
filter,
expectedAnchor: prefix,
verifyRow: e => e.PartitionKey.StartsWith(prefix, StringComparison.Ordinal),
ct);
}
/// <summary>
/// Variant B (discriminator-PK + TenantId-Property). Used for tables where rows share
/// a fixed string PartitionKey (e.g. "CodeLookup", "reports", "Feedback") and only the
/// <c>TenantId</c> property identifies tenant ownership. Verify-step MUST check the
/// property because the PK is non-anchored.
/// </summary>
public virtual Task<int> WipeByDiscriminatorAndTenantPropertyAsync(
string tableName, string discriminator, string normalizedTenantId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
if (string.IsNullOrEmpty(discriminator)) throw new ArgumentException("Discriminator required", nameof(discriminator));
var filter = OffboardingFilters.DiscriminatorWithTenantProp(discriminator, normalizedTenantId);
var expectedAnchor = $"{discriminator}+TenantId={normalizedTenantId}";
return RunFetchVerifyDeleteAsync(
tableName,
filter,
expectedAnchor: expectedAnchor,
verifyRow: e => string.Equals(e.PartitionKey, discriminator, StringComparison.Ordinal)
&& string.Equals(e.GetString("TenantId"), normalizedTenantId, StringComparison.Ordinal),
ct);
}
/// <summary>
/// Variant C (Property-only full-table filter). Reserved for low-volume tables where
/// PartitionKey is not tenant-anchored (e.g. <c>UserUsageLog</c> keyed by userOid).
/// Full-table scan — DO NOT use on high-volume tables without consideration.
/// </summary>
public virtual Task<int> WipeByTenantIdPropertyAsync(
string tableName, string normalizedTenantId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
var filter = OffboardingFilters.TenantIdProperty(normalizedTenantId);
return RunFetchVerifyDeleteAsync(
tableName,
filter,
expectedAnchor: $"TenantId={normalizedTenantId}",
verifyRow: e => string.Equals(e.GetString("TenantId"), normalizedTenantId, StringComparison.Ordinal),
ct);
}
/// <summary>
/// Blob-prefix wipe. Lists blobs under <c>{normalizedTenantId}/</c> in the named
/// container, verifies each returned name starts with the prefix, then deletes
/// per-blob (DeleteIfExistsAsync) so the helper stays idempotent across crash/resume.
/// Mismatch → abort before any delete. Returns the number of blobs deleted.
/// </summary>
public virtual async Task<int> WipeBlobsByTenantPrefixAsync(
string containerName, string normalizedTenantId, CancellationToken ct = default)
{
SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId));
if (string.IsNullOrEmpty(containerName)) throw new ArgumentException("Container required", nameof(containerName));
var prefix = normalizedTenantId + "/";
var container = _blobs.GetContainerClient(containerName);
var collected = new List<string>();
await foreach (var blob in container.GetBlobsAsync(
Azure.Storage.Blobs.Models.BlobTraits.None,
Azure.Storage.Blobs.Models.BlobStates.None,
prefix: prefix,
cancellationToken: ct))
{
collected.Add(blob.Name);
}
if (collected.Count == 0)
{
_logger.LogInformation(
"SafeWipe blobs '{Container}' tenant={Tenant}: no blobs found under '{Prefix}'",
containerName, normalizedTenantId, prefix);
return 0;
}
var mismatches = collected
.Where(name => !name.StartsWith(prefix, StringComparison.Ordinal))
.ToList();
if (mismatches.Count > 0)
{
_logger.LogCritical(
"SafeWipe blobs '{Container}' tenant={Tenant}: aborting — {Total} blob(s) listed but {Bad} do not start with '{Prefix}'. First bad: '{Sample}'",
containerName, normalizedTenantId, collected.Count, mismatches.Count, prefix, mismatches[0]);
throw new SafeWipeVerificationException(containerName, prefix, mismatches.Count);
}
int deleted = 0, missing = 0;
foreach (var name in collected)
{
ct.ThrowIfCancellationRequested();
var blobClient = container.GetBlobClient(name);
var resp = await blobClient.DeleteIfExistsAsync(cancellationToken: ct);
if (resp.Value) deleted++;
else missing++;
}
_logger.LogInformation(
"SafeWipe blobs '{Container}' tenant={Tenant}: deleted {Deleted}, already-missing {Missing}",
containerName, normalizedTenantId, deleted, missing);
return deleted;
}
// ── Internal pipeline ───────────────────────────────────────────────────
private async Task<int> RunFetchVerifyDeleteAsync(
string tableName,
string filter,
string expectedAnchor,
Func<TableEntity, bool> verifyRow,
CancellationToken ct)
{
if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName required", nameof(tableName));
var tableClient = _tables.GetTableClient(tableName);
// STEP 1 — Fetch keys + the TenantId property if any verify-rule needs it.
// We pull "PartitionKey", "RowKey", "TenantId" so Variant B/C verify works.
var fetched = new List<TableEntity>();
await foreach (var e in tableClient.QueryAsync<TableEntity>(
filter,
select: new[] { "PartitionKey", "RowKey", "TenantId" },
cancellationToken: ct))
{
fetched.Add(e);
}
if (fetched.Count == 0)
{
_logger.LogDebug(
"SafeWipe '{Table}' anchor={Anchor}: 0 rows matched filter",
tableName, expectedAnchor);
return 0;
}
// STEP 2 — Verify every row matches the expected anchor.
var mismatches = fetched.Where(e => !verifyRow(e)).ToList();
if (mismatches.Count > 0)
{
var sample = mismatches.First();
_logger.LogCritical(
"SafeWipe '{Table}' anchor={Anchor}: aborting — {Total} fetched, {Bad} fail verify. " +
"Sample bad row: PK='{Pk}' RK='{Rk}' TenantId='{Tid}'",
tableName, expectedAnchor, fetched.Count, mismatches.Count,
sample.PartitionKey, sample.RowKey, sample.GetString("TenantId") ?? "<null>");
throw new SafeWipeVerificationException(tableName, expectedAnchor, mismatches.Count);
}
// STEP 3 — Delete in batches, grouped by PartitionKey (transaction requirement).
int deleted = 0, alreadyMissing = 0;
var groupedByPk = fetched
.GroupBy(e => e.PartitionKey, StringComparer.Ordinal)
.ToList();
foreach (var group in groupedByPk)
{
var rows = group.ToList();
for (int i = 0; i < rows.Count; i += BatchActionLimit)
{
ct.ThrowIfCancellationRequested();
var chunk = rows.Skip(i).Take(BatchActionLimit).ToList();
var actions = chunk
.Select(e => new TableTransactionAction(
TableTransactionActionType.Delete,
new TableEntity(e.PartitionKey, e.RowKey) { ETag = ETag.All }))
.ToList();
try
{
await tableClient.SubmitTransactionAsync(actions, ct);
deleted += chunk.Count;
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
// Azure rolls the whole transaction back on a single 404; degrade to per-row.
foreach (var entity in chunk)
{
ct.ThrowIfCancellationRequested();
try
{
await tableClient.DeleteEntityAsync(
entity.PartitionKey, entity.RowKey, ETag.All, ct);
deleted++;
}
catch (RequestFailedException rfe) when (rfe.Status == 404)
{
alreadyMissing++;
}
}
}
}
}
// STEP 4 — Audit.
_logger.LogInformation(
"SafeWipe '{Table}' anchor={Anchor}: deleted {Deleted}, already-missing {Missing}",
tableName, expectedAnchor, deleted, alreadyMissing);
return deleted;
}
}
}
@@ -0,0 +1,28 @@
using System;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Thrown by <see cref="SafeWipeService"/> when the verify step finds rows in the fetched
/// set that do not match the expected tenant anchor (PartitionKey, TenantId-Property, or
/// blob-name prefix depending on variant). NO DATA IS DELETED at this point — the wipe is
/// aborted before any delete call so a misbehaving filter cannot cause cross-tenant data
/// loss. Operator must inspect the unexpected rows manually.
/// </summary>
public class SafeWipeVerificationException : Exception
{
public string TableOrContainer { get; }
public string ExpectedAnchor { get; }
public int MismatchCount { get; }
public SafeWipeVerificationException(string tableOrContainer, string expectedAnchor, int mismatchCount)
: base($"SafeWipe verify aborted for '{tableOrContainer}': {mismatchCount} row(s) " +
$"did not match expected anchor '{expectedAnchor}'. No deletes were issued. " +
$"Manual review required — possible filter bug or schema drift.")
{
TableOrContainer = tableOrContainer;
ExpectedAnchor = expectedAnchor;
MismatchCount = mismatchCount;
}
}
}
@@ -0,0 +1,958 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.DataAccess.TableStorage;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Deletion;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Azure.Data.Tables;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Phase 2 of the tenant-offboarding cascade plan (Rev 9). Drives a queued
/// <see cref="TenantOffboardingEnvelope"/> through:
/// <list type="number">
/// <item>2.A — History/Pointer/Marker → InProgress; Rev-9-F1 Drain-Skip-Gate when DrainCompletedAt is set.</item>
/// <item>2.B — fail-loud session enumeration + per-session cascade enqueue + Expectations-Blob upload.</item>
/// <item>2.C — drain predicate against Expectations; D2 self-re-enqueue with 2-min delay up to 60 polls.</item>
/// <item>2.D — retain-counter + SafeWipe over all tenant-scoped tables (variants A/B/C).</item>
/// <item>2.E — blob-prefix cleanup of <c>deletion-manifests/{tenantId}/</c>.</item>
/// <item>2.D-pass-2 — re-SafeWipe AuditLogs as defence-in-depth.</item>
/// <item>2.F — delete the TenantConfiguration row.</item>
/// <item>2.G — Completed state on History/Pointer/Marker; fail-soft Expectations-Blob delete.</item>
/// <item>2.H — global-tenant audit row.</item>
/// <item>2.I — <c>TenantOffboarded</c> OpsEvent.</item>
/// </list>
/// All Phase 2.D-G operations are idempotent (ETag.All + 404-fallback in SafeWipe, fail-soft
/// blob delete in 2.G, last-writer-wins on history) so the Rev-9-F1 resume from a crash
/// between 2.E and 2.G is safe even though the progress-blob predicate cannot run any more.
/// </summary>
public class TenantOffboardingHandler
{
/// <summary>Cap on drain self-re-enqueue iterations (~2h at 2-min visibility delay).</summary>
public const int MaxDrainPolls = 60;
/// <summary>Delay between drain self-polls when expectations are not yet satisfied.</summary>
internal static readonly TimeSpan DrainPollDelay = TimeSpan.FromMinutes(2);
/// <summary>Maximum CasExhausted retries per session before fail-closing the offboarding.</summary>
public const int MaxCasRetriesPerSession = 3;
/// <summary>Throttle between per-session cascade enqueues so a 10k-session tenant doesn't burst.</summary>
internal static readonly TimeSpan PerSessionEnqueueThrottle = TimeSpan.FromMilliseconds(50);
private static readonly Dictionary<string, string> EmptyDetails = new(StringComparer.Ordinal);
// Plan §6.2 — exact tenant-PK wipes (Variant A).
private static readonly string[] TenantPartitionTables =
{
Constants.TableNames.AuditLogs,
Constants.TableNames.UsageMetrics,
Constants.TableNames.UserActivity,
Constants.TableNames.AppInstallSummaries,
Constants.TableNames.TenantAdmins,
Constants.TableNames.BootstrapSessions, // main rows; CodeLookup discriminator below
Constants.TableNames.BlockedDevices,
Constants.TableNames.RuleStates, // runtime state — NOT customs (kept; §6.7)
Constants.TableNames.TenantNotifications,
Constants.TableNames.HardwareRejectionNotificationTracker,
Constants.TableNames.SlaTenantStatus,
Constants.TableNames.DeviceSnapshot,
Constants.TableNames.EventSessionIndex,
Constants.TableNames.DistressReports,
Constants.TableNames.SoftwareInventory,
Constants.TableNames.SessionInventoryContributions,
Constants.TableNames.SessionTombstones,
Constants.TableNames.PreviewWhitelist,
};
// Plan §6.4 — composite-PK "{tenantId}_..." wipes (Variant A range).
private static readonly string[] CompositePartitionTables =
{
Constants.TableNames.Events,
Constants.TableNames.RuleResults,
Constants.TableNames.VulnerabilityReports,
Constants.TableNames.Signals,
Constants.TableNames.DecisionTransitions,
Constants.TableNames.EventTypeIndex,
Constants.TableNames.CveIndex,
Constants.TableNames.SessionsByTerminal,
Constants.TableNames.SessionsByStage,
Constants.TableNames.DeadEndsByReason,
Constants.TableNames.ClassifierVerdictsByIdLevel,
Constants.TableNames.SignalsByKind,
};
// Plan §6.3.1 — Variant B (Discriminator + TenantId property).
// (table, discriminator)
private static readonly (string Table, string Discriminator)[] DiscriminatorTables =
{
(Constants.TableNames.BootstrapSessions, "CodeLookup"),
(Constants.TableNames.SessionReports, "reports"),
(Constants.TableNames.PreviewConfig, "Feedback"),
};
// Plan §6.3.2 — Variant C (TenantId property only).
private static readonly string[] PropertyOnlyTables =
{
Constants.TableNames.UserUsageLog,
};
// PR3.B plan §3 — Customs rules tables: archive each row to
// TenantOffboardingCustomsArchive (per-run partition) THEN wipe the source so the
// tenant's originals are gone. Counters in OffboardingHistory are recomputed from
// the archive table per run + table to be crash-resume safe.
private static readonly (string Table, string Field)[] ArchivedRuleTables =
{
(Constants.TableNames.GatherRules, nameof(OffboardingHistoryEntry.CustomGatherRulesArchived)),
(Constants.TableNames.AnalyzeRules, nameof(OffboardingHistoryEntry.CustomAnalyzeRulesArchived)),
(Constants.TableNames.ImeLogPatterns, nameof(OffboardingHistoryEntry.ImeLogPatternOverridesArchived)),
};
/// <summary>
/// Maximum number of archive-then-wipe iterations per rules table before the
/// post-wipe verify re-find triggers fail-closed with <c>customs_arrival_race</c>.
/// First pass handles the normal case; passes 2 + 3 absorb writes that landed
/// during the drain barrier (a warm function-host with a stale TenantConfig
/// cache could still write between Phase 1 commit and Phase 2 pickup).
/// </summary>
internal const int ArchiveIterationCap = 3;
private readonly IOffboardingAuditRepository _auditRepo;
private readonly OffboardingSessionEnumerator _enumerator;
private readonly ISessionDeletionEnqueuer _cascadeEnqueuer;
private readonly IOffboardingExpectationsStore _expectations;
private readonly IDeletionProgressDrainProbe _drainProbe;
private readonly SafeWipeService _safeWipe;
private readonly TableStorageService _storage;
private readonly IMaintenanceRepository _maintenance;
private readonly ITenantOffboardingEnqueuer _reEnqueuer;
private readonly OpsEventService _opsEvents;
private readonly ITenantCustomsArchiveRepository _customsArchive;
private readonly ILogger<TenantOffboardingHandler> _logger;
public TenantOffboardingHandler(
IOffboardingAuditRepository auditRepo,
OffboardingSessionEnumerator enumerator,
ISessionDeletionEnqueuer cascadeEnqueuer,
IOffboardingExpectationsStore expectations,
IDeletionProgressDrainProbe drainProbe,
SafeWipeService safeWipe,
TableStorageService storage,
IMaintenanceRepository maintenance,
ITenantOffboardingEnqueuer reEnqueuer,
OpsEventService opsEvents,
ITenantCustomsArchiveRepository customsArchive,
ILogger<TenantOffboardingHandler> logger)
{
_auditRepo = auditRepo;
_enumerator = enumerator;
_cascadeEnqueuer = cascadeEnqueuer;
_expectations = expectations;
_drainProbe = drainProbe;
_safeWipe = safeWipe;
_storage = storage;
_maintenance = maintenance;
_reEnqueuer = reEnqueuer;
_opsEvents = opsEvents;
_customsArchive = customsArchive;
_logger = logger;
}
public async Task HandleAsync(TenantOffboardingEnvelope envelope, CancellationToken ct = default)
{
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
var tenantId = envelope.TenantId.ToLowerInvariant();
// 2.A — Load history. Idempotent re-pickup: Completed/Failed → return early.
var history = await _auditRepo.TryGetHistoryAsync(envelope.HistoryRowKey, ct)
?? throw new InvalidOperationException(
$"OffboardingHistory row missing for envelope tenant={tenantId} history={envelope.HistoryRowKey}");
if (history.Status == "Completed")
{
_logger.LogInformation("Tenant offboarding already Completed — re-pickup is a no-op. tenant={Tenant}", tenantId);
return;
}
if (history.Status == "Failed")
{
_logger.LogWarning(
"Tenant offboarding History is Failed (phase={Phase}) — operator action required, worker returns. tenant={Tenant}",
history.ErrorMessage ?? "unknown", tenantId);
return;
}
// 2.A — first pickup transitions History/Pointer/Marker to InProgress.
if (history.Status == "Initiated")
{
history.Status = "InProgress";
await _auditRepo.UpsertHistoryAsync(history, ct);
await TransitionStatusAsync(tenantId, history.RowKey, "InProgress", ct);
}
// 2.A.5 — Rev-9-F1 Drain-Skip-Gate. After the first successful drain, DrainCompletedAt
// is stamped on the history row and all subsequent phases (2.D-G) are idempotent.
// A crash between 2.E (blob wipe) and 2.G (status flip) would otherwise leave the
// re-pickup looking for progress blobs that 2.E just deleted — fail-closed with
// expectations_missing, which is wrong (the offboard actually succeeded in spirit).
if (history.DrainCompletedAt != null)
{
_logger.LogInformation(
"Drain-Skip-Gate: History.DrainCompletedAt={DrainedAt} set; running post-drain phases idempotently. tenant={Tenant}",
history.DrainCompletedAt, tenantId);
await RunPostDrainPhasesAsync(history, tenantId, ct);
return;
}
// 2.B — Ensure Expectations blob exists. The two history markers EnumerationStartedAt
// (ESA) and EnumerationCompletedBeforeUpload (ECBU) disambiguate three resume modes
// when the blob is missing:
// • ESA null, ECBU null → first try → stamp ESA, enumerate, stamp ECBU, upload.
// • ESA set, ECBU null → mid-enumeration crash → fail-closed expectations_missing
// (re-enumerating against a mid-mutation tenant is unsafe).
// • ESA set, ECBU set → upload failure → retry enumerate+upload (idempotent,
// SessionDeletionProducer returns AlreadyInFlight).
// Full decision table lives on EnsureExpectationsBlobAsync. Plan §7.4 step 3.
var ensureFail = await EnsureExpectationsBlobAsync(history, tenantId, ct);
if (ensureFail != null)
{
await FailAsync(history, tenantId, ensureFail,
$"Drain fail-closed: {ensureFail}", ct);
throw new InvalidOperationException(
$"TenantOffboarding fail-closed for tenant={tenantId} phase={ensureFail}");
}
// 2.C — Drain probe. Three outcomes: drain OK (proceed), drain not yet (re-enqueue
// with 2-min delay up to MaxDrainPolls), fail-closed (throw → worker poisons).
var (drainOk, failReason) = await EvaluateDrainAsync(history, tenantId, ct);
if (failReason != null)
{
await FailAsync(history, tenantId, failReason, $"Drain fail-closed: {failReason}", ct);
throw new InvalidOperationException(
$"TenantOffboarding fail-closed for tenant={tenantId} phase={failReason}");
}
if (!drainOk)
{
if (envelope.DrainPollCount + 1 >= MaxDrainPolls)
{
await FailAsync(history, tenantId, "drain_timeout",
$"Drain did not settle within {MaxDrainPolls} polls (~{MaxDrainPolls * DrainPollDelay.TotalMinutes:0}min)", ct);
throw new InvalidOperationException(
$"TenantOffboarding drain_timeout for tenant={tenantId}");
}
var next = new TenantOffboardingEnvelope
{
EnvelopeVersion = envelope.EnvelopeVersion,
TenantId = envelope.TenantId,
HistoryPartitionKey = envelope.HistoryPartitionKey,
HistoryRowKey = envelope.HistoryRowKey,
InitiatedBy = envelope.InitiatedBy,
InitiatedAt = envelope.InitiatedAt,
EnqueuedAt = DateTime.UtcNow,
DrainPollCount = envelope.DrainPollCount + 1,
};
await _reEnqueuer.EnqueueAsync(next, DrainPollDelay, ct);
_logger.LogInformation(
"Drain not yet settled — re-enqueued tenant={Tenant} poll={Poll}/{Max} delay={Delay}",
tenantId, next.DrainPollCount, MaxDrainPolls, DrainPollDelay);
return;
}
// Drain OK — Rev-9-F1: stamp DrainCompletedAt BEFORE Phase 2.D begins.
history.DrainCompletedAt = DateTime.UtcNow;
await _auditRepo.UpsertHistoryAsync(history, ct);
await RunPostDrainPhasesAsync(history, tenantId, ct);
}
// ── Phase 2.A status transitions ────────────────────────────────────────
private async Task TransitionStatusAsync(string tenantId, string historyRowKey, string newStatus, CancellationToken ct)
{
// Marker — read existing fields so we don't clobber InitiatedAt/InitiatedBy.
var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct);
if (marker != null)
{
marker.Status = newStatus;
await _auditRepo.UpsertMarkerAsync(marker, ct);
}
// Pointer — read-modify-write with ETag-CAS retry (Plan §4.4).
await UpdatePointerWithRetryAsync(tenantId, historyRowKey, newStatus, ct);
}
private async Task UpdatePointerWithRetryAsync(string tenantId, string historyRowKey, string newStatus, CancellationToken ct)
{
const int maxRetries = 5;
for (int attempt = 0; attempt < maxRetries; attempt++)
{
ct.ThrowIfCancellationRequested();
var (pointer, etag) = await _auditRepo.TryGetByTenantPointerAsync(tenantId, ct);
if (pointer == null || etag == null)
{
_logger.LogWarning(
"Pointer missing for tenant={Tenant} while transitioning to {Status} — skip (graceful degradation, Plan §4.4)",
tenantId, newStatus);
return;
}
pointer.LatestStatus = newStatus;
pointer.LatestUpdatedAt = DateTime.UtcNow;
if (!string.Equals(pointer.LatestHistoryRowKey, historyRowKey, StringComparison.Ordinal))
{
// Pointer points at a different attempt — concurrent re-offboard. Skip.
_logger.LogWarning(
"Pointer.LatestHistoryRowKey={PointerHistory} != envelope.HistoryRowKey={Envelope} — skipping pointer transition",
pointer.LatestHistoryRowKey, historyRowKey);
return;
}
try
{
await _auditRepo.UpdateByTenantPointerWithEtagAsync(pointer, etag, ct);
return;
}
catch (RequestFailedException ex) when (ex.Status == 412)
{
// Concurrent writer beat us — re-read + retry.
}
}
_logger.LogWarning(
"Pointer ETag-CAS exhausted for tenant={Tenant} status={Status} — giving up (Plan §4.4 graceful degradation)",
tenantId, newStatus);
}
// ── Phase 2.B — initial enqueue + Expectations persistence ──────────────
/// <summary>
/// Returns <c>null</c> on success (blob exists / blob just created). Returns a
/// FailedPhase string when the handler must fail-closed without enumerating.
/// <para>
/// Decision table (Plan §7.4 step 3 + Review-Fix Finding 2):
/// </para>
/// <list type="table">
/// <listheader><term>blob</term><term>ESA</term><term>ECBU</term><description>Action</description></listheader>
/// <item><term>exists</term><term>—</term><term>—</term><description>Resume path; return null.</description></item>
/// <item><term>missing</term><term>null</term><term>null</term><description>First try; stamp ESA, enumerate, stamp ECBU, upload.</description></item>
/// <item><term>missing</term><term>set</term><term>null</term><description>Mid-enumeration crash → fail-closed <c>expectations_missing</c>.</description></item>
/// <item><term>missing</term><term>set</term><term>set</term><description>Upload failure → retry enumerate (idempotent — producer returns AlreadyInFlight) + re-upload.</description></item>
/// </list>
/// </summary>
private async Task<string?> EnsureExpectationsBlobAsync(
OffboardingHistoryEntry history, string tenantId, CancellationToken ct)
{
var (existing, _) = await _expectations.TryDownloadAsync(tenantId, history.RowKey, ct);
if (existing != null)
{
_logger.LogInformation(
"Expectations blob already exists for tenant={Tenant} (count={Count}, enumComplete={EnumComplete}) — resume path",
tenantId, existing.Expectations.Count, existing.EnumerationCompleted);
return null;
}
// Mid-enumeration crash: prior pickup stamped ESA but never reached ECBU. Refuse
// to re-enumerate against a possibly mid-mutation tenant.
if (history.EnumerationStartedAt != null
&& history.EnumerationCompletedBeforeUpload == null)
{
_logger.LogError(
"Expectations blob missing, EnumerationStartedAt={StartedAt}, ECBU null — fail-closed (mid-enumeration crash). tenant={Tenant}",
history.EnumerationStartedAt, tenantId);
return "expectations_missing";
}
// Either first try (ESA null) or upload-retry (ESA + ECBU set, blob missing).
// First-try only: stamp ESA before touching the enumerator so a crash here lands
// in the fail-closed branch above on the next pickup.
if (history.EnumerationStartedAt == null)
{
history.EnumerationStartedAt = DateTime.UtcNow;
await _auditRepo.UpsertHistoryAsync(history, ct);
}
else
{
_logger.LogInformation(
"Upload-retry path for tenant={Tenant} (ESA={Esa}, ECBU={Ecbu}) — re-running enumerate+upload",
tenantId, history.EnumerationStartedAt, history.EnumerationCompletedBeforeUpload);
}
// Fail-loud enumeration. Storage exceptions propagate; the queue worker poisons or
// retries via visibility timeout. NEVER catch here — that would re-introduce the
// Rev-5-F3 silent-empty bug.
var collected = new List<OffboardingExpectation>();
var actor = new DeletionActor { Type = "system", Actor = "System.TenantOffboarding" };
await foreach (var sessionId in _enumerator.EnumerateAsync(tenantId, ct))
{
ct.ThrowIfCancellationRequested();
var result = await _cascadeEnqueuer.EnqueueAsync(
tenantId, sessionId, reason: "tenant_offboard", actor: actor,
retentionContext: null, cancellationToken: ct);
collected.Add(new OffboardingExpectation
{
SessionId = sessionId,
ManifestId = result.ManifestId,
Outcome = result.Outcome.ToString(),
RetryCount = 0,
});
if (PerSessionEnqueueThrottle > TimeSpan.Zero)
{
try { await Task.Delay(PerSessionEnqueueThrottle, ct); }
catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; }
}
}
// Enumeration finished cleanly. Persist ECBU BEFORE the upload so a subsequent
// upload failure is recognized as upload-retry-eligible on the next pickup.
// On retry pickups ECBU may already be set — UpsertHistoryAsync is idempotent.
if (history.EnumerationCompletedBeforeUpload == null)
{
history.EnumerationCompletedBeforeUpload = DateTime.UtcNow;
await _auditRepo.UpsertHistoryAsync(history, ct);
}
// Plan Rev-7-F2: EnumerationCompleted on the payload = the in-blob flag the drain
// probe checks. We can only set it true here because the iterator above ran to
// completion without throwing (the throw-path skipped this whole block).
var payload = new OffboardingExpectations
{
SchemaVersion = 1,
TenantId = tenantId,
HistoryRowKey = history.RowKey,
CreatedAt = DateTime.UtcNow,
EnumerationCompleted = true,
EnumeratedSessionCount = collected.Count,
Expectations = collected,
};
var uploaded = await _expectations.TryUploadInitialAsync(payload, ct);
if (!uploaded)
{
// Race: a parallel worker beat us. Their blob is the source of truth.
_logger.LogInformation(
"Expectations blob race lost for tenant={Tenant} history={History} — using existing blob",
tenantId, history.RowKey);
}
else
{
history.CascadeSessionsEnqueued = collected.Count;
await _auditRepo.UpsertHistoryAsync(history, ct);
}
return null;
}
// ── Phase 2.C — drain predicate ─────────────────────────────────────────
private async Task<(bool DrainOk, string? FailReason)> EvaluateDrainAsync(
OffboardingHistoryEntry history, string tenantId, CancellationToken ct)
{
var (payload, etag) = await _expectations.TryDownloadAsync(tenantId, history.RowKey, ct);
if (payload == null) return (false, "expectations_missing");
if (etag == null) return (false, "expectations_missing");
if (payload.SchemaVersion != 1) return (false, "expectations_corrupt");
if (!payload.EnumerationCompleted) return (false, "enumeration_incomplete");
if (payload.Expectations.Count != payload.EnumeratedSessionCount)
return (false, "expectations_size_mismatch");
var allSatisfied = true;
var dirty = false;
var actor = new DeletionActor { Type = "system", Actor = "System.TenantOffboarding" };
for (int i = 0; i < payload.Expectations.Count; i++)
{
ct.ThrowIfCancellationRequested();
var expectation = payload.Expectations[i];
switch (expectation.Outcome)
{
case nameof(SessionDeletionEnqueueOutcome.KillSwitchActive):
return (false, "killswitch");
case nameof(SessionDeletionEnqueueOutcome.Poisoned):
return (false, "poisoned");
case nameof(SessionDeletionEnqueueOutcome.SessionNotFound):
// Session was already gone when we tried to enqueue — nothing to drain.
continue;
case nameof(SessionDeletionEnqueueOutcome.CasExhausted):
if (expectation.RetryCount >= MaxCasRetriesPerSession)
return (false, "cas_exhausted");
// Retry the per-session enqueue. Update outcome + counter in-place.
var retry = await _cascadeEnqueuer.EnqueueAsync(
tenantId, expectation.SessionId, reason: "tenant_offboard", actor: actor,
retentionContext: null, cancellationToken: ct);
expectation.Outcome = retry.Outcome.ToString();
expectation.ManifestId = retry.ManifestId;
expectation.RetryCount++;
dirty = true;
allSatisfied = false; // re-evaluate next drain cycle
break;
case nameof(SessionDeletionEnqueueOutcome.Enqueued):
case nameof(SessionDeletionEnqueueOutcome.AlreadyInFlight):
if (string.IsNullOrEmpty(expectation.ManifestId))
{
// Rev-8-F3 — AlreadyInFlight with null ManifestId is a Preparing-without-
// snapshot state. There's no progress blob to drain against; fail-closed.
return (false, "alreadyinflight_no_manifest");
}
var done = await _drainProbe.IsCascadeCompletedAsync(
tenantId, expectation.SessionId, expectation.ManifestId, ct);
if (!done) allSatisfied = false;
break;
default:
_logger.LogWarning(
"Unknown Expectation.Outcome '{Outcome}' for tenant={Tenant} session={Session}",
expectation.Outcome, tenantId, expectation.SessionId);
allSatisfied = false;
break;
}
}
if (dirty)
{
try
{
await _expectations.UpdateWithEtagCasAsync(payload, etag, ct);
}
catch (RequestFailedException ex) when (ex.Status == 412)
{
// A parallel worker re-evaluated and wrote first. Their version wins; the
// next drain cycle will pick up the latest state. We still report
// allSatisfied=false so this cycle re-enqueues.
_logger.LogInformation(
"Expectations CAS 412 for tenant={Tenant} — parallel worker won, will re-read next cycle",
tenantId);
return (false, null);
}
}
return (allSatisfied, null);
}
// ── Phases 2.D / 2.E / 2.D-pass-2 / 2.F / 2.G / 2.H / 2.I ───────────────
private async Task RunPostDrainPhasesAsync(
OffboardingHistoryEntry history, string tenantId, CancellationToken ct)
{
// 2.D-archive (PR3.B §3) — Archive then wipe GatherRules / AnalyzeRules /
// ImeLogPatterns. Fail-loud: a row that cannot be archived must NOT be wiped.
// After the third iteration if the source still has rows, throw with
// FailedPhase="customs_arrival_race" — marker stays Failed for operator action.
try
{
await ArchiveAndWipeCustomsRulesAsync(history, tenantId, ct);
}
catch (InvalidOperationException ex) when (ex.Message.StartsWith("customs_arrival_race", StringComparison.Ordinal))
{
await FailAsync(history, tenantId, "customs_arrival_race", ex.Message, ct);
throw; // worker dequeues + eventually poisons
}
// 2.D-pass-1 — SafeWipe all tenant-scoped tables, including AuditLogs.
var deletedCounts = new Dictionary<string, int>(StringComparer.Ordinal);
await SafeWipeAllTenantTablesAsync(deletedCounts, tenantId, ct);
// 2.E — Blob-prefix wipe (deletion-manifests only; offboarding-state stays).
int deletedBlobs = 0;
try
{
deletedBlobs = await _safeWipe.WipeBlobsByTenantPrefixAsync(
Constants.BlobContainers.DeletionManifests, tenantId, ct);
}
catch (SafeWipeVerificationException)
{
throw; // unrecoverable — let worker poison
}
// 2.D-pass-2 — Re-wipe AuditLogs as defence in depth against late audits that may
// have landed between Phase 2.D-pass-1 and now (the cascade-handler reorder from
// PR1.5 closes the main window, but a struggling SignalR/Audit retry could still
// bring a row in late).
var auditPass2 = await _safeWipe.WipeByExactPartitionAsync(Constants.TableNames.AuditLogs, tenantId, ct);
if (auditPass2 > 0)
{
_logger.LogInformation(
"Audit pass-2 swept {Count} late row(s) for tenant={Tenant}", auditPass2, tenantId);
deletedCounts[Constants.TableNames.AuditLogs] =
deletedCounts.TryGetValue(Constants.TableNames.AuditLogs, out var existing) ? existing + auditPass2 : auditPass2;
}
// ── 2.G — Order: side-effects FIRST, then History terminal write LAST ────
//
// Note (PR3.B review): the legacy 2.F TenantConfiguration-delete used to sit
// BEFORE 2.G. That left a tiny window where a /api/auth/me race could re-create
// a fresh Default-Disabled=false TenantConfiguration before the Completed
// state landed — letting an offboarded tenant slip back through. The delete
// has moved to AFTER 2.I (see "2.F-final" below) so the entire Phase 2 sequence
// runs against an existing Disabled=true row, and only once everything is
// committed do we drop the row. Self-service re-onboarding still works because
// the user's NEXT /api/auth/me after 2.F-final sees no row → auto-create-default.
// Plan §3 Phase 2.G + Review-Fix Finding 3: a crash between History=Completed and
// the marker/pointer/blob/audit/ops writes would otherwise make the next pickup
// return early (terminal-status check) and leave the side-effects dangling. By
// flipping History last we keep DrainCompletedAt as the resume anchor — re-pickup
// skips drain and re-runs the (idempotent) side-effect chain.
// Pre-populate the in-memory history with the final values so the LAST write below
// commits the whole snapshot atomically.
var completedAt = DateTime.UtcNow;
history.CompletedAt = completedAt;
history.DeletedRowCountsJson = JsonSerializer.Serialize(deletedCounts);
history.TotalRowsDeleted = deletedCounts.Values.Sum();
history.DeletedBlobCount = deletedBlobs;
// Side-effect 1: Marker → Completed (must happen FIRST so a crash here leaves
// History=InProgress; the next pickup re-runs the idempotent post-drain chain).
var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct);
if (marker != null)
{
marker.Status = "Completed";
marker.CompletedAt = completedAt;
await _auditRepo.UpsertMarkerAsync(marker, ct);
}
// Side-effect 2: Pointer → Completed (ETag-CAS retry).
await UpdatePointerWithRetryAsync(tenantId, history.RowKey, "Completed", ct);
// Side-effect 3: Expectations blob delete (fail-soft; MarkerCleanupFunction retries).
try
{
await _expectations.DeleteAsync(tenantId, history.RowKey, ct);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Expectations blob delete failed for tenant={Tenant} history={History} — MarkerCleanupFunction will retry",
tenantId, history.RowKey);
}
// Side-effect 4: Global-tenant audit row (persists across the wiped tenant audits).
await _maintenance.LogAuditEntryAsync(
Constants.AuditGlobalTenantId,
action: "DELETE",
entityType: "Tenant",
entityId: tenantId,
performedBy: history.InitiatedBy,
details: new Dictionary<string, string>(StringComparer.Ordinal)
{
["Action"] = "Offboard",
["Phase"] = "Completed",
["DomainName"] = history.DomainName ?? string.Empty,
["TotalRowsDeleted"] = (history.TotalRowsDeleted ?? 0).ToString(),
["DeletedBlobCount"] = (history.DeletedBlobCount ?? 0).ToString(),
["CascadeSessionsEnqueued"] = (history.CascadeSessionsEnqueued ?? 0).ToString(),
["HistoryRowKey"] = history.RowKey,
});
// Side-effect 5: OpsEvent. Reuses the existing TenantOffboarded helper so the
// dashboard category cell stays "Tenant" / event "TenantOffboarded".
await _opsEvents.RecordTenantOffboardedAsync(
tenantId, history.InitiatedBy, deletedCounts, history.DomainName);
// COMMIT: History → Completed LAST. Until this write lands the next pickup will
// see Status=InProgress + DrainCompletedAt set and re-run all of the above
// idempotently (duplicate audit/OpsEvent emit is preferable to dangling state).
history.Status = "Completed";
await _auditRepo.UpsertHistoryAsync(history, ct);
// ── 2.F-final — TenantConfiguration delete (PR3.B Codex Finding 2 reorder) ──
//
// This is the LAST step in Phase 2. By now Marker / Pointer / History all say
// "Completed" and the Disabled=true auth-gate has carried the tenant through the
// entire async wipe. Deleting the row here means the next /api/auth/me for this
// tenant will see no row → TenantConfigurationService.GetConfigurationAsync's
// auto-create-default path kicks in, giving the same user a fresh, clean tenant
// (the legitimate self-service re-onboarding case).
//
// Fail-soft: if the delete fails, Marker stays Completed and the
// OffboardingMarkerCleanupFunction's TenantConfig-sweep (defense-in-depth) runs
// every 2h and finishes the cleanup. The tenant cannot re-onboard until that
// sweep lands — acceptable, because Disabled=true still applies meanwhile.
try
{
var tenantConfigDeleted = await _safeWipe.WipeByExactPartitionAsync(
Constants.TableNames.TenantConfiguration, tenantId, ct);
if (tenantConfigDeleted > 0)
{
_logger.LogInformation(
"Phase 2.F-final: deleted TenantConfiguration row(s)={Count} for tenant={Tenant}",
tenantConfigDeleted, tenantId);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Phase 2.F-final TenantConfiguration delete failed for tenant={Tenant} — MarkerCleanupFunction's TenantConfig-sweep will retry",
tenantId);
}
_logger.LogInformation(
"Tenant offboarding Completed: tenant={Tenant} totalRows={Rows} blobs={Blobs} sessions={Sessions} archiveGather={G} archiveAnalyze={A} archivePatterns={P}",
tenantId,
history.TotalRowsDeleted, history.DeletedBlobCount, history.CascadeSessionsEnqueued,
history.CustomGatherRulesArchived, history.CustomAnalyzeRulesArchived, history.ImeLogPatternOverridesArchived);
}
// ── Helpers ─────────────────────────────────────────────────────────────
/// <summary>
/// PR3.B §3 — Phase 2.D-archive. For each rules table, archive every row to
/// <see cref="ITenantCustomsArchiveRepository"/> THEN delete that exact (PK, RK)
/// from the source. Up to <see cref="ArchiveIterationCap"/> iterations to absorb
/// late writes that may have landed during the drain barrier; if the source still
/// has rows after the cap, throw <see cref="InvalidOperationException"/> so the
/// caller fail-closes the offboarding with <c>FailedPhase="customs_arrival_race"</c>.
/// <para>
/// Crash-safety: the archive upsert is idempotent (Replace), so a worker that
/// crashed between archive-insert and source-delete will re-enumerate the same
/// row on re-pickup, re-write the (identical) archive entry, and delete the
/// source row that survived the crash. Counters are recomputed from the archive
/// table at the end so a transient crash mid-loop does not produce a stale 0.
/// </para>
/// </summary>
private async Task ArchiveAndWipeCustomsRulesAsync(
OffboardingHistoryEntry history, string tenantId, CancellationToken ct)
{
foreach (var (tableName, fieldName) in ArchivedRuleTables)
{
await ArchiveAndWipeRulesTableAsync(tableName, tenantId, history.RowKey, ct);
// Recompute the counter from the archive table (per run + table) rather than
// from in-loop accumulation. A crash between this assignment and the final
// UpsertHistoryAsync below would otherwise leave a stale 0.
var counted = await _customsArchive.CountByRunAndTableAsync(tenantId, history.RowKey, tableName, ct);
AssignArchivedCounter(history, fieldName, counted);
}
await _auditRepo.UpsertHistoryAsync(history, ct);
}
/// <summary>
/// Archive-then-wipe loop for a single rules table. Up to <see cref="ArchiveIterationCap"/>
/// iterations; final-iteration verify-read triggers <c>customs_arrival_race</c> when a
/// writer is still racing us. Marked <c>internal virtual</c> so test harnesses can
/// override it to a no-op without faking the underlying TableServiceClient.
/// </summary>
internal virtual async Task ArchiveAndWipeRulesTableAsync(
string tableName, string tenantId, string historyRowKey, CancellationToken ct)
{
var sourceClient = _storage.GetTableClient(tableName);
for (var attempt = 1; attempt <= ArchiveIterationCap; attempt++)
{
var archivedThisAttempt = 0;
var filter = OffboardingFilters.ExactPartition(tenantId);
await foreach (var entity in sourceClient.QueryAsync<TableEntity>(filter, cancellationToken: ct))
{
await _customsArchive.UpsertAsync(BuildArchiveEntry(entity, tenantId, tableName, historyRowKey), ct);
// Only AFTER the archive entry is persisted do we delete the source.
try
{
await sourceClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey, Azure.ETag.All, ct);
}
catch (Azure.RequestFailedException ex) when (ex.Status == 404)
{
// Already gone (parallel writer / re-pickup) — archive already
// captured the state we observed, so this is idempotent-safe.
}
archivedThisAttempt++;
}
if (archivedThisAttempt == 0)
{
// Source partition empty → converged.
return;
}
if (attempt == ArchiveIterationCap)
{
// Final iteration still found rows. One last verify-read; if STILL
// non-empty, a writer is racing us and we cannot guarantee complete
// archive coverage → fail-closed.
var leftover = 0;
await foreach (var _ in sourceClient.QueryAsync<TableEntity>(
filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct))
{
leftover++;
break; // existence is enough
}
if (leftover > 0)
{
_logger.LogError(
"Customs archive iteration cap exhausted for {Table} tenant={Tenant} run={History}: a writer is still producing rows after {Cap} archive-then-wipe passes — fail-closing",
tableName, tenantId, historyRowKey, ArchiveIterationCap);
throw new InvalidOperationException(
$"customs_arrival_race: table '{tableName}' still has rows after {ArchiveIterationCap} archive-then-wipe iterations");
}
}
}
}
internal TenantOffboardingCustomsArchiveEntry BuildArchiveEntry(
TableEntity source, string tenantId, string originalTable, string historyRowKey)
{
// Serialize every non-system property the source row exposes. TableEntity is
// IDictionary<string, object> internally, so we can iterate.
var snapshot = new Dictionary<string, object?>(StringComparer.Ordinal);
foreach (var kv in source)
{
// Skip pseudo-properties Azure layers add (ETag/Timestamp/odata.*).
if (kv.Key is "odata.etag" or "Timestamp") continue;
snapshot[kv.Key] = kv.Value;
}
var entityJson = JsonSerializer.Serialize(snapshot);
return new TenantOffboardingCustomsArchiveEntry
{
PartitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey(tenantId, historyRowKey),
RowKey = TableTenantCustomsArchiveRepository.BuildRowKey(originalTable, source.RowKey),
TenantId = tenantId,
OriginalTable = originalTable,
OriginalPartitionKey = source.PartitionKey,
OriginalRowKey = source.RowKey,
EntityJson = entityJson,
HistoryRowKey = historyRowKey,
ArchivedAt = DateTime.UtcNow,
ArchivedBy = "TenantOffboardingHandler",
};
}
private static void AssignArchivedCounter(OffboardingHistoryEntry history, string fieldName, int value)
{
switch (fieldName)
{
case nameof(OffboardingHistoryEntry.CustomGatherRulesArchived):
history.CustomGatherRulesArchived = value;
break;
case nameof(OffboardingHistoryEntry.CustomAnalyzeRulesArchived):
history.CustomAnalyzeRulesArchived = value;
break;
case nameof(OffboardingHistoryEntry.ImeLogPatternOverridesArchived):
history.ImeLogPatternOverridesArchived = value;
break;
default:
throw new InvalidOperationException($"Unknown archived counter field '{fieldName}'");
}
}
private async Task SafeWipeAllTenantTablesAsync(
Dictionary<string, int> counts, string tenantId, CancellationToken ct)
{
foreach (var table in TenantPartitionTables)
{
ct.ThrowIfCancellationRequested();
counts[table] = await _safeWipe.WipeByExactPartitionAsync(table, tenantId, ct);
}
foreach (var table in CompositePartitionTables)
{
ct.ThrowIfCancellationRequested();
counts[table] = await _safeWipe.WipeByCompositePartitionRangeAsync(table, tenantId, ct);
}
foreach (var (table, discriminator) in DiscriminatorTables)
{
ct.ThrowIfCancellationRequested();
var key = $"{table}/{discriminator}";
counts[key] = await _safeWipe.WipeByDiscriminatorAndTenantPropertyAsync(table, discriminator, tenantId, ct);
}
foreach (var table in PropertyOnlyTables)
{
ct.ThrowIfCancellationRequested();
counts[table] = await _safeWipe.WipeByTenantIdPropertyAsync(table, tenantId, ct);
}
}
// ── Failure path ────────────────────────────────────────────────────────
//
// Same Review-Fix Finding 3 invariant as the Completed path: write Marker/Pointer/Ops
// FIRST, then History terminal LAST. A crash between the side-effects and the History
// commit makes the next pickup re-run the side-effects idempotently — which is exactly
// what we want for the Failed path, because the marker/pointer/ops outcome is the
// operator-facing signal, not History.Status by itself.
private async Task FailAsync(
OffboardingHistoryEntry history, string tenantId,
string failedPhase, string errorMessage, CancellationToken ct)
{
// Pre-populate in-memory history with the final values so the LAST write below
// commits the snapshot.
history.ErrorMessage = errorMessage;
// Side-effect 1: Marker → Failed.
var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct);
if (marker != null)
{
marker.Status = "Failed";
marker.FailedAt = DateTime.UtcNow;
marker.FailedPhase = failedPhase;
await _auditRepo.UpsertMarkerAsync(marker, ct);
}
// Side-effect 2: Pointer → Failed.
await UpdatePointerWithRetryAsync(tenantId, history.RowKey, "Failed", ct);
// Side-effect 3: OpsEvent (Telegram-routable).
await _opsEvents.RecordTenantOffboardingFailedAsync(
tenantId, history.InitiatedBy, failedPhase, errorMessage, history.RetryCount, history.DomainName);
// COMMIT: History → Failed LAST.
history.Status = "Failed";
await _auditRepo.UpsertHistoryAsync(history, ct);
_logger.LogError(
"Tenant offboarding FAILED: tenant={Tenant} phase={Phase} message={Message}",
tenantId, failedPhase, errorMessage);
}
// ── Worker-side poison transition (Review-Fix Finding 2) ────────────────
/// <summary>
/// Called by <see cref="TenantOffboardingWorker"/> when an envelope hits max-dequeue and
/// moves to the poison queue. Idempotently transitions History/Pointer/Marker to Failed
/// so the operator dashboard reflects the dead-letter rather than leaving the tenant
/// hanging in InProgress. No-op when the row is already Completed/Failed.
/// </summary>
public async Task MarkEnvelopeFailedFromPoisonAsync(
TenantOffboardingEnvelope envelope, int dequeueCount, CancellationToken ct = default)
{
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
var tenantId = envelope.TenantId.ToLowerInvariant();
var history = await _auditRepo.TryGetHistoryAsync(envelope.HistoryRowKey, ct);
if (history == null)
{
_logger.LogWarning(
"MarkEnvelopeFailedFromPoison: history row missing for tenant={Tenant} history={History} — nothing to transition",
tenantId, envelope.HistoryRowKey);
return;
}
if (history.Status is "Completed" or "Failed")
{
_logger.LogInformation(
"MarkEnvelopeFailedFromPoison: history already terminal ({Status}) for tenant={Tenant} — skipping",
history.Status, tenantId);
return;
}
history.RetryCount = Math.Max(history.RetryCount, dequeueCount);
await FailAsync(history, tenantId,
failedPhase: "max_dequeue",
errorMessage: $"Envelope poisoned after {dequeueCount} failed handler attempts",
ct);
}
}
}
@@ -0,0 +1,345 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Azure.Identity;
using Azure.Storage.Queues;
using Azure.Storage.Queues.Models;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Offboarding;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace AutopilotMonitor.Functions.Services.Offboarding
{
/// <summary>
/// Background poll-loop for the <c>tenant-offboarding</c> queue (Plan §7.3). Modeled on
/// <see cref="Deletion.SessionDeletionWorker"/>:
/// <list type="bullet">
/// <item><see cref="BatchSize"/> = <b>1</b> — one tenant cascade per receive bounds memory
/// (per-tenant enumerate + per-session enqueue is long-running) and limits the
/// blast radius of a poison.</item>
/// <item>Heartbeat extends visibility every <see cref="DefaultHeartbeatInterval"/> while
/// the handler runs so concurrent re-delivery can't spawn a parallel offboarding
/// for the same tenant.</item>
/// <item>Max-dequeue <see cref="MaxDequeueCount"/> → poison queue + the handler's own
/// Failed-state transition (the handler emits <c>TenantOffboardingFailed</c> when
/// it fails closed; the worker emits one too on dequeue exhaustion so an envelope
/// that never reaches the handler still surfaces).</item>
/// </list>
/// </summary>
public sealed class TenantOffboardingWorker : BackgroundService
{
internal const int BatchSize = 1;
internal static readonly TimeSpan VisibilityTimeout = TimeSpan.FromMinutes(5);
internal static readonly TimeSpan DefaultHeartbeatInterval = TimeSpan.FromSeconds(60);
internal static readonly TimeSpan HeartbeatExtendBy = TimeSpan.FromMinutes(5);
internal static readonly TimeSpan DefaultPollInterval = TimeSpan.FromSeconds(10);
internal static readonly TimeSpan ErrorBackoff = TimeSpan.FromSeconds(30);
internal const int MaxDequeueCount = 5;
private const string PoisonQueueSuffix = "-poison";
private readonly QueueClient _mainQueue;
private readonly QueueClient _poisonQueue;
private readonly TenantOffboardingHandler _handler;
private readonly OpsEventService _opsEvents;
private readonly ILogger<TenantOffboardingWorker> _logger;
private readonly TimeSpan _heartbeatInterval;
private readonly TimeSpan _pollInterval;
public TenantOffboardingWorker(
IConfiguration configuration,
TenantOffboardingHandler handler,
OpsEventService opsEvents,
ILogger<TenantOffboardingWorker> logger)
{
_handler = handler ?? throw new ArgumentNullException(nameof(handler));
_opsEvents = opsEvents ?? throw new ArgumentNullException(nameof(opsEvents));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_heartbeatInterval = DefaultHeartbeatInterval;
_pollInterval = DefaultPollInterval;
var options = new QueueClientOptions { MessageEncoding = QueueMessageEncoding.Base64 };
var storageAccountName = configuration["AzureStorageAccountName"];
var connectionString = configuration["AzureTableStorageConnectionString"];
if (!string.IsNullOrEmpty(storageAccountName))
{
var mainUri = new Uri(
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboarding}");
var poisonUri = new Uri(
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboardingPoison}");
var credential = new DefaultAzureCredential();
_mainQueue = new QueueClient(mainUri, credential, options);
_poisonQueue = new QueueClient(poisonUri, credential, options);
_logger.LogInformation(
"TenantOffboardingWorker initialized with Managed Identity (account: {Account})",
storageAccountName);
}
else if (!string.IsNullOrEmpty(connectionString))
{
_mainQueue = new QueueClient(connectionString, Constants.QueueNames.TenantOffboarding, options);
_poisonQueue = new QueueClient(connectionString, Constants.QueueNames.TenantOffboardingPoison, options);
_logger.LogInformation("TenantOffboardingWorker initialized with connection string");
}
else
{
throw new InvalidOperationException(
"Queue Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureTableStorageConnectionString'.");
}
}
/// <summary>Test seam — inject mocked queues + shorter intervals for unit tests.</summary>
internal TenantOffboardingWorker(
QueueClient mainQueue,
QueueClient poisonQueue,
TenantOffboardingHandler handler,
OpsEventService opsEvents,
ILogger<TenantOffboardingWorker> logger,
TimeSpan? heartbeatInterval = null,
TimeSpan? pollInterval = null)
{
_mainQueue = mainQueue ?? throw new ArgumentNullException(nameof(mainQueue));
_poisonQueue = poisonQueue ?? throw new ArgumentNullException(nameof(poisonQueue));
_handler = handler ?? throw new ArgumentNullException(nameof(handler));
_opsEvents = opsEvents ?? throw new ArgumentNullException(nameof(opsEvents));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_heartbeatInterval = heartbeatInterval ?? DefaultHeartbeatInterval;
_pollInterval = pollInterval ?? DefaultPollInterval;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await TryCreateQueueAsync(_mainQueue, "main", stoppingToken).ConfigureAwait(false);
await TryCreateQueueAsync(_poisonQueue, "poison", stoppingToken).ConfigureAwait(false);
_logger.LogInformation("TenantOffboardingWorker: poll loop started (queue {Name})", _mainQueue.Name);
while (!stoppingToken.IsCancellationRequested)
{
try
{
var batch = await _mainQueue
.ReceiveMessagesAsync(BatchSize, VisibilityTimeout, stoppingToken)
.ConfigureAwait(false);
if (batch?.Value is null || batch.Value.Length == 0)
{
await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false);
continue;
}
foreach (var msg in batch.Value)
{
if (stoppingToken.IsCancellationRequested) break;
await ProcessOneAsync(msg, stoppingToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_logger.LogError(ex,
"TenantOffboardingWorker: poll-loop error — backing off {Backoff}",
ErrorBackoff);
try { await Task.Delay(ErrorBackoff, stoppingToken).ConfigureAwait(false); }
catch (OperationCanceledException) { break; }
}
}
_logger.LogInformation("TenantOffboardingWorker: poll loop stopped");
}
private async Task ProcessOneAsync(QueueMessage msg, CancellationToken ct)
{
if (msg.DequeueCount > MaxDequeueCount)
{
await MoveToPoisonAsync(msg, ct).ConfigureAwait(false);
return;
}
TenantOffboardingEnvelope? envelope;
try
{
envelope = JsonConvert.DeserializeObject<TenantOffboardingEnvelope>(msg.Body.ToString());
}
catch (JsonException ex)
{
_logger.LogWarning(ex,
"TenantOffboardingWorker: malformed envelope JSON — dropping (msg {Id})",
msg.MessageId);
await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false);
return;
}
if (envelope is null
|| string.IsNullOrEmpty(envelope.TenantId)
|| string.IsNullOrEmpty(envelope.HistoryRowKey))
{
_logger.LogWarning(
"TenantOffboardingWorker: envelope missing required fields — dropping (msg {Id})",
msg.MessageId);
await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false);
return;
}
using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
var heartbeatState = new HeartbeatState(msg.MessageId, msg.PopReceipt);
var heartbeatTask = Task.Run(() => HeartbeatLoopAsync(heartbeatState, heartbeatCts.Token), heartbeatCts.Token);
try
{
await _handler.HandleAsync(envelope, ct).ConfigureAwait(false);
heartbeatCts.Cancel();
try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { }
await SafeDeleteAsync(msg.MessageId, heartbeatState.PopReceipt, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
heartbeatCts.Cancel();
throw;
}
catch (Exception ex)
{
heartbeatCts.Cancel();
try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { }
_logger.LogWarning(ex,
"TenantOffboardingWorker: handler failed (tenant={Tenant} history={History} dequeue={N}) — visibility-timeout retry",
envelope.TenantId, envelope.HistoryRowKey, msg.DequeueCount);
}
}
private async Task HeartbeatLoopAsync(HeartbeatState state, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
try { await Task.Delay(_heartbeatInterval, ct).ConfigureAwait(false); }
catch (OperationCanceledException) { return; }
try
{
var updated = await _mainQueue.UpdateMessageAsync(
state.MessageId, state.PopReceipt,
visibilityTimeout: HeartbeatExtendBy,
cancellationToken: ct).ConfigureAwait(false);
state.PopReceipt = updated.Value.PopReceipt;
_logger.LogDebug(
"TenantOffboardingWorker: heartbeat extended visibility for message {Id} by {Extend}",
state.MessageId, HeartbeatExtendBy);
}
catch (OperationCanceledException) { return; }
catch (Exception ex)
{
_logger.LogWarning(ex,
"TenantOffboardingWorker: heartbeat UpdateMessageAsync failed for message {Id} — exiting heartbeat",
state.MessageId);
return;
}
}
}
private async Task MoveToPoisonAsync(QueueMessage msg, CancellationToken ct)
{
TenantOffboardingEnvelope? envelope = null;
try { envelope = JsonConvert.DeserializeObject<TenantOffboardingEnvelope>(msg.Body.ToString()); }
catch (JsonException) { /* still poison the malformed envelope below */ }
// Review-Fix (second-pass) Finding 1: the durable Failed-state transition MUST land
// BEFORE the poison-queue send + main-queue delete. Otherwise a transient storage /
// OpsEvent / pointer-CAS failure on the transition would leave the message in poison
// but the Tenant stuck at InProgress. By doing the transition first we keep the
// message visible on the main queue when the transition fails — the worker will
// retry on the next dequeue. The transition itself is idempotent
// (MarkEnvelopeFailedFromPoisonAsync no-ops on terminal History.Status) so retries
// are safe.
if (envelope != null
&& !string.IsNullOrEmpty(envelope.TenantId)
&& !string.IsNullOrEmpty(envelope.HistoryRowKey))
{
try
{
await _handler.MarkEnvelopeFailedFromPoisonAsync(
envelope, (int)(msg.DequeueCount - 1), ct);
}
catch (Exception ex)
{
// Critical: leave the message visible so the next dequeue can retry the
// transition. We do NOT proceed to the poison send / SafeDelete here —
// otherwise the operator would see a queue with nothing in it and a tenant
// hanging on InProgress.
_logger.LogError(ex,
"TenantOffboardingWorker: MarkEnvelopeFailedFromPoisonAsync failed for tenant={Tenant} history={History} — message left visible for retry (durable Failed state has higher priority than poison move)",
envelope.TenantId, envelope.HistoryRowKey);
return;
}
}
// Failed state is now durable (or envelope was malformed and couldn't transition).
// Best-effort poison move + SafeDelete; transient failure here leaves the message
// visible but the next pickup will see History.Status=Failed and SafeDelete cleanly.
try
{
await _poisonQueue.SendMessageAsync(msg.Body.ToString(), ct).ConfigureAwait(false);
await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false);
_logger.LogWarning(
"TenantOffboardingWorker: moved message {Id} to poison queue after {N} failed attempts",
msg.MessageId, msg.DequeueCount - 1);
}
catch (Exception ex)
{
_logger.LogError(ex,
"TenantOffboardingWorker: poison move failed for message {Id} (will retry; durable Failed state already written)",
msg.MessageId);
}
}
private async Task SafeDeleteAsync(string messageId, string popReceipt, CancellationToken ct)
{
try
{
await _mainQueue.DeleteMessageAsync(messageId, popReceipt, ct).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"TenantOffboardingWorker: DeleteMessageAsync failed for message {Id} — message will reappear after visibility expires",
messageId);
}
}
private async Task TryCreateQueueAsync(QueueClient queue, string label, CancellationToken ct)
{
try
{
await queue.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"TenantOffboardingWorker: CreateIfNotExists failed on {Label} queue {Name}",
label, queue.Name);
}
}
private sealed class HeartbeatState
{
public string MessageId { get; }
public string PopReceipt { get; set; }
public HeartbeatState(string messageId, string popReceipt)
{
MessageId = messageId;
PopReceipt = popReceipt;
}
}
}
}
@@ -275,6 +275,26 @@ namespace AutopilotMonitor.Functions.Services
tenantId, performedBy, details);
}
/// <summary>
/// Fired by the tenant-offboarding worker when the cascade fails closed (kill-switch
/// active mid-enqueue, drain timeout, expectations blob corruption, ETag/CAS exhaustion,
/// SafeWipe verify abort, …). Marker stays Failed until operator action; this event is
/// the Telegram-routable signal that something needs human attention. Plan Rev-4 Q2.
/// </summary>
public Task RecordTenantOffboardingFailedAsync(
string tenantId, string performedBy, string failedPhase, string errorMessage,
int retryCount, string? domainName = null)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName)
? tenantId
: $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, "TenantOffboardingFailed", OpsEventSeverity.Error,
$"Tenant {tenantLabel} offboarding failed at phase '{failedPhase}': {errorMessage}",
tenantId, performedBy,
new { domainName, failedPhase, errorMessage, retryCount });
}
// ── Agent ──────────────────────────────────────────────────────────────
public Task RecordSessionTimeoutsAsync(string tenantId, int sessionCount, int timeoutHours)
@@ -124,7 +124,7 @@ namespace AutopilotMonitor.Functions.Services
/// Returns (config, exists). exists=false when no row was found — does NOT auto-create.
/// Use for agent security gates where unknown tenants must be rejected.
/// </summary>
public async Task<(TenantConfiguration config, bool exists)> TryGetConfigurationAsync(string tenantId)
public virtual async Task<(TenantConfiguration config, bool exists)> TryGetConfigurationAsync(string tenantId)
{
if (string.IsNullOrEmpty(tenantId))
return (TenantConfiguration.CreateDefault("unknown"), false);
@@ -493,6 +493,25 @@ namespace AutopilotMonitor.Shared
public const string ClassifierVerdictsByIdLevel = "ClassifierVerdictsByIdLevel";
public const string SignalsByKind = "SignalsByKind";
// Tenant offboarding audit (cascade-worker plan, Rev 9). Houses three
// PartitionKey patterns side-by-side:
// - "OffboardingMarker" (1 row/tenant, RK=tenantId) active 403-gate marker
// - "OffboardingHistory" (N rows/tenant, RK="{yyyyMMddHHmmssfff}_{tenantId}") audit trail
// - "OffboardingByTenant" (1 row/tenant, RK=tenantId) O(1) re-onboarding pointer
// Marker survives the Completed transition for at least 15min so that warm
// function-instance caches of TenantConfiguration cannot leak auth after wipe.
// See OffboardingPartitionKeys for the canonical PK string literals.
public const string OffboardingAudit = "OffboardingAudit";
// Tenant offboarding customs archive (PR3.B). Phase 2.D-archive writes a
// snapshot of every GatherRules / AnalyzeRules / ImeLogPatterns row keyed by
// PK=tenantId before the originals are safe-wiped, so a Global Admin can
// review them post-offboarding from the /admin/customs-archive page.
// - PartitionKey = "{normalizedTenantId}_{historyRowKey}" (one partition per offboarding run)
// - RowKey = "{originalTable}_{base64url(originalRowKey)}"
// Lives forever; cleanup is operator-driven via the admin UI.
public const string TenantOffboardingCustomsArchive = "TenantOffboardingCustomsArchive";
/// <summary>
/// Returns all table names for initialization
/// </summary>
@@ -546,10 +565,54 @@ namespace AutopilotMonitor.Shared
SessionsByStage,
DeadEndsByReason,
ClassifierVerdictsByIdLevel,
SignalsByKind
SignalsByKind,
OffboardingAudit,
TenantOffboardingCustomsArchive
};
}
/// <summary>
/// Canonical PartitionKey strings stored in <see cref="TableNames.OffboardingAudit"/>.
/// All three patterns coexist in the same table; consumers must always read by
/// (PartitionKey, RowKey) point-lookup, never by table scan.
/// </summary>
public static class OffboardingPartitionKeys
{
/// <summary>Active 403 sperrmarker. RowKey = normalized tenantId. 1 row per active offboarding.</summary>
public const string Marker = "OffboardingMarker";
/// <summary>Chronological audit history. RowKey = "{yyyyMMddHHmmssfff}_{normalizedTenantId}". N rows per tenant.</summary>
public const string History = "OffboardingHistory";
/// <summary>O(1) pointer index for re-onboarding lookup. RowKey = normalized tenantId. 1 row per tenant.</summary>
public const string ByTenant = "OffboardingByTenant";
}
/// <summary>
/// Azure Blob Storage container names. Single source of truth for container strings
/// so that storage helpers, cascade workers, and offboarding workers all agree.
/// </summary>
public static class BlobContainers
{
/// <summary>
/// Houses cascade-deletion manifests (snapshot + progress blobs) under prefix
/// <c>{tenantId}/{sessionId}/{manifestId}.{snapshot.json.gz | progress.json}</c>.
/// 30d lifecycle delete + 3d soft delete = ~33d effective retention.
/// Phase 2.E of tenant offboarding wipes this container by <c>{tenantId}/</c> prefix.
/// </summary>
public const string DeletionManifests = "deletion-manifests";
/// <summary>
/// Houses tenant-offboarding state blobs that MUST survive the deletion-manifests wipe.
/// Currently: per-tenant Expectations-Blob written at the first handler pickup
/// (<c>{tenantId}/{historyRowKey}.expectations.json</c>). Phase 2.E intentionally
/// skips this container; Phase 2.G deletes the blob explicitly as its last step,
/// with <see cref="Functions.Maintenance.OffboardingMarkerCleanupFunction"/> as a
/// defense-in-depth second-pass.
/// </summary>
public const string OffboardingState = "offboarding-state";
}
/// <summary>
/// Azure Storage Queue names. Queues feed async fan-out / reconciliation paths
/// that would otherwise block the hot ingest loop.
@@ -593,6 +656,20 @@ namespace AutopilotMonitor.Shared
/// the Sessions row. Poison suffix <c>-poison</c>, max-dequeue 5.
/// </summary>
public const string SessionDeletion = "session-deletion";
/// <summary>
/// Tenant offboarding cascade-worker queue. Producer (TenantOffboardFunction) writes
/// one envelope per tenant after inserting the History/Pointer/Marker rows in
/// <see cref="TableNames.OffboardingAudit"/>. Worker (TenantOffboardingWorker)
/// enumerates sessions, enqueues per-session cascades, drains against an
/// Expectations-Blob in <see cref="BlobContainers.OffboardingState"/>, then safe-wipes
/// all remaining tenant-scoped rows. Max-dequeue 5, paired with
/// <see cref="TenantOffboardingPoison"/>.
/// </summary>
public const string TenantOffboarding = "tenant-offboarding";
/// <summary>Poison sibling of <see cref="TenantOffboarding"/>.</summary>
public const string TenantOffboardingPoison = "tenant-offboarding-poison";
}
}
}
@@ -69,7 +69,16 @@ namespace AutopilotMonitor.Shared.DataAccess
Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId);
// --- Tenant Offboarding ---
Task<Dictionary<string, int>> DeleteAllTenantDataAsync(string tenantId);
/// <summary>
/// Fail-loud iterator over every session belonging to <paramref name="tenantId"/>.
/// Used by the tenant-offboarding worker to drive per-session cascade enqueue.
/// Unlike <see cref="GetSessionsByDateRangeAsync"/>, this method MUST NOT swallow
/// storage exceptions — a transient failure during enumeration must propagate so
/// the queue worker can retry / poison, instead of silently returning zero sessions
/// which would trigger a same-cycle wipe without cascade backup.
/// </summary>
IAsyncEnumerable<string> EnumerateSessionsForOffboardingAsync(string tenantId, System.Threading.CancellationToken ct = default);
}
public class OrphanedEventSession
@@ -0,0 +1,74 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.Models.Offboarding;
namespace AutopilotMonitor.Shared.DataAccess
{
/// <summary>
/// Access to the <c>OffboardingAudit</c> table. Owns three PartitionKey patterns
/// (Marker / History / ByTenant). All write paths fail-loud — Queue retry / poison
/// owns recovery semantics.
/// </summary>
public interface IOffboardingAuditRepository
{
// ── Marker (offboarding lifecycle anchor; idempotency + cleanup) ───────
/// <summary>
/// Uncached point lookup used by <c>TenantOffboardFunction</c>'s idempotency check
/// + <c>OffboardingMarkerCleanupFunction</c>'s timer sweep. NOT read in the agent
/// or web auth hotpath any more — the active auth-gate is
/// <c>TenantConfiguration.Disabled=true</c>. Returns null when no marker exists.
/// </summary>
Task<OffboardingMarkerEntry?> TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default);
/// <summary>Inserts a fresh marker. Throws if a marker for the tenant already exists.</summary>
Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default);
/// <summary>Idempotent upsert; preferred path from worker phase transitions.</summary>
Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default);
/// <summary>Removes a marker by tenant id. 404 silently swallowed (idempotent).</summary>
Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default);
/// <summary>Enumerates ALL active markers — used by <c>OffboardingMarkerCleanupFunction</c>.</summary>
IAsyncEnumerable<OffboardingMarkerEntry> QueryMarkersAsync(CancellationToken ct = default);
// ── History (audit trail, one row per offboarding attempt) ──────────────
Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default);
Task<OffboardingHistoryEntry?> TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default);
Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default);
// ── Pointer (O(1) re-onboarding index) ──────────────────────────────────
//
// Plan §4.4 update protocol is strictly Read-Modify-Write with ETag-CAS:
// 1. TryGet → null + null ETag → Insert with OffboardCount=1
// 2. TryGet → (pointer, etag) → modify in-memory + UpdateWithEtag(etag),
// 412 on mismatch ⇒ re-read and retry.
// The "blind upsert" path was removed because it cannot satisfy OffboardCount=existing+1
// under concurrent writers — the read-modify-write window is the whole point of the
// pointer's design.
/// <summary>
/// Reads the pointer row plus its ETag for CAS. Returns <c>(null, null)</c> on 404.
/// </summary>
Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync(
string normalizedTenantId, CancellationToken ct = default);
/// <summary>
/// Inserts a fresh pointer row. Throws <see cref="Azure.RequestFailedException"/> with
/// status 409 when a pointer for the tenant already exists — caller is expected to fall
/// back to <see cref="UpdateByTenantPointerWithEtagAsync"/> in that case.
/// </summary>
Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default);
/// <summary>
/// Conditional replace: succeeds only when the stored row's ETag still matches
/// <paramref name="ifMatchEtag"/>. Throws <see cref="Azure.RequestFailedException"/> with
/// status 412 on mismatch so the caller can re-read + retry within their bounded loop.
/// </summary>
Task UpdateByTenantPointerWithEtagAsync(
OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default);
}
}
@@ -0,0 +1,55 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.Models.Offboarding;
namespace AutopilotMonitor.Shared.DataAccess
{
/// <summary>
/// Access to <c>TenantOffboardingCustomsArchive</c>. Writes happen during Phase 2.D-archive
/// of the offboarding worker; reads / deletes are driven by the Global Admin
/// <c>/admin/customs-archive</c> UI.
/// <para>
/// All write paths fail-loud — when an Azure storage call throws, exceptions propagate
/// to the caller (queue worker), which owns retry/poison semantics. Reads + deletes
/// from the admin API similarly surface errors to the HTTP caller.
/// </para>
/// </summary>
public interface ITenantCustomsArchiveRepository
{
/// <summary>
/// Idempotent upsert (Replace). Used during Phase 2.D-archive so a worker that
/// crashed mid-archive can re-pickup without duplicate-key errors.
/// </summary>
Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default);
/// <summary>Count entries within a single offboarding run filtered by source table.</summary>
Task<int> CountByRunAndTableAsync(
string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default);
/// <summary>List one offboarding run's archive partition (all 3 source tables interleaved).</summary>
IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByRunAsync(
string normalizedTenantId, string historyRowKey, CancellationToken ct = default);
/// <summary>List every archive partition for a tenant (one entry per row across all runs).</summary>
IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryByTenantAsync(
string normalizedTenantId, CancellationToken ct = default);
/// <summary>List every archived run as a (tenantId, historyRowKey)-pair summary across all tenants.</summary>
IAsyncEnumerable<TenantOffboardingCustomsArchiveEntry> QueryAllAsync(CancellationToken ct = default);
/// <summary>Fetch a single archive entry by (PK, RK).</summary>
Task<TenantOffboardingCustomsArchiveEntry?> TryGetEntryAsync(
string partitionKey, string rowKey, CancellationToken ct = default);
/// <summary>Delete a single archived entry. 404 silently swallowed (idempotent).</summary>
Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default);
/// <summary>
/// Delete every entry in a single run's archive partition. Used by the Global Admin
/// "Delete run" action after they have reviewed the contents.
/// </summary>
Task<int> DeleteRunAsync(
string normalizedTenantId, string historyRowKey, CancellationToken ct = default);
}
}
@@ -0,0 +1,42 @@
using System;
namespace AutopilotMonitor.Shared.Models.Offboarding
{
/// <summary>
/// O(1) point-lookup pointer used by the re-onboarding hook to find the latest history
/// row for a given tenant without scanning the <c>OffboardingHistory</c> partition.
/// Azure Tables cannot serve <c>RowKey endswith '_{tenantId}'</c> efficiently
/// server-side (full partition scan + client-side filter), so we maintain a sidecar
/// row keyed by tenantId.
/// <para>
/// Lives in <c>OffboardingAudit</c> under
/// <c>PartitionKey = OffboardingPartitionKeys.ByTenant</c> with
/// <c>RowKey = normalizedTenantId</c>. Upserted with two-step writes (history-first,
/// pointer-second) and updated as the worker progresses through Status transitions.
/// </para>
/// </summary>
public sealed class OffboardingByTenantPointer
{
/// <summary>Always <c>OffboardingPartitionKeys.ByTenant</c>.</summary>
public string PartitionKey { get; set; } = string.Empty;
/// <summary>Normalized (lowercase GUID) tenant id.</summary>
public string RowKey { get; set; } = string.Empty;
public string TenantId { get; set; } = string.Empty;
/// <summary>"{yyyyMMddHHmmssfff}_{tenantId}" of the current/latest history row.</summary>
public string LatestHistoryRowKey { get; set; } = string.Empty;
/// <summary>Mirrors the matching history row's Status:
/// <c>Initiated | InProgress | Completed | Failed | Reonboarded</c>.</summary>
public string LatestStatus { get; set; } = "Initiated";
/// <summary>UTC of the most recent pointer update; monotonically increasing.</summary>
public DateTime LatestUpdatedAt { get; set; }
/// <summary>How many distinct offboarding rounds this tenant has been through.
/// Incremented on each new Initiate. Useful for "Tenant ging und kam zurück"-Reporting.</summary>
public int OffboardCount { get; set; } = 1;
}
}
@@ -0,0 +1,78 @@
using System;
using System.Collections.Generic;
namespace AutopilotMonitor.Shared.Models.Offboarding
{
/// <summary>
/// Per-tenant snapshot of the cascade enqueue outcome captured at the first handler pickup.
/// Drives the drain predicate: every entry is matched against either a progress blob
/// (Enqueued/AlreadyInFlight), a no-op (SessionNotFound), a fail-closed condition
/// (KillSwitchActive/Poisoned), or a bounded retry loop (CasExhausted).
/// <para>
/// Stored as a single blob at
/// <c>{BlobContainers.OffboardingState}/{tenantId}/{historyRowKey}.expectations.json</c>.
/// The container is deliberately separate from <c>deletion-manifests</c> because the latter
/// is wiped in Phase 2.E; storing this blob there would make crash-recovery between 2.E
/// and 2.G impossible.
/// </para>
/// </summary>
public sealed class OffboardingExpectations
{
/// <summary>Bump on breaking-schema changes. Drain-probe fail-closed if mismatch.</summary>
public int SchemaVersion { get; set; } = 1;
public string TenantId { get; set; } = string.Empty;
/// <summary>Matching history row's RowKey, so a blob can be paired back to its run.</summary>
public string HistoryRowKey { get; set; } = string.Empty;
public DateTime CreatedAt { get; set; }
/// <summary>
/// True only after the session enumerator iterated to completion without exception AND
/// the upload is being written. Default false; drain-probe treats <c>false</c> as
/// "this blob was uploaded with incomplete data" → fail-closed with
/// <c>FailedPhase="enumeration_incomplete"</c>. Critical disambiguation: lets the
/// probe tell apart "0 sessions, happy path" from "0 entries because enumerator
/// crashed before yielding any".
/// </summary>
public bool EnumerationCompleted { get; set; }
/// <summary>
/// Number of sessions the enumerator yielded BEFORE per-session enqueue ran. Must equal
/// <see cref="Expectations"/>.Count on upload; mismatch → fail-closed.
/// </summary>
public int EnumeratedSessionCount { get; set; }
public List<OffboardingExpectation> Expectations { get; set; } = new();
}
/// <summary>One per session that the enumerator returned.</summary>
public sealed class OffboardingExpectation
{
public string SessionId { get; set; } = string.Empty;
/// <summary>
/// Manifest id returned by <c>SessionDeletionProducer.EnqueueAsync</c>. Null for outcomes
/// that did not produce a cascade (SessionNotFound, KillSwitchActive, CasExhausted) or
/// when the producer returned <c>AlreadyInFlight</c> against a Preparing-without-snapshot
/// state. The drain probe fails closed on Enqueued/AlreadyInFlight with a null/empty
/// manifest id.
/// </summary>
public string? ManifestId { get; set; }
/// <summary>
/// String form of <c>SessionDeletionProducer.EnqueueOutcome</c>:
/// <c>Enqueued | AlreadyInFlight | SessionNotFound | KillSwitchActive | Poisoned | CasExhausted</c>.
/// Stored as string so the shared assembly doesn't need to reference the
/// Functions-only enum and so future enum additions don't break already-written blobs.
/// </summary>
public string Outcome { get; set; } = string.Empty;
/// <summary>
/// CasExhausted retry counter. Drain-probe re-enqueues this session and increments;
/// after 3 retries the offboarding fails closed with <c>FailedPhase="cas_exhausted"</c>.
/// </summary>
public int RetryCount { get; set; }
}
}
@@ -0,0 +1,110 @@
using System;
namespace AutopilotMonitor.Shared.Models.Offboarding
{
/// <summary>
/// Chronological audit-trail row for one tenant-offboarding attempt. Multiple rows per
/// tenant when the tenant offboarded, re-onboarded, and offboarded again. Lives in
/// <c>OffboardingAudit</c> under <c>PartitionKey = OffboardingPartitionKeys.History</c>
/// with <c>RowKey = "{yyyyMMddHHmmssfff}_{normalizedTenantId}"</c> so rows sort
/// newest-last lexicographically within the partition.
/// </summary>
public sealed class OffboardingHistoryEntry
{
/// <summary>Always <c>OffboardingPartitionKeys.History</c>.</summary>
public string PartitionKey { get; set; } = string.Empty;
/// <summary>"{yyyyMMddHHmmssfff}_{normalizedTenantId}". Unique per offboarding attempt.</summary>
public string RowKey { get; set; } = string.Empty;
public string TenantId { get; set; } = string.Empty;
public string DomainName { get; set; } = string.Empty;
public string InitiatedBy { get; set; } = string.Empty;
public DateTime OffboardedAt { get; set; }
public DateTime? CompletedAt { get; set; }
/// <summary><c>Initiated</c> | <c>InProgress</c> | <c>Completed</c> | <c>Failed</c>.</summary>
public string Status { get; set; } = "Initiated";
/// <summary>
/// Cache-Drain-Barrier (plan v2 §2). The worker MUST NOT begin destructive Phase 2
/// before this UTC timestamp. <c>TenantOffboardFunction</c> sets it to
/// <c>OffboardedAt + DrainBarrier</c> (default 6 min) and enqueues the envelope
/// with the matching <c>visibilityDelay</c>, so the message is invisible to the
/// worker until then. All warm function-host instances will have hit their 5-min
/// <c>TenantConfigurationService</c> cache TTL by the time it expires, so the
/// existing <c>Disabled=true</c> gate carries the 403 instead of the marker.
/// <para>
/// Also rendered by the Web UI as the "data deletion starts in mm ss" countdown
/// during the drain barrier state, and used by the resume-path to compute the
/// remaining delay when an admin re-clicks mid-barrier.
/// </para>
/// </summary>
public DateTime? EarliestProcessingAt { get; set; }
// ── Diagnostics ────────────────────────────────────────────────────────
/// <summary>JSON: <c>{ tableName: deletedRowCount }</c> across SafeWipe + Cascade phases.</summary>
public string? DeletedRowCountsJson { get; set; }
public int? TotalRowsDeleted { get; set; }
public int? DeletedBlobCount { get; set; }
public int? CascadeSessionsEnqueued { get; set; }
public string? ErrorMessage { get; set; }
public int RetryCount { get; set; }
/// <summary>
/// Rev-9-F1 phase marker. Set once by the handler immediately after the drain predicate
/// returns true, BEFORE Phase 2.D begins wiping tables. Re-pickup after crash skips the
/// drain predicate when this is non-null because Phase 2.E may have wiped the cascade
/// progress blobs that the drain predicate reads from; the post-drain phases (2.D-G)
/// are all idempotent so a clean resume requires no progress blobs.
/// </summary>
public DateTime? DrainCompletedAt { get; set; }
/// <summary>
/// Stamped by the handler the first time it enters <c>EnsureExpectationsBlobAsync</c>,
/// BEFORE the session enumerator starts iterating. On re-pickup after an enumerator
/// crash this field is non-null while the Expectations blob is still missing — the
/// handler then fail-closes with <c>FailedPhase="expectations_missing"</c> instead of
/// re-enumerating against a tenant whose underlying Sessions table may already be
/// mid-mutation. Plan §7.4 step 3 (Rev-5-F3 + Rev-7-F2 crash-after-enumerate-start).
/// </summary>
public DateTime? EnumerationStartedAt { get; set; }
/// <summary>
/// Stamped after the session enumerator loop finishes cleanly, BEFORE the Expectations
/// blob upload. Disambiguates two failure modes that both leave the blob missing with
/// <see cref="EnumerationStartedAt"/> set:
/// <list type="bullet">
/// <item><b>Mid-enumeration crash</b>: ECBU is null → fail-closed
/// (<c>FailedPhase="expectations_missing"</c>); the tenant may be mid-mutation
/// and re-enumerating is unsafe.</item>
/// <item><b>Upload failure</b>: ECBU is non-null → retry the enumerate+upload pair
/// on the next pickup. The session-deletion producer is idempotent
/// (<c>AlreadyInFlight</c> outcome with the existing ManifestId), so re-running
/// the loop produces the same cascade-state set.</item>
/// </list>
/// Plan §7.4 step 3 + Review-Fix (second-pass) Finding 2.
/// </summary>
public DateTime? EnumerationCompletedBeforeUpload { get; set; }
// ── Archived Customs (PR3.B §3) ────────────────────────────────────────
//
// Counts of rows that were snapshotted into TenantOffboardingCustomsArchive
// during Phase 2.D-archive, immediately before the originals were safe-wiped.
// Recomputed from the archive table by post-archive query so a crash between
// archive-insert and counter-write does not leave a stale 0.
public int? CustomGatherRulesArchived { get; set; }
public int? CustomAnalyzeRulesArchived { get; set; }
public int? ImeLogPatternOverridesArchived { get; set; }
// ── Re-onboarding tracking (legacy, kept for audit history continuity) ─
public DateTime? ReonboardedAt { get; set; }
public string? ReonboardedBy { get; set; }
public int? CustomsAutoWipedOnReonboard { get; set; }
}
}
@@ -0,0 +1,67 @@
using System;
namespace AutopilotMonitor.Shared.Models.Offboarding
{
/// <summary>
/// Lifecycle anchor + audit row for a tenant currently (or recently) being offboarded.
/// Lives in <c>OffboardingAudit</c> table under
/// <c>PartitionKey = OffboardingPartitionKeys.Marker</c> with <c>RowKey = normalizedTenantId</c>.
/// <para>
/// In the PR3-revised architecture this row is NO LONGER read in the agent or web
/// auth hotpath. The active gate is <c>TenantConfiguration.Disabled=true</c> + the
/// 6-minute cache-drain barrier on the worker envelope. The marker exists to:
/// </para>
/// <list type="bullet">
/// <item>Provide an idempotency anchor for the <c>TenantOffboardFunction</c>
/// admin endpoint (re-clicks resolve to the existing marker and resume).</item>
/// <item>Track the offboarding lifecycle status (<see cref="Status"/>) across
/// Phase 1 → Phase 2 → Completed/Failed for ops visibility.</item>
/// <item>Anchor the post-completion cleanup sweeps run by
/// <c>OffboardingMarkerCleanupFunction</c> (Expectations blob delete +
/// defense-in-depth TenantConfiguration sweep if Phase 2.F-final failed).</item>
/// </list>
/// <para>
/// Marker survives the Completed transition: Phase 2.G sets <see cref="Status"/> =
/// <c>"Completed"</c> + <see cref="CompletedAt"/>. <c>OffboardingMarkerCleanupFunction</c>
/// removes Completed markers after 15min (post-cleanup-sweep settle buffer). Failed
/// markers are never auto-deleted — operator action required.
/// </para>
/// </summary>
public sealed class OffboardingMarkerEntry
{
/// <summary>Always <c>OffboardingPartitionKeys.Marker</c>.</summary>
public string PartitionKey { get; set; } = string.Empty;
/// <summary>Normalized (lowercase GUID) tenant id.</summary>
public string RowKey { get; set; } = string.Empty;
public string TenantId { get; set; } = string.Empty;
/// <summary>Cross-reference to the matching <c>OffboardingHistory</c> row's RowKey.</summary>
public string OffboardingHistoryRowKey { get; set; } = string.Empty;
public DateTime InitiatedAt { get; set; }
/// <summary>UPN of the admin (or "System.*") that triggered the offboarding.</summary>
public string InitiatedBy { get; set; } = string.Empty;
/// <summary>
/// Lifecycle: <c>Initiated</c> → <c>InProgress</c> → <c>Completed</c> | <c>Failed</c>.
/// Marker remains active in all four states; only the cleanup function removes
/// Completed markers (and never Failed).
/// </summary>
public string Status { get; set; } = "Initiated";
/// <summary>UTC timestamp set by Phase 2.G; gates 15-min cleanup window.</summary>
public DateTime? CompletedAt { get; set; }
/// <summary>UTC timestamp set after max-dequeue → Failed; marker stays active until operator action.</summary>
public DateTime? FailedAt { get; set; }
/// <summary>Phase that failed: <c>drain</c>, <c>wipe</c>, <c>blobs</c>, <c>tenantconfig</c>,
/// <c>killswitch</c>, <c>poisoned</c>, <c>cas_exhausted</c>, <c>drain_timeout</c>,
/// <c>expectations_missing</c>, <c>expectations_corrupt</c>, <c>enumeration_incomplete</c>,
/// <c>expectations_size_mismatch</c>, <c>alreadyinflight_no_manifest</c>.</summary>
public string? FailedPhase { get; set; }
}
}
@@ -0,0 +1,58 @@
using System;
namespace AutopilotMonitor.Shared.Models.Offboarding
{
/// <summary>
/// Snapshot of a tenant's custom rule row written by the offboarding handler during
/// Phase 2.D-archive, BEFORE the original row is safe-wiped. Survives forever so a
/// Global Admin can review (and selectively delete) the tenant's custom rules from
/// <c>/admin/customs-archive</c> after offboarding completes.
/// <para>
/// Storage layout (see PR3.B plan §3.2):
/// <list type="bullet">
/// <item><b>PartitionKey</b>: <c>"{normalizedTenantId}_{historyRowKey}"</c>. One
/// partition per offboarding run — Re-Re-Offboarding produces a fresh, immutable
/// partition each time so multiple runs co-exist without RowKey collisions.</item>
/// <item><b>RowKey</b>: <c>"{originalTable}_{base64url(originalRowKey)}"</c>. The
/// table-name prefix disambiguates collisions across the three rules tables; the
/// base64url encoding of <see cref="OriginalRowKey"/> avoids the Azure-Tables
/// RowKey-forbidden characters (<c>#</c>, <c>?</c>, <c>/</c>, <c>\</c>, control chars).</item>
/// </list>
/// </para>
/// </summary>
public sealed class TenantOffboardingCustomsArchiveEntry
{
/// <summary>"{normalizedTenantId}_{historyRowKey}".</summary>
public string PartitionKey { get; set; } = string.Empty;
/// <summary>"{originalTable}_{base64url(originalRowKey)}".</summary>
public string RowKey { get; set; } = string.Empty;
public string TenantId { get; set; } = string.Empty;
/// <summary>One of <c>GatherRules</c> / <c>AnalyzeRules</c> / <c>ImeLogPatterns</c>.</summary>
public string OriginalTable { get; set; } = string.Empty;
public string OriginalPartitionKey { get; set; } = string.Empty;
/// <summary>Original RowKey verbatim — base64url-encoded into <see cref="RowKey"/>'s suffix.</summary>
public string OriginalRowKey { get; set; } = string.Empty;
/// <summary>
/// JSON dump of every property on the source row (excluding system properties).
/// The full body is small enough to fit in a single Azure Table string property
/// (64 KB limit; observed rule bodies are well under 4 KB). If a future rule type
/// ever exceeds that, an EntityJsonBlobUrl overflow pointer can be added — not
/// part of this PR.
/// </summary>
public string EntityJson { get; set; } = string.Empty;
/// <summary>Cross-reference to the <c>OffboardingHistory</c> row for the run.</summary>
public string HistoryRowKey { get; set; } = string.Empty;
public DateTime ArchivedAt { get; set; }
/// <summary>Constant <c>"TenantOffboardingHandler"</c> in production. Test fakes may override.</summary>
public string ArchivedBy { get; set; } = "TenantOffboardingHandler";
}
}
@@ -0,0 +1,37 @@
using System;
namespace AutopilotMonitor.Shared.Models.Offboarding
{
/// <summary>
/// Queue message envelope carried on <c>tenant-offboarding</c> (and its poison sibling).
/// Producer (<c>TenantOffboardFunction</c>) writes one envelope per tenant after the
/// History/Pointer/Marker rows are committed. Worker (<c>TenantOffboardingWorker</c>)
/// dequeues, runs <c>TenantOffboardingHandler.HandleAsync</c>, and either acks on
/// success or re-enqueues itself with <see cref="DrainPollCount"/> incremented when the
/// drain predicate is still in flight.
/// </summary>
public sealed class TenantOffboardingEnvelope
{
/// <summary>Schema version — bump on breaking envelope changes.</summary>
public string EnvelopeVersion { get; set; } = "1";
public string TenantId { get; set; } = string.Empty;
/// <summary>Always <c>OffboardingPartitionKeys.History</c>; kept for self-documentation.</summary>
public string HistoryPartitionKey { get; set; } = string.Empty;
/// <summary>"{yyyyMMddHHmmssfff}_{tenantId}" pointing at the matching history row.</summary>
public string HistoryRowKey { get; set; } = string.Empty;
public string InitiatedBy { get; set; } = string.Empty;
public DateTime InitiatedAt { get; set; }
public DateTime EnqueuedAt { get; set; }
/// <summary>
/// Drain self-poll counter. Handler re-enqueues with this incremented when the
/// cascade-progress drain predicate has not yet settled. Capped at 60 (~2h) before
/// failing the offboarding with <c>FailedPhase="drain_timeout"</c>.
/// </summary>
public int DrainPollCount { get; set; }
}
}
@@ -44,7 +44,11 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
"PoisonQueueBacklogHigh",
"PoisonQueueBacklogCritical",
],
Tenant: ["TenantOffboarded"],
// TenantOffboardingFailed is dispatched by TenantOffboardingWorker when the async cascade fails
// closed (drain timeout, KillSwitch active, Poisoned session, ETag/CAS exhaustion, SafeWipe verify
// abort, etc.). Marker remains in Failed state until operator action — this event is the
// Telegram-routable signal that something needs human attention. Plan Rev-4 Q2 + Rev-9 §11 PR3.
Tenant: ["TenantOffboarded", "TenantOffboardingFailed"],
Agent: ["BlobStorageMissing", "BlobStorageUnreachable", "NewImeVersionDetected", "ExcessiveSessionEvents"],
};
@@ -0,0 +1,243 @@
"use client";
import { useCallback, useEffect, useState, use as usePromise } from "react";
import Link from "next/link";
import {
api,
type CustomsArchiveEntrySummary,
type CustomsArchiveFullEntry,
type CustomsArchiveListEntriesResponse,
} from "@/lib/api";
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
import { useAdminConfig } from "../../../AdminConfigContext";
import { AdminNotifications } from "../../../AdminNotifications";
import { DeleteConfirmModal } from "../../components/DeleteConfirmModal";
interface PageProps {
params: Promise<{ tenantId: string; historyRowKey: string }>;
}
export default function CustomsArchiveDetailPage({ params }: PageProps) {
// Next.js 15: route params are async. Resolve once.
const resolved = usePromise(params);
const tenantId = decodeURIComponent(resolved.tenantId);
const historyRowKey = decodeURIComponent(resolved.historyRowKey);
const { getAccessToken, setError, setSuccessMessage } = useAdminConfig();
const [entries, setEntries] = useState<CustomsArchiveEntrySummary[]>([]);
const [loading, setLoading] = useState<boolean>(false);
const [expanded, setExpanded] = useState<Record<string, CustomsArchiveFullEntry | "loading" | undefined>>({});
const [deletingRk, setDeletingRk] = useState<string | null>(null);
const [pendingDelete, setPendingDelete] = useState<CustomsArchiveEntrySummary | null>(null);
const load = useCallback(async () => {
try {
setLoading(true);
const url = api.customsArchive.listEntries(tenantId, historyRowKey);
const res = await authenticatedFetch(url, getAccessToken);
if (!res.ok) {
throw new Error(`List entries failed: ${res.status} ${res.statusText}`);
}
const body = (await res.json()) as CustomsArchiveListEntriesResponse;
setEntries(body.entries ?? []);
} catch (err) {
if (err instanceof TokenExpiredError) {
setError(err.message);
} else {
setError(err instanceof Error ? err.message : String(err));
}
} finally {
setLoading(false);
}
}, [tenantId, historyRowKey, getAccessToken, setError]);
useEffect(() => {
void load();
}, [load]);
const toggleExpand = useCallback(async (entry: CustomsArchiveEntrySummary) => {
const key = entry.rowKey;
if (expanded[key]) {
// Collapse
setExpanded((prev) => {
const next = { ...prev };
delete next[key];
return next;
});
return;
}
setExpanded((prev) => ({ ...prev, [key]: "loading" }));
try {
const res = await authenticatedFetch(
api.customsArchive.getEntry(tenantId, historyRowKey, entry.rowKey),
getAccessToken,
);
if (!res.ok) throw new Error(`Get entry failed: ${res.status}`);
const body = await res.json();
setExpanded((prev) => ({ ...prev, [key]: body.entry as CustomsArchiveFullEntry }));
} catch (err) {
setExpanded((prev) => {
const next = { ...prev };
delete next[key];
return next;
});
setError(err instanceof Error ? err.message : String(err));
}
}, [expanded, tenantId, historyRowKey, getAccessToken, setError]);
const confirmDelete = useCallback(async () => {
const entry = pendingDelete;
if (!entry) return;
try {
setDeletingRk(entry.rowKey);
const res = await authenticatedFetch(
api.customsArchive.deleteEntry(tenantId, historyRowKey, entry.rowKey),
getAccessToken,
{ method: "DELETE" },
);
if (!res.ok) throw new Error(`Delete failed: ${res.status}`);
setSuccessMessage(`Deleted ${entry.originalTable} / ${entry.originalRowKey}`);
setPendingDelete(null);
await load();
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
} finally {
setDeletingRk(null);
}
}, [pendingDelete, tenantId, historyRowKey, getAccessToken, load, setError, setSuccessMessage]);
return (
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
<header className="bg-white dark:bg-gray-800 shadow dark:shadow-gray-700">
<div className="py-6 px-4 sm:px-6 lg:px-8">
<Link
href="/admin/customs-archive"
className="text-sm text-blue-600 dark:text-blue-400 hover:underline"
>
← Back to runs
</Link>
<h1 className="text-2xl font-normal text-gray-900 dark:text-white mt-2">
Customs Archive — Run detail
</h1>
<p className="text-xs font-mono text-gray-600 dark:text-gray-400 mt-1">
Tenant: {tenantId} · Run: {historyRowKey}
</p>
</div>
</header>
<main className="max-w-6xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-4">
<AdminNotifications />
<div className="flex items-center justify-between">
<span className="text-sm text-gray-600 dark:text-gray-300">
{entries.length} entry/entries{loading ? " — loading…" : ""}
</span>
<button
type="button"
onClick={load}
disabled={loading}
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
>
{loading ? "Loading…" : "Refresh"}
</button>
</div>
{entries.length === 0 && !loading && (
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 text-sm text-gray-600 dark:text-gray-400">
No archived entries for this run.
</div>
)}
<div className="space-y-2">
{entries.map((entry) => {
const detail = expanded[entry.rowKey];
const isOpen = !!detail;
const isLoadingFull = detail === "loading";
const fullEntry = typeof detail === "object" ? (detail as CustomsArchiveFullEntry) : null;
return (
<div
key={entry.rowKey}
className="bg-white dark:bg-gray-800 rounded-lg shadow border border-gray-200 dark:border-gray-700"
>
<div className="p-3 flex flex-wrap items-center justify-between gap-2">
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<span className="inline-block px-2 py-0.5 rounded text-xs font-medium bg-indigo-100 text-indigo-800 dark:bg-indigo-900/40 dark:text-indigo-200">
{entry.originalTable}
</span>
<span className="text-sm font-mono text-gray-900 dark:text-white truncate">
{entry.originalRowKey}
</span>
</div>
<div className="text-xs text-gray-500 dark:text-gray-400 mt-1">
Archived {new Date(entry.archivedAt).toLocaleString()}
</div>
</div>
<div className="flex items-center gap-2">
<button
type="button"
onClick={() => toggleExpand(entry)}
className="px-3 py-1.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-600 text-gray-900 dark:text-white text-sm rounded-md transition-colors"
>
{isOpen ? "Collapse" : "Inspect"}
</button>
<button
type="button"
onClick={() => setPendingDelete(entry)}
disabled={deletingRk === entry.rowKey}
className="px-3 py-1.5 bg-red-600 hover:bg-red-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
>
{deletingRk === entry.rowKey ? "Deleting…" : "Delete entry"}
</button>
</div>
</div>
{isOpen && (
<div className="border-t border-gray-200 dark:border-gray-700 p-3 bg-gray-50 dark:bg-gray-900/50">
{isLoadingFull ? (
<div className="text-sm text-gray-500">Loading…</div>
) : fullEntry ? (
<pre className="text-xs font-mono whitespace-pre-wrap break-all text-gray-800 dark:text-gray-200">
{prettyJson(fullEntry.entityJson)}
</pre>
) : null}
</div>
)}
</div>
);
})}
</div>
</main>
<DeleteConfirmModal
open={pendingDelete !== null}
title="Delete archive entry"
description={pendingDelete && (
<>
<p>
Delete this archived entry from{" "}
<span className="font-mono">{pendingDelete.originalTable}</span>?
</p>
<p className="mt-2 font-mono text-xs break-all">
{pendingDelete.originalRowKey}
</p>
<p className="mt-2">This action is irreversible.</p>
</>
)}
busy={deletingRk !== null}
onCancel={() => setPendingDelete(null)}
onConfirm={confirmDelete}
/>
</div>
);
}
function prettyJson(raw: string): string {
try {
return JSON.stringify(JSON.parse(raw), null, 2);
} catch {
return raw;
}
}
@@ -0,0 +1,83 @@
"use client";
import { useEffect, useState } from "react";
interface DeleteConfirmModalProps {
open: boolean;
title: string;
description: React.ReactNode;
busy?: boolean;
onCancel: () => void;
onConfirm: () => void;
}
const CONFIRM_WORD = "DELETE";
/**
* Lightweight typed-confirmation modal for destructive customs-archive actions.
* Replaces the native window.confirm() so the dialog can render rich context
* (tenant id, table/row, count) and require a typed "DELETE" before the
* primary button activates.
*/
export function DeleteConfirmModal({
open,
title,
description,
busy = false,
onCancel,
onConfirm,
}: DeleteConfirmModalProps) {
const [text, setText] = useState("");
useEffect(() => {
if (!open) setText("");
}, [open]);
if (!open) return null;
const canConfirm = text === CONFIRM_WORD && !busy;
return (
<div className="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 p-4">
<div className="bg-white dark:bg-gray-800 rounded-lg shadow-xl max-w-md w-full p-6">
<h3 className="text-lg font-bold text-gray-900 dark:text-white mb-2">{title}</h3>
<div className="text-sm text-gray-700 dark:text-gray-300 mb-4">{description}</div>
<div className="mb-4">
<label className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">
Type <span className="font-bold text-red-600 dark:text-red-400">{CONFIRM_WORD}</span> to confirm
</label>
<input
type="text"
value={text}
onChange={(e) => setText(e.target.value)}
placeholder={CONFIRM_WORD}
disabled={busy}
autoFocus
autoComplete="off"
className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-900 text-gray-900 dark:text-white focus:outline-none focus:ring-2 focus:ring-red-500 focus:border-red-500"
/>
</div>
<div className="flex space-x-3">
<button
type="button"
onClick={onCancel}
disabled={busy}
className="flex-1 px-4 py-2 border border-gray-300 dark:border-gray-600 rounded-md text-gray-700 dark:text-gray-200 bg-white dark:bg-gray-700 hover:bg-gray-50 dark:hover:bg-gray-600 disabled:opacity-50 transition-colors"
>
Cancel
</button>
<button
type="button"
onClick={onConfirm}
disabled={!canConfirm}
className="flex-1 px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
>
{busy ? "Deleting…" : "Delete"}
</button>
</div>
</div>
</div>
);
}
@@ -0,0 +1,182 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import Link from "next/link";
import {
api,
type CustomsArchiveListRunsResponse,
type CustomsArchiveRunSummary,
} from "@/lib/api";
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
import { useAdminConfig } from "../AdminConfigContext";
import { AdminNotifications } from "../AdminNotifications";
import { DeleteConfirmModal } from "./components/DeleteConfirmModal";
export default function CustomsArchivePage() {
const { getAccessToken, setError, setSuccessMessage } = useAdminConfig();
const [filter, setFilter] = useState<string>("");
const [runs, setRuns] = useState<CustomsArchiveRunSummary[]>([]);
const [loading, setLoading] = useState<boolean>(false);
const [deletingPk, setDeletingPk] = useState<string | null>(null);
const [pendingDelete, setPendingDelete] = useState<CustomsArchiveRunSummary | null>(null);
const load = useCallback(async () => {
try {
setLoading(true);
const url = api.customsArchive.listRuns({ tenantId: filter.trim() || undefined });
const res = await authenticatedFetch(url, getAccessToken);
if (!res.ok) {
throw new Error(`List runs failed: ${res.status} ${res.statusText}`);
}
const body = (await res.json()) as CustomsArchiveListRunsResponse;
setRuns(body.runs ?? []);
} catch (err) {
if (err instanceof TokenExpiredError) {
setError(err.message);
} else {
setError(err instanceof Error ? err.message : String(err));
}
} finally {
setLoading(false);
}
}, [filter, getAccessToken, setError]);
useEffect(() => {
void load();
}, [load]);
const confirmDelete = useCallback(async () => {
const run = pendingDelete;
if (!run) return;
try {
setDeletingPk(run.partitionKey);
const res = await authenticatedFetch(
api.customsArchive.deleteRun(run.tenantId, run.historyRowKey),
getAccessToken,
{ method: "DELETE" }
);
if (!res.ok) {
throw new Error(`Delete run failed: ${res.status} ${res.statusText}`);
}
const body = await res.json();
setSuccessMessage(`Deleted ${body.deleted ?? 0} archived rules.`);
setPendingDelete(null);
await load();
} catch (err) {
if (err instanceof TokenExpiredError) {
setError(err.message);
} else {
setError(err instanceof Error ? err.message : String(err));
}
} finally {
setDeletingPk(null);
}
}, [pendingDelete, getAccessToken, load, setError, setSuccessMessage]);
return (
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
<header className="bg-white dark:bg-gray-800 shadow dark:shadow-gray-700">
<div className="py-6 px-4 sm:px-6 lg:px-8">
<h1 className="text-2xl font-normal text-gray-900 dark:text-white">Customs Archive</h1>
<p className="text-sm text-gray-600 dark:text-gray-400 mt-1">
Snapshots of each tenant&apos;s custom <strong>GatherRules</strong> / <strong>AnalyzeRules</strong> /{" "}
<strong>ImeLogPatterns</strong> taken during offboarding (Phase 2.D-archive). Review the content,
then delete entries selectively when no longer needed. No auto-cleanup.
</p>
</div>
</header>
<main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
<AdminNotifications />
<div className="flex flex-wrap items-center gap-2">
<input
type="text"
value={filter}
onChange={(e) => setFilter(e.target.value)}
placeholder="Filter by tenant ID (optional)"
className="px-3 py-1.5 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm text-gray-900 dark:text-white"
/>
<button
type="button"
onClick={load}
disabled={loading}
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
>
{loading ? "Loading…" : "Refresh"}
</button>
<span className="text-sm text-gray-500 dark:text-gray-400 ml-2">{runs.length} run(s)</span>
</div>
{runs.length === 0 && !loading && (
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 text-sm text-gray-600 dark:text-gray-400">
No archived offboarding runs found{filter ? ` for tenant ${filter}.` : "."}
</div>
)}
<div className="space-y-3">
{runs.map((run) => (
<div
key={run.partitionKey}
className="bg-white dark:bg-gray-800 rounded-lg shadow p-4 flex flex-wrap items-center justify-between gap-3"
>
<div className="flex-1 min-w-0">
<div className="text-sm font-mono text-gray-900 dark:text-white truncate">{run.tenantId}</div>
<div className="text-xs text-gray-500 dark:text-gray-400 mt-0.5">
Run: <span className="font-mono">{run.historyRowKey}</span> · Archived{" "}
{new Date(run.archivedAt).toLocaleString()}
</div>
<div className="text-xs text-gray-600 dark:text-gray-300 mt-1 space-x-3">
<span>Gather: <strong>{run.gatherRulesCount}</strong></span>
<span>Analyze: <strong>{run.analyzeRulesCount}</strong></span>
<span>ImePatterns: <strong>{run.imeLogPatternsCount}</strong></span>
</div>
</div>
<div className="flex items-center gap-2">
<Link
href={`/admin/customs-archive/${encodeURIComponent(run.tenantId)}/${encodeURIComponent(run.historyRowKey)}`}
className="px-3 py-1.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-600 text-gray-900 dark:text-white text-sm rounded-md transition-colors"
>
Inspect
</Link>
<button
type="button"
onClick={() => setPendingDelete(run)}
disabled={deletingPk === run.partitionKey}
className="px-3 py-1.5 bg-red-600 hover:bg-red-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
>
{deletingPk === run.partitionKey ? "Deleting…" : "Delete run"}
</button>
</div>
</div>
))}
</div>
</main>
<DeleteConfirmModal
open={pendingDelete !== null}
title="Delete archive run"
description={pendingDelete && (
<>
<p>
Delete the entire archive run for tenant{" "}
<span className="font-mono">{pendingDelete.tenantId}</span>?
</p>
<p className="mt-2">
This permanently removes{" "}
<strong>
{pendingDelete.gatherRulesCount +
pendingDelete.analyzeRulesCount +
pendingDelete.imeLogPatternsCount}
</strong>{" "}
archived rules. This action is irreversible.
</p>
</>
)}
busy={deletingPk !== null}
onCancel={() => setPendingDelete(null)}
onConfirm={confirmDelete}
/>
</div>
);
}
@@ -15,6 +15,18 @@ import { type BootstrapSessionItem } from "./components/BootstrapSessionsSection
// ---------------------------------------------------------------------------
// Context value interface
// ---------------------------------------------------------------------------
/**
* State surfaced by handleOffboard once the DELETE returns 202. Drives the post-confirm
* banner in OffboardingSection. EarliestProcessingAt is rendered as a live countdown;
* when it elapses, handleDrainBarrierElapsed() runs and logs the user out.
*/
export interface OffboardingInProgressInfo {
status: string;
historyRowKey: string;
earliestProcessingAt?: string | null;
message: string;
}
interface TenantConfigContextValue {
// Core
config: TenantConfiguration | null;
@@ -200,6 +212,12 @@ interface TenantConfigContextValue {
setOffboardError: (v: string | null) => void;
handleOffboard: () => Promise<void>;
/** Set after the DELETE returns 202; drives the post-confirm drain-barrier banner. */
offboardingInProgress: OffboardingInProgressInfo | null;
/** Called by the banner countdown when the cache-drain barrier elapses → triggers logout. */
handleDrainBarrierElapsed: () => void;
// Auth helpers
user: ReturnType<typeof useAuth>["user"];
getAccessToken: () => Promise<string | null>;
@@ -243,6 +261,7 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode }
const [offboardConfirmText, setOffboardConfirmText] = useState("");
const [offboarding, setOffboarding] = useState(false);
const [offboardError, setOffboardError] = useState<string | null>(null);
const [offboardingInProgress, setOffboardingInProgress] = useState<OffboardingInProgressInfo | null>(null);
// Bootstrap sessions
const [bootstrapSessions, setBootstrapSessions] = useState<BootstrapSessionItem[]>([]);
@@ -1163,18 +1182,40 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode }
throw new Error(data?.error || `Offboard failed: ${response.statusText}`);
}
// Offboard successful – sign out the user as their admin access is gone
// Backend returns 202 (or 200 for idempotent re-clicks) with the History row pointer
// and EarliestProcessingAt (cache-drain barrier deadline). Switch the UI into the
// drain-barrier banner state; the banner's countdown will auto-logout once the
// barrier elapses (by then the worker has started Phase 2 and the auth pipeline
// returns 403 via the existing Disabled-flag gate).
const body = await response.json().catch(() => ({}));
trackEvent("tenant_offboarded");
logout();
setOffboardingInProgress({
status: body?.status ?? "Queued",
historyRowKey: body?.historyRowKey ?? "",
earliestProcessingAt: body?.earliestProcessingAt ?? null,
message: body?.message ?? "Tenant offboarding queued.",
});
// Dismiss the confirmation dialog now that the banner has taken over.
setShowOffboardDialog(false);
} catch (err) {
if (err instanceof TokenExpiredError) {
addNotification('error', 'Session Expired', err.message, 'session-expired-error');
} else {
setOffboardError(err instanceof Error ? err.message : 'Offboard failed');
}
} finally {
setOffboarding(false);
}
}, [tenantId, getAccessToken, addNotification, logout]);
}, [tenantId, getAccessToken, addNotification]);
const handleDrainBarrierElapsed = useCallback(() => {
// The cache-drain barrier has expired. The worker is starting Phase 2 right now and
// all function-host instances have refreshed their TenantConfiguration cache to see
// Disabled=true. Sign the user out — any further authenticated call will fail with
// 403 anyway.
logout();
}, [logout]);
// -----------------------------------------------------------------------
// Provider value
@@ -1281,6 +1322,7 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode }
offboardConfirmText, setOffboardConfirmText,
offboarding, offboardError, setOffboardError,
handleOffboard,
offboardingInProgress, handleDrainBarrierElapsed,
// Auth
user, getAccessToken,
@@ -1,5 +1,14 @@
"use client";
import { useEffect, useRef, useState } from "react";
interface OffboardingInProgressInfo {
status: string; // "Queued" | "Initiated" | "InProgress" | "Completed" | "Failed"
historyRowKey: string;
earliestProcessingAt?: string | null; // ISO UTC; drives countdown in drain-barrier state
message: string;
}
interface OffboardingSectionProps {
showOffboardDialog: boolean;
setShowOffboardDialog: (value: boolean) => void;
@@ -9,6 +18,58 @@ interface OffboardingSectionProps {
offboardError: string | null;
setOffboardError: (value: string | null) => void;
onOffboard: () => void;
/** Set after the DELETE returns 202; switches the section into the drain-barrier banner state. */
offboardingInProgress: OffboardingInProgressInfo | null;
/** Invoked by the banner's auto-logout timer once the drain barrier has fully elapsed. */
onDrainBarrierElapsed: () => void;
}
function formatRemaining(totalSeconds: number): string {
if (totalSeconds <= 0) return "0 min";
return `~${Math.ceil(totalSeconds / 60)} min`;
}
/** Live countdown — re-renders every second until EarliestProcessingAt passes, then calls onElapsed once. */
function useCountdown(targetIso: string | null | undefined, onElapsed: () => void): number {
const [secondsLeft, setSecondsLeft] = useState<number>(() => {
if (!targetIso) return 0;
return Math.max(0, Math.floor((new Date(targetIso).getTime() - Date.now()) / 1000));
});
const firedRef = useRef(false);
useEffect(() => {
if (!targetIso) return;
firedRef.current = false;
// Anchor the remaining-time once at mount, then drive subsequent ticks off
// performance.now() — a monotonic clock that is NOT affected by NTP-syncs
// or manual clock changes during the ~6 min drain barrier. The previous
// implementation re-read Date.now() on every tick, so a wall-clock jump
// forward would fire onElapsed (auto-logout) prematurely.
const initialRemainingMs = Math.max(
0,
new Date(targetIso).getTime() - Date.now(),
);
const monoAtMount = performance.now();
const tick = () => {
const elapsedMs = performance.now() - monoAtMount;
const remaining = Math.max(0, Math.floor((initialRemainingMs - elapsedMs) / 1000));
setSecondsLeft(remaining);
if (remaining === 0 && !firedRef.current) {
firedRef.current = true;
onElapsed();
}
};
tick(); // immediate sync
const interval = setInterval(tick, 1000);
return () => clearInterval(interval);
}, [targetIso, onElapsed]);
return secondsLeft;
}
export default function OffboardingSection({
@@ -20,7 +81,20 @@ export default function OffboardingSection({
offboardError,
setOffboardError,
onOffboard,
offboardingInProgress,
onDrainBarrierElapsed,
}: OffboardingSectionProps) {
// When the offboarding has been queued, replace the section entirely with the
// drain-barrier banner. The dialog is dismissed by handleOffboard on success.
if (offboardingInProgress) {
return (
<OffboardingInProgressBanner
info={offboardingInProgress}
onDrainBarrierElapsed={onDrainBarrierElapsed}
/>
);
}
return (
<>
{/* Danger Zone: Tenant Offboarding */}
@@ -79,7 +153,10 @@ export default function OffboardingSection({
<li>Tenant configuration</li>
<li>All admin accounts (including yours)</li>
</ul>
<p className="mt-3 font-semibold">After offboarding you will be signed out and lose all access.</p>
<p className="mt-3 font-semibold">
Data deletion will start after a ~6 minute preparation window and then run in the background.
You can close this tab once the request has been sent.
</p>
</div>
<div className="mb-4">
@@ -118,10 +195,10 @@ export default function OffboardingSection({
{offboarding ? (
<>
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-white"></div>
<span>Offboarding...</span>
<span>Queuing...</span>
</>
) : (
<span>Permanently Delete All Data</span>
<span>Queue Permanent Deletion</span>
)}
</button>
</div>
@@ -131,3 +208,95 @@ export default function OffboardingSection({
</>
);
}
/**
* Banner shown after the DELETE returns 202. Replaces the entire Danger Zone section.
* Renders a live countdown until `earliestProcessingAt`, then auto-logs the user out
* (the worker has by then started Phase 2 and the auth pipeline will return 403
* anyway via the Disabled-flag gate).
*/
function OffboardingInProgressBanner({
info,
onDrainBarrierElapsed,
}: {
info: OffboardingInProgressInfo;
onDrainBarrierElapsed: () => void;
}) {
const secondsLeft = useCountdown(info.earliestProcessingAt ?? null, onDrainBarrierElapsed);
const inDrainBarrier = (info.earliestProcessingAt && secondsLeft > 0) === true;
// Single 4-state visual; status text drives the badge color + label.
// "Preparing" is derived from earliestProcessingAt + countdown rather than from status,
// because the backend status stays "Queued" during the cache-drain window.
const stateLabel = inDrainBarrier
? "Preparing"
: info.status === "Failed"
? "Failed"
: info.status === "Completed"
? "Completed"
: "In Progress";
const stateColors: Record<string, string> = {
"Preparing": "bg-amber-100 text-amber-800 border-amber-300",
"In Progress": "bg-blue-100 text-blue-800 border-blue-300",
"Completed": "bg-green-100 text-green-800 border-green-300",
"Failed": "bg-red-100 text-red-800 border-red-300",
};
return (
<div className="bg-white rounded-lg shadow border-2 border-red-200 p-6 space-y-4">
<div className="flex items-center space-x-3">
<div className="w-12 h-12 bg-red-100 rounded-full flex items-center justify-center flex-shrink-0">
<svg className="w-6 h-6 text-red-600" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
</svg>
</div>
<div>
<h2 className="text-xl font-semibold text-red-900">Tenant Offboarding queued</h2>
<p className="text-sm text-gray-600 mt-1">
This tenant will be permanently deleted. You do not need to keep this page open.
</p>
</div>
</div>
<div className="flex items-center space-x-3">
<span className={`inline-flex items-center px-3 py-1 rounded-full text-sm font-medium border ${stateColors[stateLabel] ?? stateColors["In Progress"]}`}>
{stateLabel}
</span>
{inDrainBarrier && (
<span className="text-sm text-gray-700">
Data deletion starts in <span className="font-mono font-semibold">{formatRemaining(secondsLeft)}</span>
</span>
)}
</div>
{inDrainBarrier ? (
<div className="bg-amber-50 border border-amber-200 rounded-lg p-4 text-sm text-amber-900 space-y-1">
<p className="font-semibold">Preparing for deletion</p>
<p>
Your tenant has been disabled and is being prepared for permanent deletion. The wipe will start
automatically once the countdown reaches zero.
</p>
<p className="mt-2">
Everything needed for the deletion is already recorded — you can sign out and close this tab.
The deletion continues in the background.
</p>
</div>
) : (
<div className="bg-blue-50 border border-blue-200 rounded-lg p-4 text-sm text-blue-900 space-y-1">
<p className="font-semibold">Deletion in progress</p>
<p>
Your tenant is being permanently deleted. You can close this tab safely.
</p>
</div>
)}
<div className="text-xs text-gray-500 space-y-0.5">
<p>History row: <span className="font-mono">{info.historyRowKey}</span></p>
{info.earliestProcessingAt && (
<p>Earliest processing at: <span className="font-mono">{info.earliestProcessingAt}</span></p>
)}
</div>
</div>
);
}
@@ -10,6 +10,7 @@ export function SectionOffboarding() {
offboardConfirmText, setOffboardConfirmText,
offboarding, offboardError, setOffboardError,
handleOffboard,
offboardingInProgress, handleDrainBarrierElapsed,
} = useTenantConfig();
return (
@@ -24,6 +25,8 @@ export function SectionOffboarding() {
offboardError={offboardError}
setOffboardError={setOffboardError}
onOffboard={handleOffboard}
offboardingInProgress={offboardingInProgress}
onDrainBarrierElapsed={handleDrainBarrierElapsed}
/>
</>
);
+57
View File
@@ -402,6 +402,21 @@ export const api = {
})}`,
},
// ── Customs Archive (PR3.B Tenant offboarding archive) ────────────────────
// Response types defined below (CustomsArchive*).
customsArchive: {
listRuns: (opts?: { tenantId?: string }) =>
`${API_BASE_URL}/api/global/customs-archive${qs({ tenantId: opts?.tenantId })}`,
listEntries: (tenantId: string, historyRowKey: string) =>
`${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}`,
getEntry: (tenantId: string, historyRowKey: string, archiveRowKey: string) =>
`${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}/${encodeURIComponent(archiveRowKey)}`,
deleteEntry: (tenantId: string, historyRowKey: string, archiveRowKey: string) =>
`${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}/${encodeURIComponent(archiveRowKey)}`,
deleteRun: (tenantId: string, historyRowKey: string) =>
`${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}`,
},
// ── Feedback ──────────────────────────────────────────────────────────────
feedback: {
status: () => `${API_BASE_URL}/api/feedback/status`,
@@ -484,3 +499,45 @@ export const api = {
leaveGroup: () => `${API_BASE_URL}/api/realtime/groups/leave`,
},
};
// ── Customs Archive response types ────────────────────────────────────────
// Shared by /admin/customs-archive (list runs) and
// /admin/customs-archive/[tenantId]/[historyRowKey] (list/inspect entries).
export interface CustomsArchiveRunSummary {
partitionKey: string;
tenantId: string;
historyRowKey: string;
archivedAt: string;
gatherRulesCount: number;
analyzeRulesCount: number;
imeLogPatternsCount: number;
}
export interface CustomsArchiveListRunsResponse {
success: boolean;
count: number;
runs: CustomsArchiveRunSummary[];
}
export interface CustomsArchiveEntrySummary {
partitionKey: string;
rowKey: string;
originalTable: string;
originalRowKey: string;
archivedAt: string;
entityJsonPreview: string;
}
export interface CustomsArchiveListEntriesResponse {
success: boolean;
count: number;
entries: CustomsArchiveEntrySummary[];
}
export interface CustomsArchiveFullEntry extends CustomsArchiveEntrySummary {
tenantId: string;
originalPartitionKey: string;
entityJson: string;
historyRowKey: string;
archivedBy: string;
}
@@ -263,6 +263,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [
items: [
{ id: "ga-maintenance", label: "Maintenance", href: "/admin/ops" },
{ id: "ga-session-cleanup", label: "Session Cleanup", href: "/admin/ops/session-cleanup" },
{ id: "ga-customs-archive", label: "Customs Archive", href: "/admin/customs-archive" },
],
},
{