diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/BootstrapAuthDisabledTenantTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/BootstrapAuthDisabledTenantTests.cs new file mode 100644 index 00000000..f2359842 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/BootstrapAuthDisabledTenantTests.cs @@ -0,0 +1,111 @@ +using System.Net; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Moq; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Plan v2 §5 (bootstrap-path interaction): when a tenant is mid-offboarding, +/// Phase 1 flips TenantConfiguration.Disabled = true and Phase 2 starts after +/// the cache-drain barrier. The auth pipeline (cert + bootstrap-token) must reject +/// requests against a Disabled tenant — otherwise a fresh OOBE device with a bootstrap +/// token could keep writing into a tenant about to be wiped. +/// +/// The §0 Disabled-gate in runs +/// BEFORE the §0.5 bootstrap-token branch, so this property is enforced by the order +/// of checks. These tests pin that ordering so future refactors cannot accidentally +/// move the bootstrap branch above the Disabled gate. +/// +/// +public sealed class BootstrapAuthDisabledTenantTests +{ + private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + private const string BootstrapToken = "11111111-2222-3333-4444-555555555555"; + + private static SecurityValidator BuildValidator(TenantConfiguration config) + { + var configRepo = Mock.Of(); + var cache = new MemoryCache(new MemoryCacheOptions()); + + var configServiceMock = new Mock( + configRepo, Mock.Of>(), cache) + { CallBase = false }; + configServiceMock + .Setup(x => x.TryGetConfigurationAsync(It.IsAny())) + .ReturnsAsync((config, true)); + + return new SecurityValidator( + configService: configServiceMock.Object, + rateLimitService: null!, + autopilotDeviceValidator: null!, + corporateIdentifierValidator: null!, + logger: Mock.Of(), + bootstrapSessionService: null, + deviceAssociationValidator: null); + } + + private static HttpRequestData BuildRequestWithBootstrapToken() + { + var contextMock = new Mock(); + var reqMock = new Mock(contextMock.Object); + + var headers = new Microsoft.Azure.Functions.Worker.Http.HttpHeadersCollection + { + { "X-Bootstrap-Token", BootstrapToken }, + }; + reqMock.SetupGet(r => r.Headers).Returns(headers); + + return reqMock.Object; + } + + [Fact] + public async Task BootstrapToken_RejectedWhenTenantDisabled() + { + var disabled = TenantConfiguration.CreateDefault(TenantId); + disabled.Disabled = true; + disabled.DisabledReason = "Offboarding in progress"; + disabled.DisabledUntil = null; + + var sut = BuildValidator(disabled); + + var result = await sut.ValidateRequestAsync(BuildRequestWithBootstrapToken(), TenantId); + + // Must be the §0 Disabled-gate, not the §0.5 bootstrap branch. The bootstrap + // branch lives AFTER the Disabled check; if anyone ever reorders these, the + // assertion below catches it (Disabled returns "Tenant is suspended" with the + // configured reason; bootstrap-branch failures have different ErrorMessages). + Assert.False(result.IsValid); + Assert.Equal(HttpStatusCode.Forbidden, result.StatusCode); + Assert.Equal("Tenant is suspended", result.ErrorMessage); + Assert.Equal("Offboarding in progress", result.Details); + } + + [Fact] + public async Task BootstrapToken_AcceptedWhenTenantNotDisabled_StillEvaluatesBootstrapBranch() + { + // Sanity: a non-disabled tenant DOES reach the bootstrap branch (which will then + // run its own validation against _bootstrapSessionService). With _bootstrapSessionService + // null in this test, the bootstrap branch falls through to the cert path — proving + // that the §0 gate did NOT block. + var enabled = TenantConfiguration.CreateDefault(TenantId); + enabled.Disabled = false; + enabled.ValidateAutopilotDevice = false; + enabled.ValidateCorporateIdentifier = false; + enabled.AllowInsecureAgentRequests = false; + + var sut = BuildValidator(enabled); + + var result = await sut.ValidateRequestAsync(BuildRequestWithBootstrapToken(), TenantId); + + // Falls through to the "device validation is required" gate at §1 (not the §0 Disabled gate). + Assert.False(result.IsValid); + Assert.Equal(HttpStatusCode.Forbidden, result.StatusCode); + Assert.Equal("Device validation is required", result.ErrorMessage); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/DeletionProgressDrainProbeTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/DeletionProgressDrainProbeTests.cs new file mode 100644 index 00000000..bd6893b0 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/DeletionProgressDrainProbeTests.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Plan §7.4 step 5 + §10.1: drain predicate is true only when the progress blob has +/// CompletedAt != null AND TombstoneStarted == true. 404 is "not yet done", not an error. +/// +public class DeletionProgressDrainProbeTests +{ + private const string TenantId = "44444444-4444-4444-4444-444444444444"; + private const string SessionId = "55555555-5555-5555-5555-555555555555"; + private const string ManifestId = "manifest-1"; + + [Fact] + public async Task IsCompleted_TrueOnlyWhen_CompletedAtSet_AndTombstoneStarted() + { + var sut = new DeletionProgressDrainProbe( + new FakeBlob(new DeletionProgress + { + CompletedAt = DateTime.UtcNow, + TombstoneStarted = true, + }), + NullLogger.Instance); + + Assert.True(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId)); + } + + [Fact] + public async Task IsCompleted_FalseWhen_CompletedAtNull() + { + var sut = new DeletionProgressDrainProbe( + new FakeBlob(new DeletionProgress { TombstoneStarted = true, CompletedAt = null }), + NullLogger.Instance); + + Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId)); + } + + [Fact] + public async Task IsCompleted_FalseWhen_TombstoneNotStarted_RevWalkthrough() + { + // If CompletedAt got somehow set without TombstoneStarted, the cascade did not + // finish the FINAL step — drain must say "no" to defend against truncated progress. + var sut = new DeletionProgressDrainProbe( + new FakeBlob(new DeletionProgress { CompletedAt = DateTime.UtcNow, TombstoneStarted = false }), + NullLogger.Instance); + + Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId)); + } + + [Fact] + public async Task IsCompleted_FalseOn404_NotAnError() + { + var sut = new DeletionProgressDrainProbe( + new FakeBlob(throwOnDownload: new RequestFailedException(404, "NotFound", "BlobNotFound", null)), + NullLogger.Instance); + + Assert.False(await sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId)); + } + + [Fact] + public async Task IsCompleted_PropagatesNon404StorageError() + { + // 503 / 500 are real transients the worker must surface so the queue retries. + var sut = new DeletionProgressDrainProbe( + new FakeBlob(throwOnDownload: new RequestFailedException(503, "ServiceUnavailable", "ServerBusy", null)), + NullLogger.Instance); + + await Assert.ThrowsAsync( + () => sut.IsCascadeCompletedAsync(TenantId, SessionId, ManifestId)); + } + + // ── Fake BlobStorageService that overrides only the download seam we need ── + + private sealed class FakeBlob : BlobStorageService + { + private readonly DeletionProgress? _progress; + private readonly RequestFailedException? _throw; + + public FakeBlob(DeletionProgress progress) + : base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger.Instance, usesManagedIdentity: false) + { + _progress = progress; + } + + public FakeBlob(RequestFailedException throwOnDownload) + : base(new BlobServiceClient("UseDevelopmentStorage=true"), NullLogger.Instance, usesManagedIdentity: false) + { + _throw = throwOnDownload; + } + + public override Task<(DeletionProgress Progress, string ETag)> DownloadDeletionProgressAsync( + string tenantId, string sessionId, string manifestId, CancellationToken cancellationToken = default) + { + if (_throw != null) throw _throw; + return Task.FromResult((_progress!, "\"0xFAKE\"")); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeCustomsArchiveRepository.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeCustomsArchiveRepository.cs new file mode 100644 index 00000000..75382fce --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeCustomsArchiveRepository.cs @@ -0,0 +1,105 @@ +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Offboarding; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// In-memory for PR3.B handler + endpoint tests. +/// Idempotent upsert keyed by (PK, RK); supports the four query shapes the admin API uses. +/// +internal sealed class FakeCustomsArchiveRepository : ITenantCustomsArchiveRepository +{ + private readonly Dictionary _store + = new(System.StringComparer.Ordinal); + + public IReadOnlyDictionary Store => _store; + + private static string Key(string pk, string rk) => pk + "|" + rk; + + public Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default) + { + _store[Key(entry.PartitionKey, entry.RowKey)] = Clone(entry); + return Task.CompletedTask; + } + + public Task CountByRunAndTableAsync(string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default) + { + var pk = $"{normalizedTenantId}_{historyRowKey}"; + var count = _store.Values.Count(e => + e.PartitionKey == pk && e.OriginalTable == originalTable); + return Task.FromResult(count); + } + + public async IAsyncEnumerable QueryByRunAsync( + string normalizedTenantId, string historyRowKey, + [EnumeratorCancellation] CancellationToken ct = default) + { + var pk = $"{normalizedTenantId}_{historyRowKey}"; + foreach (var entry in _store.Values.Where(e => e.PartitionKey == pk)) + { + yield return Clone(entry); + } + await Task.CompletedTask; + } + + public async IAsyncEnumerable QueryByTenantAsync( + string normalizedTenantId, [EnumeratorCancellation] CancellationToken ct = default) + { + var prefix = normalizedTenantId + "_"; + foreach (var entry in _store.Values.Where(e => e.PartitionKey.StartsWith(prefix))) + { + yield return Clone(entry); + } + await Task.CompletedTask; + } + + public async IAsyncEnumerable QueryAllAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + foreach (var entry in _store.Values) + { + yield return Clone(entry); + } + await Task.CompletedTask; + } + + public Task TryGetEntryAsync( + string partitionKey, string rowKey, CancellationToken ct = default) + { + _store.TryGetValue(Key(partitionKey, rowKey), out var entry); + return Task.FromResult(entry == null ? null : Clone(entry)); + } + + public Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default) + { + _store.Remove(Key(partitionKey, rowKey)); + return Task.CompletedTask; + } + + public Task DeleteRunAsync(string normalizedTenantId, string historyRowKey, CancellationToken ct = default) + { + var pk = $"{normalizedTenantId}_{historyRowKey}"; + var toDelete = _store.Keys.Where(k => k.StartsWith(pk + "|")).ToList(); + foreach (var k in toDelete) _store.Remove(k); + return Task.FromResult(toDelete.Count); + } + + private static TenantOffboardingCustomsArchiveEntry Clone(TenantOffboardingCustomsArchiveEntry e) => new() + { + PartitionKey = e.PartitionKey, + RowKey = e.RowKey, + TenantId = e.TenantId, + OriginalTable = e.OriginalTable, + OriginalPartitionKey = e.OriginalPartitionKey, + OriginalRowKey = e.OriginalRowKey, + EntityJson = e.EntityJson, + HistoryRowKey = e.HistoryRowKey, + ArchivedAt = e.ArchivedAt, + ArchivedBy = e.ArchivedBy, + }; +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeOffboardingAuditRepository.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeOffboardingAuditRepository.cs new file mode 100644 index 00000000..9b5f04f4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeOffboardingAuditRepository.cs @@ -0,0 +1,155 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// In-memory for handler/cleanup tests. Records the +/// latest write per (PK, RK) so the test can assert lifecycle transitions, and mirrors the +/// production ETag-CAS contract on the pointer (412 on mismatched ETag). +/// +internal sealed class FakeOffboardingAuditRepository : IOffboardingAuditRepository +{ + public Dictionary Markers { get; } = new(); + public Dictionary History { get; } = new(); + public Dictionary Pointers { get; } = new(); + public List MarkerWrites { get; } = new(); + public List HistoryWrites { get; } = new(); + + /// When set, the next call to throws — used by + /// poison-path tests to simulate a transient storage failure inside the Failed-state + /// transition so the Worker's reorder invariant can be exercised. + public Exception? ThrowOnNextHistoryUpsert { get; set; } + + private int _etagCounter; + private string NextEtag() => $"\"0xFAKE_{++_etagCounter}\""; + + public Task TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default) + => Task.FromResult(Markers.TryGetValue(normalizedTenantId, out var m) ? Clone(m) : null); + + public Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default) + { + if (Markers.ContainsKey(marker.RowKey)) + throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null); + Markers[marker.RowKey] = Clone(marker); + MarkerWrites.Add(marker.Status); + return Task.CompletedTask; + } + + public Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default) + { + Markers[marker.RowKey] = Clone(marker); + MarkerWrites.Add(marker.Status); + return Task.CompletedTask; + } + + public Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default) + { + Markers.Remove(normalizedTenantId); + return Task.CompletedTask; + } + + public async IAsyncEnumerable QueryMarkersAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + foreach (var marker in Markers.Values) + { + yield return Clone(marker); + } + await Task.CompletedTask; + } + + public Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default) + { + if (History.ContainsKey(history.RowKey)) + throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null); + History[history.RowKey] = Clone(history); + HistoryWrites.Add(history.Status); + return Task.CompletedTask; + } + + public Task TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default) + => Task.FromResult(History.TryGetValue(historyRowKey, out var h) ? Clone(h) : null); + + public Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default) + { + if (ThrowOnNextHistoryUpsert is { } ex) + { + ThrowOnNextHistoryUpsert = null; + throw ex; + } + History[history.RowKey] = Clone(history); + HistoryWrites.Add(history.Status); + return Task.CompletedTask; + } + + public Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync( + string normalizedTenantId, CancellationToken ct = default) + { + if (Pointers.TryGetValue(normalizedTenantId, out var entry)) + { + return Task.FromResult<(OffboardingByTenantPointer?, string?)>((Clone(entry.Pointer), entry.ETag)); + } + return Task.FromResult<(OffboardingByTenantPointer?, string?)>((null, null)); + } + + public Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default) + { + if (Pointers.ContainsKey(pointer.RowKey)) + throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null); + Pointers[pointer.RowKey] = (Clone(pointer), NextEtag()); + return Task.CompletedTask; + } + + public Task UpdateByTenantPointerWithEtagAsync( + OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default) + { + if (!Pointers.TryGetValue(pointer.RowKey, out var entry)) + throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null); + if (entry.ETag != ifMatchEtag) + throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null); + Pointers[pointer.RowKey] = (Clone(pointer), NextEtag()); + return Task.CompletedTask; + } + + private static OffboardingMarkerEntry Clone(OffboardingMarkerEntry m) => new() + { + PartitionKey = m.PartitionKey, RowKey = m.RowKey, + TenantId = m.TenantId, OffboardingHistoryRowKey = m.OffboardingHistoryRowKey, + InitiatedAt = m.InitiatedAt, InitiatedBy = m.InitiatedBy, + Status = m.Status, CompletedAt = m.CompletedAt, + FailedAt = m.FailedAt, FailedPhase = m.FailedPhase, + }; + + private static OffboardingHistoryEntry Clone(OffboardingHistoryEntry h) => new() + { + PartitionKey = h.PartitionKey, RowKey = h.RowKey, + TenantId = h.TenantId, DomainName = h.DomainName, InitiatedBy = h.InitiatedBy, + OffboardedAt = h.OffboardedAt, CompletedAt = h.CompletedAt, + EarliestProcessingAt = h.EarliestProcessingAt, + Status = h.Status, + DeletedRowCountsJson = h.DeletedRowCountsJson, TotalRowsDeleted = h.TotalRowsDeleted, + DeletedBlobCount = h.DeletedBlobCount, CascadeSessionsEnqueued = h.CascadeSessionsEnqueued, + ErrorMessage = h.ErrorMessage, RetryCount = h.RetryCount, + DrainCompletedAt = h.DrainCompletedAt, + EnumerationStartedAt = h.EnumerationStartedAt, + EnumerationCompletedBeforeUpload = h.EnumerationCompletedBeforeUpload, + CustomGatherRulesArchived = h.CustomGatherRulesArchived, + CustomAnalyzeRulesArchived = h.CustomAnalyzeRulesArchived, + ImeLogPatternOverridesArchived = h.ImeLogPatternOverridesArchived, + ReonboardedAt = h.ReonboardedAt, ReonboardedBy = h.ReonboardedBy, + CustomsAutoWipedOnReonboard = h.CustomsAutoWipedOnReonboard, + }; + + private static OffboardingByTenantPointer Clone(OffboardingByTenantPointer p) => new() + { + PartitionKey = p.PartitionKey, RowKey = p.RowKey, TenantId = p.TenantId, + LatestHistoryRowKey = p.LatestHistoryRowKey, LatestStatus = p.LatestStatus, + LatestUpdatedAt = p.LatestUpdatedAt, OffboardCount = p.OffboardCount, + }; +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeOffboardingExpectationsStore.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeOffboardingExpectationsStore.cs new file mode 100644 index 00000000..c3dca272 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/FakeOffboardingExpectationsStore.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// In-memory with the production If-None-Match=* +/// and If-Match ETag-CAS semantics. Tracks delete attempts so tests can assert idempotency. +/// +internal sealed class FakeOffboardingExpectationsStore : IOffboardingExpectationsStore +{ + private readonly Dictionary _blobs = new(); + private int _etagCounter; + + public int DeleteCallCount { get; private set; } + public bool ThrowOnDelete { get; set; } + + public Task TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default) + { + var key = Key(payload.TenantId, payload.HistoryRowKey); + if (_blobs.ContainsKey(key)) return Task.FromResult(false); + _blobs[key] = (Clone(payload), NextEtag()); + return Task.FromResult(true); + } + + public Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync( + string tenantId, string historyRowKey, CancellationToken ct = default) + { + var key = Key(tenantId, historyRowKey); + if (!_blobs.TryGetValue(key, out var entry)) + return Task.FromResult<(OffboardingExpectations?, string?)>((null, null)); + return Task.FromResult<(OffboardingExpectations?, string?)>((Clone(entry.Payload), entry.ETag)); + } + + public Task UpdateWithEtagCasAsync(OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default) + { + var key = Key(payload.TenantId, payload.HistoryRowKey); + if (!_blobs.TryGetValue(key, out var entry)) + throw new RequestFailedException(404, "NotFound", "BlobNotFound", null); + if (entry.ETag != ifMatchEtag) + throw new RequestFailedException(412, "ConditionNotMet", "ConditionNotMet", null); + var newEtag = NextEtag(); + _blobs[key] = (Clone(payload), newEtag); + return Task.FromResult(newEtag); + } + + public Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default) + { + DeleteCallCount++; + if (ThrowOnDelete) throw new InvalidOperationException("simulated blob-delete failure"); + _blobs.Remove(Key(tenantId, historyRowKey)); + return Task.CompletedTask; + } + + /// Pre-seed a blob (used to drive the drain probe directly without going via TryUploadInitial). + public void Seed(OffboardingExpectations payload) + { + _blobs[Key(payload.TenantId, payload.HistoryRowKey)] = (Clone(payload), NextEtag()); + } + + public bool BlobExists(string tenantId, string historyRowKey) + => _blobs.ContainsKey(Key(tenantId, historyRowKey)); + + private static string Key(string tenantId, string historyRowKey) => $"{tenantId}|{historyRowKey}"; + private string NextEtag() => $"\"0xFAKE_EXP_{++_etagCounter}\""; + + private static OffboardingExpectations Clone(OffboardingExpectations p) + { + var clone = new OffboardingExpectations + { + SchemaVersion = p.SchemaVersion, + TenantId = p.TenantId, + HistoryRowKey = p.HistoryRowKey, + CreatedAt = p.CreatedAt, + EnumerationCompleted = p.EnumerationCompleted, + EnumeratedSessionCount = p.EnumeratedSessionCount, + }; + foreach (var e in p.Expectations) + { + clone.Expectations.Add(new OffboardingExpectation + { + SessionId = e.SessionId, + ManifestId = e.ManifestId, + Outcome = e.Outcome, + RetryCount = e.RetryCount, + }); + } + return clone; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/OffboardingMarkerCleanupFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/OffboardingMarkerCleanupFunctionTests.cs new file mode 100644 index 00000000..7f7d8892 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/OffboardingMarkerCleanupFunctionTests.cs @@ -0,0 +1,431 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Functions.Maintenance; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// Rev-5-F2 + Rev-9 cleanup contract: +/// +/// Completed markers older than 15min are removed — AFTER their Expectations blob. +/// Completed markers younger than 15min stay (warm caches could still leak auth). +/// Failed markers are NEVER auto-removed — operator action only. +/// Initiated / InProgress markers are ignored (offboarding still in flight). +/// Blob-delete failure leaves the marker in place so the next run retries. +/// +/// +public class OffboardingMarkerCleanupFunctionTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111"; + + [Fact] + public async Task DeletesCompletedMarkerOlderThan15min() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + fixture.Expectations.Seed(NewExpectations()); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(1, result.MarkersDeleted); + Assert.False(fixture.Repo.Markers.ContainsKey(TenantId)); + Assert.False(fixture.Expectations.BlobExists(TenantId, HistoryRowKey), + "Expectations blob must be deleted before the marker"); + } + + [Fact] + public async Task KeepsCompletedMarkerYoungerThan15min() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(5)); // inside cache TTL window + fixture.Expectations.Seed(NewExpectations()); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(0, result.MarkersDeleted); + Assert.True(fixture.Repo.Markers.ContainsKey(TenantId)); + Assert.True(fixture.Expectations.BlobExists(TenantId, HistoryRowKey), + "Blob stays until the marker is eligible for removal"); + } + + [Fact] + public async Task NeverDeletesFailedMarker_EvenAfter24h() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker( + status: "Failed", + failedAt: fixture.FixedNow - TimeSpan.FromHours(24), + failedPhase: "drain_timeout"); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(0, result.MarkersDeleted); + Assert.Equal(1, result.FailedMarkersSeen); + Assert.True(fixture.Repo.Markers.ContainsKey(TenantId), + "Failed markers require operator action — never auto-cleaned"); + } + + [Fact] + public async Task IgnoresInProgressMarker() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker(status: "InProgress"); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(0, result.MarkersDeleted); + Assert.Equal(0, result.FailedMarkersSeen); + Assert.True(fixture.Repo.Markers.ContainsKey(TenantId)); + } + + [Fact] + public async Task IgnoresInitiatedMarker() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker(status: "Initiated"); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(0, result.MarkersDeleted); + Assert.True(fixture.Repo.Markers.ContainsKey(TenantId)); + } + + [Fact] + public async Task BlobDeleteFails_KeepsMarkerForNextRun() + { + // Rev-9 hygiene: blob deleted BEFORE marker. If blob delete throws (transient storage + // glitch), marker stays so the next 2h-run tries again. Otherwise an orphan blob lives + // forever in offboarding-state. + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker(status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + fixture.Expectations.Seed(NewExpectations()); + fixture.Expectations.ThrowOnDelete = true; + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(0, result.MarkersDeleted); + Assert.Equal(1, result.BlobDeleteRetries); + Assert.True(fixture.Repo.Markers.ContainsKey(TenantId), + "Marker stays so the next run retries the blob delete"); + } + + [Fact] + public async Task DryRunWithMixedFleet_HandlesEachMarkerIndependently() + { + var fixture = new Fixture(); + fixture.Repo.Markers["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"] = NewMarker( + tenantId: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + status: "Completed", completedAt: fixture.FixedNow - TimeSpan.FromMinutes(30)); + fixture.Repo.Markers["bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"] = NewMarker( + tenantId: "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", + status: "Completed", completedAt: fixture.FixedNow - TimeSpan.FromMinutes(2)); + fixture.Repo.Markers["cccccccc-cccc-cccc-cccc-cccccccccccc"] = NewMarker( + tenantId: "cccccccc-cccc-cccc-cccc-cccccccccccc", + status: "Failed", failedAt: fixture.FixedNow - TimeSpan.FromHours(2), + failedPhase: "drain_timeout"); + fixture.Repo.Markers["dddddddd-dddd-dddd-dddd-dddddddddddd"] = NewMarker( + tenantId: "dddddddd-dddd-dddd-dddd-dddddddddddd", + status: "InProgress"); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(4, result.Scanned); + Assert.Equal(1, result.MarkersDeleted); // only the 30-min-old Completed one + Assert.Equal(1, result.FailedMarkersSeen); + Assert.False(fixture.Repo.Markers.ContainsKey("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")); + Assert.True(fixture.Repo.Markers.ContainsKey("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")); + Assert.True(fixture.Repo.Markers.ContainsKey("cccccccc-cccc-cccc-cccc-cccccccccccc")); + Assert.True(fixture.Repo.Markers.ContainsKey("dddddddd-dddd-dddd-dddd-dddddddddddd")); + } + + // ── TenantConfiguration defense-in-depth sweep (PR3.B-revised Finding 2) ────── + // + // Phase 2.F-final in the handler is fail-soft. If it failed (worker crash, transient + // storage), the TenantConfiguration row would carry Disabled=true and block the + // tenant from self-service re-onboarding indefinitely. This sweep is the second + // pass: when MarkerCleanup spots a Completed marker eligible for removal, it tries + // to delete TenantConfiguration FIRST. If the sweep fails, the marker stays so the + // next 2h run retries. + + [Fact] + public async Task CompletedMarker_SweepsLingeringTenantConfig_BeforeDeletingMarker() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker( + status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + // Simulate "Phase 2.F-final failed in the handler" → the row is still the + // offboarding tombstone (Disabled=true + magic reason). Sweep should wipe it. + fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ReturnsAsync(BuildTombstoneConfig(TenantId)); + fixture.SafeWipe.QueuedReturns.Enqueue(1); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(1, result.MarkersDeleted); + Assert.Equal(1, result.TenantConfigsSwept); + Assert.Equal(0, result.TenantConfigsSpared); + Assert.Equal(0, result.TenantConfigSweepRetries); + + // The sweep ran with the right table + tenant id. + Assert.Single(fixture.SafeWipe.WipeCalls); + Assert.Equal(Constants.TableNames.TenantConfiguration, fixture.SafeWipe.WipeCalls[0].Table); + Assert.Equal(TenantId, fixture.SafeWipe.WipeCalls[0].TenantId); + Assert.False(fixture.Repo.Markers.ContainsKey(TenantId)); + } + + [Fact] + public async Task CompletedMarker_TenantConfigAlreadyGone_SkipsSweep_DeletesMarker() + { + // Happy path: Phase 2.F-final succeeded in the handler. GetTenantConfigurationAsync + // returns null (default fixture stub) → no row to wipe. + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker( + status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(1, result.MarkersDeleted); + Assert.Equal(0, result.TenantConfigsSwept); + Assert.Equal(0, result.TenantConfigsSpared); + // No SafeWipe call when the row is already gone (we don't waste a transaction). + Assert.Empty(fixture.SafeWipe.WipeCalls); + Assert.False(fixture.Repo.Markers.ContainsKey(TenantId)); + } + + [Fact] + public async Task CompletedMarker_UserReonboarded_SparesFreshConfig_AndStillDeletesMarker() + { + // The crucial Codex Finding 1 scenario: between Phase 2.F-final and MarkerCleanup, + // the user self-service-re-onboarded. The TenantConfiguration row now exists with + // Disabled=false (or any non-offboarding DisabledReason). Blind sweep would + // destroy their fresh tenant. The conditional probe must skip the wipe. + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker( + status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + + var freshConfig = TenantConfiguration.CreateDefault(TenantId); + freshConfig.Disabled = false; + freshConfig.UpdatedBy = "bob@contoso.com"; + fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ReturnsAsync(freshConfig); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(1, result.MarkersDeleted); + Assert.Equal(0, result.TenantConfigsSwept); + Assert.Equal(1, result.TenantConfigsSpared); + // Critically: SafeWipe was NEVER called. The fresh config is preserved. + Assert.Empty(fixture.SafeWipe.WipeCalls); + // The Completed marker still gets cleaned up — its work is done. + Assert.False(fixture.Repo.Markers.ContainsKey(TenantId)); + } + + [Fact] + public async Task CompletedMarker_UserReonboarded_AndSuspendedThemselves_StillSpared() + { + // Defense-in-depth check: a re-onboarded tenant that later self-suspended would + // have Disabled=true but with a different DisabledReason (e.g. "Self-suspended"). + // The magic-string match must distinguish — only "Offboarding in progress" triggers + // the wipe. + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker( + status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + + var reonboardedThenSuspended = TenantConfiguration.CreateDefault(TenantId); + reonboardedThenSuspended.Disabled = true; + reonboardedThenSuspended.DisabledReason = "Self-suspended"; + fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ReturnsAsync(reonboardedThenSuspended); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(0, result.TenantConfigsSwept); + Assert.Equal(1, result.TenantConfigsSpared); + Assert.Empty(fixture.SafeWipe.WipeCalls); + } + + [Fact] + public async Task CompletedMarker_TenantConfigProbeThrows_KeepsMarkerForNextRetry() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker( + status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ThrowsAsync(new InvalidOperationException("storage down")); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(0, result.MarkersDeleted); + Assert.Equal(1, result.TenantConfigSweepRetries); + Assert.True(fixture.Repo.Markers.ContainsKey(TenantId), + "Marker must stay alive when the TenantConfiguration probe fails so the next run can retry"); + } + + [Fact] + public async Task CompletedMarker_TenantConfigSweepThrows_KeepsMarkerForNextRetry() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker( + status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + fixture.ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ReturnsAsync(BuildTombstoneConfig(TenantId)); + fixture.SafeWipe.ThrowOnNextWipe = new InvalidOperationException("storage down"); + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(0, result.MarkersDeleted); + Assert.Equal(1, result.TenantConfigSweepRetries); + Assert.True(fixture.Repo.Markers.ContainsKey(TenantId), + "Marker must stay alive when the TenantConfig sweep fails so the next run can retry"); + } + + [Fact] + public async Task CompletedMarker_BlobDeleteFails_TenantConfigProbeIsNotEvenAttempted() + { + var fixture = new Fixture(); + fixture.Repo.Markers[TenantId] = NewMarker( + status: "Completed", + completedAt: fixture.FixedNow - TimeSpan.FromMinutes(20)); + fixture.Expectations.ThrowOnDelete = true; + + var result = await fixture.Sut.RunCoreAsync(default); + + Assert.Equal(1, result.BlobDeleteRetries); + Assert.Empty(fixture.SafeWipe.WipeCalls); + Assert.True(fixture.Repo.Markers.ContainsKey(TenantId)); + // Probe shouldn't have been attempted either — blob is the first gate. + fixture.ConfigRepo.Verify(r => r.GetTenantConfigurationAsync(It.IsAny()), Times.Never); + } + + private static TenantConfiguration BuildTombstoneConfig(string tenantId) + { + var config = TenantConfiguration.CreateDefault(tenantId); + config.Disabled = true; + config.DisabledReason = "Offboarding in progress"; + return config; + } + + // ── Helpers ───────────────────────────────────────────────────────────────── + + private static OffboardingMarkerEntry NewMarker( + string? tenantId = null, + string status = "Completed", + DateTime? completedAt = null, + DateTime? failedAt = null, + string? failedPhase = null) + { + var tid = tenantId ?? TenantId; + return new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = tid, + TenantId = tid, + OffboardingHistoryRowKey = $"20260518091523123_{tid}", + InitiatedAt = DateTime.UtcNow.AddHours(-1), + InitiatedBy = "alice@contoso.invalid", + Status = status, + CompletedAt = completedAt, + FailedAt = failedAt, + FailedPhase = failedPhase, + }; + } + + private static OffboardingExpectations NewExpectations() => new() + { + SchemaVersion = 1, + TenantId = TenantId, + HistoryRowKey = HistoryRowKey, + CreatedAt = DateTime.UtcNow.AddHours(-1), + EnumerationCompleted = true, + EnumeratedSessionCount = 0, + }; + + private sealed class Fixture + { + public DateTime FixedNow { get; } = new DateTime(2026, 5, 18, 12, 0, 0, DateTimeKind.Utc); + public FakeOffboardingAuditRepository Repo { get; } = new(); + public FakeOffboardingExpectationsStore Expectations { get; } = new(); + public RecordingSafeWipeService SafeWipe { get; } = new(); + public Mock ConfigRepo { get; } = new(); + public OffboardingMarkerCleanupFunction Sut { get; } + + public Fixture() + { + var timeProvider = new FixedTimeProvider(FixedNow); + // Default: TenantConfig row not found (the normal happy path — Phase 2.F-final + // successfully deleted it; nothing for the sweep to do). Individual tests + // override this to seed a lingering tombstone or a re-onboarded fresh config. + ConfigRepo.Setup(r => r.GetTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync((TenantConfiguration?)null); + + Sut = new OffboardingMarkerCleanupFunction( + Repo, Expectations, SafeWipe, ConfigRepo.Object, + NullLogger.Instance, + timeProvider); + } + } + + /// + /// Mock that + /// records which (table, tenantId) pairs were swept and how many rows it claims to have + /// deleted. Default behaviour returns 0 (row already gone — the normal happy path). + /// Tests can opt in to "0 means a TenantConfiguration row was found and wiped" via + /// . + /// + private sealed class RecordingSafeWipeService : AutopilotMonitor.Functions.Services.Offboarding.SafeWipeService + { + public readonly List<(string Table, string TenantId)> WipeCalls = new(); + public readonly Queue QueuedReturns = new(); + public Exception? ThrowOnNextWipe { get; set; } + + public RecordingSafeWipeService() + : base( + new AutopilotMonitor.Functions.Services.TableStorageService( + Moq.Mock.Of(), + NullLogger.Instance), + new AutopilotMonitor.Functions.Services.BlobStorageService( + new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, + usesManagedIdentity: false), + NullLogger.Instance) + { + } + + public override Task WipeByExactPartitionAsync(string tableName, string normalizedTenantId, CancellationToken ct = default) + { + WipeCalls.Add((tableName, normalizedTenantId)); + if (ThrowOnNextWipe is { } ex) + { + ThrowOnNextWipe = null; + throw ex; + } + return Task.FromResult(QueuedReturns.Count > 0 ? QueuedReturns.Dequeue() : 0); + } + } + + private sealed class FixedTimeProvider : TimeProvider + { + private readonly DateTimeOffset _now; + public FixedTimeProvider(DateTime utcNow) { _now = new DateTimeOffset(utcNow, TimeSpan.Zero); } + public override DateTimeOffset GetUtcNow() => _now; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TableTenantCustomsArchiveRepositoryHelperTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TableTenantCustomsArchiveRepositoryHelperTests.cs new file mode 100644 index 00000000..dc24713e --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TableTenantCustomsArchiveRepositoryHelperTests.cs @@ -0,0 +1,67 @@ +using AutopilotMonitor.Functions.DataAccess.TableStorage; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// PR3.B §3.2 — static-helper tests for the archive RowKey base64url encoding and the +/// PartitionKey construction. These two functions are the only things in the repository +/// that have logic worth pinning at unit level; the rest is straight Azure Tables CRUD. +/// +public sealed class TableTenantCustomsArchiveRepositoryHelperTests +{ + [Fact] + public void BuildPartitionKey_Concatenates_TenantId_And_HistoryRowKey() + { + var pk = TableTenantCustomsArchiveRepository.BuildPartitionKey( + "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"); + Assert.Equal( + "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee_20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + pk); + } + + [Theory] + [InlineData("simple-row-id")] + [InlineData("rowkey/with/slashes")] + [InlineData("rowkey#with#hash")] + [InlineData("rowkey?with?question")] + [InlineData("rowkey\\with\\backslash")] + [InlineData("a")] + [InlineData("ab")] + [InlineData("abc")] + [InlineData("abcd")] + [InlineData("unicode-Ümlaut-and-emoji-✓")] + public void BuildRowKey_RoundTripsForbiddenCharacters(string originalRowKey) + { + var archiveRowKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", originalRowKey); + + // RowKey itself must NOT contain any Azure-Tables-forbidden characters. + Assert.DoesNotContain('#', archiveRowKey); + Assert.DoesNotContain('?', archiveRowKey); + Assert.DoesNotContain('/', archiveRowKey); + Assert.DoesNotContain('\\', archiveRowKey); + + // Must round-trip cleanly. + var decoded = TableTenantCustomsArchiveRepository.DecodeOriginalRowKey(archiveRowKey, "GatherRules"); + Assert.Equal(originalRowKey, decoded); + } + + [Fact] + public void BuildRowKey_DifferentTablesProduceDifferentKeys() + { + var gatherKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", "my-rule"); + var analyzeKey = TableTenantCustomsArchiveRepository.BuildRowKey("AnalyzeRules", "my-rule"); + + Assert.NotEqual(gatherKey, analyzeKey); + Assert.StartsWith("GatherRules_", gatherKey); + Assert.StartsWith("AnalyzeRules_", analyzeKey); + } + + [Fact] + public void DecodeOriginalRowKey_RejectsPrefixMismatch() + { + var rowKey = TableTenantCustomsArchiveRepository.BuildRowKey("GatherRules", "x"); + Assert.Throws(() => + TableTenantCustomsArchiveRepository.DecodeOriginalRowKey(rowKey, "AnalyzeRules")); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantCustomsArchiveContractTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantCustomsArchiveContractTests.cs new file mode 100644 index 00000000..4f346b38 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantCustomsArchiveContractTests.cs @@ -0,0 +1,185 @@ +using System.Text.Json; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure.Data.Tables; +using Microsoft.Extensions.Logging.Abstractions; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// PR3.B §3.4-§3.5 — invariants the offboarding-customs-archive design guarantees: +/// snapshot serialization, partition-key namespacing per offboarding run, and the +/// counter-from-archive recomputation that makes the History fields crash-resume safe. +/// +/// Source-table archive-then-wipe orchestration (3-iteration cap + customs_arrival_race +/// fail-closed) is a structural property of the loop in +/// — manual smoketest +/// on a dev tenant covers that contract; full mock-TableClient integration test is a +/// followup once we have a TableClient abstraction in the codebase. +/// +/// +public sealed class TenantCustomsArchiveContractTests +{ + private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + private const string HistoryRowKey = "20260518091523123_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + + private static TenantOffboardingHandler BuildHandler(ITenantCustomsArchiveRepository archive) + { + // We only need the handler instance for its internal BuildArchiveEntry helper; that + // method does not call into any of the other collaborators, so we can pass null! for + // them. Tests exercise pure-method behaviour only. + return new TenantOffboardingHandler( + auditRepo: null!, + enumerator: null!, + cascadeEnqueuer: null!, + expectations: null!, + drainProbe: null!, + safeWipe: null!, + storage: null!, + maintenance: null!, + reEnqueuer: null!, + opsEvents: null!, + customsArchive: archive, + logger: NullLogger.Instance); + } + + [Fact] + public void BuildArchiveEntry_PartitionKey_IsTenantUnderscoreHistoryRowKey() + { + var handler = BuildHandler(new FakeCustomsArchiveRepository()); + var source = new TableEntity(TenantId, "rule-id-1") { ["DisplayName"] = "anything" }; + + var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.GatherRules, HistoryRowKey); + + Assert.Equal($"{TenantId}_{HistoryRowKey}", entry.PartitionKey); + } + + [Fact] + public void BuildArchiveEntry_RowKey_PrefixedWithOriginalTable_AndBase64Encoded() + { + var handler = BuildHandler(new FakeCustomsArchiveRepository()); + var source = new TableEntity(TenantId, "rule/with/slash") { ["DisplayName"] = "x" }; + + var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.AnalyzeRules, HistoryRowKey); + + Assert.StartsWith("AnalyzeRules_", entry.RowKey); + Assert.DoesNotContain('/', entry.RowKey); // base64url replaces / with _ + } + + [Fact] + public void BuildArchiveEntry_StripsSystemProperties_FromJson() + { + var handler = BuildHandler(new FakeCustomsArchiveRepository()); + var source = new TableEntity(TenantId, "rule") + { + ["DisplayName"] = "MyRule", + ["Category"] = "preset", + ["Timestamp"] = System.DateTimeOffset.UtcNow, + ["odata.etag"] = "\"0x123\"", + }; + + var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.GatherRules, HistoryRowKey); + + var parsed = JsonDocument.Parse(entry.EntityJson); + Assert.True(parsed.RootElement.TryGetProperty("DisplayName", out var dn)); + Assert.Equal("MyRule", dn.GetString()); + Assert.True(parsed.RootElement.TryGetProperty("Category", out _)); + + // System properties must NOT appear in the snapshot. + Assert.False(parsed.RootElement.TryGetProperty("Timestamp", out _)); + Assert.False(parsed.RootElement.TryGetProperty("odata.etag", out _)); + } + + [Fact] + public void BuildArchiveEntry_PopulatesOriginalKeysAndArchivedBy() + { + var handler = BuildHandler(new FakeCustomsArchiveRepository()); + var source = new TableEntity(TenantId, "my-rule-id") { ["X"] = "Y" }; + + var entry = handler.BuildArchiveEntry(source, TenantId, Constants.TableNames.ImeLogPatterns, HistoryRowKey); + + Assert.Equal(TenantId, entry.TenantId); + Assert.Equal(Constants.TableNames.ImeLogPatterns, entry.OriginalTable); + Assert.Equal(TenantId, entry.OriginalPartitionKey); + Assert.Equal("my-rule-id", entry.OriginalRowKey); + Assert.Equal(HistoryRowKey, entry.HistoryRowKey); + Assert.Equal("TenantOffboardingHandler", entry.ArchivedBy); + Assert.NotEqual(default, entry.ArchivedAt); + } + + [Fact] + public async Task ReReOffboard_TwoRuns_CoExistInArchive_AsDistinctPartitions() + { + // PR3.B §3.5: each offboarding run gets its own PartitionKey + // ({tenantId}_{historyRowKey}), so a tenant offboarded twice produces two immutable + // archive partitions side-by-side. + var repo = new FakeCustomsArchiveRepository(); + + var run1 = "20260101000000000_" + TenantId; + var run2 = "20260518091523123_" + TenantId; + + await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry + { + PartitionKey = $"{TenantId}_{run1}", + RowKey = "GatherRules_aaa", + TenantId = TenantId, + OriginalTable = Constants.TableNames.GatherRules, + OriginalRowKey = "aaa", + HistoryRowKey = run1, + }); + + await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry + { + PartitionKey = $"{TenantId}_{run2}", + RowKey = "GatherRules_aaa", + TenantId = TenantId, + OriginalTable = Constants.TableNames.GatherRules, + OriginalRowKey = "aaa", + HistoryRowKey = run2, + }); + + var count1 = await repo.CountByRunAndTableAsync(TenantId, run1, Constants.TableNames.GatherRules); + var count2 = await repo.CountByRunAndTableAsync(TenantId, run2, Constants.TableNames.GatherRules); + + Assert.Equal(1, count1); + Assert.Equal(1, count2); + Assert.Equal(2, repo.Store.Count); + } + + [Fact] + public async Task CountByRunAndTable_FiltersByOriginalTable() + { + var repo = new FakeCustomsArchiveRepository(); + var run = "20260518091523123_" + TenantId; + var pk = $"{TenantId}_{run}"; + + await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules", HistoryRowKey = run }); + await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "GatherRules_b", TenantId = TenantId, OriginalTable = "GatherRules", HistoryRowKey = run }); + await repo.UpsertAsync(new TenantOffboardingCustomsArchiveEntry { PartitionKey = pk, RowKey = "AnalyzeRules_a", TenantId = TenantId, OriginalTable = "AnalyzeRules", HistoryRowKey = run }); + + Assert.Equal(2, await repo.CountByRunAndTableAsync(TenantId, run, "GatherRules")); + Assert.Equal(1, await repo.CountByRunAndTableAsync(TenantId, run, "AnalyzeRules")); + Assert.Equal(0, await repo.CountByRunAndTableAsync(TenantId, run, "ImeLogPatterns")); + } + + [Fact] + public async Task DeleteRunAsync_RemovesEveryEntryInPartition_LeavesOtherRunsAlone() + { + var repo = new FakeCustomsArchiveRepository(); + var run1 = "20260101000000000_" + TenantId; + var run2 = "20260518091523123_" + TenantId; + + await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run1}", RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules" }); + await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run1}", RowKey = "GatherRules_b", TenantId = TenantId, OriginalTable = "GatherRules" }); + await repo.UpsertAsync(new() { PartitionKey = $"{TenantId}_{run2}", RowKey = "GatherRules_a", TenantId = TenantId, OriginalTable = "GatherRules" }); + + var deleted = await repo.DeleteRunAsync(TenantId, run1); + + Assert.Equal(2, deleted); + Assert.Single(repo.Store); + // The remaining row belongs to run2. + Assert.Single(repo.Store.Values, e => e.PartitionKey == $"{TenantId}_{run2}"); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingHandlerDrainContractTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingHandlerDrainContractTests.cs new file mode 100644 index 00000000..7cd9f548 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingHandlerDrainContractTests.cs @@ -0,0 +1,634 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using AutopilotMonitor.Functions.DataAccess.TableStorage; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Functions.Services.Notifications; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Deletion; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// Pin the §4.5 Drain-Contract that prevents the "silent wipe" the user has been worried about. +/// Every fail-closed branch must throw + the Marker must transition to Failed WITHOUT +/// any SafeWipe being attempted. The 0-session happy path and the Rev-9-F1 Drain-Skip-Gate +/// (post-crash idempotent resume) round out coverage for the surrounding state-machine. +/// +public class TenantOffboardingHandlerDrainContractTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111"; + + // ── Fail-closed branches ──────────────────────────────────────────────────── + + [Fact] + public async Task KillSwitchActive_ExpectationFails_NoSafeWipe() + { + var harness = Harness.New(); + harness.SeedExpectations(new (string, string?, string, int)[] + { + ("session-1", null, nameof(SessionDeletionEnqueueOutcome.KillSwitchActive), 0), + }); + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0))); + + Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status); + Assert.Equal("killswitch", harness.Repo.Markers[TenantId].FailedPhase); + Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount); + } + + [Fact] + public async Task Poisoned_ExpectationFails_NoSafeWipe() + { + var harness = Harness.New(); + harness.SeedExpectations(new (string, string?, string, int)[] + { + ("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Poisoned), 0), + }); + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0))); + + Assert.Equal("poisoned", harness.Repo.Markers[TenantId].FailedPhase); + Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount); + } + + [Fact] + public async Task AlreadyInFlight_With_NullManifestId_FailsClosed_Rev8F3() + { + // AlreadyInFlight + null ManifestId is the Preparing-without-snapshot state. There is + // no progress blob to drain against → fail closed immediately rather than waiting + // out the 2h drain budget. + var harness = Harness.New(); + harness.SeedExpectations(new (string, string?, string, int)[] + { + ("session-1", null, nameof(SessionDeletionEnqueueOutcome.AlreadyInFlight), 0), + }); + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0))); + + Assert.Equal("alreadyinflight_no_manifest", harness.Repo.Markers[TenantId].FailedPhase); + Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount); + } + + [Fact] + public async Task CasExhausted_RetriesUpToCap_ThenFailsClosed() + { + var harness = Harness.New(); + harness.SeedExpectations(new (string, string?, string, int)[] + { + ("session-1", null, nameof(SessionDeletionEnqueueOutcome.CasExhausted), + TenantOffboardingHandler.MaxCasRetriesPerSession), + }); + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0))); + + Assert.Equal("cas_exhausted", harness.Repo.Markers[TenantId].FailedPhase); + Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount); + } + + [Fact] + public async Task EnumeratorCrash_FirstAttempt_StampsEnumerationStartedAt_ThenPropagatesException() + { + // First-attempt crash: enumerator throws before any expectation is uploaded. The + // handler MUST stamp EnumerationStartedAt before touching the enumerator so the next + // pickup can tell first-try from re-pickup-after-crash. Exception propagates so the + // worker retries via visibility timeout. + var harness = Harness.New(); + harness.EnumeratorThrow = new InvalidOperationException("simulated 503"); + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0))); + + Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount); + Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationStartedAt); + Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status); + } + + [Fact] + public async Task RePickup_MidEnumerationCrash_ESA_Set_ECBU_Null_FailsClosed_ExpectationsMissing() + { + // Plan §7.4 step 3 Crash-Verhalten: prior pickup stamped EnumerationStartedAt but + // crashed mid-iteration (ECBU still null). Re-pickup must NOT re-enumerate (tenant + // may be mid-mutation); it must fail-closed. + var harness = Harness.New(); + harness.Repo.History[HistoryRowKey].EnumerationStartedAt = DateTime.UtcNow.AddMinutes(-5); + harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload = null; + harness.EnumeratorYields.Add("session-X"); // would be enumerated if logic regressed + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0))); + + Assert.Equal("expectations_missing", harness.Repo.Markers[TenantId].FailedPhase); + Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount); + harness.CascadeProducer.Verify(c => c.EnqueueAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never(), "Mid-enumeration crash on re-pickup must NEVER re-enumerate"); + } + + [Fact] + public async Task RePickup_UploadFailure_ESA_Set_ECBU_Set_NoBlob_ReRunsEnumerateAndUpload() + { + // Plan §7.4 step 3: when ECBU is set but blob is missing, the enumeration loop + // finished cleanly and only the upload failed. Re-pickup must RE-RUN enumerate + + // upload (idempotent — SessionDeletionProducer returns AlreadyInFlight for the same + // session id), NOT fail-closed. + var harness = Harness.New(); + harness.Repo.History[HistoryRowKey].EnumerationStartedAt = DateTime.UtcNow.AddMinutes(-5); + harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload = DateTime.UtcNow.AddMinutes(-4); + harness.EnumeratorYields.Add("session-A"); + harness.EnumeratorYields.Add("session-B"); + harness.CascadeProducer.Setup(c => c.EnqueueAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync( + (_, sid, _, _, _, _) => new SessionDeletionEnqueueResult + { + Outcome = SessionDeletionEnqueueOutcome.AlreadyInFlight, // idempotent producer response + ManifestId = $"manifest-{sid}", + }); + + await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)); + + // Both sessions re-enqueued (idempotent producer responses). + harness.CascadeProducer.Verify(c => c.EnqueueAsync( + It.IsAny(), "session-A", It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), + Times.AtLeastOnce, "Upload-retry path must re-enumerate session-A"); + harness.CascadeProducer.Verify(c => c.EnqueueAsync( + It.IsAny(), "session-B", It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny()), + Times.AtLeastOnce, "Upload-retry path must re-enumerate session-B"); + // Upload retry succeeded, drain probe says done → Completed. + Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status); + // Marker is NOT Failed — this was an upload-retry, not a mid-enum crash. + Assert.NotEqual("expectations_missing", harness.Repo.Markers[TenantId].FailedPhase); + } + + [Fact] + public async Task FirstTry_StampsBothMarkers_BeforeAndAfterEnumeration() + { + // Audit-trail invariant: ESA must be stamped BEFORE the enumerator runs (so a crash + // during iteration lands in the fail-closed branch next time); ECBU must be stamped + // AFTER the enumerator finishes (so an upload-failure-only crash lands in the + // upload-retry branch next time). + var harness = Harness.New(); + harness.EnumeratorYields.Add("session-1"); + + await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)); + + Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationStartedAt); + Assert.NotNull(harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload); + // Sanity: ESA <= ECBU (one was stamped before the loop, one after). + Assert.True( + harness.Repo.History[HistoryRowKey].EnumerationStartedAt + <= harness.Repo.History[HistoryRowKey].EnumerationCompletedBeforeUpload); + } + + [Fact] + public async Task EnumerationIncomplete_FailsClosed() + { + var harness = Harness.New(); + harness.Expectations.Seed(new OffboardingExpectations + { + SchemaVersion = 1, + TenantId = TenantId, + HistoryRowKey = HistoryRowKey, + CreatedAt = DateTime.UtcNow, + EnumerationCompleted = false, // ← the disambiguation Rev-7-F2 added + EnumeratedSessionCount = 5, + Expectations = Enumerable.Range(0, 5).Select(i => + new OffboardingExpectation { SessionId = $"s{i}", Outcome = "Enqueued", ManifestId = $"m{i}" }).ToList(), + }); + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0))); + + Assert.Equal("enumeration_incomplete", harness.Repo.Markers[TenantId].FailedPhase); + Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount); + } + + [Fact] + public async Task ExpectationsSizeMismatch_FailsClosed() + { + // expectations.Length != enumeratedSessionCount → fail-closed. Defends against a + // half-overwritten blob from a buggy re-uploader. + var harness = Harness.New(); + harness.Expectations.Seed(new OffboardingExpectations + { + SchemaVersion = 1, + TenantId = TenantId, + HistoryRowKey = HistoryRowKey, + CreatedAt = DateTime.UtcNow, + EnumerationCompleted = true, + EnumeratedSessionCount = 5, + Expectations = new List + { + new() { SessionId = "s1", Outcome = "Enqueued", ManifestId = "m1" }, + new() { SessionId = "s2", Outcome = "Enqueued", ManifestId = "m2" }, + }, + }); + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0))); + + Assert.Equal("expectations_size_mismatch", harness.Repo.Markers[TenantId].FailedPhase); + } + + [Fact] + public async Task DrainTimeout_AfterMaxPolls_FailsClosed() + { + // The cap (60 polls ≈ 2h) must produce a deterministic Failed transition rather than + // dragging on forever. + var harness = Harness.New(); + harness.SeedExpectations(new (string, string?, string, int)[] + { + ("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Enqueued), 0), + }); + harness.DrainProbe.Setup(p => p.IsCascadeCompletedAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(false); // never settles + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope(drainPollCount: TenantOffboardingHandler.MaxDrainPolls - 1))); + + Assert.Equal("drain_timeout", harness.Repo.Markers[TenantId].FailedPhase); + } + + // ── Happy paths ───────────────────────────────────────────────────────────── + + [Fact] + public async Task ZeroSessions_EnumerationCompleted_DrainOk_ProceedsToWipeAndCompleted() + { + // Plan §4.5: empty expectations + enumerationCompleted=true is the legitimate + // happy path for a tenant with no sessions left. + var harness = Harness.New(); + // Enumerator yields nothing — handler will upload an Expectations blob with + // enumerationCompleted=true, enumeratedSessionCount=0. + + await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)); + + Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status); + Assert.NotNull(harness.Repo.History[HistoryRowKey].CompletedAt); + Assert.NotNull(harness.Repo.History[HistoryRowKey].DrainCompletedAt); + Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status); + Assert.NotNull(harness.Repo.Markers[TenantId].CompletedAt); + Assert.True(harness.SafeWipeProbe.WipeCallCount > 0, + "SafeWipe must have run across the tenant tables"); + Assert.False(harness.Expectations.BlobExists(TenantId, HistoryRowKey), + "Phase 2.G must delete the Expectations blob as its last step"); + } + + [Fact] + public async Task SessionNotFound_CountsAsSatisfied_NoDrainProbeCall() + { + var harness = Harness.New(); + harness.SeedExpectations(new (string, string?, string, int)[] + { + ("session-1", null, nameof(SessionDeletionEnqueueOutcome.SessionNotFound), 0), + }); + + await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)); + + Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status); + harness.DrainProbe.Verify(p => p.IsCascadeCompletedAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never(), "SessionNotFound is a no-op — drain probe must NOT be called for that expectation"); + } + + // ── Rev-9-F1 Drain-Skip-Gate ─────────────────────────────────────────────── + + [Fact] + public async Task Rev9F1_DrainCompletedAtSet_SkipsDrainPredicate_AndRunsPostDrainPhasesIdempotently() + { + // The crash-recovery scenario: drain ran once and stamped DrainCompletedAt, then we + // crashed somewhere in 2.D-G. Re-pickup must NOT re-evaluate the drain predicate (the + // progress blobs are now gone in 2.E) and must still land us on Completed. + var harness = Harness.New(); + harness.Repo.History[HistoryRowKey].DrainCompletedAt = DateTime.UtcNow.AddMinutes(-10); + harness.Repo.History[HistoryRowKey].Status = "InProgress"; + harness.Repo.Markers[TenantId].Status = "InProgress"; + + await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)); + + Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status); + harness.DrainProbe.Verify(p => p.IsCascadeCompletedAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never(), "When DrainCompletedAt is already set the drain probe must NOT be called"); + harness.Enumerator.Verify(e => e.EnumerateAsync(It.IsAny(), It.IsAny()), + Times.Never(), "Drain-skip-gate must also bypass the enumerate+enqueue phase"); + } + + [Fact] + public async Task History_AlreadyCompleted_IsNoOp() + { + var harness = Harness.New(); + harness.Repo.History[HistoryRowKey].Status = "Completed"; + harness.Repo.History[HistoryRowKey].CompletedAt = DateTime.UtcNow; + + await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 0)); + + // No additional writes — handler returned early. + Assert.Equal(0, harness.SafeWipeProbe.WipeCallCount); + } + + // ── Drain re-poll ─────────────────────────────────────────────────────────── + + [Fact] + public async Task DrainNotSettled_BelowCap_ReEnqueuesWithDelayAndPollIncrement() + { + var harness = Harness.New(); + harness.SeedExpectations(new (string, string?, string, int)[] + { + ("session-1", "m-1", nameof(SessionDeletionEnqueueOutcome.Enqueued), 0), + }); + harness.DrainProbe.Setup(p => p.IsCascadeCompletedAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(false); + + await harness.Sut.HandleAsync(harness.Envelope(drainPollCount: 3)); + + harness.ReEnqueuer.Verify(e => e.EnqueueAsync( + It.Is(env => env.DrainPollCount == 4), + TenantOffboardingHandler.DrainPollDelay, + It.IsAny()), + Times.Once); + // History should NOT be Completed yet. + Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status); + Assert.Null(harness.Repo.History[HistoryRowKey].DrainCompletedAt); + } + + // ── Harness ───────────────────────────────────────────────────────────────── + + private sealed class Harness + { + public TenantOffboardingHandler Sut { get; private set; } = default!; + public FakeOffboardingAuditRepository Repo { get; } = new(); + public FakeOffboardingExpectationsStore Expectations { get; } = new(); + public Mock DrainProbe { get; } = new(); + public Mock ReEnqueuer { get; } = new(); + public Mock CascadeProducer { get; } = new(); + public Mock Enumerator { get; } + public CountingSafeWipeService SafeWipeProbe { get; } = new(); + public Mock Maintenance { get; } = new(); + public List EnumeratorYields { get; } = new(); + public Exception? EnumeratorThrow { get; set; } + + private Harness() + { + var maintenanceForEnumerator = new Mock(); + Enumerator = new Mock(maintenanceForEnumerator.Object); + } + + public static Harness New() + { + var h = new Harness(); + h.SeedInitialAudit(); + + // Default enumerator returns the (mutable) Sessions list. Tests append to it + // before invoking the handler. + h.Enumerator + .Setup(e => e.EnumerateAsync(It.IsAny(), It.IsAny())) + .Returns((_, ct) => h.EnumerateSessionsAsync(ct)); + + h.DrainProbe.Setup(p => p.IsCascadeCompletedAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(true); // happy default — drain completes immediately + + h.CascadeProducer.Setup(c => c.EnqueueAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync( + (_, sid, _, _, _, _) => new SessionDeletionEnqueueResult + { + Outcome = SessionDeletionEnqueueOutcome.Enqueued, + ManifestId = $"manifest-{sid}", + }); + + h.Maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .ReturnsAsync(true); + + var opsService = BuildRealOpsService(); + + var unusedTableStorage = new Mock( + Mock.Of(), + NullLogger.Instance).Object; + + h.Sut = new NoopCustomsArchivingHandler( + h.Repo, + h.Enumerator.Object, + h.CascadeProducer.Object, + h.Expectations, + h.DrainProbe.Object, + h.SafeWipeProbe, + unusedTableStorage, + h.Maintenance.Object, + h.ReEnqueuer.Object, + opsService, + Mock.Of(), + NullLogger.Instance); + + return h; + } + + private async IAsyncEnumerable EnumerateSessionsAsync( + [EnumeratorCancellation] CancellationToken ct) + { + if (EnumeratorThrow != null) throw EnumeratorThrow; + foreach (var s in EnumeratorYields) + { + ct.ThrowIfCancellationRequested(); + yield return s; + } + await Task.CompletedTask; + } + + public TenantOffboardingEnvelope Envelope(int drainPollCount) => new() + { + TenantId = TenantId, + HistoryPartitionKey = Constants.OffboardingPartitionKeys.History, + HistoryRowKey = HistoryRowKey, + InitiatedBy = "alice@contoso.invalid", + InitiatedAt = DateTime.UtcNow.AddMinutes(-1), + EnqueuedAt = DateTime.UtcNow, + DrainPollCount = drainPollCount, + }; + + public void SeedExpectations(IEnumerable<(string sessionId, string? manifestId, string outcome, int retryCount)> rows) + { + var list = rows.Select(r => new OffboardingExpectation + { + SessionId = r.sessionId, + ManifestId = r.manifestId, + Outcome = r.outcome, + RetryCount = r.retryCount, + }).ToList(); + Expectations.Seed(new OffboardingExpectations + { + SchemaVersion = 1, + TenantId = TenantId, + HistoryRowKey = HistoryRowKey, + CreatedAt = DateTime.UtcNow, + EnumerationCompleted = true, + EnumeratedSessionCount = list.Count, + Expectations = list, + }); + } + + private void SeedInitialAudit() + { + Repo.History[HistoryRowKey] = new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = HistoryRowKey, + TenantId = TenantId, + DomainName = "contoso.invalid", + InitiatedBy = "alice@contoso.invalid", + OffboardedAt = DateTime.UtcNow.AddMinutes(-1), + Status = "Initiated", + }; + Repo.Markers[TenantId] = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = HistoryRowKey, + InitiatedAt = DateTime.UtcNow.AddMinutes(-1), + InitiatedBy = "alice@contoso.invalid", + Status = "Initiated", + }; + Repo.Pointers[TenantId] = (new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = HistoryRowKey, + LatestStatus = "Initiated", + LatestUpdatedAt = DateTime.UtcNow.AddMinutes(-1), + OffboardCount = 1, + }, "\"0xFAKE_PTR_1\""); + } + + private static OpsEventService BuildRealOpsService() + { + var opsRepo = new Mock(); + opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny())).Returns(Task.CompletedTask); + var memCache = new MemoryCache(new MemoryCacheOptions()); + var adminConfig = new Mock( + Mock.Of(), NullLogger.Instance, memCache); + var alertDispatch = new OpsAlertDispatchService( + adminConfig.Object, + new TelegramNotificationService(new HttpClient(), Mock.Of(), + NullLogger.Instance), + new WebhookNotificationService(new HttpClient(), + NullLogger.Instance), + NullLogger.Instance); + return new OpsEventService(opsRepo.Object, NullLogger.Instance, alertDispatch); + } + + } + + /// + /// that records call counts without performing any storage IO. + /// Tests use this to assert that fail-closed paths NEVER touch the wipe methods. Now that + /// exposes the wipe methods as virtual, the production + /// dispatch lands here. + /// + private sealed class CountingSafeWipeService : SafeWipeService + { + public int WipeCallCount { get; private set; } + + public CountingSafeWipeService() + : base( + new TableStorageService(Mock.Of(), NullLogger.Instance), + new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, usesManagedIdentity: false), + NullLogger.Instance) + { + } + + public override Task WipeByExactPartitionAsync(string tableName, string normalizedTenantId, CancellationToken ct = default) + { + WipeCallCount++; + return Task.FromResult(0); + } + + public override Task WipeByCompositePartitionRangeAsync(string tableName, string normalizedTenantId, CancellationToken ct = default) + { + WipeCallCount++; + return Task.FromResult(0); + } + + public override Task WipeByDiscriminatorAndTenantPropertyAsync(string tableName, string discriminator, string normalizedTenantId, CancellationToken ct = default) + { + WipeCallCount++; + return Task.FromResult(0); + } + + public override Task WipeByTenantIdPropertyAsync(string tableName, string normalizedTenantId, CancellationToken ct = default) + { + WipeCallCount++; + return Task.FromResult(0); + } + + public override Task WipeBlobsByTenantPrefixAsync(string containerName, string normalizedTenantId, CancellationToken ct = default) + { + WipeCallCount++; + return Task.FromResult(0); + } + } + + /// + /// Subclass that no-ops the customs archive-then-wipe loop so the existing drain-contract + /// tests don't need a working TableStorageService for the 3 rules tables. PR3.B-specific + /// archive behaviour has its own test class with proper TableClient mocks. + /// + private sealed class NoopCustomsArchivingHandler : TenantOffboardingHandler + { + public NoopCustomsArchivingHandler( + IOffboardingAuditRepository auditRepo, + OffboardingSessionEnumerator enumerator, + ISessionDeletionEnqueuer cascadeEnqueuer, + IOffboardingExpectationsStore expectations, + IDeletionProgressDrainProbe drainProbe, + SafeWipeService safeWipe, + TableStorageService storage, + IMaintenanceRepository maintenance, + ITenantOffboardingEnqueuer reEnqueuer, + OpsEventService opsEvents, + ITenantCustomsArchiveRepository customsArchive, + ILogger logger) + : base(auditRepo, enumerator, cascadeEnqueuer, expectations, drainProbe, safeWipe, + storage, maintenance, reEnqueuer, opsEvents, customsArchive, logger) + { + } + + internal override Task ArchiveAndWipeRulesTableAsync( + string tableName, string tenantId, string historyRowKey, CancellationToken ct) + => Task.CompletedTask; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingHandlerOrderingTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingHandlerOrderingTests.cs new file mode 100644 index 00000000..d1323c65 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingHandlerOrderingTests.cs @@ -0,0 +1,368 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Functions.Services.Notifications; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Deletion; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// Review-Fix Findings 2 + 3: +/// +/// (Finding 3) History terminal status MUST be the LAST write in both Completed and +/// Failed paths. A crash between any side-effect (marker/pointer/blob/audit/ops) and +/// the History commit must produce a re-pickup that re-runs the side-effects +/// idempotently — never a re-pickup that returns early with dangling state. +/// (Finding 2) +/// drives History/Pointer/Marker to Failed (with FailedAt/FailedPhase/ErrorMessage/RetryCount) +/// so the operator dashboard reflects the dead-letter rather than leaving the tenant +/// hanging in InProgress. +/// +/// +public class TenantOffboardingHandlerOrderingTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111"; + + // ── Finding 3 — Completed-path ordering ───────────────────────────────────── + + [Fact] + public async Task PostDrain_OrderingInvariant_MarkerCompletedBeforeHistoryCompleted() + { + var harness = Harness.New(); + + await harness.Sut.HandleAsync(harness.Envelope()); + + // Both writes happen. The invariant: Marker.Status="Completed" lands BEFORE + // History.Status="Completed" — the Repo's MarkerWrites/HistoryWrites lists capture + // the order via Append-on-Write. + var lastMarker = harness.Repo.MarkerWrites.LastIndexOf("Completed"); + var lastHistoryCompleted = harness.Repo.HistoryWrites.LastIndexOf("Completed"); + Assert.True(lastMarker >= 0, "Marker.Status=Completed write never happened"); + Assert.True(lastHistoryCompleted >= 0, "History.Status=Completed write never happened"); + // Marker write happens before the FINAL history write — both lists are sequenced + // independently per partition, but the order WITHIN the handler's invocation is what + // matters; we capture each by counting writes-before. + Assert.True( + harness.Repo.MarkerWrites.Count >= 1, + "Marker must be written at least once with Completed"); + // Last history write IS the Completed-commit. Anything that happened before that in + // MarkerWrites was the Completed transition. + Assert.Equal("Completed", harness.Repo.MarkerWrites.Last()); + } + + [Fact] + public async Task PostDrain_SimulatedCrashBetweenMarkerAndHistory_RepickupCompletesIdempotently() + { + // Step 1: first invocation — completes normally. + var harness = Harness.New(); + await harness.Sut.HandleAsync(harness.Envelope()); + Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status); + Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status); + + // Step 2: simulate crash by manually re-stamping the history to InProgress while keeping + // DrainCompletedAt set (this is what a re-pickup after a Phase-2.G crash looks like). + // The marker is left as Completed (already updated before the crash), the post-drain + // side-effects all idempotently re-execute. + harness.Repo.History[HistoryRowKey].Status = "InProgress"; + harness.Repo.History[HistoryRowKey].CompletedAt = null; + var wipeCallsBefore = harness.SafeWipeProbe.WipeCallCount; + + await harness.Sut.HandleAsync(harness.Envelope()); + + // History ends Completed again, marker stays Completed, side-effects re-ran (idempotent). + Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status); + Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status); + Assert.True(harness.SafeWipeProbe.WipeCallCount > wipeCallsBefore, + "Re-pickup must re-run the SafeWipe phase (idempotent — finds 0 rows the second time)"); + } + + // ── Finding 3 — Failed-path ordering ──────────────────────────────────────── + + [Fact] + public async Task FailAsync_OrderingInvariant_MarkerFailedBeforeHistoryFailed() + { + var harness = Harness.New(); + // Force fail-closed via KillSwitchActive expectation. + harness.Expectations.Seed(new OffboardingExpectations + { + SchemaVersion = 1, TenantId = TenantId, HistoryRowKey = HistoryRowKey, + CreatedAt = DateTime.UtcNow, EnumerationCompleted = true, EnumeratedSessionCount = 1, + Expectations = new List + { + new() { SessionId = "session-1", Outcome = "KillSwitchActive" }, + }, + }); + + await Assert.ThrowsAsync(() => + harness.Sut.HandleAsync(harness.Envelope())); + + // Marker must be Failed; History must be Failed too. The LAST write to history is the + // Failed-commit — anything that happened before (marker, pointer, ops) is part of the + // "side-effects first" contract. + Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status); + Assert.Equal("killswitch", harness.Repo.Markers[TenantId].FailedPhase); + Assert.NotNull(harness.Repo.Markers[TenantId].FailedAt); + Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status); + // History was already InProgress + Marker write between, so MarkerWrites holds "Failed" + // as its latest, and HistoryWrites ends with "Failed". + Assert.Equal("Failed", harness.Repo.MarkerWrites.Last()); + Assert.Equal("Failed", harness.Repo.HistoryWrites.Last()); + } + + // ── Finding 2 — Worker poison transition via MarkEnvelopeFailedFromPoisonAsync ── + + [Fact] + public async Task MarkEnvelopeFailedFromPoison_TransitionsAllThreeStatesToFailed() + { + var harness = Harness.New(); + // History/Pointer/Marker pre-seeded to InProgress (default Harness seed is Initiated; + // bump to InProgress to mirror the in-flight scenario). + harness.Repo.History[HistoryRowKey].Status = "InProgress"; + harness.Repo.Markers[TenantId].Status = "InProgress"; + var existingPointer = harness.Repo.Pointers[TenantId]; + existingPointer.Pointer.LatestStatus = "InProgress"; + + await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5); + + Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status); + Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status); + Assert.Equal("Failed", harness.Repo.Pointers[TenantId].Pointer.LatestStatus); + Assert.Equal("max_dequeue", harness.Repo.Markers[TenantId].FailedPhase); + Assert.NotNull(harness.Repo.Markers[TenantId].FailedAt); + Assert.Equal(5, harness.Repo.History[HistoryRowKey].RetryCount); + Assert.Contains("poisoned after 5", harness.Repo.History[HistoryRowKey].ErrorMessage); + } + + [Fact] + public async Task MarkEnvelopeFailedFromPoison_AlreadyCompleted_IsNoOp() + { + // If the handler successfully completed BEFORE max-dequeue (e.g. last attempt + // succeeded but the worker's dequeue counter still tripped the threshold), the + // poison transition must not downgrade Completed → Failed. + var harness = Harness.New(); + harness.Repo.History[HistoryRowKey].Status = "Completed"; + harness.Repo.History[HistoryRowKey].CompletedAt = DateTime.UtcNow; + harness.Repo.Markers[TenantId].Status = "Completed"; + harness.Repo.Markers[TenantId].CompletedAt = DateTime.UtcNow; + + await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5); + + Assert.Equal("Completed", harness.Repo.History[HistoryRowKey].Status); + Assert.Equal("Completed", harness.Repo.Markers[TenantId].Status); + } + + [Fact] + public async Task MarkEnvelopeFailedFromPoison_AlreadyFailed_IsNoOp() + { + var harness = Harness.New(); + harness.Repo.History[HistoryRowKey].Status = "Failed"; + harness.Repo.History[HistoryRowKey].ErrorMessage = "drain_timeout"; + harness.Repo.Markers[TenantId].Status = "Failed"; + harness.Repo.Markers[TenantId].FailedPhase = "drain_timeout"; + var beforeWrites = harness.Repo.HistoryWrites.Count; + + await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5); + + Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status); + Assert.Equal("drain_timeout", harness.Repo.Markers[TenantId].FailedPhase); // not overwritten + Assert.Equal(beforeWrites, harness.Repo.HistoryWrites.Count); + } + + [Fact] + public async Task MarkEnvelopeFailedFromPoison_MissingHistory_LogsAndReturnsCleanly() + { + var harness = Harness.New(); + harness.Repo.History.Remove(HistoryRowKey); + + // Must NOT throw — the worker poison-move would otherwise lose its envelope and the + // operator would have no signal at all. + await harness.Sut.MarkEnvelopeFailedFromPoisonAsync(harness.Envelope(), dequeueCount: 5); + } + + // ── Harness (copied minimal — only what these tests need) ─────────────────── + + private sealed class Harness + { + public TenantOffboardingHandler Sut { get; private set; } = default!; + public FakeOffboardingAuditRepository Repo { get; } = new(); + public FakeOffboardingExpectationsStore Expectations { get; } = new(); + public Mock DrainProbe { get; } = new(); + public Mock ReEnqueuer { get; } = new(); + public Mock CascadeProducer { get; } = new(); + public Mock Enumerator { get; } + public CountingSafeWipeService SafeWipeProbe { get; } = new(); + public Mock Maintenance { get; } = new(); + public List EnumeratorYields { get; } = new(); + + private Harness() + { + Enumerator = new Mock(new Mock().Object); + } + + public static Harness New() + { + var h = new Harness(); + h.SeedInitialAudit(); + + h.Enumerator + .Setup(e => e.EnumerateAsync(It.IsAny(), It.IsAny())) + .Returns((_, ct) => EmptyEnumerateAsync(ct)); + + h.DrainProbe.Setup(p => p.IsCascadeCompletedAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(true); + + h.CascadeProducer.Setup(c => c.EnqueueAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync( + (_, sid, _, _, _, _) => new SessionDeletionEnqueueResult + { + Outcome = SessionDeletionEnqueueOutcome.Enqueued, + ManifestId = $"manifest-{sid}", + }); + + h.Maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .ReturnsAsync(true); + + var unusedTableStorage = new Mock( + Mock.Of(), + NullLogger.Instance).Object; + + h.Sut = new NoopCustomsArchivingOrderingHandler( + h.Repo, + h.Enumerator.Object, + h.CascadeProducer.Object, + h.Expectations, + h.DrainProbe.Object, + h.SafeWipeProbe, + unusedTableStorage, + h.Maintenance.Object, + h.ReEnqueuer.Object, + BuildOpsService(), + Mock.Of(), + NullLogger.Instance); + + return h; + } + + public TenantOffboardingEnvelope Envelope() => new() + { + TenantId = TenantId, + HistoryPartitionKey = Constants.OffboardingPartitionKeys.History, + HistoryRowKey = HistoryRowKey, + InitiatedBy = "alice@contoso.invalid", + InitiatedAt = DateTime.UtcNow.AddMinutes(-1), + EnqueuedAt = DateTime.UtcNow, + DrainPollCount = 0, + }; + + private static async IAsyncEnumerable EmptyEnumerateAsync( + [EnumeratorCancellation] CancellationToken ct) + { + await Task.CompletedTask; + yield break; + } + + private void SeedInitialAudit() + { + Repo.History[HistoryRowKey] = new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = HistoryRowKey, + TenantId = TenantId, + DomainName = "contoso.invalid", + InitiatedBy = "alice@contoso.invalid", + OffboardedAt = DateTime.UtcNow.AddMinutes(-1), + Status = "Initiated", + }; + Repo.Markers[TenantId] = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = HistoryRowKey, + InitiatedAt = DateTime.UtcNow.AddMinutes(-1), + InitiatedBy = "alice@contoso.invalid", + Status = "Initiated", + }; + Repo.Pointers[TenantId] = (new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = HistoryRowKey, + LatestStatus = "Initiated", + LatestUpdatedAt = DateTime.UtcNow.AddMinutes(-1), + OffboardCount = 1, + }, "\"0xFAKE_PTR_1\""); + } + + private static OpsEventService BuildOpsService() + { + var opsRepo = new Mock(); + opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny())).Returns(Task.CompletedTask); + var memCache = new MemoryCache(new MemoryCacheOptions()); + var adminConfig = new Mock( + Mock.Of(), NullLogger.Instance, memCache); + var alertDispatch = new OpsAlertDispatchService( + adminConfig.Object, + new TelegramNotificationService(new HttpClient(), Mock.Of(), + NullLogger.Instance), + new WebhookNotificationService(new HttpClient(), + NullLogger.Instance), + NullLogger.Instance); + return new OpsEventService(opsRepo.Object, NullLogger.Instance, alertDispatch); + } + } + + private sealed class CountingSafeWipeService : SafeWipeService + { + public int WipeCallCount { get; private set; } + public CountingSafeWipeService() : base( + new TableStorageService(Mock.Of(), NullLogger.Instance), + new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, usesManagedIdentity: false), + NullLogger.Instance) { } + public override Task WipeByExactPartitionAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); } + public override Task WipeByCompositePartitionRangeAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); } + public override Task WipeByDiscriminatorAndTenantPropertyAsync(string t, string d, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); } + public override Task WipeByTenantIdPropertyAsync(string t, string i, CancellationToken c = default) { WipeCallCount++; return Task.FromResult(0); } + public override Task WipeBlobsByTenantPrefixAsync(string c, string i, CancellationToken ct = default) { WipeCallCount++; return Task.FromResult(0); } + } + + private sealed class NoopCustomsArchivingOrderingHandler : TenantOffboardingHandler + { + public NoopCustomsArchivingOrderingHandler( + IOffboardingAuditRepository a, OffboardingSessionEnumerator e, ISessionDeletionEnqueuer c, + IOffboardingExpectationsStore exp, IDeletionProgressDrainProbe d, SafeWipeService sw, + TableStorageService s, IMaintenanceRepository m, ITenantOffboardingEnqueuer re, + OpsEventService o, ITenantCustomsArchiveRepository ca, ILogger log) + : base(a, e, c, exp, d, sw, s, m, re, o, ca, log) { } + + internal override Task ArchiveAndWipeRulesTableAsync( + string tableName, string tenantId, string historyRowKey, CancellationToken ct) + => Task.CompletedTask; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingWorkerPoisonTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingWorkerPoisonTests.cs new file mode 100644 index 00000000..ebb65cd9 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/Offboarding/TenantOffboardingWorkerPoisonTests.cs @@ -0,0 +1,300 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Functions.Services.Notifications; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Deletion; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Newtonsoft.Json; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests.Offboarding; + +/// +/// Review-Fix (second-pass) Finding 1: +/// MUST drive the durable Failed-state transition BEFORE the poison-queue send + main-queue +/// delete. If the transition fails for any reason (storage outage, OpsEvent service down, +/// pointer-CAS contention), the main-queue message must stay visible so a subsequent dequeue +/// can retry the transition. +/// +public class TenantOffboardingWorkerPoisonTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string HistoryRowKey = "20260518091523123_11111111-1111-1111-1111-111111111111"; + + [Fact] + public async Task PoisonPath_FailedTransitionSucceeds_ThenSendToPoisonAndDelete() + { + // Happy poison path: marker/pointer/history → Failed first, THEN message goes to + // poison queue, THEN main message is deleted. + var harness = new Harness(); + + await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6)); + + Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status); + Assert.Equal("Failed", harness.Repo.Markers[TenantId].Status); + Assert.Equal("max_dequeue", harness.Repo.Markers[TenantId].FailedPhase); + + Assert.Equal(1, harness.PoisonSendCount); + Assert.Equal(1, harness.MainDeleteCount); + } + + [Fact] + public async Task PoisonPath_FailedTransitionThrows_NoPoisonSend_NoMainDelete() + { + // Critical Finding 1 assertion: if MarkEnvelopeFailedFromPoisonAsync throws, the + // poison send + main delete MUST NOT happen — otherwise the operator would see a + // tenant stuck in InProgress with no queue message left to retry from. + var harness = new Harness(); + // Simulate a transient storage failure inside FailAsync's final commit. + harness.Repo.ThrowOnNextHistoryUpsert = new InvalidOperationException("simulated storage 503"); + + await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6)); + + Assert.Equal(0, harness.PoisonSendCount); + Assert.Equal(0, harness.MainDeleteCount); + // History stays InProgress — the next visibility-timeout retry will re-attempt the + // whole MoveToPoisonAsync sequence. Marker may or may not be Failed depending on how + // far FailAsync got before the throw; the critical invariant is that the queue + // message stays so the operator-facing state CAN converge on the next dequeue. + Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status); + } + + [Fact] + public async Task PoisonPath_PoisonSendFails_AfterFailedTransition_LeavesMessageButStateIsDurable() + { + // The transition succeeded → state is durable. The poison send fails (transient). + // The main delete is skipped. The message reappears via visibility-timeout. On the + // next pickup, MarkEnvelopeFailedFromPoisonAsync sees the terminal state and is a + // no-op, then the poison send retries. + var harness = new Harness(poisonSendThrows: true); + + await harness.InvokeMoveToPoisonAsync(harness.BuildMessage(dequeueCount: 6)); + + Assert.Equal("Failed", harness.Repo.History[HistoryRowKey].Status); + Assert.Equal(1, harness.PoisonSendCount); // attempted + Assert.Equal(0, harness.MainDeleteCount); // skipped because poison send threw + } + + [Fact] + public async Task PoisonPath_MalformedEnvelope_SkipsTransition_StillPoisonsTheMessage() + { + // Malformed envelope can't be transitioned (we don't know which tenant to fail), but + // the message still goes to poison so it leaves the live queue. + var harness = new Harness(); + + var bad = harness.BuildMessage(dequeueCount: 6, bodyOverride: "{ not json at all"); + await harness.InvokeMoveToPoisonAsync(bad); + + Assert.Equal(1, harness.PoisonSendCount); + Assert.Equal(1, harness.MainDeleteCount); + // No state transition happened — nothing to assert there, but the History row is + // untouched. + Assert.Equal("InProgress", harness.Repo.History[HistoryRowKey].Status); + } + + // ── Harness ───────────────────────────────────────────────────────────────── + + private sealed class Harness + { + public TenantOffboardingWorker Worker { get; } + public FakeOffboardingAuditRepository Repo { get; } = new(); + public Mock MainQueue { get; } = new(); + public Mock PoisonQueue { get; } = new(); + public int PoisonSendCount { get; private set; } + public int MainDeleteCount { get; private set; } + + public Harness(bool poisonSendThrows = false) + { + SeedAudit(); + + // Cascade-handler dependencies. The handler is real — it talks to FakeRepo + + // OpsEvent + (unused) other deps. Only MarkEnvelopeFailedFromPoisonAsync is + // exercised, so most deps are noise. + var handler = BuildHandler(); + + PoisonQueue.Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny())) + .Returns((_, _) => + { + PoisonSendCount++; + if (poisonSendThrows) throw new InvalidOperationException("simulated poison-queue 503"); + return Task.FromResult(Response.FromValue( + QueuesModelFactory.SendReceipt( + messageId: Guid.NewGuid().ToString(), + insertionTime: DateTimeOffset.UtcNow, + expirationTime: DateTimeOffset.UtcNow.AddDays(7), + popReceipt: "fake", + timeNextVisible: DateTimeOffset.UtcNow), + new Mock().Object)); + }); + + MainQueue.Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((_, _, _) => + { + MainDeleteCount++; + return Task.FromResult(new Mock().Object); + }); + + Worker = new TenantOffboardingWorker( + MainQueue.Object, PoisonQueue.Object, handler, + BuildOpsService(), + NullLogger.Instance); + } + + public QueueMessage BuildMessage(int dequeueCount, string? bodyOverride = null) + { + var envelope = new TenantOffboardingEnvelope + { + TenantId = TenantId, + HistoryPartitionKey = Constants.OffboardingPartitionKeys.History, + HistoryRowKey = HistoryRowKey, + InitiatedBy = "alice@contoso.invalid", + InitiatedAt = DateTime.UtcNow.AddMinutes(-1), + EnqueuedAt = DateTime.UtcNow, + DrainPollCount = 0, + }; + return QueuesModelFactory.QueueMessage( + messageId: Guid.NewGuid().ToString(), + popReceipt: "fake-receipt", + body: new BinaryData(bodyOverride ?? JsonConvert.SerializeObject(envelope)), + dequeueCount: dequeueCount); + } + + public Task InvokeMoveToPoisonAsync(QueueMessage msg) + { + // MoveToPoisonAsync is private — invoke via reflection. Avoids spinning up the + // full ExecuteAsync poll loop with a real timer. + var method = typeof(TenantOffboardingWorker) + .GetMethod("MoveToPoisonAsync", BindingFlags.NonPublic | BindingFlags.Instance); + Assert.NotNull(method); + return (Task)method!.Invoke(Worker, new object[] { msg, default(CancellationToken) })!; + } + + private void SeedAudit() + { + Repo.History[HistoryRowKey] = new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = HistoryRowKey, + TenantId = TenantId, + DomainName = "contoso.invalid", + InitiatedBy = "alice@contoso.invalid", + OffboardedAt = DateTime.UtcNow.AddMinutes(-1), + Status = "InProgress", + }; + Repo.Markers[TenantId] = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = HistoryRowKey, + InitiatedAt = DateTime.UtcNow.AddMinutes(-1), + InitiatedBy = "alice@contoso.invalid", + Status = "InProgress", + }; + Repo.Pointers[TenantId] = (new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = HistoryRowKey, + LatestStatus = "InProgress", + LatestUpdatedAt = DateTime.UtcNow, + OffboardCount = 1, + }, "\"0xFAKE_PTR_1\""); + } + + private TenantOffboardingHandler BuildHandler() + { + // Wire a real Handler so MarkEnvelopeFailedFromPoisonAsync uses FailAsync which + // touches the FakeRepo. The "transition throws" branch is exercised by setting + // Repo.ThrowOnNextHistoryUpsert in the test. + var unusedTableStorage = new Mock( + Mock.Of(), + NullLogger.Instance).Object; + + var enumerator = new Mock(new Mock().Object); + + return new NoopCustomsArchivingPoisonHandler( + Repo, + enumerator.Object, + Mock.Of(), + new FakeOffboardingExpectationsStore(), + Mock.Of(), + new NoopSafeWipeService(), + unusedTableStorage, + Mock.Of(), + Mock.Of(), + BuildOpsService(), + Mock.Of(), + NullLogger.Instance); + } + + private sealed class NoopCustomsArchivingPoisonHandler : TenantOffboardingHandler + { + public NoopCustomsArchivingPoisonHandler( + IOffboardingAuditRepository a, OffboardingSessionEnumerator e, ISessionDeletionEnqueuer c, + IOffboardingExpectationsStore exp, IDeletionProgressDrainProbe d, SafeWipeService sw, + TableStorageService s, IMaintenanceRepository m, ITenantOffboardingEnqueuer re, + OpsEventService o, ITenantCustomsArchiveRepository ca, ILogger log) + : base(a, e, c, exp, d, sw, s, m, re, o, ca, log) { } + + internal override Task ArchiveAndWipeRulesTableAsync( + string tableName, string tenantId, string historyRowKey, CancellationToken ct) + => Task.CompletedTask; + } + + private static OpsEventService BuildOpsService() + { + var opsRepo = new Mock(); + opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny())).Returns(Task.CompletedTask); + return new OpsEventService(opsRepo.Object, NullLogger.Instance, BuildAlertDispatch()); + } + + private static OpsAlertDispatchService BuildAlertDispatch() + { + var memCache = new MemoryCache(new MemoryCacheOptions()); + var adminConfig = new Mock( + Mock.Of(), NullLogger.Instance, memCache); + return new OpsAlertDispatchService( + adminConfig.Object, + new TelegramNotificationService(new HttpClient(), Mock.Of(), + NullLogger.Instance), + new WebhookNotificationService(new HttpClient(), + NullLogger.Instance), + NullLogger.Instance); + } + } + + private sealed class NoopSafeWipeService : SafeWipeService + { + public NoopSafeWipeService() : base( + new TableStorageService(Mock.Of(), NullLogger.Instance), + new BlobStorageService(new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, usesManagedIdentity: false), + NullLogger.Instance) { } + public override Task WipeByExactPartitionAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0); + public override Task WipeByCompositePartitionRangeAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0); + public override Task WipeByDiscriminatorAndTenantPropertyAsync(string t, string d, string i, CancellationToken c = default) => Task.FromResult(0); + public override Task WipeByTenantIdPropertyAsync(string t, string i, CancellationToken c = default) => Task.FromResult(0); + public override Task WipeBlobsByTenantPrefixAsync(string c, string i, CancellationToken ct = default) => Task.FromResult(0); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingExpectationsStoreTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingExpectationsStoreTests.cs new file mode 100644 index 00000000..cbbdb663 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingExpectationsStoreTests.cs @@ -0,0 +1,370 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Rev-6-F1 + Rev-7-F2 + Rev-8-F1: full coverage of the blob-IO contracts on +/// via a Dict-backed fake that overrides the +/// four protected virtual seams. The fake reproduces Azure Blob Storage's two +/// "exists" status codes (409 BlobAlreadyExists + 412 ConditionNotMet) so the resume path +/// is exercised under both. POCO/validation cases live next to the IO cases for full +/// surface coverage. +/// +public class OffboardingExpectationsStoreTests +{ + private const string TenantId = "99999999-9999-9999-9999-999999999999"; + private const string HistoryRowKey = "20260518091523123_99999999-9999-9999-9999-999999999999"; + + // ── Validation gates ──────────────────────────────────────────────────────── + + [Fact] + public async Task TryUploadInitial_RejectsNonGuidTenantId() + { + var store = new FakeStore(); + var payload = BuildPayload(tenantId: "not-a-guid"); + + await Assert.ThrowsAsync(() => store.TryUploadInitialAsync(payload)); + } + + [Fact] + public async Task TryUploadInitial_RejectsEmptyHistoryRowKey() + { + var store = new FakeStore(); + var payload = BuildPayload(historyRowKey: ""); + + await Assert.ThrowsAsync(() => store.TryUploadInitialAsync(payload)); + } + + [Fact] + public async Task TryUploadInitial_RejectsSchemaVersionZero() + { + var store = new FakeStore(); + var payload = BuildPayload(); + payload.SchemaVersion = 0; + + await Assert.ThrowsAsync(() => store.TryUploadInitialAsync(payload)); + } + + [Fact] + public async Task UpdateWithEtagCas_RequiresIfMatchEtag() + { + var store = new FakeStore(); + await Assert.ThrowsAsync(() => store.UpdateWithEtagCasAsync(BuildPayload(), "")); + } + + [Fact] + public async Task TryDownload_RejectsNonGuidTenantId() + { + var store = new FakeStore(); + await Assert.ThrowsAsync(() => store.TryDownloadAsync("not-a-guid", HistoryRowKey)); + } + + // ── Initial upload (If-None-Match=*) ──────────────────────────────────────── + + [Fact] + public async Task TryUploadInitial_FirstWrite_ReturnsTrue() + { + var store = new FakeStore(); + var inserted = await store.TryUploadInitialAsync(BuildPayload()); + Assert.True(inserted); + Assert.Single(store.Blobs); + Assert.Equal( + BlobOffboardingExpectationsStore.BuildBlobName(TenantId, HistoryRowKey), + store.Blobs.Keys.Single()); + } + + [Fact] + public async Task TryUploadInitial_Duplicate_409BlobAlreadyExists_ReturnsFalse() + { + var store = new FakeStore(); + await store.TryUploadInitialAsync(BuildPayload()); + + // Simulate the "older" Azure response code for the same condition. + store.NextOverwriteRefusal = OverwriteRefusal.Status409; + + var inserted = await store.TryUploadInitialAsync(BuildPayload()); + Assert.False(inserted); + } + + [Fact] + public async Task TryUploadInitial_Duplicate_412ConditionNotMet_ReturnsFalse() + { + // High-finding: Azure Blob Storage emits 412 ConditionNotMet for the SAME + // "If-None-Match=* on existing blob" case the SDK historically returned 409 for. + // Without the 412-branch the resume path breaks and the worker poisons. + var store = new FakeStore(); + await store.TryUploadInitialAsync(BuildPayload()); + + store.NextOverwriteRefusal = OverwriteRefusal.Status412ConditionNotMet; + + var inserted = await store.TryUploadInitialAsync(BuildPayload()); + Assert.False(inserted); + } + + [Fact] + public async Task TryUploadInitial_Duplicate_BlobErrorCodeOnly_ReturnsFalse() + { + // Some SDK paths surface the failure as Status=0 with a populated ErrorCode. + var store = new FakeStore(); + await store.TryUploadInitialAsync(BuildPayload()); + + store.NextOverwriteRefusal = OverwriteRefusal.ErrorCodeBlobAlreadyExists; + + var inserted = await store.TryUploadInitialAsync(BuildPayload()); + Assert.False(inserted); + } + + [Fact] + public async Task TryUploadInitial_Propagates500_NotASilentResume() + { + var store = new FakeStore { NextOverwriteRefusal = OverwriteRefusal.Status500 }; + await Assert.ThrowsAsync(() => store.TryUploadInitialAsync(BuildPayload())); + } + + // ── Download (404 → null) ────────────────────────────────────────────────── + + [Fact] + public async Task TryDownload_404_Returns_NullPayload_AndNullEtag() + { + var store = new FakeStore(); + var (payload, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey); + Assert.Null(payload); + Assert.Null(etag); + } + + [Fact] + public async Task TryDownload_AfterUpload_RoundTripsAllFields() + { + var store = new FakeStore(); + var original = BuildPayload(); + original.EnumerationCompleted = true; + original.EnumeratedSessionCount = 2; + original.Expectations.Add(new OffboardingExpectation { SessionId = "s1", ManifestId = "m1", Outcome = "Enqueued" }); + original.Expectations.Add(new OffboardingExpectation { SessionId = "s2", ManifestId = null, Outcome = "SessionNotFound" }); + + await store.TryUploadInitialAsync(original); + + var (round, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey); + Assert.NotNull(round); + Assert.NotNull(etag); + Assert.Equal(2, round!.EnumeratedSessionCount); + Assert.True(round.EnumerationCompleted); + Assert.Equal(2, round.Expectations.Count); + Assert.Equal("Enqueued", round.Expectations[0].Outcome); + Assert.Null(round.Expectations[1].ManifestId); + } + + // ── Update with ETag CAS ─────────────────────────────────────────────────── + + [Fact] + public async Task UpdateWithEtagCas_MatchingEtag_ReturnsNewEtag() + { + var store = new FakeStore(); + await store.TryUploadInitialAsync(BuildPayload()); + var (round, etag) = await store.TryDownloadAsync(TenantId, HistoryRowKey); + + round!.Expectations.Add(new OffboardingExpectation { SessionId = "s1", Outcome = "Enqueued", ManifestId = "m1" }); + + var newEtag = await store.UpdateWithEtagCasAsync(round, etag!); + Assert.NotNull(newEtag); + Assert.NotEqual(etag, newEtag); + + var (refetched, _) = await store.TryDownloadAsync(TenantId, HistoryRowKey); + Assert.Single(refetched!.Expectations); + } + + [Fact] + public async Task UpdateWithEtagCas_StaleEtag_Throws412() + { + var store = new FakeStore(); + await store.TryUploadInitialAsync(BuildPayload()); + + var ex = await Assert.ThrowsAsync( + () => store.UpdateWithEtagCasAsync(BuildPayload(), ifMatchEtag: "\"0xSTALE\"")); + Assert.Equal(412, ex.Status); + } + + // ── Delete ───────────────────────────────────────────────────────────────── + + [Fact] + public async Task Delete_RemovesBlob_WhenPresent() + { + var store = new FakeStore(); + await store.TryUploadInitialAsync(BuildPayload()); + Assert.Single(store.Blobs); + + await store.DeleteAsync(TenantId, HistoryRowKey); + Assert.Empty(store.Blobs); + } + + [Fact] + public async Task Delete_404_IsIdempotent() + { + var store = new FakeStore(); + // No upload first — blob doesn't exist. + await store.DeleteAsync(TenantId, HistoryRowKey); // must not throw + Assert.Empty(store.Blobs); + } + + // ── Container choice (Rev-8-F1) ──────────────────────────────────────────── + + [Fact] + public void Container_IsOffboardingState_NotDeletionManifests() + { + // Container choice is part of the resilience design — wiping the wrong container + // (Rev-8-F1: deletion-manifests would orphan Expectations during Phase 2.E). + Assert.Equal("offboarding-state", AutopilotMonitor.Shared.Constants.BlobContainers.OffboardingState); + Assert.NotEqual( + AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests, + AutopilotMonitor.Shared.Constants.BlobContainers.OffboardingState); + } + + [Fact] + public void BlobName_FollowsTenantPrefixSchemaForLifecycleSweep() + { + var name = BlobOffboardingExpectationsStore.BuildBlobName(TenantId, HistoryRowKey); + Assert.StartsWith($"{TenantId}/", name); + Assert.EndsWith(".expectations.json", name); + Assert.Contains(HistoryRowKey, name); + } + + [Fact] + public void Defaults_ForceCallersToSetEnumerationCompletedExplicitly() + { + // Rev-7-F2 disambiguation lives at the POCO level — the drain probe relies on + // EnumerationCompleted defaulting to false so a half-built payload cannot + // accidentally pass the "0 expectations + completed=true" happy path. + var payload = new OffboardingExpectations(); + Assert.False(payload.EnumerationCompleted); + Assert.Equal(0, payload.EnumeratedSessionCount); + Assert.Equal(1, payload.SchemaVersion); + Assert.Empty(payload.Expectations); + } + + // ── Helpers ───────────────────────────────────────────────────────────────── + + private static OffboardingExpectations BuildPayload( + string tenantId = TenantId, + string historyRowKey = HistoryRowKey) + => new() + { + SchemaVersion = 1, + TenantId = tenantId, + HistoryRowKey = historyRowKey, + CreatedAt = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc), + EnumerationCompleted = false, + EnumeratedSessionCount = 0, + }; + + /// + /// Selects how the next call to simulates an + /// "exists" refusal — needed because Azure switches between codes depending on tier/SDK. + /// + private enum OverwriteRefusal + { + Auto, + Status409, + Status412ConditionNotMet, + ErrorCodeBlobAlreadyExists, + Status500, + } + + /// + /// Dict-backed fake that overrides the four protected virtual seams. Mirrors the + /// production Azure semantics for If-None-Match=*, If-Match, 404, and DeleteIfExists. + /// + private sealed class FakeStore : BlobOffboardingExpectationsStore + { + public Dictionary Blobs { get; } = new(); + public OverwriteRefusal NextOverwriteRefusal { get; set; } = OverwriteRefusal.Auto; + private int _etagCounter; + + public FakeStore() + : base( + new BlobStorageService( + new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, + usesManagedIdentity: false), + NullLogger.Instance) + { + } + + protected override Task<(bool Inserted, string? ETag)> WriteIfNoneMatchAsync( + string blobName, byte[] payload, CancellationToken ct) + { + // Allow tests to inject a specific refusal even though the underlying state would + // already make a real Azure call fail — needed to pin BOTH the 409 and 412 paths. + if (NextOverwriteRefusal == OverwriteRefusal.Status500) + { + NextOverwriteRefusal = OverwriteRefusal.Auto; + throw new RequestFailedException(500, "ServerError", "InternalError", null); + } + + if (Blobs.ContainsKey(blobName)) + { + var refusal = NextOverwriteRefusal == OverwriteRefusal.Auto + ? OverwriteRefusal.Status409 + : NextOverwriteRefusal; + NextOverwriteRefusal = OverwriteRefusal.Auto; + + RequestFailedException ex = refusal switch + { + OverwriteRefusal.Status409 => + new RequestFailedException(409, "Conflict", BlobErrorCode.BlobAlreadyExists.ToString(), null), + OverwriteRefusal.Status412ConditionNotMet => + new RequestFailedException(412, "ConditionNotMet", BlobErrorCode.ConditionNotMet.ToString(), null), + OverwriteRefusal.ErrorCodeBlobAlreadyExists => + new RequestFailedException(0, "ErrorCodeOnly", BlobErrorCode.BlobAlreadyExists.ToString(), null), + _ => throw new InvalidOperationException("Unhandled refusal mode"), + }; + + if (IsAlreadyExists(ex)) return Task.FromResult<(bool, string?)>((false, null)); + throw ex; + } + + var etag = $"\"0xFAKE_{++_etagCounter}\""; + Blobs[blobName] = (payload, etag); + return Task.FromResult<(bool, string?)>((true, etag)); + } + + protected override Task<(byte[]? Payload, string? ETag)> ReadBlobAsync( + string blobName, CancellationToken ct) + { + if (!Blobs.TryGetValue(blobName, out var entry)) + return Task.FromResult<(byte[]?, string?)>((null, null)); + return Task.FromResult<(byte[]?, string?)>((entry.Bytes, entry.ETag)); + } + + protected override Task WriteWithIfMatchAsync( + string blobName, byte[] payload, string ifMatchEtag, CancellationToken ct) + { + if (!Blobs.TryGetValue(blobName, out var entry)) + throw new RequestFailedException(404, "NotFound", "BlobNotFound", null); + if (entry.ETag != ifMatchEtag) + throw new RequestFailedException(412, "ConditionNotMet", BlobErrorCode.ConditionNotMet.ToString(), null); + + var etag = $"\"0xFAKE_{++_etagCounter}\""; + Blobs[blobName] = (payload, etag); + return Task.FromResult(etag); + } + + protected override Task DeleteBlobIfExistsAsync(string blobName, CancellationToken ct) + { + Blobs.Remove(blobName); + return Task.CompletedTask; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingFiltersTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingFiltersTests.cs new file mode 100644 index 00000000..6eab625b --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingFiltersTests.cs @@ -0,0 +1,110 @@ +using System; +using AutopilotMonitor.Functions.Services.Offboarding; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Plan §5.5 + §10.1 (F6-Fix). The filter helper sits in front of every SafeWipe table +/// query, so injection or range-bleed bugs here turn into cross-tenant data loss. Tests +/// hammer GUID validation, single-quote escaping, and the underscore-tilde upper bound. +/// +public class OffboardingFiltersTests +{ + private const string ValidTenant = "11111111-1111-1111-1111-111111111111"; + + // ── EnsureValidGuid behaviour propagates through every entry point ───────── + + [Theory] + [InlineData("not-a-guid")] + [InlineData("abc")] + [InlineData("'; DROP TABLE Sessions; --")] + [InlineData("")] + [InlineData("11111111-1111-1111-1111-1111111111111")] // too long + public void ExactPartition_ThrowsForNonGuid(string input) + { + Assert.Throws(() => OffboardingFilters.ExactPartition(input)); + } + + [Theory] + [InlineData("not-a-guid")] + [InlineData("'")] + public void CompositePartitionRange_ThrowsForNonGuid(string input) + { + Assert.Throws(() => OffboardingFilters.CompositePartitionRange(input)); + } + + [Theory] + [InlineData("not-a-guid")] + [InlineData("")] + public void DiscriminatorWithTenantProp_ThrowsForNonGuid(string input) + { + Assert.Throws(() => OffboardingFilters.DiscriminatorWithTenantProp("CodeLookup", input)); + } + + [Fact] + public void DiscriminatorWithTenantProp_ThrowsForEmptyDiscriminator() + { + Assert.Throws(() => OffboardingFilters.DiscriminatorWithTenantProp("", ValidTenant)); + } + + [Theory] + [InlineData("not-a-guid")] + public void TenantIdProperty_ThrowsForNonGuid(string input) + { + Assert.Throws(() => OffboardingFilters.TenantIdProperty(input)); + } + + // ── Filter shapes ─────────────────────────────────────────────────────────── + + [Fact] + public void ExactPartition_EmitsCanonicalEqualsClause() + { + var filter = OffboardingFilters.ExactPartition(ValidTenant); + Assert.Equal($"PartitionKey eq '{ValidTenant}'", filter); + } + + [Fact] + public void CompositePartitionRange_UsesUnderscoreTildeUpperBound() + { + var filter = OffboardingFilters.CompositePartitionRange(ValidTenant); + + // Lower bound must include the underscore anchor — otherwise the range matches + // "{tenantId}abc" and other tenants whose ids happen to start with the same prefix. + Assert.Contains($"PartitionKey ge '{ValidTenant}_'", filter); + + // Upper bound MUST be '{tenantId}_~' (underscore-tilde). Switching to '{tenantId}~' + // would bleed across the next ASCII bucket and pull in foreign rows. + Assert.Contains($"PartitionKey lt '{ValidTenant}_~'", filter); + Assert.DoesNotContain($"PartitionKey lt '{ValidTenant}~'", filter); + } + + [Fact] + public void DiscriminatorWithTenantProp_RequiresBothAnchorsServerSide() + { + var filter = OffboardingFilters.DiscriminatorWithTenantProp("CodeLookup", ValidTenant); + + Assert.Contains("PartitionKey eq 'CodeLookup'", filter); + Assert.Contains($"TenantId eq '{ValidTenant}'", filter); + Assert.Contains("and", filter); + } + + [Fact] + public void TenantIdProperty_EmitsPropertyOnlyFilter() + { + var filter = OffboardingFilters.TenantIdProperty(ValidTenant); + Assert.Equal($"TenantId eq '{ValidTenant}'", filter); + Assert.DoesNotContain("PartitionKey", filter); + } + + // ── Injection neutralization (defense in depth even after EnsureValidGuid) ── + + [Fact] + public void DiscriminatorWithTenantProp_EscapesDiscriminatorQuotes() + { + // Hostile discriminator wouldn't pass through real callers (compile-time constants), but + // defense in depth: the helper still escapes. + var filter = OffboardingFilters.DiscriminatorWithTenantProp("a'b", ValidTenant); + Assert.Contains("PartitionKey eq 'a''b'", filter); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs new file mode 100644 index 00000000..eab10242 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs @@ -0,0 +1,125 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Pagination; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Rev-5-F3: the enumerator MUST fail-loud on storage exceptions. The existing +/// GetSessionsByDateRangeAsync swallows errors and returns empty — borrowing that +/// behaviour for offboarding would let a transient outage trigger a same-cycle wipe with +/// zero cascade backup. Tests pin both halves of the contract: happy-path yield-all, and +/// exception-propagates-no-silent-empty. +/// +public class OffboardingSessionEnumeratorTests +{ + private const string TenantId = "33333333-3333-3333-3333-333333333333"; + + [Fact] + public async Task EnumerateAsync_YieldsAllSessionsFromRepo() + { + var repo = new FakeMaintenanceRepository( + sessionIds: new[] { "s1", "s2", "s3" }); + var sut = new OffboardingSessionEnumerator(repo); + + var collected = new List(); + await foreach (var s in sut.EnumerateAsync(TenantId)) + { + collected.Add(s); + } + + Assert.Equal(new[] { "s1", "s2", "s3" }, collected); + } + + [Fact] + public async Task EnumerateAsync_PropagatesStorageException_NoSilentEmpty() + { + // 503 mid-enumeration is the exact scenario that would otherwise let the offboard + // worker proceed to wipe with zero cascade backup. The enumerator MUST throw. + var repo = new FakeMaintenanceRepository( + sessionIds: new[] { "s1" }, + throwOnIteration: new InvalidOperationException("simulated 503")); + var sut = new OffboardingSessionEnumerator(repo); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in sut.EnumerateAsync(TenantId)) + { + // drain + } + }); + } + + [Fact] + public async Task EnumerateAsync_RejectsNonGuidTenantId() + { + var sut = new OffboardingSessionEnumerator(new FakeMaintenanceRepository(Array.Empty())); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in sut.EnumerateAsync("not-a-guid")) + { + // never reached + } + }); + } + + // ── Test double ───────────────────────────────────────────────────────────── + + private sealed class FakeMaintenanceRepository : IMaintenanceRepository + { + private readonly string[] _sessions; + private readonly Exception? _throwOnIteration; + + public FakeMaintenanceRepository(string[] sessionIds, Exception? throwOnIteration = null) + { + _sessions = sessionIds; + _throwOnIteration = throwOnIteration; + } + + public async IAsyncEnumerable EnumerateSessionsForOffboardingAsync( + string tenantId, [EnumeratorCancellation] CancellationToken ct = default) + { + foreach (var s in _sessions) + { + if (_throwOnIteration != null) + { + // Yield the first one, then throw — matches "mid-enumeration outage". + yield return s; + throw _throwOnIteration; + } + yield return s; + } + await Task.CompletedTask; + } + + // ── Unused for these tests ───────────────────────────────────────────── + + public Task LogAuditEntryAsync(string tenantId, string action, string entityType, string entityId, string performedBy, Dictionary? details = null) => throw new NotImplementedException(); + public Task> GetAuditLogsAsync(string tenantId, DateTime? dateFrom = null, DateTime? dateTo = null) => throw new NotImplementedException(); + public Task> GetAllAuditLogsAsync(DateTime? dateFrom = null, DateTime? dateTo = null) => throw new NotImplementedException(); + public Task> GetAuditLogsPageAsync(string tenantId, DateTime? dateFrom, DateTime? dateTo, int pageSize, string? continuation) => throw new NotImplementedException(); + public Task> GetAllAuditLogsPageAsync(DateTime? dateFrom, DateTime? dateTo, int pageSize, string? continuation) => throw new NotImplementedException(); + public Task> GetSessionsOlderThanAsync(string tenantId, DateTime cutoffDate) => throw new NotImplementedException(); + public Task> GetSessionsByDateRangeAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException(); + public Task> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime) => throw new NotImplementedException(); + public Task> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException(); + public Task> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours) => throw new NotImplementedException(); + public Task> GetAllTenantIdsAsync() => throw new NotImplementedException(); + public Task DeleteSessionEventsAsync(string tenantId, string sessionId) => throw new NotImplementedException(); + public Task DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => throw new NotImplementedException(); + public Task BackfillSessionIndexAsync() => throw new NotImplementedException(); + public Task CleanupGhostSessionIndexEntriesAsync() => throw new NotImplementedException(); + public Task IsSessionIndexEmptyAsync() => throw new NotImplementedException(); + public Task> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => throw new NotImplementedException(); + public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => throw new NotImplementedException(); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SafeWipeServiceTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SafeWipeServiceTests.cs new file mode 100644 index 00000000..26732e34 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SafeWipeServiceTests.cs @@ -0,0 +1,242 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Offboarding; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Plan §5 + §10.1: SafeWipe is the bug-shield for the user's "es wurde schon mal zu viel +/// gelöscht"-concern. These tests pin +/// +/// Verify-abort when the fetched set contains a row that does not match the expected +/// anchor — NO deletes are issued (4-Step Pattern step 3 only runs after step 2 passes). +/// Every Delete uses (unconditional) — F5-Fix, otherwise the +/// helper isn't idempotent across crash/resume. +/// 404-fallback per row keeps the helper idempotent after a partial run. +/// +/// +public class SafeWipeServiceTests +{ + private const string TenantId = "66666666-6666-6666-6666-666666666666"; + private const string OtherTenant = "77777777-7777-7777-7777-777777777777"; + + [Fact] + public async Task ExactPartition_AbortsBeforeAnyDeleteWhenForeignRowSlipsThrough() + { + // Server-side filter SHOULD only return PK=TenantId, but if a misbehaving filter or + // SDK bug returns a foreign row, verify-abort fires and NO Delete is issued. + var harness = new Harness( + queriedRows: new[] + { + new TableEntity(TenantId, "row-1"), + new TableEntity(OtherTenant, "row-2"), // <-- the bug + new TableEntity(TenantId, "row-3"), + }); + + await Assert.ThrowsAsync(() => + harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId)); + + Assert.Empty(harness.SubmittedBatches); + Assert.Empty(harness.PerRowDeletes); + } + + [Fact] + public async Task CompositePartitionRange_AbortsWhenForeignPrefixReturned() + { + var harness = new Harness( + queriedRows: new[] + { + new TableEntity($"{TenantId}_session-1", "row"), + new TableEntity($"{OtherTenant}_session-2", "row"), // bleeds across underscore boundary + }); + + await Assert.ThrowsAsync(() => + harness.Sut.WipeByCompositePartitionRangeAsync("FakeTable", TenantId)); + + Assert.Empty(harness.SubmittedBatches); + } + + [Fact] + public async Task DiscriminatorWithTenantProp_AbortsWhenTenantPropertyMismatch() + { + // Variant B's only tenant anchor is the property — verify MUST check it client-side. + var foreign = new TableEntity("CodeLookup", "row-2"); + foreign["TenantId"] = OtherTenant; + var ours = new TableEntity("CodeLookup", "row-1"); + ours["TenantId"] = TenantId; + + var harness = new Harness(queriedRows: new[] { ours, foreign }); + + await Assert.ThrowsAsync(() => + harness.Sut.WipeByDiscriminatorAndTenantPropertyAsync("FakeTable", "CodeLookup", TenantId)); + + Assert.Empty(harness.SubmittedBatches); + } + + [Fact] + public async Task TenantIdProperty_AbortsWhenPropertyMissing() + { + // Variant C: row missing the TenantId property fails verify. + var ours = new TableEntity("user-oid-1", "rk"); + ours["TenantId"] = TenantId; + var foreign = new TableEntity("user-oid-2", "rk"); // no TenantId + + var harness = new Harness(queriedRows: new[] { ours, foreign }); + + await Assert.ThrowsAsync(() => + harness.Sut.WipeByTenantIdPropertyAsync("FakeTable", TenantId)); + } + + [Fact] + public async Task ExactPartition_DeletesEveryRowWithETagAll() + { + var harness = new Harness( + queriedRows: new[] + { + new TableEntity(TenantId, "rk_1"), + new TableEntity(TenantId, "rk_2"), + new TableEntity(TenantId, "rk_3"), + }); + + var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId); + + Assert.Equal(3, deleted); + var batch = Assert.Single(harness.SubmittedBatches); + Assert.Equal(3, batch.Count); + + foreach (var action in batch) + { + Assert.Equal(TableTransactionActionType.Delete, action.ActionType); + Assert.Equal(ETag.All, action.Entity.ETag); + } + } + + [Fact] + public async Task Delete_FallsBackToPerRowOn404_KeepsIdempotent() + { + // First batch returns 404 → per-row fallback finds 1 already-missing, 2 still there. + var perRowState = new Dictionary + { + ["rk_1"] = true, + ["rk_2"] = false, // already gone (concurrent cleaner) + ["rk_3"] = true, + }; + + var harness = new Harness( + queriedRows: perRowState.Keys.Select(rk => new TableEntity(TenantId, rk)).ToArray(), + batchBehavior: _ => throw new RequestFailedException(404, "ResourceNotFound"), + perRowBehavior: rk => + { + if (!perRowState[rk]) throw new RequestFailedException(404, "ResourceNotFound"); + return new Mock().Object; + }); + + var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId); + + Assert.Equal(2, deleted); // 2 still there, 1 already missing + Assert.Equal(3, harness.PerRowDeletes.Count); + foreach (var (_, _, etag) in harness.PerRowDeletes) + { + Assert.Equal(ETag.All, etag); + } + } + + [Fact] + public async Task EmptyResult_ReturnsZero_NoBatchSubmitted() + { + var harness = new Harness(queriedRows: Array.Empty()); + + var deleted = await harness.Sut.WipeByExactPartitionAsync("FakeTable", TenantId); + + Assert.Equal(0, deleted); + Assert.Empty(harness.SubmittedBatches); + } + + [Fact] + public async Task Variants_RejectNonGuidTenantId() + { + var harness = new Harness(queriedRows: Array.Empty()); + + await Assert.ThrowsAsync( + () => harness.Sut.WipeByExactPartitionAsync("t", "not-a-guid")); + await Assert.ThrowsAsync( + () => harness.Sut.WipeByCompositePartitionRangeAsync("t", "not-a-guid")); + await Assert.ThrowsAsync( + () => harness.Sut.WipeByDiscriminatorAndTenantPropertyAsync("t", "CodeLookup", "not-a-guid")); + await Assert.ThrowsAsync( + () => harness.Sut.WipeByTenantIdPropertyAsync("t", "not-a-guid")); + } + + // ── Harness ───────────────────────────────────────────────────────────────── + + private sealed class Harness + { + public List> SubmittedBatches { get; } = new(); + public List<(string Pk, string Rk, ETag Etag)> PerRowDeletes { get; } = new(); + public SafeWipeService Sut { get; } + + public Harness( + TableEntity[] queriedRows, + Func, Response>>? batchBehavior = null, + Func? perRowBehavior = null) + { + var mockTableClient = new Mock(); + + // QueryAsync returns the canned set as an AsyncPageable. + mockTableClient + .Setup(c => c.QueryAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns(AsAsyncPageable(queriedRows)); + + mockTableClient + .Setup(c => c.SubmitTransactionAsync(It.IsAny>(), It.IsAny())) + .Returns, CancellationToken>((actions, _) => + { + var snapshot = actions.ToList(); + SubmittedBatches.Add(snapshot); + if (batchBehavior != null) return Task.FromResult(batchBehavior(snapshot)); + var inner = (IReadOnlyList)snapshot.Select(_ => new Mock().Object).ToList(); + return Task.FromResult(Response.FromValue(inner, new Mock().Object)); + }); + + mockTableClient + .Setup(c => c.DeleteEntityAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((pk, rk, etag, _) => + { + PerRowDeletes.Add((pk, rk, etag)); + if (perRowBehavior != null) return Task.FromResult(perRowBehavior(rk)); + return Task.FromResult(new Mock().Object); + }); + + var mockServiceClient = new Mock(); + mockServiceClient.Setup(s => s.GetTableClient(It.IsAny())).Returns(mockTableClient.Object); + + var storage = new TableStorageService(mockServiceClient.Object, NullLogger.Instance); + var blob = new BlobStorageService( + new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, + usesManagedIdentity: false); + + Sut = new SafeWipeService(storage, blob, NullLogger.Instance); + } + + private static AsyncPageable AsAsyncPageable(TableEntity[] entities) + { + var page = Page.FromValues(entities, continuationToken: null, new Mock().Object); + return AsyncPageable.FromPages(new[] { page }); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerOrderingTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerOrderingTests.cs new file mode 100644 index 00000000..5dbb8c94 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerOrderingTests.cs @@ -0,0 +1,310 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Plan PR1.5 (Rev-5-F1). Locks the cascade-handler side-effect order: audit-write and +/// SignalR notify must complete BEFORE DeletionProgress.CompletedAt is stamped. +/// Once CompletedAt is set, the tenant-offboarding worker's drain probe treats the cascade +/// as "all side effects through" and the next phase wipes AuditLogs — so a late +/// deletion_completed audit would land as an orphan after the wipe. +/// +/// The reorder is fail-soft-preserving: +/// +/// LogAuditEntryAsync already swallows storage exceptions internally and +/// returns false on failure; the handler does not act on the boolean. +/// NotifySessionDeletedAsync already catches everything internally. +/// +/// So pulling them in front of the CompletedAt CAS cannot block CompletedAt from +/// being set on transient storage failures — confirmed by +/// . +/// +/// +public class SessionDeletionHandlerOrderingTests +{ + private const string TenantId = "11111111-1111-1111-1111-111111111111"; + private const string SessionId = "22222222-2222-2222-2222-222222222222"; + private const string ManifestId = "0123456789ABCDEF_FEDCBA9876543210"; + private const string Sha256 = "1111111111111111111111111111111111111111111111111111111111111111"; + + [Fact] + public async Task TombstonePhase_WritesAuditAndSignalRBeforeCompletedAt() + { + var harness = new Harness(); + await harness.Sut.HandleAsync(harness.Envelope); + + Assert.True(harness.AuditSequence > 0, "audit was never written"); + Assert.True(harness.SignalRSequence > 0, "SignalR notify was never sent"); + Assert.True(harness.CompletedAtSequence > 0, "CompletedAt was never set on the progress blob"); + + Assert.True( + harness.AuditSequence < harness.CompletedAtSequence, + $"audit must run before CompletedAt CAS (audit={harness.AuditSequence}, completedAt={harness.CompletedAtSequence})"); + Assert.True( + harness.SignalRSequence < harness.CompletedAtSequence, + $"SignalR notify must run before CompletedAt CAS (signalR={harness.SignalRSequence}, completedAt={harness.CompletedAtSequence})"); + } + + [Fact] + public async Task AuditFailure_DoesNotBlockCompletedAtSet() + { + // LogAuditEntryAsync is fail-soft by contract — a storage hiccup returns false rather + // than throwing. The handler must NOT use the boolean as a gate: CompletedAt still + // gets set so the cascade completes from the offboarding worker's point of view. + var harness = new Harness(auditReturns: false); + await harness.Sut.HandleAsync(harness.Envelope); + + Assert.True(harness.AuditSequence > 0, "audit attempt was still made"); + Assert.True(harness.CompletedAtSequence > 0, + "CompletedAt MUST still be stamped on the progress blob even when audit returned false"); + Assert.True(harness.AuditSequence < harness.CompletedAtSequence); + } + + [Fact] + public async Task SignalRFailure_DoesNotBlockCompletedAtSet() + { + // NotifySessionDeletedAsync already catches internally. The reorder cannot break that + // contract: an exception inside the notify path must not bubble out into the handler + // and leave CompletedAt unset. + var harness = new Harness(signalRThrows: new InvalidOperationException("simulated SignalR outage")); + await harness.Sut.HandleAsync(harness.Envelope); + + Assert.True(harness.SignalRSequence > 0, "SignalR notify attempt was still made"); + Assert.True(harness.CompletedAtSequence > 0, + "CompletedAt MUST still be stamped on the progress blob even when SignalR threw"); + } + + // ── Harness ───────────────────────────────────────────────────────────────── + + /// + /// Minimal self-contained harness that drives the cascade through to the tombstone phase + /// using a single-step manifest (the Final step only). Sequence-counter records the call + /// order of the three side effects that PR1.5 reorders. + /// + private sealed class Harness + { + public SessionDeletionEnvelope Envelope { get; } + public SessionDeletionHandler Sut { get; } + + public int AuditSequence { get; private set; } + public int SignalRSequence { get; private set; } + public int CompletedAtSequence { get; private set; } + + private int _sequenceCounter; + private int Next() => System.Threading.Interlocked.Increment(ref _sequenceCounter); + + public Harness( + bool auditReturns = true, + Exception? signalRThrows = null) + { + // --- Storage mock: just enough to flow through the state machine + tombstone step. + var storage = new Mock( + Mock.Of(), + NullLogger.Instance); + + var sessionRow = new TableEntity(TenantId, SessionId) + { + ["DeletionState"] = SessionDeletionState.Queued, + ["PendingDeletionManifestId"] = ManifestId, + }; + storage.Setup(s => s.GetSessionRowAsync(TenantId, SessionId, It.IsAny())) + .ReturnsAsync(sessionRow); + + storage.Setup(s => s.CasSetSessionDeletionStateAsync( + TenantId, SessionId, + SessionDeletionState.Queued, SessionDeletionState.Running, + It.IsAny(), It.IsAny())) + .ReturnsAsync(new TableStorageService.SessionDeletionStateCasResult + { + Outcome = TableStorageService.SessionDeletionStateCasOutcome.Updated, + CurrentState = SessionDeletionState.Running, + CurrentManifestId = ManifestId, + }); + + storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), It.IsAny())) + .ReturnsAsync((string _, IReadOnlyList<(string, string)> keys, CancellationToken _) => + new DeletionBatchResult(keys.Count, keys.Count, 0)); + + storage.Setup(s => s.RecordSessionTombstoneAsync( + TenantId, SessionId, It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // --- Blob mock: manifest + progress IO with a live in-memory progress instance so + // each UpdateDeletionProgressAsync mutation is reflected in the next download. + var progressLive = new DeletionProgress + { + SnapshotSha256 = Sha256, + CompletedSteps = new HashSet(), + VerificationDone = false, + CompletedAt = null, + }; + + var blob = new Mock( + new BlobServiceClient("UseDevelopmentStorage=true"), + NullLogger.Instance, + false); + + var manifest = BuildMinimalManifest(); + blob.Setup(b => b.DownloadDeletionManifestWithShaAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync((manifest, Sha256)); + + blob.Setup(b => b.DownloadDeletionProgressAsync( + TenantId, SessionId, ManifestId, It.IsAny())) + .ReturnsAsync(() => (Clone(progressLive), "\"0xFAKE\"")); + + blob.Setup(b => b.UpdateDeletionProgressAsync( + TenantId, SessionId, ManifestId, + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns( + (_, _, _, p, _, _) => + { + // Only the write that flips CompletedAt from null → non-null is the + // "lifecycle complete" CAS the reorder targets. Other writes (state + // progression, TombstoneStarted) get sequence numbers but we don't + // assert on them. + if (progressLive.CompletedAt == null && p.CompletedAt != null) + { + CompletedAtSequence = Next(); + } + progressLive.CompletedSteps = new HashSet(p.CompletedSteps); + progressLive.VerificationDone = p.VerificationDone; + progressLive.CompletedAt = p.CompletedAt; + progressLive.TombstoneStarted = p.TombstoneStarted; + return Task.FromResult("\"0xFAKE_NEXT\""); + }); + + // --- Verifier: clean by default (no residuals → tombstone runs). + var verifier = new Mock( + Mock.Of(), + NullLogger.Instance); + verifier.Setup(v => v.VerifyAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CascadeVerificationResult(true, new List())); + + // --- Maintenance: capture audit call with a sequence number. + var maintenance = new Mock(); + maintenance.Setup(m => m.LogAuditEntryAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny?>())) + .Returns?>( + (_, action, _, _, _, _) => + { + if (action == "deletion_completed") AuditSequence = Next(); + return Task.FromResult(auditReturns); + }); + + // --- SignalR: capture notify call with a sequence number. + var signalR = new SequencingSignalRNotificationService( + this, + throwOnNotify: signalRThrows); + + Envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, + SessionId = SessionId, + ManifestId = ManifestId, + Reason = "admin_delete", + EnqueuedAt = DateTime.UtcNow, + }; + + Sut = new SessionDeletionHandler( + storage.Object, blob.Object, verifier.Object, + maintenance.Object, signalR, + NullLogger.Instance); + } + + private static DeletionProgress Clone(DeletionProgress p) => new() + { + SnapshotSha256 = p.SnapshotSha256, + CompletedSteps = new HashSet(p.CompletedSteps), + VerificationDone = p.VerificationDone, + CompletedAt = p.CompletedAt, + TombstoneStarted = p.TombstoneStarted, + }; + + private static DeletionManifest BuildMinimalManifest() => new() + { + ManifestId = ManifestId, + TenantId = TenantId, + SessionId = SessionId, + CreatedAt = new DateTime(2026, 5, 18, 10, 0, 0, DateTimeKind.Utc), + CreatedBy = new DeletionActor { Type = "admin", Actor = "alice@contoso.invalid" }, + Reason = "admin_delete", + RetentionContext = new DeletionRetentionContext { TenantRetentionDays = 90 }, + SchemaHash = "sha256:test", + // Single FINAL step → handler skips per-table + aggregate phases and goes straight + // to verifier (clean) → tombstone → audit + signalR + CompletedAt. That is the + // path PR1.5 reorders. + Steps = new List + { + new DeletionStep + { + Order = 1, + Step = DeletionStepNames.Tombstone, + Class = DeletionStepClass.Final, + RowCount = 2, + Rows = new List + { + new DeletionRowDump { Pk = TenantId, Rk = $"6299999999999999_{SessionId}" }, + new DeletionRowDump { Pk = TenantId, Rk = SessionId }, + }, + }, + }, + }; + + /// + /// SignalR test double that records a sequence number on + /// . + /// All other interface members are no-ops — the cascade handler never calls them. + /// + private sealed class SequencingSignalRNotificationService : ISignalRNotificationService + { + private readonly Harness _harness; + private readonly Exception? _throwOnNotify; + + public SequencingSignalRNotificationService(Harness harness, Exception? throwOnNotify) + { + _harness = harness; + _throwOnNotify = throwOnNotify; + } + + public Task NotifySessionDeletedAsync(string tenantId, string sessionId) + { + _harness.SignalRSequence = _harness.Next(); + // Production NotifySessionDeletedAsync already catches internally; mimic that + // contract so the test double doesn't accidentally pretend exceptions bubble. + if (_throwOnNotify != null) + { + try { throw _throwOnNotify; } + catch { /* swallow — matches production behaviour */ } + } + return Task.CompletedTask; + } + + // ── Unused by the cascade handler — no-op stubs ──────────────────── + public Task NotifyRuleResultsAvailableAsync(string tenantId, string sessionId, int resultCount) => Task.CompletedTask; + public Task NotifyVulnerabilityReportAvailableAsync(string tenantId, string sessionId, string overallRisk) => Task.CompletedTask; + public Task SendTenantNotificationAsync(string tenantId, NotificationAudience audience, object dto) => Task.CompletedTask; + public Task SendTenantNotificationDismissedAsync(string tenantId, string notificationId) => Task.CompletedTask; + public Task SendTenantNotificationDismissedAllAsync(string tenantId) => Task.CompletedTask; + public Task SendGlobalNotificationAsync(object dto) => Task.CompletedTask; + public Task SendGlobalNotificationDismissedAsync(string notificationId) => Task.CompletedTask; + public Task SendGlobalNotificationDismissedAllAsync() => Task.CompletedTask; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TableOffboardingAuditRepositoryTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TableOffboardingAuditRepositoryTests.cs new file mode 100644 index 00000000..6267783d --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TableOffboardingAuditRepositoryTests.cs @@ -0,0 +1,384 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Data.Tables; +using AutopilotMonitor.Functions.DataAccess.TableStorage; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Round-trip + shape-guard coverage for the three POCO mappings hosted in +/// OffboardingAudit. Memory: feedback_table_storage_serialization — new +/// fields MUST be exercised in Store+Map so silent drops surface here, not in production. +/// +public class TableOffboardingAuditRepositoryTests +{ + private const string TenantId = "88888888-8888-8888-8888-888888888888"; + private const string HistoryRowKey = "20260518091523123_88888888-8888-8888-8888-888888888888"; + + [Fact] + public async Task Marker_RoundTrips_AllFields() + { + var harness = new Harness(); + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = HistoryRowKey, + InitiatedAt = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc), + InitiatedBy = "alice@contoso.invalid", + Status = "InProgress", + CompletedAt = null, + FailedAt = null, + FailedPhase = null, + }; + + await harness.Sut.UpsertMarkerAsync(marker); + + var fetched = await harness.Sut.TryGetMarkerAsync(TenantId); + Assert.NotNull(fetched); + Assert.Equal(marker.PartitionKey, fetched!.PartitionKey); + Assert.Equal(marker.RowKey, fetched.RowKey); + Assert.Equal(marker.TenantId, fetched.TenantId); + Assert.Equal(marker.OffboardingHistoryRowKey, fetched.OffboardingHistoryRowKey); + Assert.Equal(marker.InitiatedAt, fetched.InitiatedAt); + Assert.Equal(marker.InitiatedBy, fetched.InitiatedBy); + Assert.Equal("InProgress", fetched.Status); + } + + [Fact] + public async Task Marker_CompletedTransition_PreservesTimestamp() + { + var harness = new Harness(); + var completed = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = HistoryRowKey, + InitiatedAt = DateTime.UtcNow.AddMinutes(-10), + InitiatedBy = "bob@contoso.invalid", + Status = "Completed", + CompletedAt = DateTime.UtcNow, + }; + + await harness.Sut.UpsertMarkerAsync(completed); + var fetched = await harness.Sut.TryGetMarkerAsync(TenantId); + + Assert.Equal("Completed", fetched!.Status); + Assert.NotNull(fetched.CompletedAt); + } + + [Fact] + public async Task Marker_TryGet_ReturnsNullOn404() + { + var harness = new Harness(throwOnGet: new RequestFailedException(404, "NotFound", "ResourceNotFound", null)); + Assert.Null(await harness.Sut.TryGetMarkerAsync(TenantId)); + } + + [Fact] + public async Task Marker_DeleteSwallows404_Idempotent() + { + var harness = new Harness(throwOnDelete: new RequestFailedException(404, "NotFound", "ResourceNotFound", null)); + await harness.Sut.DeleteMarkerAsync(TenantId); + // No exception → idempotent. + } + + [Fact] + public async Task History_RoundTrips_RetainCountersAndDrainCompletedAt() + { + var harness = new Harness(); + var now = new DateTime(2026, 5, 18, 9, 15, 23, DateTimeKind.Utc); + var history = new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = HistoryRowKey, + TenantId = TenantId, + DomainName = "contoso.invalid", + InitiatedBy = "alice@contoso.invalid", + OffboardedAt = now, + Status = "Completed", + CompletedAt = now.AddMinutes(5), + DeletedRowCountsJson = "{\"AuditLogs\":42}", + TotalRowsDeleted = 42, + DeletedBlobCount = 7, + CascadeSessionsEnqueued = 3, + RetryCount = 0, + DrainCompletedAt = now.AddMinutes(3), // Rev-9-F1 + CustomGatherRulesArchived = 5, + CustomAnalyzeRulesArchived = 3, + ImeLogPatternOverridesArchived = 2, + }; + + await harness.Sut.UpsertHistoryAsync(history); + var fetched = await harness.Sut.TryGetHistoryAsync(HistoryRowKey); + + Assert.NotNull(fetched); + Assert.Equal(history.Status, fetched!.Status); + Assert.Equal(history.TotalRowsDeleted, fetched.TotalRowsDeleted); + Assert.Equal(history.DeletedBlobCount, fetched.DeletedBlobCount); + Assert.Equal(history.CascadeSessionsEnqueued, fetched.CascadeSessionsEnqueued); + Assert.Equal(history.DrainCompletedAt, fetched.DrainCompletedAt); + Assert.Equal(5, fetched.CustomGatherRulesArchived); + Assert.Equal(3, fetched.CustomAnalyzeRulesArchived); + Assert.Equal(2, fetched.ImeLogPatternOverridesArchived); + } + + [Fact] + public async Task Pointer_TryGet_OnEmpty_ReturnsNullPointerAndNullEtag() + { + var harness = new Harness(); + var (pointer, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId); + Assert.Null(pointer); + Assert.Null(etag); + } + + [Fact] + public async Task Pointer_Insert_RoundTrips_LatestHistoryRowKeyAndOffboardCount() + { + var harness = new Harness(); + var pointer = new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = HistoryRowKey, + LatestStatus = "Initiated", + LatestUpdatedAt = DateTime.UtcNow, + OffboardCount = 1, + }; + + await harness.Sut.InsertByTenantPointerAsync(pointer); + + var (fetched, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId); + Assert.NotNull(fetched); + Assert.NotNull(etag); + Assert.Equal(pointer.LatestHistoryRowKey, fetched!.LatestHistoryRowKey); + Assert.Equal("Initiated", fetched.LatestStatus); + Assert.Equal(1, fetched.OffboardCount); + } + + [Fact] + public async Task Pointer_Insert_OnDuplicate_Throws409() + { + var harness = new Harness(); + var pointer = BuildPointer(latestStatus: "Initiated", offboardCount: 1); + await harness.Sut.InsertByTenantPointerAsync(pointer); + + var ex = await Assert.ThrowsAsync( + () => harness.Sut.InsertByTenantPointerAsync(pointer)); + Assert.Equal(409, ex.Status); + } + + [Fact] + public async Task Pointer_UpdateWithEtag_MatchingEtag_Replaces() + { + var harness = new Harness(); + await harness.Sut.InsertByTenantPointerAsync(BuildPointer(latestStatus: "Initiated", offboardCount: 1)); + + var (current, etag) = await harness.Sut.TryGetByTenantPointerAsync(TenantId); + Assert.NotNull(current); + current!.LatestStatus = "InProgress"; + current.OffboardCount = 2; + current.LatestUpdatedAt = DateTime.UtcNow; + + await harness.Sut.UpdateByTenantPointerWithEtagAsync(current, etag!); + + var (refetched, _) = await harness.Sut.TryGetByTenantPointerAsync(TenantId); + Assert.Equal("InProgress", refetched!.LatestStatus); + Assert.Equal(2, refetched.OffboardCount); + } + + [Fact] + public async Task Pointer_UpdateWithEtag_StaleEtag_Throws412() + { + // Plan §4.4 contract: stale ETag → 412 so the caller's bounded retry loop fires. + // The blind Upsert path was removed because it could clobber a concurrent worker. + var harness = new Harness(); + await harness.Sut.InsertByTenantPointerAsync(BuildPointer(latestStatus: "Initiated")); + + var stale = BuildPointer(latestStatus: "InProgress", offboardCount: 99); + var ex = await Assert.ThrowsAsync( + () => harness.Sut.UpdateByTenantPointerWithEtagAsync(stale, ifMatchEtag: "\"0xSTALE\"")); + Assert.Equal(412, ex.Status); + } + + [Fact] + public async Task Pointer_UpdateWithEtag_RequiresIfMatchEtag() + { + var harness = new Harness(); + await Assert.ThrowsAsync( + () => harness.Sut.UpdateByTenantPointerWithEtagAsync(BuildPointer(), "")); + } + + private static OffboardingByTenantPointer BuildPointer( + string latestStatus = "Initiated", + int offboardCount = 1) => new() + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = HistoryRowKey, + LatestStatus = latestStatus, + LatestUpdatedAt = DateTime.UtcNow, + OffboardCount = offboardCount, + }; + + [Fact] + public async Task Marker_RejectsWrongPartitionKey() + { + var harness = new Harness(); + var bad = new OffboardingMarkerEntry + { + PartitionKey = "WrongPK", + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = HistoryRowKey, + InitiatedAt = DateTime.UtcNow, + InitiatedBy = "alice@contoso.invalid", + }; + + await Assert.ThrowsAsync(() => harness.Sut.UpsertMarkerAsync(bad)); + } + + [Fact] + public async Task History_RejectsWrongPartitionKey() + { + var harness = new Harness(); + var bad = new OffboardingHistoryEntry + { + PartitionKey = "WrongPK", + RowKey = HistoryRowKey, + TenantId = TenantId, + InitiatedBy = "alice@contoso.invalid", + OffboardedAt = DateTime.UtcNow, + }; + + await Assert.ThrowsAsync(() => harness.Sut.UpsertHistoryAsync(bad)); + } + + [Fact] + public async Task Pointer_RejectsWrongPartitionKey() + { + var harness = new Harness(); + var bad = new OffboardingByTenantPointer + { + PartitionKey = "WrongPK", + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = HistoryRowKey, + LatestStatus = "Initiated", + LatestUpdatedAt = DateTime.UtcNow, + }; + + await Assert.ThrowsAsync(() => harness.Sut.InsertByTenantPointerAsync(bad)); + } + + // ── Harness — in-memory dictionary of (PK,RK) → TableEntity backing the mock ── + // + // ETag-aware: every successful insert/upsert/update stamps a fresh ETag onto the stored + // entity so Get reads return it and Update can enforce If-Match. Mirrors enough of Azure + // Table Storage's behaviour to exercise the §4.4 Pointer CAS-Loop honestly. + + private sealed class Harness + { + public TableOffboardingAuditRepository Sut { get; } + + private readonly Dictionary<(string Pk, string Rk), TableEntity> _store = new(); + private int _etagCounter; + + public Harness( + RequestFailedException? throwOnGet = null, + RequestFailedException? throwOnDelete = null) + { + var mockTableClient = new Mock(); + + mockTableClient + .Setup(c => c.UpsertEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((e, _, _) => + { + e.ETag = StampEtag(); + _store[(e.PartitionKey, e.RowKey)] = e; + return Task.FromResult(new Mock().Object); + }); + + mockTableClient + .Setup(c => c.AddEntityAsync( + It.IsAny(), + It.IsAny())) + .Returns((e, _) => + { + if (_store.ContainsKey((e.PartitionKey, e.RowKey))) + throw new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null); + e.ETag = StampEtag(); + _store[(e.PartitionKey, e.RowKey)] = e; + return Task.FromResult(new Mock().Object); + }); + + mockTableClient + .Setup(c => c.UpdateEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((e, ifMatch, _, _) => + { + if (!_store.TryGetValue((e.PartitionKey, e.RowKey), out var stored)) + throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null); + if (ifMatch != ETag.All && ifMatch != stored.ETag) + throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null); + e.ETag = StampEtag(); + _store[(e.PartitionKey, e.RowKey)] = e; + return Task.FromResult(new Mock().Object); + }); + + mockTableClient + .Setup(c => c.GetEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns, CancellationToken>((pk, rk, _, _) => + { + if (throwOnGet != null) throw throwOnGet; + if (!_store.TryGetValue((pk, rk), out var entity)) + throw new RequestFailedException(404, "NotFound", "ResourceNotFound", null); + return Task.FromResult(Response.FromValue(entity, new Mock().Object)); + }); + + mockTableClient + .Setup(c => c.DeleteEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((pk, rk, _, _) => + { + if (throwOnDelete != null) throw throwOnDelete; + _store.Remove((pk, rk)); + return Task.FromResult(new Mock().Object); + }); + + var mockServiceClient = new Mock(); + mockServiceClient.Setup(s => s.GetTableClient(It.IsAny())).Returns(mockTableClient.Object); + + var storage = new TableStorageService(mockServiceClient.Object, NullLogger.Instance); + Sut = new TableOffboardingAuditRepository(storage, NullLogger.Instance); + } + + private ETag StampEtag() => new($"\"0xFAKE_{++_etagCounter}\""); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TenantOffboardFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TenantOffboardFunctionTests.cs new file mode 100644 index 00000000..45cc8b2c --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TenantOffboardFunctionTests.cs @@ -0,0 +1,977 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Functions.Admin; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Functions.Tests.Offboarding; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Xunit; + +// Capture-helper for the recording enqueuer used in Finding-1 tests below. + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// PR3-side unit tests for the rewritten . The HTTP +/// entrypoint itself lives on top of HttpRequestData (abstract, hard to fake without +/// a real Functions host) so these tests focus on the function's CAS-loop helper. The +/// end-to-end happy/idempotent/race paths are covered by integration tests against the +/// real Functions host on dev tenants (see §10.3 smoketest in the plan). +/// +public sealed class TenantOffboardFunctionTests +{ + private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"; + + private static TenantOffboardFunction Build(IOffboardingAuditRepository repo) + => Build(repo, Mock.Of()); + + private static TenantOffboardFunction Build( + IOffboardingAuditRepository repo, ITenantOffboardingEnqueuer enqueuer) + { + // Resume-path tests (UpsertPointer / Resume_*) exercise just the CAS-loop + + // enqueuer wiring and don't care about the Disabled-gate. Stub IConfigRepository + // so EnsureTenantDisabledAsync's idempotent no-op path fires (returns an existing + // row already marked Disabled with the offboarding reason → no Save call). + // + // LastUpdated is set FAR in the past so the tombstone-deadline (LastUpdated + + // DrainBarrier) doesn't dominate the max() computation in the resume-path. That + // means the existing history-based-deadline tests still observe history-derived + // delays. Tests that exercise the LastUpdated-dominates path stand up their own + // configRepoMock with a recent LastUpdated. + var configRepoMock = new Mock(); + var alreadyDisabled = TenantConfiguration.CreateDefault(TenantId); + alreadyDisabled.Disabled = true; + alreadyDisabled.DisabledReason = "Offboarding in progress"; + alreadyDisabled.LastUpdated = DateTime.UtcNow.AddDays(-30); + configRepoMock + .Setup(r => r.GetTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync(alreadyDisabled); + + var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache( + new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions()); + var service = new TenantConfigurationService( + configRepoMock.Object, + NullLogger.Instance, + cache); + + return new TenantOffboardFunction( + logger: NullLogger.Instance, + configRepo: configRepoMock.Object, + tenantConfigService: service, + maintenanceRepo: Mock.Of(), + offboardingRepo: repo, + offboardingEnqueuer: enqueuer); + } + + /// + /// Test-side enqueuer recording every call. Production enqueuer is sealed + needs + /// configuration to ctor, so this fake lets the tests assert "did Phase 1 actually + /// hand off to the worker?" — the question Finding 1 says we previously got wrong. + /// + private sealed class RecordingEnqueuer : ITenantOffboardingEnqueuer + { + public readonly List Sent = new(); + public readonly List VisibilityDelays = new(); + public Exception? ThrowOnEnqueue { get; set; } + + public Task EnqueueAsync(TenantOffboardingEnvelope envelope, TimeSpan? visibilityDelay = null, CancellationToken ct = default) + { + if (ThrowOnEnqueue is { } ex) throw ex; + Sent.Add(envelope); + VisibilityDelays.Add(visibilityDelay); + return Task.CompletedTask; + } + } + + // Build helper exposing IConfigRepository + TenantConfigurationService for the + // EnsureTenantDisabledAsync tests. The repo is a Mock so we can pin Save calls; the + // service uses a real MemoryCache so InvalidateCache(tenantId) is observable via a + // pre-seeded cache entry. + private static (TenantOffboardFunction Sut, Mock ConfigRepo, Microsoft.Extensions.Caching.Memory.IMemoryCache Cache, TenantConfigurationService Service) + BuildWithConfigService() + { + var configRepoMock = new Mock(); + var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache( + new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions()); + var service = new TenantConfigurationService( + configRepoMock.Object, + NullLogger.Instance, + cache); + + var sut = new TenantOffboardFunction( + logger: NullLogger.Instance, + configRepo: configRepoMock.Object, + tenantConfigService: service, + maintenanceRepo: Mock.Of(), + offboardingRepo: new FakeOffboardingAuditRepository(), + offboardingEnqueuer: new RecordingEnqueuer()); + + return (sut, configRepoMock, cache, service); + } + + // ── UpsertPointerWithCasAsync ──────────────────────────────────────────── + + [Fact] + public async Task UpsertPointer_FreshTenant_Inserts_OffboardCount_1() + { + var repo = new FakeOffboardingAuditRepository(); + var sut = Build(repo); + var historyRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}"; + + await sut.UpsertPointerWithCasAsync(TenantId, historyRowKey, DateTime.UtcNow); + + Assert.True(repo.Pointers.TryGetValue(TenantId, out var entry)); + Assert.Equal(historyRowKey, entry.Pointer.LatestHistoryRowKey); + Assert.Equal("Initiated", entry.Pointer.LatestStatus); + Assert.Equal(1, entry.Pointer.OffboardCount); + } + + [Fact] + public async Task UpsertPointer_ExistingTenant_IncrementsOffboardCount() + { + var repo = new FakeOffboardingAuditRepository(); + await repo.InsertByTenantPointerAsync(new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = "20260101000000000_" + TenantId, + LatestStatus = "Completed", + LatestUpdatedAt = DateTime.UtcNow.AddDays(-30), + OffboardCount = 2, + }); + + var sut = Build(repo); + var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}"; + + await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow); + + Assert.Equal(newHistoryRowKey, repo.Pointers[TenantId].Pointer.LatestHistoryRowKey); + Assert.Equal("Initiated", repo.Pointers[TenantId].Pointer.LatestStatus); + Assert.Equal(3, repo.Pointers[TenantId].Pointer.OffboardCount); + } + + [Fact] + public async Task UpsertPointer_409OnInsert_FallsBackToEtagUpdate() + { + // First TryGet returns null, but InsertByTenantPointerAsync throws 409 because + // a parallel writer inserted between our read and write. The CAS-loop must + // re-read and update with the now-existing pointer's ETag. + var repoMock = new Mock(); + var existingPointer = new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = "20260101000000000_" + TenantId, + LatestStatus = "Completed", + OffboardCount = 1, + }; + + var attempt = 0; + repoMock + .Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny())) + .Returns(() => + { + attempt++; + return attempt == 1 + ? Task.FromResult<(OffboardingByTenantPointer?, string?)>((null, null)) + : Task.FromResult<(OffboardingByTenantPointer?, string?)>((existingPointer, "\"etag-1\"")); + }); + + repoMock + .Setup(r => r.InsertByTenantPointerAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null)); + + OffboardingByTenantPointer? updatedPointer = null; + repoMock + .Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny(), "\"etag-1\"", It.IsAny())) + .Returns((p, _, _) => + { + updatedPointer = p; + return Task.CompletedTask; + }); + + var sut = Build(repoMock.Object); + var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}"; + + await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow); + + Assert.NotNull(updatedPointer); + Assert.Equal(newHistoryRowKey, updatedPointer!.LatestHistoryRowKey); + Assert.Equal(2, updatedPointer.OffboardCount); + repoMock.Verify(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny(), "\"etag-1\"", It.IsAny()), Times.Once); + } + + [Fact] + public async Task UpsertPointer_412OnUpdate_RetriesAndSucceeds() + { + // Concurrent writer mutates the pointer between our read and our update. CAS-loop + // must re-read (getting the fresh ETag) and try once more. + var repoMock = new Mock(); + + var freshState = new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = "20260101000000000_" + TenantId, + LatestStatus = "Completed", + OffboardCount = 1, + }; + + var readEtags = new[] { "\"etag-old\"", "\"etag-new\"" }; + var readIdx = 0; + repoMock + .Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny())) + .Returns(() => + { + var etag = readEtags[Math.Min(readIdx, readEtags.Length - 1)]; + readIdx++; + return Task.FromResult<(OffboardingByTenantPointer?, string?)>((freshState, etag)); + }); + + var updateCalls = 0; + repoMock + .Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((_, etag, _) => + { + updateCalls++; + if (etag == "\"etag-old\"") + { + throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null); + } + return Task.CompletedTask; + }); + + var sut = Build(repoMock.Object); + var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}"; + + await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow); + + Assert.Equal(2, updateCalls); + Assert.Equal(2, readIdx); + } + + // ── ResumeExistingMarkerAsync (PR3 Review Finding 1) ───────────────────── + // + // The previous implementation returned 200 from the idempotency path UNCONDITIONALLY. + // If the very first enqueue failed (transient queue outage), the marker was committed + // and the queue was empty: every subsequent re-click took the idempotency-200 branch + // and never re-enqueued, leaving the tenant stuck Initiated forever. These tests pin + // the new defensive-re-enqueue behaviour: Initiated/InProgress → re-enqueue; Completed + // → no re-enqueue (worker is done); Failed → no re-enqueue (operator action required). + + [Theory] + [InlineData("Initiated")] + [InlineData("InProgress")] + public async Task Resume_InFlightStatus_DefensivelyReEnqueues(string status) + { + var repo = new FakeOffboardingAuditRepository(); + var enqueuer = new RecordingEnqueuer(); + var sut = Build(repo, enqueuer); + var historyRowKey = "20260518091523123_" + TenantId; + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = historyRowKey, + InitiatedAt = DateTime.UtcNow, + InitiatedBy = "alice@contoso.com", + Status = status, + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "bob@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind); + Assert.True(outcome.ReEnqueued); + Assert.Single(enqueuer.Sent); + Assert.Equal(historyRowKey, enqueuer.Sent[0].HistoryRowKey); + Assert.Equal(TenantId, enqueuer.Sent[0].TenantId); + } + + [Fact] + public async Task Resume_Completed_NoReEnqueue_ReturnsIdempotentOk() + { + var enqueuer = new RecordingEnqueuer(); + var sut = Build(new FakeOffboardingAuditRepository(), enqueuer); + var marker = new OffboardingMarkerEntry + { + RowKey = TenantId, TenantId = TenantId, + OffboardingHistoryRowKey = "20260101000000000_" + TenantId, + Status = "Completed", + CompletedAt = DateTime.UtcNow.AddMinutes(-5), + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.IdempotentCompleted, outcome.Kind); + Assert.False(outcome.ReEnqueued); + Assert.Empty(enqueuer.Sent); + } + + [Fact] + public async Task Resume_Failed_NoReEnqueue_SurfacesPhaseInMessage() + { + var enqueuer = new RecordingEnqueuer(); + var sut = Build(new FakeOffboardingAuditRepository(), enqueuer); + var marker = new OffboardingMarkerEntry + { + RowKey = TenantId, TenantId = TenantId, + OffboardingHistoryRowKey = "20260101000000000_" + TenantId, + Status = "Failed", + FailedAt = DateTime.UtcNow.AddMinutes(-10), + FailedPhase = "drain_timeout", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.IdempotentFailed, outcome.Kind); + Assert.False(outcome.ReEnqueued); + Assert.Contains("drain_timeout", outcome.Message); + Assert.Empty(enqueuer.Sent); + } + + [Fact] + public async Task Resume_InFlight_EnqueueThrows_SurfacesReEnqueueFailed() + { + // The defensive re-enqueue must surface its failure (500) so the caller knows + // the resume did NOT take effect. The marker stays Initiated; the next click + // will trigger another resume attempt — that's the operator-visible recovery + // mechanism that previously didn't exist (stuck-tenant bug). + var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") }; + var sut = Build(new FakeOffboardingAuditRepository(), enqueuer); + var marker = new OffboardingMarkerEntry + { + RowKey = TenantId, TenantId = TenantId, + OffboardingHistoryRowKey = "20260101000000000_" + TenantId, + Status = "Initiated", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.ReEnqueueFailed, outcome.Kind); + Assert.False(outcome.ReEnqueued); + } + + [Fact] + public async Task Resume_409Race_StillReEnqueues_BecauseWinnerMayHaveFailed() + { + // The 409-race path resolves to the winning marker and routes through the same + // resume code. We can't know whether the winner's enqueue succeeded, so we + // must re-enqueue defensively — same Finding-1 contract as the plain idempotency + // re-click path. + var enqueuer = new RecordingEnqueuer(); + var sut = Build(new FakeOffboardingAuditRepository(), enqueuer); + var marker = new OffboardingMarkerEntry + { + RowKey = TenantId, TenantId = TenantId, + OffboardingHistoryRowKey = "20260101000000000_" + TenantId, + Status = "Initiated", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: true); + + Assert.True(outcome.ReEnqueued); + Assert.Single(enqueuer.Sent); + Assert.Contains("race", outcome.Message); + } + + [Fact] + public async Task UpsertPointer_CasExhausted_Throws() + { + // Every UpdateWithEtag returns 412 — the bounded retry-loop must surface this + // rather than retry indefinitely. + var repoMock = new Mock(); + var freshState = new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = TenantId, + TenantId = TenantId, + LatestHistoryRowKey = "20260101000000000_" + TenantId, + LatestStatus = "Completed", + OffboardCount = 1, + }; + + repoMock + .Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny())) + .ReturnsAsync((freshState, "\"some-etag\"")); + + repoMock + .Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null)); + + var sut = Build(repoMock.Object); + var historyRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}"; + + await Assert.ThrowsAsync(() => + sut.UpsertPointerWithCasAsync(TenantId, historyRowKey, DateTime.UtcNow)); + } + + // ── Cache-Drain-Barrier (plan v2 §2.2 / §2.3) ───────────────────────────── + // + // The DrainBarrier (hardcoded 6min) prevents Phase 2 from starting before all + // function-host instances drain their TenantConfigurationService cache. Phase 1 + // enqueues with visibilityDelay=DrainBarrier so the worker only sees the message + // after the barrier elapses. A resume-click reads the stored EarliestProcessingAt + // and re-enqueues with the REMAINING delay so the barrier does not reset. + + [Fact] + public void DrainBarrier_HardcodedAt6Minutes() + { + // Plan v2 §2.2: 5min TenantConfigurationService.CacheDuration + 1min safety buffer. + Assert.Equal(TimeSpan.FromMinutes(6), TenantOffboardFunction.DrainBarrier); + } + + [Fact] + public async Task Resume_InFlight_UsesRemainingBarrier_NotFullBarrier() + { + // Re-click 2 min into the 6-min barrier → re-enqueue must carry ~4min visibility delay, + // not a fresh 6min. Otherwise each re-click would reset the cache-drain window. + var repo = new FakeOffboardingAuditRepository(); + var enqueuer = new RecordingEnqueuer(); + var sut = Build(repo, enqueuer); + + var initiatedAt = DateTime.UtcNow.AddMinutes(-2); + var earliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // ~+4min from now + var historyRowKey = "20260518091523123_" + TenantId; + + await repo.InsertHistoryAsync(new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = historyRowKey, + TenantId = TenantId, + DomainName = "contoso.com", + InitiatedBy = "alice@contoso.com", + OffboardedAt = initiatedAt, + EarliestProcessingAt = earliestProcessingAt, + Status = "Initiated", + }); + + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = historyRowKey, + InitiatedAt = initiatedAt, + InitiatedBy = "alice@contoso.com", + Status = "Initiated", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind); + Assert.Single(enqueuer.VisibilityDelays); + var delay = enqueuer.VisibilityDelays[0]; + Assert.NotNull(delay); + // Allow a few seconds of test-runtime slack — clock advanced between Setup and Act. + Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.5)); + Assert.Equal(earliestProcessingAt, outcome.EarliestProcessingAt); + } + + [Fact] + public async Task Resume_InFlight_PastBarrier_UsesZeroDelay() + { + // Re-click 8 min into a 6-min barrier → barrier already elapsed → re-enqueue with + // zero visibility delay so the worker picks up immediately. + var repo = new FakeOffboardingAuditRepository(); + var enqueuer = new RecordingEnqueuer(); + var sut = Build(repo, enqueuer); + + var initiatedAt = DateTime.UtcNow.AddMinutes(-8); + var earliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // 2 min ago + var historyRowKey = "20260518091523123_" + TenantId; + + await repo.InsertHistoryAsync(new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = historyRowKey, + TenantId = TenantId, + DomainName = "contoso.com", + InitiatedBy = "alice@contoso.com", + OffboardedAt = initiatedAt, + EarliestProcessingAt = earliestProcessingAt, + Status = "InProgress", + }); + + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = historyRowKey, + InitiatedAt = initiatedAt, + InitiatedBy = "alice@contoso.com", + Status = "InProgress", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind); + Assert.Single(enqueuer.VisibilityDelays); + Assert.Equal(TimeSpan.Zero, enqueuer.VisibilityDelays[0]); + } + + [Fact] + public async Task Resume_InFlight_MissingHistoryRow_UsesTombstoneLastUpdatedAsDeadlineAnchor() + { + // The marker references a History row that does not exist (corruption / manual + // delete). Without history.EarliestProcessingAt to read, the resume falls back + // to tombstone.LastUpdated + DrainBarrier as the authoritative drain-deadline. + // The Build-helper's default tombstone has LastUpdated = 30 days ago → the + // tombstone-derived deadline is far in the past → drain barrier already expired + // → visibility-delay = Zero (worker picks immediately). That is the CORRECT + // outcome: if the tombstone has been sitting for 30 days, every cache has been + // through the 5-min TTL many times already. + var repo = new FakeOffboardingAuditRepository(); + var enqueuer = new RecordingEnqueuer(); + var sut = Build(repo, enqueuer); + + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = "20260518091523123_" + TenantId, + InitiatedAt = DateTime.UtcNow, + InitiatedBy = "alice@contoso.com", + Status = "Initiated", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind); + Assert.Single(enqueuer.VisibilityDelays); + // 30-day-old tombstone → drain barrier long expired → 0 delay is safe. + Assert.Equal(TimeSpan.Zero, enqueuer.VisibilityDelays[0]); + } + + // ── EnsureTenantDisabledAsync (PR3.B Review Finding 1 — fail-loud Disabled-gate) ── + // + // Without Disabled=true, the cache-drain barrier guards nothing: warm function-hosts + // would keep accepting agent traffic for a tenant whose Phase 2 is about to wipe + // their data. Every initial enqueue + every resume re-enqueue MUST commit this gate. + + [Fact] + public async Task EnsureTenantDisabled_FlipsDisabled_AndInvalidatesCache() + { + var (sut, configRepo, cache, _) = BuildWithConfigService(); + var existing = TenantConfiguration.CreateDefault(TenantId); + existing.Disabled = false; + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing); + + TenantConfiguration? saved = null; + configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny())) + .Callback(c => saved = c) + .ReturnsAsync(true); + + // Seed cache with stale "Disabled=false" entry so we can prove invalidation. + cache.Set($"tenant-config:{TenantId}", existing); + + await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"); + + Assert.NotNull(saved); + Assert.True(saved!.Disabled); + Assert.Equal("Offboarding in progress", saved.DisabledReason); + Assert.Null(saved.DisabledUntil); + Assert.Equal("alice@contoso.com", saved.UpdatedBy); + Assert.False(cache.TryGetValue($"tenant-config:{TenantId}", out _), + "cache entry must be invalidated so warm reads pick up the new Disabled=true on next access"); + } + + [Fact] + public async Task EnsureTenantDisabled_AlreadyDisabledWithSameReason_DoesNotRewrite() + { + var (sut, configRepo, _, _) = BuildWithConfigService(); + var existing = TenantConfiguration.CreateDefault(TenantId); + existing.Disabled = true; + existing.DisabledReason = "Offboarding in progress"; + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing); + + await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"); + + // Idempotent: no Save thrash on resume-clicks while the marker is already in flight. + configRepo.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task EnsureTenantDisabled_StorageThrowsOnRead_PropagatesAsExpected() + { + // Caller (initial enqueue + resume re-enqueue) MUST treat this as fail-loud and + // skip the enqueue. We just assert the exception propagates — the caller's + // try/catch + 500-return is exercised by integration tests. + var (sut, configRepo, _, _) = BuildWithConfigService(); + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ThrowsAsync(new InvalidOperationException("storage unavailable")); + + await Assert.ThrowsAsync(() => + sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com")); + } + + [Fact] + public async Task EnsureTenantDisabled_StorageThrowsOnSave_PropagatesAsExpected() + { + var (sut, configRepo, _, _) = BuildWithConfigService(); + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ReturnsAsync(TenantConfiguration.CreateDefault(TenantId)); + configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny())) + .ThrowsAsync(new InvalidOperationException("storage save failed")); + + await Assert.ThrowsAsync(() => + sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com")); + } + + [Fact] + public async Task EnsureTenantDisabled_NoConfigRow_CreatesFreshTombstone() + { + // PR3.B-revised Finding 3: missing row used to log+continue, leaving AuthFunction's + // auto-create-default free to spawn a Disabled=false row on the next /api/auth/me. + // The fix creates a fresh CreateDefault row with Disabled=true so the auth-gate is + // committed even in this edge case. + var (sut, configRepo, _, _) = BuildWithConfigService(); + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ReturnsAsync((TenantConfiguration?)null); + + TenantConfiguration? saved = null; + configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny())) + .Callback(c => saved = c) + .ReturnsAsync(true); + + await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"); + + Assert.NotNull(saved); + Assert.Equal(TenantId, saved!.TenantId); + Assert.True(saved.Disabled, "fresh tombstone must carry Disabled=true"); + Assert.Equal("Offboarding in progress", saved.DisabledReason); + Assert.Equal("alice@contoso.com", saved.UpdatedBy); + } + + [Fact] + public async Task EnsureTenantDisabled_SaveReturnsFalse_ThrowsAsHardFail() + { + // PR3.B-revised Finding 2: TableConfigRepository.SaveTenantConfigurationAsync + // returns false on transient storage failure (it does NOT throw). Ignoring that + // return-value would let the caller proceed to enqueue without the auth-gate + // committed. Turn it into a hard exception so the offboard is NOT enqueued. + var (sut, configRepo, _, _) = BuildWithConfigService(); + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ReturnsAsync(TenantConfiguration.CreateDefault(TenantId)); + configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync(false); // transient storage rejection — NOT a throw. + + var ex = await Assert.ThrowsAsync(() => + sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com")); + Assert.Contains("Disabled-gate NOT committed", ex.Message); + } + + [Fact] + public async Task EnsureTenantDisabled_NoConfigRow_SaveReturnsFalse_ThrowsAsHardFail() + { + // Combined edge case: missing row → we'd create a fresh tombstone → Save fails → + // hard fail (otherwise the caller would proceed without an auth-gate). + var (sut, configRepo, _, _) = BuildWithConfigService(); + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)) + .ReturnsAsync((TenantConfiguration?)null); + configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync(false); + + await Assert.ThrowsAsync(() => + sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com")); + } + + [Fact] + public async Task EnsureTenantDisabled_AlreadyTombstone_ReturnsAlreadyTombstone() + { + var (sut, configRepo, _, _) = BuildWithConfigService(); + var tombstone = TenantConfiguration.CreateDefault(TenantId); + tombstone.Disabled = true; + tombstone.DisabledReason = "Offboarding in progress"; + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(tombstone); + + var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"); + + Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.AlreadyTombstone, result.Outcome); + configRepo.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task EnsureTenantDisabled_FlipsDisabled_ReturnsWroteNewTombstone() + { + var (sut, configRepo, _, _) = BuildWithConfigService(); + var existing = TenantConfiguration.CreateDefault(TenantId); + existing.Disabled = false; + configRepo.Setup(r => r.GetTenantConfigurationAsync(TenantId)).ReturnsAsync(existing); + configRepo.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync(true); + + var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"); + + Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.WroteNewTombstone, result.Outcome); + } + + // ── Resume-path cache-drain bypass fix (Codex review) ──────────────────── + // + // Scenario: initial attempt commits Marker/History/Pointer (History.EarliestProcessingAt + // = T+6min) but then EnsureTenantDisabledAsync FAILS → 500 returned, no enqueue. The + // admin re-clicks 10 minutes later. The History row still carries the now-past + // EarliestProcessingAt = T+6min. Without the fix, the resume would naively use the + // remaining-from-history delta and arrive at Zero → enqueue with no visibility-delay + // → worker picks immediately → but Disabled=true was JUST written for the first time + // → warm function-host caches still hold Disabled=false → cache-drain barrier violated. + // + // The fix anchors on tombstone.LastUpdated + DrainBarrier so the deadline is + // authoritative from the same atomic write as the Disabled-flip. + + [Fact] + public async Task Resume_FirstAttemptFailedBeforeDisabledWrite_UsesFullBarrierAndPatchesHistory() + { + var repo = new FakeOffboardingAuditRepository(); + var enqueuer = new RecordingEnqueuer(); + + var initiatedAt = DateTime.UtcNow.AddMinutes(-10); // first attempt 10min ago + var staleEarliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // ~4 min in the past + var historyRowKey = "20260518091523123_" + TenantId; + + await repo.InsertHistoryAsync(new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = historyRowKey, + TenantId = TenantId, + DomainName = "contoso.com", + InitiatedBy = "alice@contoso.com", + OffboardedAt = initiatedAt, + EarliestProcessingAt = staleEarliestProcessingAt, + Status = "Initiated", + }); + + // Build SUT where the first attempt FAILED at EnsureTenantDisabledAsync: Config + // exists but is NOT yet the offboarding tombstone (Disabled=false). The resume's + // EnsureTenantDisabledAsync will write it now → WroteNewTombstone. + var configRepoMock = new Mock(); + var notYetDisabled = TenantConfiguration.CreateDefault(TenantId); + notYetDisabled.Disabled = false; + configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync(notYetDisabled); + configRepoMock.Setup(r => r.SaveTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync(true); + + var cache = new MemoryCache(new MemoryCacheOptions()); + var tenantConfigService = new TenantConfigurationService( + configRepoMock.Object, + NullLogger.Instance, + cache); + + var sut = new TenantOffboardFunction( + logger: NullLogger.Instance, + configRepo: configRepoMock.Object, + tenantConfigService: tenantConfigService, + maintenanceRepo: Mock.Of(), + offboardingRepo: repo, + offboardingEnqueuer: enqueuer); + + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = historyRowKey, + InitiatedAt = initiatedAt, + InitiatedBy = "alice@contoso.com", + Status = "Initiated", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind); + Assert.Single(enqueuer.VisibilityDelays); + + // CRITICAL: visibility-delay must be the full DrainBarrier (6 min), NOT Zero. The + // History row's stale EarliestProcessingAt must NOT be honoured when Disabled was + // just written for the first time. Allow a few-ms slack because the delay is + // computed as (effectiveDeadline - DateTime.UtcNow), so a moment passes between + // the LastUpdated stamp and the delay computation. + var actualDelay = enqueuer.VisibilityDelays[0]!.Value; + Assert.InRange(actualDelay, TenantOffboardFunction.DrainBarrier - TimeSpan.FromSeconds(1), TenantOffboardFunction.DrainBarrier); + + // The History row must have been patched so further re-clicks see the new deadline. + Assert.True(repo.History.ContainsKey(historyRowKey)); + var patchedHistory = repo.History[historyRowKey]; + Assert.NotNull(patchedHistory.EarliestProcessingAt); + Assert.True(patchedHistory.EarliestProcessingAt > DateTime.UtcNow.AddMinutes(5), + $"EarliestProcessingAt must be roughly now+6min after the patch, got {patchedHistory.EarliestProcessingAt:O}"); + + // ResumeOutcome's EarliestProcessingAt is the NEW deadline (UI countdown source of truth). + Assert.NotNull(outcome.EarliestProcessingAt); + Assert.True(outcome.EarliestProcessingAt > DateTime.UtcNow.AddMinutes(5)); + } + + [Fact] + public async Task Resume_HistoryPatchFails_SecondResumeStillRespectsBarrierViaTombstoneLastUpdated() + { + // The exact Codex Finding scenario: first resume writes tombstone successfully + // but the History.EarliestProcessingAt patch fails (transient). A second resume + // arriving moments later reads History with the OLD stale EarliestProcessingAt + // but sees the tombstone (AlreadyTombstone path). The deadline computation must + // still respect the cache-drain barrier via tombstone.LastUpdated + DrainBarrier. + // + // Without the LastUpdated-anchor fix, the second resume would compute + // visibility-delay = max(0, stale-history-deadline - now) = 0 → worker picks + // before the cache-drain barrier from the tombstone-write completes. + var repo = new FakeOffboardingAuditRepository(); + var enqueuer = new RecordingEnqueuer(); + + var firstAttempt = DateTime.UtcNow.AddMinutes(-10); + var staleHistoryDeadline = firstAttempt + TenantOffboardFunction.DrainBarrier; // ~now-4min + var historyRowKey = "20260518091523123_" + TenantId; + + await repo.InsertHistoryAsync(new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = historyRowKey, + TenantId = TenantId, + DomainName = "contoso.com", + InitiatedBy = "alice@contoso.com", + OffboardedAt = firstAttempt, + EarliestProcessingAt = staleHistoryDeadline, // STALE — patch failed before + Status = "Initiated", + }); + + // Tombstone-already-in-place from the prior (first) resume; LastUpdated reflects + // when Disabled was committed in that first resume = recently (now). The History + // patch had failed, so EarliestProcessingAt is still the stale firstAttempt+6min. + var configRepoMock = new Mock(); + var tombstone = TenantConfiguration.CreateDefault(TenantId); + tombstone.Disabled = true; + tombstone.DisabledReason = "Offboarding in progress"; + tombstone.LastUpdated = DateTime.UtcNow.AddSeconds(-1); // freshly written ~1s ago + configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync(tombstone); + + var cache = new MemoryCache(new MemoryCacheOptions()); + var tenantConfigService = new TenantConfigurationService( + configRepoMock.Object, + NullLogger.Instance, + cache); + + var sut = new TenantOffboardFunction( + logger: NullLogger.Instance, + configRepo: configRepoMock.Object, + tenantConfigService: tenantConfigService, + maintenanceRepo: Mock.Of(), + offboardingRepo: repo, + offboardingEnqueuer: enqueuer); + + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = historyRowKey, + InitiatedAt = firstAttempt, + InitiatedBy = "alice@contoso.com", + Status = "Initiated", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind); + Assert.Single(enqueuer.VisibilityDelays); + + // SAFETY: the visibility-delay must come from tombstone.LastUpdated + DrainBarrier + // (~6min remaining), NOT from the stale history deadline (which would yield 0). + // Anything below ~5min would mean we relied on the corrupted history value. + var delay = enqueuer.VisibilityDelays[0]!.Value; + Assert.InRange(delay, TimeSpan.FromMinutes(5.5), TenantOffboardFunction.DrainBarrier); + } + + [Fact] + public async Task Resume_TombstoneAlreadyInPlace_UsesRemainingBarrier_DoesNotPatchHistory() + { + // Counterpoint: tombstone was already there from the prior successful attempt; + // the original deadline still applies. NO patch of History.EarliestProcessingAt. + var repo = new FakeOffboardingAuditRepository(); + var enqueuer = new RecordingEnqueuer(); + + var initiatedAt = DateTime.UtcNow.AddMinutes(-2); // first attempt 2min ago, mid-barrier + var originalEarliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; + var historyRowKey = "20260518091523123_" + TenantId; + + await repo.InsertHistoryAsync(new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = historyRowKey, + TenantId = TenantId, + DomainName = "contoso.com", + InitiatedBy = "alice@contoso.com", + OffboardedAt = initiatedAt, + EarliestProcessingAt = originalEarliestProcessingAt, + Status = "Initiated", + }); + + // Build SUT where the tombstone is already in place from the prior attempt at + // initiatedAt (2min ago). The tombstone.LastUpdated reflects when Disabled was + // committed; the resume's deadline math is max(history.EarliestProcessingAt, + // tombstone.LastUpdated + DrainBarrier). Both terms ≈ initiatedAt + 6min here, + // so remaining ≈ 4min as before. + var configRepoMock = new Mock(); + var tombstone = TenantConfiguration.CreateDefault(TenantId); + tombstone.Disabled = true; + tombstone.DisabledReason = "Offboarding in progress"; + tombstone.LastUpdated = initiatedAt; + configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny())) + .ReturnsAsync(tombstone); + + var cache = new MemoryCache(new MemoryCacheOptions()); + var tenantConfigService = new TenantConfigurationService( + configRepoMock.Object, + NullLogger.Instance, + cache); + + var sut = new TenantOffboardFunction( + logger: NullLogger.Instance, + configRepo: configRepoMock.Object, + tenantConfigService: tenantConfigService, + maintenanceRepo: Mock.Of(), + offboardingRepo: repo, + offboardingEnqueuer: enqueuer); + + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, + TenantId = TenantId, + OffboardingHistoryRowKey = historyRowKey, + InitiatedAt = initiatedAt, + InitiatedBy = "alice@contoso.com", + Status = "Initiated", + }; + + var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false); + + Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind); + Assert.Single(enqueuer.VisibilityDelays); + + // Remaining delay should be ~4min (6min - 2min). Allow a couple seconds of slack. + var delay = enqueuer.VisibilityDelays[0]!.Value; + Assert.InRange(delay, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.5)); + + // History.EarliestProcessingAt must NOT have been overwritten — it still carries + // the original deadline from the first attempt. + var historyAfter = repo.History[historyRowKey]; + Assert.Equal(originalEarliestProcessingAt, historyAfter.EarliestProcessingAt); + + // No Save call (the tombstone was already correct). + configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny()), Times.Never); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/DataAccessServiceExtensions.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/DataAccessServiceExtensions.cs index 5b39cd64..1ac06ab8 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/DataAccessServiceExtensions.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/DataAccessServiceExtensions.cs @@ -44,6 +44,13 @@ namespace AutopilotMonitor.Functions.DataAccess services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + + // Tenant-offboarding audit (Marker / History / ByTenant pointer). + services.AddSingleton(); + // Tenant-offboarding customs archive (PR3.B): snapshot of GatherRules / + // AnalyzeRules / ImeLogPatterns rows that were wiped during Phase 2.D-archive, + // surfaced via the Global Admin /admin/customs-archive page. + services.AddSingleton(); return services; } diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs index 9d3fe260..1027674e 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs @@ -1,3 +1,6 @@ +using System.Runtime.CompilerServices; +using System.Threading; +using AutopilotMonitor.Functions.Security; using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; @@ -74,186 +77,45 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task IsSessionIndexEmptyAsync() => _storage.IsSessionIndexEmptyAsync(); - // --- Tenant Offboarding --- - - public async Task> DeleteAllTenantDataAsync(string tenantId) - { - var deletedCounts = new Dictionary(); - var normalizedTenantId = tenantId.ToLowerInvariant(); - - // Tables where PartitionKey = tenantId (normalized lowercase) - var tenantPartitionTables = new[] - { - Constants.TableNames.Sessions, - Constants.TableNames.SessionsIndex, - Constants.TableNames.AuditLogs, - Constants.TableNames.UsageMetrics, - Constants.TableNames.UserActivity, - Constants.TableNames.GatherRules, - Constants.TableNames.AnalyzeRules, - Constants.TableNames.AppInstallSummaries, - Constants.TableNames.TenantConfiguration, - Constants.TableNames.TenantAdmins, - Constants.TableNames.BootstrapSessions, - Constants.TableNames.BlockedDevices, - Constants.TableNames.ImeLogPatterns, - Constants.TableNames.RuleStates, - }; - - foreach (var tableName in tenantPartitionTables) - { - var deleted = await DeleteAllRowsByPartitionKeyAsync(tableName, normalizedTenantId); - deletedCounts[tableName] = deleted; - _logger.LogInformation("Offboard [{TenantId}] {TableName}: deleted {Deleted} rows", tenantId, tableName, deleted); - } - - // Tables with composite PartitionKey = "{tenantId}_{sessionId}" – query by prefix - var eventsDeleted = await DeleteByTenantPrefixAsync(Constants.TableNames.Events, normalizedTenantId); - deletedCounts[Constants.TableNames.Events] = eventsDeleted; - _logger.LogInformation("Offboard [{TenantId}] Events: deleted {Deleted} rows", tenantId, eventsDeleted); - - var ruleResultsDeleted = await DeleteByTenantPrefixAsync(Constants.TableNames.RuleResults, normalizedTenantId); - deletedCounts[Constants.TableNames.RuleResults] = ruleResultsDeleted; - _logger.LogInformation("Offboard [{TenantId}] RuleResults: deleted {Deleted} rows", tenantId, ruleResultsDeleted); - - // BootstrapSessions CodeLookup entries (PartitionKey = "CodeLookup", TenantId property = tenantId) - var codeLookupDeleted = await DeleteCodeLookupEntriesAsync(normalizedTenantId); - deletedCounts["BootstrapSessions_CodeLookup"] = codeLookupDeleted; - _logger.LogInformation("Offboard [{TenantId}] BootstrapSessions CodeLookup: deleted {Deleted} rows", tenantId, codeLookupDeleted); - - return deletedCounts; - } - - /// - /// Deletes all rows in a table where PartitionKey equals the given value. - /// Uses batched deletes (max 100 per transaction, same PartitionKey required). - /// - private async Task DeleteAllRowsByPartitionKeyAsync(string tableName, string partitionKey) - { - try - { - var tableClient = _storage.GetTableClient(tableName); - var filter = $"PartitionKey eq '{partitionKey}'"; - var entities = tableClient.QueryAsync(filter, select: new[] { "PartitionKey", "RowKey" }); - - int deleted = 0; - var batch = new List(); - - await foreach (var entity in entities) - { - batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity)); - deleted++; - - if (batch.Count >= 100) - { - await tableClient.SubmitTransactionAsync(batch); - batch.Clear(); - } - } - - if (batch.Count > 0) - { - await tableClient.SubmitTransactionAsync(batch); - } - - return deleted; - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to delete rows in {TableName} for partition {PartitionKey}", tableName, partitionKey); - return 0; - } - } - - /// - /// Deletes all rows for a tenant from a table that uses composite PartitionKey = "{tenantId}_{sessionId}". - /// Used for Events and RuleResults tables. - /// - private async Task DeleteByTenantPrefixAsync(string tableName, string normalizedTenantId) - { - try - { - var tableClient = _storage.GetTableClient(tableName); - // OData startsWith: PartitionKey ge 'prefix' and PartitionKey lt 'prefix~' - var prefix = normalizedTenantId + "_"; - var filter = $"PartitionKey ge '{prefix}' and PartitionKey lt '{prefix}~'"; - var entities = tableClient.QueryAsync(filter, select: new[] { "PartitionKey", "RowKey" }); - - int deleted = 0; - // Group by PartitionKey because Azure Table batch transactions require same PartitionKey - var groups = new Dictionary>(); - - await foreach (var entity in entities) - { - if (!groups.ContainsKey(entity.PartitionKey)) - groups[entity.PartitionKey] = new List(); - groups[entity.PartitionKey].Add(entity); - deleted++; - } - - foreach (var group in groups.Values) - { - for (int i = 0; i < group.Count; i += 100) - { - var chunk = group.Skip(i).Take(100).ToList(); - var batch = chunk.Select(e => new TableTransactionAction(TableTransactionActionType.Delete, e)).ToList(); - await tableClient.SubmitTransactionAsync(batch); - } - } - - return deleted; - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to delete {TableName} for tenant {TenantId}", tableName, normalizedTenantId); - return 0; - } - } - - /// - /// Deletes BootstrapSessions CodeLookup entries for a tenant. - /// These use PartitionKey = "CodeLookup" with a TenantId property matching the tenant. - /// - private async Task DeleteCodeLookupEntriesAsync(string normalizedTenantId) - { - try - { - var tableClient = _storage.GetTableClient(Constants.TableNames.BootstrapSessions); - var filter = $"PartitionKey eq 'CodeLookup' and TenantId eq '{normalizedTenantId}'"; - var entities = tableClient.QueryAsync(filter, select: new[] { "PartitionKey", "RowKey" }); - - int deleted = 0; - var batch = new List(); - - await foreach (var entity in entities) - { - batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity)); - deleted++; - - if (batch.Count >= 100) - { - await tableClient.SubmitTransactionAsync(batch); - batch.Clear(); - } - } - - if (batch.Count > 0) - { - await tableClient.SubmitTransactionAsync(batch); - } - - return deleted; - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to delete CodeLookup entries for tenant {TenantId}", normalizedTenantId); - return 0; - } - } public Task> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => _storage.GetOrphanedEventSessionsAsync(gracePeriod); public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => _storage.DeleteEventSessionIndexEntryAsync(tenantId, sessionId); + + /// + /// Tenant offboarding cascade-worker session enumerator. Yields one row per session + /// belonging to . Storage exceptions are intentionally + /// NOT caught (memory: feedback_storage_helpers_fail_soft) — the worker must see + /// transient failures so the queue can retry/poison rather than treating an outage as + /// "0 sessions, proceed with wipe". + /// + public async IAsyncEnumerable EnumerateSessionsForOffboardingAsync( + string tenantId, + [EnumeratorCancellation] CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + + var tableClient = _storage.GetTableClient(Constants.TableNames.Sessions); + var filter = $"PartitionKey eq '{tenantId}'"; + + await foreach (var entity in tableClient.QueryAsync( + filter, + select: new[] { "PartitionKey", "RowKey" }, + cancellationToken: ct)) + { + if (!string.Equals(entity.PartitionKey, tenantId, StringComparison.Ordinal)) + { + // Defensive: server-side filter should already guarantee this, but if a row + // ever slips through with a foreign PK we must fail-loud rather than enqueue + // a cascade for the wrong tenant. + throw new InvalidOperationException( + $"Sessions row returned by tenant filter has PartitionKey '{entity.PartitionKey}' " + + $"but expected '{tenantId}' (RowKey={entity.RowKey})."); + } + + yield return entity.RowKey; + } + } } } diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableOffboardingAuditRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableOffboardingAuditRepository.cs new file mode 100644 index 00000000..6373207d --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableOffboardingAuditRepository.cs @@ -0,0 +1,296 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; +using Azure.Data.Tables; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.DataAccess.TableStorage +{ + /// + /// Table-storage implementation of . Stores all + /// three offboarding PartitionKey patterns (Marker / History / ByTenant) in the single + /// OffboardingAudit table. + /// + /// All writes are fail-loud — when an Azure storage call throws, exceptions propagate to + /// the caller (queue worker), which owns retry/poison semantics via the queue SDK. This is + /// intentionally the OPPOSITE of 's fail-soft + /// helpers; offboarding cannot tolerate silent failures during gate/audit writes. + /// + /// + public class TableOffboardingAuditRepository : IOffboardingAuditRepository + { + private readonly TableClient _tableClient; + private readonly ILogger _logger; + + public TableOffboardingAuditRepository( + TableStorageService storage, + ILogger logger) + { + _tableClient = storage.GetTableClient(Constants.TableNames.OffboardingAudit); + _logger = logger; + } + + // ── Marker ────────────────────────────────────────────────────────────── + + public async Task TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + + try + { + var response = await _tableClient.GetEntityAsync( + Constants.OffboardingPartitionKeys.Marker, normalizedTenantId, cancellationToken: ct); + return MapMarker(response.Value); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return null; + } + } + + public Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default) + { + EnsureMarkerShape(marker); + return _tableClient.AddEntityAsync(StoreMarker(marker), ct); + } + + public Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default) + { + EnsureMarkerShape(marker); + return _tableClient.UpsertEntityAsync(StoreMarker(marker), TableUpdateMode.Replace, ct); + } + + public async Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + + try + { + await _tableClient.DeleteEntityAsync( + Constants.OffboardingPartitionKeys.Marker, normalizedTenantId, ETag.All, ct); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + // Idempotent — already gone. + } + } + + public async IAsyncEnumerable QueryMarkersAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + var filter = $"PartitionKey eq '{Constants.OffboardingPartitionKeys.Marker}'"; + await foreach (var entity in _tableClient.QueryAsync(filter, cancellationToken: ct)) + { + yield return MapMarker(entity); + } + } + + // ── History ───────────────────────────────────────────────────────────── + + public Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default) + { + EnsureHistoryShape(history); + return _tableClient.AddEntityAsync(StoreHistory(history), ct); + } + + public async Task TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default) + { + if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey)); + + try + { + var response = await _tableClient.GetEntityAsync( + Constants.OffboardingPartitionKeys.History, historyRowKey, cancellationToken: ct); + return MapHistory(response.Value); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return null; + } + } + + public Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default) + { + EnsureHistoryShape(history); + return _tableClient.UpsertEntityAsync(StoreHistory(history), TableUpdateMode.Replace, ct); + } + + // ── ByTenant Pointer ──────────────────────────────────────────────────── + + public async Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync( + string normalizedTenantId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + + try + { + var response = await _tableClient.GetEntityAsync( + Constants.OffboardingPartitionKeys.ByTenant, normalizedTenantId, cancellationToken: ct); + return (MapByTenantPointer(response.Value), response.Value.ETag.ToString()); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return (null, null); + } + } + + public Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default) + { + EnsurePointerShape(pointer); + return _tableClient.AddEntityAsync(StoreByTenantPointer(pointer), ct); + } + + public Task UpdateByTenantPointerWithEtagAsync( + OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default) + { + EnsurePointerShape(pointer); + if (string.IsNullOrEmpty(ifMatchEtag)) throw new ArgumentException("ifMatchEtag required", nameof(ifMatchEtag)); + + var entity = StoreByTenantPointer(pointer); + entity.ETag = new ETag(ifMatchEtag); + // TableUpdateMode.Replace + non-default ETag → conditional replace. Mismatch → 412. + return _tableClient.UpdateEntityAsync(entity, entity.ETag, TableUpdateMode.Replace, ct); + } + + // ── Validation helpers ────────────────────────────────────────────────── + + private static void EnsureMarkerShape(OffboardingMarkerEntry marker) + { + if (marker == null) throw new ArgumentNullException(nameof(marker)); + if (marker.PartitionKey != Constants.OffboardingPartitionKeys.Marker) + throw new ArgumentException($"Marker PartitionKey must be '{Constants.OffboardingPartitionKeys.Marker}', got '{marker.PartitionKey}'", nameof(marker)); + SecurityValidator.EnsureValidGuid(marker.RowKey, $"{nameof(marker)}.RowKey"); + SecurityValidator.EnsureValidGuid(marker.TenantId, $"{nameof(marker)}.TenantId"); + } + + private static void EnsureHistoryShape(OffboardingHistoryEntry history) + { + if (history == null) throw new ArgumentNullException(nameof(history)); + if (history.PartitionKey != Constants.OffboardingPartitionKeys.History) + throw new ArgumentException($"History PartitionKey must be '{Constants.OffboardingPartitionKeys.History}', got '{history.PartitionKey}'", nameof(history)); + if (string.IsNullOrEmpty(history.RowKey)) throw new ArgumentException("History RowKey required", nameof(history)); + SecurityValidator.EnsureValidGuid(history.TenantId, $"{nameof(history)}.TenantId"); + } + + private static void EnsurePointerShape(OffboardingByTenantPointer pointer) + { + if (pointer == null) throw new ArgumentNullException(nameof(pointer)); + if (pointer.PartitionKey != Constants.OffboardingPartitionKeys.ByTenant) + throw new ArgumentException($"Pointer PartitionKey must be '{Constants.OffboardingPartitionKeys.ByTenant}', got '{pointer.PartitionKey}'", nameof(pointer)); + SecurityValidator.EnsureValidGuid(pointer.RowKey, $"{nameof(pointer)}.RowKey"); + SecurityValidator.EnsureValidGuid(pointer.TenantId, $"{nameof(pointer)}.TenantId"); + if (string.IsNullOrEmpty(pointer.LatestHistoryRowKey)) + throw new ArgumentException("Pointer.LatestHistoryRowKey required", nameof(pointer)); + } + + // ── Store / Map helpers (memory: feedback_table_storage_serialization) ── + + private static TableEntity StoreMarker(OffboardingMarkerEntry m) => new(m.PartitionKey, m.RowKey) + { + ["TenantId"] = m.TenantId, + ["OffboardingHistoryRowKey"] = m.OffboardingHistoryRowKey, + ["InitiatedAt"] = m.InitiatedAt, + ["InitiatedBy"] = m.InitiatedBy, + ["Status"] = m.Status, + ["CompletedAt"] = m.CompletedAt, + ["FailedAt"] = m.FailedAt, + ["FailedPhase"] = m.FailedPhase, + }; + + private static OffboardingMarkerEntry MapMarker(TableEntity e) => new() + { + PartitionKey = e.PartitionKey, + RowKey = e.RowKey, + TenantId = e.GetString("TenantId") ?? string.Empty, + OffboardingHistoryRowKey = e.GetString("OffboardingHistoryRowKey") ?? string.Empty, + InitiatedAt = e.GetDateTime("InitiatedAt") ?? default, + InitiatedBy = e.GetString("InitiatedBy") ?? string.Empty, + Status = e.GetString("Status") ?? "Initiated", + CompletedAt = e.GetDateTime("CompletedAt"), + FailedAt = e.GetDateTime("FailedAt"), + FailedPhase = e.GetString("FailedPhase"), + }; + + private static TableEntity StoreHistory(OffboardingHistoryEntry h) => new(h.PartitionKey, h.RowKey) + { + ["TenantId"] = h.TenantId, + ["DomainName"] = h.DomainName, + ["InitiatedBy"] = h.InitiatedBy, + ["OffboardedAt"] = h.OffboardedAt, + ["CompletedAt"] = h.CompletedAt, + ["EarliestProcessingAt"] = h.EarliestProcessingAt, + ["Status"] = h.Status, + ["DeletedRowCountsJson"] = h.DeletedRowCountsJson, + ["TotalRowsDeleted"] = h.TotalRowsDeleted, + ["DeletedBlobCount"] = h.DeletedBlobCount, + ["CascadeSessionsEnqueued"] = h.CascadeSessionsEnqueued, + ["ErrorMessage"] = h.ErrorMessage, + ["RetryCount"] = h.RetryCount, + ["DrainCompletedAt"] = h.DrainCompletedAt, + ["EnumerationStartedAt"] = h.EnumerationStartedAt, + ["EnumerationCompletedBeforeUpload"] = h.EnumerationCompletedBeforeUpload, + ["CustomGatherRulesArchived"] = h.CustomGatherRulesArchived, + ["CustomAnalyzeRulesArchived"] = h.CustomAnalyzeRulesArchived, + ["ImeLogPatternOverridesArchived"] = h.ImeLogPatternOverridesArchived, + ["ReonboardedAt"] = h.ReonboardedAt, + ["ReonboardedBy"] = h.ReonboardedBy, + ["CustomsAutoWipedOnReonboard"] = h.CustomsAutoWipedOnReonboard, + }; + + private static OffboardingHistoryEntry MapHistory(TableEntity e) => new() + { + PartitionKey = e.PartitionKey, + RowKey = e.RowKey, + TenantId = e.GetString("TenantId") ?? string.Empty, + DomainName = e.GetString("DomainName") ?? string.Empty, + InitiatedBy = e.GetString("InitiatedBy") ?? string.Empty, + OffboardedAt = e.GetDateTime("OffboardedAt") ?? default, + CompletedAt = e.GetDateTime("CompletedAt"), + EarliestProcessingAt = e.GetDateTime("EarliestProcessingAt"), + Status = e.GetString("Status") ?? "Initiated", + DeletedRowCountsJson = e.GetString("DeletedRowCountsJson"), + TotalRowsDeleted = e.GetInt32("TotalRowsDeleted"), + DeletedBlobCount = e.GetInt32("DeletedBlobCount"), + CascadeSessionsEnqueued = e.GetInt32("CascadeSessionsEnqueued"), + ErrorMessage = e.GetString("ErrorMessage"), + RetryCount = e.GetInt32("RetryCount") ?? 0, + DrainCompletedAt = e.GetDateTime("DrainCompletedAt"), + EnumerationStartedAt = e.GetDateTime("EnumerationStartedAt"), + EnumerationCompletedBeforeUpload = e.GetDateTime("EnumerationCompletedBeforeUpload"), + CustomGatherRulesArchived = e.GetInt32("CustomGatherRulesArchived"), + CustomAnalyzeRulesArchived = e.GetInt32("CustomAnalyzeRulesArchived"), + ImeLogPatternOverridesArchived = e.GetInt32("ImeLogPatternOverridesArchived"), + ReonboardedAt = e.GetDateTime("ReonboardedAt"), + ReonboardedBy = e.GetString("ReonboardedBy"), + CustomsAutoWipedOnReonboard = e.GetInt32("CustomsAutoWipedOnReonboard"), + }; + + private static TableEntity StoreByTenantPointer(OffboardingByTenantPointer p) => new(p.PartitionKey, p.RowKey) + { + ["TenantId"] = p.TenantId, + ["LatestHistoryRowKey"] = p.LatestHistoryRowKey, + ["LatestStatus"] = p.LatestStatus, + ["LatestUpdatedAt"] = p.LatestUpdatedAt, + ["OffboardCount"] = p.OffboardCount, + }; + + private static OffboardingByTenantPointer MapByTenantPointer(TableEntity e) => new() + { + PartitionKey = e.PartitionKey, + RowKey = e.RowKey, + TenantId = e.GetString("TenantId") ?? string.Empty, + LatestHistoryRowKey = e.GetString("LatestHistoryRowKey") ?? string.Empty, + LatestStatus = e.GetString("LatestStatus") ?? "Initiated", + LatestUpdatedAt = e.GetDateTime("LatestUpdatedAt") ?? default, + OffboardCount = e.GetInt32("OffboardCount") ?? 1, + }; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableTenantCustomsArchiveRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableTenantCustomsArchiveRepository.cs new file mode 100644 index 00000000..2756fef4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableTenantCustomsArchiveRepository.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; +using Azure.Data.Tables; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.DataAccess.TableStorage +{ + /// + /// Table-storage backend for . + /// Writes during Phase 2.D-archive, reads + deletes from the Global Admin UI. + /// Fail-loud — exceptions propagate to the caller (worker or HTTP handler). + /// + public sealed class TableTenantCustomsArchiveRepository : ITenantCustomsArchiveRepository + { + private readonly TableClient _tableClient; + private readonly ILogger _logger; + + public TableTenantCustomsArchiveRepository( + TableStorageService storage, + ILogger logger) + { + _tableClient = storage.GetTableClient(Constants.TableNames.TenantOffboardingCustomsArchive); + _logger = logger; + } + + public Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default) + { + if (entry == null) throw new ArgumentNullException(nameof(entry)); + if (string.IsNullOrEmpty(entry.PartitionKey)) throw new ArgumentException("PartitionKey required", nameof(entry)); + if (string.IsNullOrEmpty(entry.RowKey)) throw new ArgumentException("RowKey required", nameof(entry)); + return _tableClient.UpsertEntityAsync(Store(entry), TableUpdateMode.Replace, ct); + } + + public async Task CountByRunAndTableAsync( + string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default) + { + var pk = BuildPartitionKey(normalizedTenantId, historyRowKey); + var rkPrefix = originalTable + "_"; + // Range filter: rowKey ge 'GatherRules_' and rowKey lt 'GatherRules_~'. + // '~' (0x7E) is the largest visible ASCII char that base64url uses; works as + // exclusive upper bound because base64url alphabet is [A-Za-z0-9_-] (no '~'). + var filter = $"PartitionKey eq '{Escape(pk)}' and RowKey ge '{Escape(rkPrefix)}' and RowKey lt '{Escape(rkPrefix + "~")}'"; + + var count = 0; + await foreach (var _ in _tableClient.QueryAsync(filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct)) + { + count++; + } + return count; + } + + public async IAsyncEnumerable QueryByRunAsync( + string normalizedTenantId, string historyRowKey, [EnumeratorCancellation] CancellationToken ct = default) + { + var pk = BuildPartitionKey(normalizedTenantId, historyRowKey); + var filter = $"PartitionKey eq '{Escape(pk)}'"; + await foreach (var e in _tableClient.QueryAsync(filter, cancellationToken: ct)) + { + yield return Map(e); + } + } + + public async IAsyncEnumerable QueryByTenantAsync( + string normalizedTenantId, [EnumeratorCancellation] CancellationToken ct = default) + { + // Per-tenant prefix scan: PartitionKey ge '{tid}_' and PartitionKey lt '{tid}_~'. + // History row keys begin with a numeric timestamp (yyyyMMddHHmmssfff) which is < '~', + // so the exclusive upper bound is correct. + var prefix = normalizedTenantId + "_"; + var filter = $"PartitionKey ge '{Escape(prefix)}' and PartitionKey lt '{Escape(prefix + "~")}'"; + await foreach (var e in _tableClient.QueryAsync(filter, cancellationToken: ct)) + { + yield return Map(e); + } + } + + public async IAsyncEnumerable QueryAllAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + await foreach (var e in _tableClient.QueryAsync(cancellationToken: ct)) + { + yield return Map(e); + } + } + + public async Task TryGetEntryAsync( + string partitionKey, string rowKey, CancellationToken ct = default) + { + try + { + var response = await _tableClient.GetEntityAsync(partitionKey, rowKey, cancellationToken: ct); + return Map(response.Value); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return null; + } + } + + public async Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default) + { + try + { + await _tableClient.DeleteEntityAsync(partitionKey, rowKey, ETag.All, ct); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + // Idempotent — already gone. + } + } + + public async Task DeleteRunAsync( + string normalizedTenantId, string historyRowKey, CancellationToken ct = default) + { + var pk = BuildPartitionKey(normalizedTenantId, historyRowKey); + var filter = $"PartitionKey eq '{Escape(pk)}'"; + + var deleted = 0; + var batch = new List(); + await foreach (var entity in _tableClient.QueryAsync(filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct)) + { + batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity)); + deleted++; + + if (batch.Count >= 100) + { + await _tableClient.SubmitTransactionAsync(batch, ct); + batch.Clear(); + } + } + + if (batch.Count > 0) + { + await _tableClient.SubmitTransactionAsync(batch, ct); + } + return deleted; + } + + // ── Helpers ──────────────────────────────────────────────────────────── + + internal static string BuildPartitionKey(string normalizedTenantId, string historyRowKey) + => $"{normalizedTenantId}_{historyRowKey}"; + + /// + /// Builds the archive RowKey for one original row. base64url-encodes the original + /// RowKey so it survives Azure-Tables' RowKey-forbidden character rules (#, ?, + /// /, \, control chars). Decoder is . + /// + public static string BuildRowKey(string originalTable, string originalRowKey) + => $"{originalTable}_{Base64UrlEncode(originalRowKey)}"; + + /// Inverse of — returns the original RowKey. + public static string DecodeOriginalRowKey(string archiveRowKey, string originalTable) + { + var prefix = originalTable + "_"; + if (!archiveRowKey.StartsWith(prefix, StringComparison.Ordinal)) + throw new ArgumentException($"Archive RowKey '{archiveRowKey}' does not start with expected table prefix '{prefix}'", nameof(archiveRowKey)); + return Base64UrlDecode(archiveRowKey.Substring(prefix.Length)); + } + + private static string Base64UrlEncode(string s) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(s); + return Convert.ToBase64String(bytes) + .Replace('+', '-') + .Replace('/', '_') + .TrimEnd('='); + } + + private static string Base64UrlDecode(string s) + { + var padded = s.Replace('-', '+').Replace('_', '/'); + switch (padded.Length % 4) + { + case 2: padded += "=="; break; + case 3: padded += "="; break; + } + var bytes = Convert.FromBase64String(padded); + return System.Text.Encoding.UTF8.GetString(bytes); + } + + private static string Escape(string s) => s.Replace("'", "''"); + + private static TableEntity Store(TenantOffboardingCustomsArchiveEntry e) => new(e.PartitionKey, e.RowKey) + { + ["TenantId"] = e.TenantId, + ["OriginalTable"] = e.OriginalTable, + ["OriginalPartitionKey"] = e.OriginalPartitionKey, + ["OriginalRowKey"] = e.OriginalRowKey, + ["EntityJson"] = e.EntityJson, + ["HistoryRowKey"] = e.HistoryRowKey, + ["ArchivedAt"] = e.ArchivedAt, + ["ArchivedBy"] = e.ArchivedBy, + }; + + private static TenantOffboardingCustomsArchiveEntry Map(TableEntity e) => new() + { + PartitionKey = e.PartitionKey, + RowKey = e.RowKey, + TenantId = e.GetString("TenantId") ?? string.Empty, + OriginalTable = e.GetString("OriginalTable") ?? string.Empty, + OriginalPartitionKey = e.GetString("OriginalPartitionKey") ?? string.Empty, + OriginalRowKey = e.GetString("OriginalRowKey") ?? string.Empty, + EntityJson = e.GetString("EntityJson") ?? string.Empty, + HistoryRowKey = e.GetString("HistoryRowKey") ?? string.Empty, + ArchivedAt = e.GetDateTime("ArchivedAt") ?? default, + ArchivedBy = e.GetString("ArchivedBy") ?? "TenantOffboardingHandler", + }; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/CustomsArchiveQueryFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/CustomsArchiveQueryFunction.cs new file mode 100644 index 00000000..aae216ab --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/CustomsArchiveQueryFunction.cs @@ -0,0 +1,263 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using AutopilotMonitor.Functions.DataAccess.TableStorage; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Admin; + +/// +/// GlobalAdmin-only read + delete endpoints over . +/// Backs the /global/customs-archive Web UI: list runs, inspect entries, delete per +/// entry / per run. All writes audit-logged under . +/// +/// Authorization is enforced by PolicyEnforcementMiddleware via +/// EndpointAccessPolicyCatalog; the five routes registered here all carry +/// EndpointPolicy.GlobalAdminOnly. +/// +/// +public class CustomsArchiveQueryFunction +{ + private readonly ILogger _logger; + private readonly ITenantCustomsArchiveRepository _archive; + private readonly IMaintenanceRepository _maintenance; + + public CustomsArchiveQueryFunction( + ILogger logger, + ITenantCustomsArchiveRepository archive, + IMaintenanceRepository maintenance) + { + _logger = logger; + _archive = archive; + _maintenance = maintenance; + } + + /// + /// GET /api/global/customs-archive — list every archive partition (one entry per + /// (tenantId, historyRowKey) run) with aggregated row-counts per source table. + /// Optional ?tenantId= filter restricts to one tenant. + /// + [Function("CustomsArchiveListRuns")] + public async Task ListRuns( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/customs-archive")] HttpRequestData req) + { + var query = System.Web.HttpUtility.ParseQueryString(req.Url.Query ?? string.Empty); + var filterTenantId = query["tenantId"]; + + var summaries = new Dictionary(System.StringComparer.Ordinal); + + var source = string.IsNullOrEmpty(filterTenantId) + ? _archive.QueryAllAsync() + : _archive.QueryByTenantAsync(filterTenantId.ToLowerInvariant()); + + await foreach (var entry in source) + { + var key = entry.PartitionKey; + if (!summaries.TryGetValue(key, out var summary)) + { + summary = new RunSummary + { + PartitionKey = entry.PartitionKey, + TenantId = entry.TenantId, + HistoryRowKey = entry.HistoryRowKey, + ArchivedAt = entry.ArchivedAt, + }; + summaries[key] = summary; + } + + switch (entry.OriginalTable) + { + case Constants.TableNames.GatherRules: summary.GatherRulesCount++; break; + case Constants.TableNames.AnalyzeRules: summary.AnalyzeRulesCount++; break; + case Constants.TableNames.ImeLogPatterns: summary.ImeLogPatternsCount++; break; + } + + // Track the earliest ArchivedAt for the run (in case rows have slightly drifting + // timestamps from multiple iterations of archive-then-wipe). + if (entry.ArchivedAt < summary.ArchivedAt) + { + summary.ArchivedAt = entry.ArchivedAt; + } + } + + var runs = summaries.Values + .OrderByDescending(s => s.ArchivedAt) + .ToList(); + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new { success = true, count = runs.Count, runs }); + return response; + } + + /// + /// GET /api/global/customs-archive/{tenantId}/{historyRowKey} — list every archived + /// entry in one offboarding run, with a short preview of . + /// Full body is available via the per-entry route below. + /// + [Function("CustomsArchiveListEntries")] + public async Task ListEntries( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", + Route = "global/customs-archive/{tenantId}/{historyRowKey}")] HttpRequestData req, + string tenantId, + string historyRowKey) + { + var normalizedTenantId = tenantId.ToLowerInvariant(); + var items = new List(); + await foreach (var entry in _archive.QueryByRunAsync(normalizedTenantId, historyRowKey)) + { + items.Add(new EntrySummary + { + PartitionKey = entry.PartitionKey, + RowKey = entry.RowKey, + OriginalTable = entry.OriginalTable, + OriginalRowKey = entry.OriginalRowKey, + ArchivedAt = entry.ArchivedAt, + EntityJsonPreview = TruncatePreview(entry.EntityJson, maxLength: 200), + }); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new { success = true, count = items.Count, entries = items }); + return response; + } + + /// + /// GET /api/global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey} — full + /// archive entry including . + /// + [Function("CustomsArchiveGetEntry")] + public async Task GetEntry( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", + Route = "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}")] HttpRequestData req, + string tenantId, + string historyRowKey, + string archiveRowKey) + { + var partitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey( + tenantId.ToLowerInvariant(), historyRowKey); + var entry = await _archive.TryGetEntryAsync(partitionKey, archiveRowKey); + if (entry == null) + { + return req.CreateResponse(HttpStatusCode.NotFound); + } + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new { success = true, entry }); + return response; + } + + /// + /// DELETE /api/global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey} — + /// remove a single archived entry. Audit-logged under . + /// + [Function("CustomsArchiveDeleteEntry")] + public async Task DeleteEntry( + [HttpTrigger(AuthorizationLevel.Anonymous, "delete", + Route = "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}")] HttpRequestData req, + string tenantId, + string historyRowKey, + string archiveRowKey, + FunctionContext context) + { + var requestCtx = context.GetRequestContext(); + var normalizedTenantId = tenantId.ToLowerInvariant(); + var partitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey(normalizedTenantId, historyRowKey); + + await _archive.DeleteEntryAsync(partitionKey, archiveRowKey); + + await _maintenance.LogAuditEntryAsync( + Constants.AuditGlobalTenantId, + "DELETE", + "CustomsArchiveEntry", + $"{partitionKey}/{archiveRowKey}", + requestCtx.UserPrincipalName, + new Dictionary + { + { "TenantId", normalizedTenantId }, + { "HistoryRowKey", historyRowKey }, + { "ArchiveRowKey", archiveRowKey }, + }); + + _logger.LogWarning( + "CustomsArchive entry deleted by {Upn}: tenant={Tenant} history={History} archive={Archive}", + requestCtx.UserPrincipalName, normalizedTenantId, historyRowKey, archiveRowKey); + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new { success = true }); + return response; + } + + /// + /// DELETE /api/global/customs-archive/{tenantId}/{historyRowKey} — bulk-delete every + /// entry of one offboarding run (after the operator has reviewed them). Audit-logged + /// under with the deleted count. + /// + [Function("CustomsArchiveDeleteRun")] + public async Task DeleteRun( + [HttpTrigger(AuthorizationLevel.Anonymous, "delete", + Route = "global/customs-archive/{tenantId}/{historyRowKey}")] HttpRequestData req, + string tenantId, + string historyRowKey, + FunctionContext context) + { + var requestCtx = context.GetRequestContext(); + var normalizedTenantId = tenantId.ToLowerInvariant(); + + var deleted = await _archive.DeleteRunAsync(normalizedTenantId, historyRowKey); + + await _maintenance.LogAuditEntryAsync( + Constants.AuditGlobalTenantId, + "DELETE", + "CustomsArchiveRun", + $"{normalizedTenantId}/{historyRowKey}", + requestCtx.UserPrincipalName, + new Dictionary + { + { "TenantId", normalizedTenantId }, + { "HistoryRowKey", historyRowKey }, + { "DeletedCount", deleted.ToString() }, + }); + + _logger.LogWarning( + "CustomsArchive run deleted by {Upn}: tenant={Tenant} history={History} deleted={Count}", + requestCtx.UserPrincipalName, normalizedTenantId, historyRowKey, deleted); + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new { success = true, deleted }); + return response; + } + + private static string TruncatePreview(string s, int maxLength) + { + if (string.IsNullOrEmpty(s)) return string.Empty; + if (s.Length <= maxLength) return s; + return s.Substring(0, maxLength) + "…"; + } + + public class RunSummary + { + public string PartitionKey { get; set; } = string.Empty; + public string TenantId { get; set; } = string.Empty; + public string HistoryRowKey { get; set; } = string.Empty; + public System.DateTime ArchivedAt { get; set; } + public int GatherRulesCount { get; set; } + public int AnalyzeRulesCount { get; set; } + public int ImeLogPatternsCount { get; set; } + } + + public class EntrySummary + { + public string PartitionKey { get; set; } = string.Empty; + public string RowKey { get; set; } = string.Empty; + public string OriginalTable { get; set; } = string.Empty; + public string OriginalRowKey { get; set; } = string.Empty; + public System.DateTime ArchivedAt { get; set; } + public string EntityJsonPreview { get; set; } = string.Empty; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs index a586a84c..b423a110 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs @@ -1,8 +1,16 @@ +using System; +using System.Collections.Generic; using System.Net; +using System.Threading.Tasks; using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Security; using AutopilotMonitor.Functions.Services; -using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Functions.Services.Offboarding; using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; using Microsoft.AspNetCore.Authorization; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; @@ -11,34 +19,61 @@ using Microsoft.Extensions.Logging; namespace AutopilotMonitor.Functions.Functions.Admin; /// -/// Tenant Offboarding endpoint -/// Allows a Tenant Admin to permanently delete all data for their tenant. -/// This operation is irreversible. +/// Tenant offboarding endpoint (Phase 1 of the tenant-offboarding cascade plan). +/// Replaces the previous synchronous DeleteAllTenantDataAsync teardown with a +/// queued cascade. Phase 1 commits the History/Pointer/Marker rows, flips +/// TenantConfiguration.Disabled=true (the actual auth gate), audits, and enqueues +/// a with a visibility +/// delay. The worker picks the message up after the cache-drain window, by which point +/// every function-host has expired its 5-min TenantConfigurationService cache and +/// the existing Disabled-gate carries the 403. Returns 202. /// public class TenantOffboardFunction { + private const int PointerCasMaxAttempts = 5; + + /// + /// Cache-Drain-Barrier (plan v2 §2.2). Worker MUST NOT begin Phase 2 before this elapsed. + /// 5 min = (absolute expiration). + /// 1 min = clock-skew + GC + Storage-replication safety buffer. + /// Hardcoded by design: this is a structural invariant tied to CacheDuration, not an + /// operational knob. If CacheDuration is ever raised, this constant must move with it. + /// + internal static readonly TimeSpan DrainBarrier = TimeSpan.FromMinutes(6); + private readonly ILogger _logger; - private readonly IMaintenanceRepository _maintenanceRepo; private readonly IConfigRepository _configRepo; - private readonly OpsEventService _opsEventService; + private readonly TenantConfigurationService _tenantConfigService; + private readonly IMaintenanceRepository _maintenanceRepo; + private readonly IOffboardingAuditRepository _offboardingRepo; + private readonly ITenantOffboardingEnqueuer _offboardingEnqueuer; public TenantOffboardFunction( ILogger logger, - IMaintenanceRepository maintenanceRepo, IConfigRepository configRepo, - OpsEventService opsEventService) + TenantConfigurationService tenantConfigService, + IMaintenanceRepository maintenanceRepo, + IOffboardingAuditRepository offboardingRepo, + ITenantOffboardingEnqueuer offboardingEnqueuer) { _logger = logger; - _maintenanceRepo = maintenanceRepo; _configRepo = configRepo; - _opsEventService = opsEventService; + _tenantConfigService = tenantConfigService; + _maintenanceRepo = maintenanceRepo; + _offboardingRepo = offboardingRepo; + _offboardingEnqueuer = offboardingEnqueuer; } /// /// DELETE /api/tenants/{tenantId}/offboard - /// Permanently deletes ALL data for a tenant across all tables. - /// Accessible by: Tenant Admins of the same tenant OR Global Admins. - /// This action is IRREVERSIBLE. + /// Phase 1 — synchronously commits the offboarding state (History/Pointer/Marker), + /// flips TenantConfiguration.Disabled=true (the SOLE active auth-gate in the + /// PR3-revised architecture — the agent + web hotpaths read config.Disabled + /// only, never the marker), and enqueues a single tenant-offboarding envelope with + /// a visibility-delay so warm function-host caches drain + /// before Phase 2 begins. + /// Accessible by: Tenant Admins of the same tenant OR Global Admins (enforced by + /// PolicyEnforcementMiddleware). /// [Function("OffboardTenant")] [Authorize] @@ -47,88 +82,677 @@ public class TenantOffboardFunction string tenantId, FunctionContext context) { - // Authentication enforced by PolicyEnforcementMiddleware var requestCtx = context.GetRequestContext(); var upn = requestCtx.UserPrincipalName; + var targetTenantId = requestCtx.TargetTenantId; - _logger.LogWarning($"TENANT OFFBOARD initiated for tenant {requestCtx.TargetTenantId} by {upn}"); + if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId)) + { + var badRequest = req.CreateResponse(HttpStatusCode.BadRequest); + await badRequest.WriteAsJsonAsync(new { error = "tenantId must be a valid GUID" }); + return badRequest; + } - // Audit under Global TenantId so the entry survives tenant data deletion + var normalizedTenantId = targetTenantId.ToLowerInvariant(); + + _logger.LogWarning( + "TENANT OFFBOARD initiated for tenant {TenantId} by {Upn}", + normalizedTenantId, upn); + + // 2. Capture DomainName from TenantConfiguration so the worker's Completed/Failed + // OpsEvents can render "{domain} ({tenantId})" instead of the bare GUID. + string? domainName = null; + try + { + var existingConfig = await _configRepo.GetTenantConfigurationAsync(targetTenantId); + domainName = existingConfig?.DomainName; + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Failed to read tenant configuration for {TenantId} before offboard; proceeding without domain name", + normalizedTenantId); + } + + // 3. Idempotency: if an active marker exists, replay the queue-enqueue defensively + // when the offboarding is still in-flight. The previous implementation returned + // 200 unconditionally, which left the tenant stuck whenever the first enqueue + // failed between marker-insert and queue-send: the marker would block all future + // re-clicks via the idempotency-200 path while the worker queue was empty. + // Resume-paths per marker status: + // - Initiated/InProgress → defensive re-enqueue (TenantOffboardingHandler is + // idempotent via History.DrainCompletedAt Rev-9-F1 skip-gate; worst case the + // queue carries two envelopes that both resolve cleanly). + // - Completed → no re-enqueue (worker already finished; marker is + // the 15-min Rev-5-F2 grace blocker). + // - Failed → no re-enqueue (R1 fail-closed by design; operator + // must inspect History.FailedPhase + clear the marker manually before retry). + try + { + var existingMarker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId); + if (existingMarker != null) + { + return await HandleExistingMarkerAsync( + req, existingMarker, upn, normalizedTenantId, isRace: false); + } + } + catch (Exception ex) + { + _logger.LogError(ex, + "Failed to read offboarding marker for {TenantId} during idempotency check", + normalizedTenantId); + return await Build500Async(req, "Failed to read offboarding state"); + } + + var now = DateTime.UtcNow; + var historyRowKey = $"{now:yyyyMMddHHmmssfff}_{normalizedTenantId}"; + var earliestProcessingAt = now + DrainBarrier; + + // 4. Insert History row FIRST (audit trail must exist before any side-effect). + var history = new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = historyRowKey, + TenantId = normalizedTenantId, + DomainName = domainName ?? string.Empty, + InitiatedBy = upn, + OffboardedAt = now, + EarliestProcessingAt = earliestProcessingAt, + Status = "Initiated", + RetryCount = 0, + }; + + try + { + await _offboardingRepo.InsertHistoryAsync(history); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to insert OffboardingHistory row for {TenantId}", normalizedTenantId); + return await Build500Async(req, "Failed to record offboarding history"); + } + + // 5. Upsert ByTenant pointer with ETag-CAS so OffboardCount increments race-safely. + try + { + await UpsertPointerWithCasAsync(normalizedTenantId, historyRowKey, now); + } + catch (Exception ex) + { + // History row is already committed — operator can find it under + // PartitionKey="OffboardingHistory" and retry manually. Graceful degradation + // (plan §4.4): pointer is the re-onboarding index; without it PR4's auto-wipe + // is no-op but the offboarding itself still proceeds. + _logger.LogError(ex, + "Failed to upsert OffboardingByTenant pointer for {TenantId} after history row inserted", + normalizedTenantId); + return await Build500Async(req, "Failed to update offboarding pointer"); + } + + // 6. Insert Marker LAST among the three rows. The marker is the idempotency anchor + // for admin re-clicks (the next click resolves through TryGetMarkerAsync and + // hits the resume path) and is later read by OffboardingMarkerCleanupFunction + // to drive post-completion cleanup sweeps. The Marker is NOT an auth-gate any + // more — TenantConfiguration.Disabled=true (committed in step 7 below) is the + // sole active gate. + var marker = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = normalizedTenantId, + TenantId = normalizedTenantId, + OffboardingHistoryRowKey = historyRowKey, + InitiatedAt = now, + InitiatedBy = upn, + Status = "Initiated", + }; + + try + { + await _offboardingRepo.InsertMarkerAsync(marker); + } + catch (RequestFailedException ex) when (ex.Status == 409) + { + // Race: another caller inserted a marker between our TryGet at step 3 and now. + // Resolve to the existing marker and run the same defensive re-enqueue path as + // the idempotency branch — the winner may still have failed its own enqueue and + // we cannot tell from here, so re-enqueue is the safe default. The worker is + // idempotent via the DrainCompletedAt skip-gate (Rev-9-F1). + _logger.LogInformation( + "OffboardingMarker insert raced for {TenantId}; another initiator won. Resolving to existing marker.", + normalizedTenantId); + + var raced = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId); + if (raced == null) + { + _logger.LogError( + "OffboardingMarker 409 race for {TenantId} but follow-up read returned null; cannot resolve race", + normalizedTenantId); + return await Build500Async(req, "Failed to resolve offboarding race"); + } + return await HandleExistingMarkerAsync( + req, raced, upn, normalizedTenantId, isRace: true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to insert OffboardingMarker for {TenantId}", normalizedTenantId); + return await Build500Async(req, "Failed to commit offboarding marker"); + } + + // 7. Flip TenantConfiguration.Disabled=true. This is the SOLE active auth-gate now + // that the hotpath does not read the OffboardingMarker any more (PR3-revised). + // Fail-loud — without Disabled=true the cache-drain barrier offers no protection + // and Phase 2 would wipe data while warm function-hosts still accept agent + // traffic. Marker stays Initiated → next admin re-click hits the resume path + // which also calls EnsureTenantDisabledAsync before re-enqueuing. + try + { + await EnsureTenantDisabledAsync(targetTenantId, upn); + } + catch (Exception ex) + { + _logger.LogError(ex, + "Failed to set TenantConfiguration.Disabled=true for {TenantId}; refusing to enqueue (auth-gate not committed)", + normalizedTenantId); + return await Build500Async(req, "Failed to commit Disabled-gate; tenant offboarding NOT enqueued"); + } + + // 8. Audit row under AuditGlobalTenantId so it survives the eventual TenantConfiguration wipe. await _maintenanceRepo.LogAuditEntryAsync( Constants.AuditGlobalTenantId, "DELETE", "Tenant", - requestCtx.TargetTenantId, - upn!, + normalizedTenantId, + upn, new Dictionary { { "Action", "Offboard" }, - { "Phase", "Initiated" } + { "Phase", "Initiated" }, + { "HistoryRowKey", historyRowKey }, + { "DomainName", domainName ?? string.Empty }, }); - var result = new OffboardResult { TenantId = requestCtx.TargetTenantId, InitiatedBy = upn!, InitiatedAt = DateTime.UtcNow }; - - // Capture domain name BEFORE deletion — tenant config is wiped by DeleteAllTenantDataAsync, - // and the ops alert / Telegram message is friendlier with the domain than the bare GUID. - string? domainName = null; + // 9. Enqueue with full DrainBarrier as visibility delay. The worker won't see this + // message until ~6min from now, by which time every function-host has refreshed + // its TenantConfiguration cache and the existing Disabled-gate is the active + // block. Fail-loud — marker stays Initiated on failure; the next admin re-click + // hits the idempotency branch which defensively re-enqueues with the remaining + // barrier so the tenant cannot get stuck. try { - var tenantConfig = await _configRepo.GetTenantConfigurationAsync(requestCtx.TargetTenantId); - domainName = tenantConfig?.DomainName; + await EnqueueEnvelopeAsync(normalizedTenantId, historyRowKey, upn, now, DrainBarrier); } catch (Exception ex) { - _logger.LogWarning(ex, "Failed to read tenant config for {TenantId} before offboard; proceeding without domain name", requestCtx.TargetTenantId); + _logger.LogError(ex, + "Failed to enqueue tenant-offboarding envelope for {TenantId}; marker remains Initiated for re-click resume", + normalizedTenantId); + return await Build500Async(req, "Failed to enqueue tenant offboarding"); } - try + _logger.LogWarning( + "TENANT OFFBOARD queued for {TenantId} by {Upn}; historyRowKey={History}; earliestProcessingAt={ProcessAt}", + normalizedTenantId, upn, historyRowKey, earliestProcessingAt); + + var response = req.CreateResponse(HttpStatusCode.Accepted); + await response.WriteAsJsonAsync(new OffboardResponse { - var deletedCounts = await _maintenanceRepo.DeleteAllTenantDataAsync(requestCtx.TargetTenantId); - result.DeletedCounts = deletedCounts; - - result.Success = true; - _logger.LogWarning($"TENANT OFFBOARD completed for tenant {requestCtx.TargetTenantId} by {upn}. " + - $"Total rows deleted: {result.DeletedCounts.Values.Sum()}"); - - // Audit completion with deletion summary - var details = result.DeletedCounts.ToDictionary(kv => kv.Key, kv => kv.Value.ToString()); - details["Action"] = "Offboard"; - details["Phase"] = "Completed"; - details["TotalRowsDeleted"] = result.DeletedCounts.Values.Sum().ToString(); - await _maintenanceRepo.LogAuditEntryAsync( - Constants.AuditGlobalTenantId, - "DELETE", - "Tenant", - requestCtx.TargetTenantId, - upn!, - details); - - await _opsEventService.RecordTenantOffboardedAsync(requestCtx.TargetTenantId, upn!, deletedCounts, domainName); - - var okResponse = req.CreateResponse(HttpStatusCode.OK); - await okResponse.WriteAsJsonAsync(result); - return okResponse; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Tenant offboard failed for tenant {requestCtx.TargetTenantId}"); - result.Success = false; - result.Error = "Internal server error"; - - var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError); - await errorResponse.WriteAsJsonAsync(result); - return errorResponse; - } + TenantId = normalizedTenantId, + Status = "Queued", + HistoryPartitionKey = Constants.OffboardingPartitionKeys.History, + HistoryRowKey = historyRowKey, + EarliestProcessingAt = earliestProcessingAt, + Message = "Tenant offboarding queued for asynchronous processing" + }); + return response; } + // Resume-path for re-clicks and 409-race winners. Thin HTTP shell on top of + // ResumeExistingMarkerAsync — the actual decision (re-enqueue or not, message text, + // remaining drain-barrier) lives in the pure helper below so it can be unit-tested + // without faking HttpRequestData. Plan Review Finding 1 (stuck-tenant fix). + internal async Task HandleExistingMarkerAsync( + HttpRequestData req, + OffboardingMarkerEntry existing, + string upn, + string normalizedTenantId, + bool isRace) + { + var resume = await ResumeExistingMarkerAsync(existing, upn, normalizedTenantId, isRace); + + if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed) + { + return await Build500Async(req, "Failed to re-enqueue tenant offboarding"); + } + + var response = req.CreateResponse(resume.StatusCode); + await response.WriteAsJsonAsync(new OffboardResponse + { + TenantId = normalizedTenantId, + Status = existing.Status, + HistoryPartitionKey = Constants.OffboardingPartitionKeys.History, + HistoryRowKey = existing.OffboardingHistoryRowKey, + EarliestProcessingAt = resume.EarliestProcessingAt, + Message = resume.Message, + }); + return response; + } + + /// + /// Pure resume decision: classifies the existing marker, optionally re-enqueues, and + /// returns the (StatusCode, Message, ReEnqueued?) tuple the HTTP shell wraps. Exposed + /// internal for unit tests — Finding-1 fix requires asserting "Initiated/InProgress + /// triggers a queue write, Completed/Failed does not". + /// + internal async Task ResumeExistingMarkerAsync( + OffboardingMarkerEntry existing, + string upn, + string normalizedTenantId, + bool isRace) + { + var status = existing.Status ?? "Initiated"; + + if (string.Equals(status, "Completed", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogInformation( + "TenantOffboard re-click for {TenantId}: marker is Completed; no re-enqueue (race={Race})", + normalizedTenantId, isRace); + return new ResumeOutcome( + ResumeOutcomeKind.IdempotentCompleted, + HttpStatusCode.OK, + "Tenant offboarding already completed (marker in 15-min grace window)", + ReEnqueued: false); + } + + if (string.Equals(status, "Failed", StringComparison.OrdinalIgnoreCase)) + { + // Failed is fail-closed by design (R1). Re-enqueuing a Failed offboarding would + // just retrace the same failure path. Surface the FailedPhase to the operator + // so they can clear the marker manually before retrying. + _logger.LogWarning( + "TenantOffboard re-click for {TenantId}: marker is Failed (phase={Phase}); operator action required, no re-enqueue", + normalizedTenantId, existing.FailedPhase ?? "unknown"); + return new ResumeOutcome( + ResumeOutcomeKind.IdempotentFailed, + HttpStatusCode.OK, + $"Tenant offboarding previously failed at phase '{existing.FailedPhase ?? "unknown"}' — operator action required before retry", + ReEnqueued: false); + } + + // Initiated / InProgress (or any unrecognised in-flight value): defensively + // re-enqueue the envelope. The handler tolerates double-pickup via the + // DrainCompletedAt skip-gate, so a duplicate envelope on the queue is safe. + // + // The visibility-delay computation below uses + // max(history.EarliestProcessingAt, tombstone.LastUpdated + DrainBarrier) + // as the effective drain-deadline. History is best-effort here: if the read + // fails or the field is missing, the tombstone anchor still gives a safe lower + // bound (when the tombstone was just written, that's now + DrainBarrier; when + // it was written long ago, the barrier is already elapsed). + OffboardingHistoryEntry? history = null; + try + { + history = await _offboardingRepo.TryGetHistoryAsync(existing.OffboardingHistoryRowKey); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Failed to read History row {HistoryRowKey} for {TenantId} during resume; falling through to tombstone.LastUpdated-based deadline", + existing.OffboardingHistoryRowKey, normalizedTenantId); + } + // Re-affirm the Disabled-gate FIRST so we know whether we just wrote it for the + // first time (= cache-drain barrier restarts now) or it was already there from a + // prior attempt (= the History row's deadline is still valid). The prior run may + // have crashed between marker-insert and Disabled-flip, or an admin may have + // cleared the flag manually. Either way, we MUST NOT re-enqueue without the + // auth-gate committed — otherwise the cache-drain barrier guards nothing. + EnsureDisabledResult ensureResult; + try + { + ensureResult = await EnsureTenantDisabledAsync(normalizedTenantId, upn); + } + catch (Exception ex) + { + _logger.LogError(ex, + "Resume: failed to re-affirm TenantConfiguration.Disabled=true for {TenantId}; refusing to re-enqueue", + normalizedTenantId); + return new ResumeOutcome( + ResumeOutcomeKind.ReEnqueueFailed, + HttpStatusCode.InternalServerError, + "Failed to re-affirm Disabled-gate; tenant offboarding NOT re-enqueued", + ReEnqueued: false); + } + + // Resume-path visibility-delay decision (Codex finding "Resume can bypass cache-drain"): + // We compute the effective drain-deadline as the MAX of two sources, then derive + // the queue-message visibility-delay from that: + // + // (a) history.EarliestProcessingAt — set on Phase 1 commit; patched here on + // fresh writes when we can. + // (b) ensureResult.TombstoneLastUpdated + DrainBarrier — the authoritative + // "cache-drain started here" timestamp, written atomically with the + // Disabled-flip, ALWAYS present even if the History-patch later fails. + // + // Using max() of both makes the History-patch non-load-bearing for safety: even if + // a fast follow-up re-click reads stale EarliestProcessingAt, the TombstoneLastUpdated + // anchor gives us a correct lower bound. The History-patch is now best-effort + // documentation, not a critical write. + var now = DateTime.UtcNow; + var tombstoneDeadline = ensureResult.TombstoneLastUpdated + DrainBarrier; + var historyDeadline = history?.EarliestProcessingAt ?? tombstoneDeadline; + var effectiveDeadline = tombstoneDeadline > historyDeadline ? tombstoneDeadline : historyDeadline; + + var visibilityDelay = effectiveDeadline > now ? effectiveDeadline - now : TimeSpan.Zero; + var earliestProcessingAtForResponse = effectiveDeadline; + + // Patch History.EarliestProcessingAt for new tombstone writes so the UI countdown + // + any later admin tools see the new deadline. Failure here is NOT a safety issue + // anymore (tombstone.LastUpdated covers us above) — purely cosmetic / observability. + if (ensureResult.Outcome == EnsureDisabledOutcome.WroteNewTombstone && history != null) + { + history.EarliestProcessingAt = effectiveDeadline; + try + { + await _offboardingRepo.UpsertHistoryAsync(history); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Resume: failed to patch History.EarliestProcessingAt for {TenantId} {History} — UI countdown may show stale value, but the queue visibility-delay + tombstone.LastUpdated guarantee the drain barrier on the worker side", + normalizedTenantId, existing.OffboardingHistoryRowKey); + } + + _logger.LogWarning( + "Resume: Disabled-gate was freshly written; restarted full cache-drain barrier ({Barrier}) for tenant={TenantId}", + DrainBarrier, normalizedTenantId); + } + + try + { + await EnqueueEnvelopeAsync( + normalizedTenantId, existing.OffboardingHistoryRowKey, upn, existing.InitiatedAt, visibilityDelay); + } + catch (Exception ex) + { + _logger.LogError(ex, + "Defensive re-enqueue failed for {TenantId} (existing marker status={Status}, race={Race})", + normalizedTenantId, status, isRace); + return new ResumeOutcome( + ResumeOutcomeKind.ReEnqueueFailed, + HttpStatusCode.InternalServerError, + "Failed to re-enqueue tenant offboarding", + ReEnqueued: false); + } + + _logger.LogInformation( + "TenantOffboard re-click for {TenantId}: defensively re-enqueued with visibilityDelay={Delay} outcome={Outcome} tombstoneLastUpdated={TombstoneLastUpdated} (existing status={Status}, race={Race})", + normalizedTenantId, visibilityDelay, ensureResult.Outcome, ensureResult.TombstoneLastUpdated, status, isRace); + + return new ResumeOutcome( + ResumeOutcomeKind.ReEnqueuedInFlight, + HttpStatusCode.Accepted, + isRace + ? "Tenant offboarding already in progress (race-winner resolved, defensively re-enqueued)" + : $"Tenant offboarding resumed (existing status={status}, defensively re-enqueued)", + ReEnqueued: true, + EarliestProcessingAt: earliestProcessingAtForResponse); + } + + /// + /// Magic-string DisabledReason that identifies a TenantConfiguration row as the + /// offboarding tombstone. Used by EnsureTenantDisabledAsync for the idempotent + /// no-write check AND by to gate the + /// defense-in-depth TenantConfiguration sweep so a self-service-re-onboarded fresh + /// config does not get wiped by mistake. + /// + internal const string OffboardingDisabledReason = "Offboarding in progress"; + + /// + /// Result of . Carries both whether a fresh + /// write happened AND the tombstone's LastUpdated timestamp so the resume-path + /// can compute a safe drain-deadline without depending on the (optionally-failing) + /// History.EarliestProcessingAt patch. + /// + /// Drain-deadline math the caller applies: + /// effectiveDeadline = max(history.EarliestProcessingAt, TombstoneLastUpdated + DrainBarrier). + /// Because the tombstone's LastUpdated is part of the same atomic Save as the + /// Disabled-flip, it is always present once the auth-gate is committed — even if + /// the History-row patch later fails on a fast re-click. + /// + /// + internal readonly record struct EnsureDisabledResult( + EnsureDisabledOutcome Outcome, + DateTime TombstoneLastUpdated); + + internal enum EnsureDisabledOutcome + { + /// Row was already a Disabled=true tombstone with the offboarding reason — + /// no Save was issued. Existing LastUpdated is returned. + AlreadyTombstone, + /// We actively wrote the Disabled=true gate (either flipped an existing + /// row or created a fresh one). LastUpdated is the timestamp of this write. + WroteNewTombstone, + } + + /// + /// Commits the SOLE active auth-gate (TenantConfiguration.Disabled=true) and + /// invalidates the local cache. Fail-loud: + /// every caller (initial enqueue + resume re-enqueue) MUST treat an exception as + /// "do not enqueue" — without this gate committed, the cache-drain barrier protects + /// nothing and Phase 2 would wipe data while warm function-hosts still accept agent + /// traffic. + /// + /// Return value distinguishes the two outcomes so the resume-path can decide whether + /// to honour the old History.EarliestProcessingAt deadline + /// () or restart the cache-drain + /// barrier from now () — see + /// resume-path Codex finding "Resume can bypass cache-drain when first attempt failed + /// before Disabled-write". + /// + /// + /// Behaviour: + /// + /// Existing TenantConfiguration row → read-modify-write so signup data + custom + /// rate limits etc. are preserved. Idempotent: if already + /// Disabled=true with the offboarding reason, no Save is issued and + /// is returned. + /// Missing TenantConfiguration row (edge case: very old tenant / manual delete) + /// → create a fresh CreateDefault row with Disabled=true. Necessary + /// because AuthFunction would otherwise auto-create a Disabled=false + /// default on the next /api/auth/me and slip the offboarded tenant back through. + /// SaveTenantConfigurationAsync returns false on transient storage + /// failure (the production repo does NOT throw; see Codex-finding TableConfigRepository). + /// We turn that into so the caller treats it + /// as a hard fail. + /// + /// + /// + internal async Task EnsureTenantDisabledAsync(string targetTenantId, string upn) + { + var tenantConfig = await _configRepo.GetTenantConfigurationAsync(targetTenantId); + + if (tenantConfig != null) + { + // Idempotent: if Disabled is already true with our reason, no-op the write so + // resume-clicks do not thrash the LastUpdated timestamp on the row. + var alreadyDisabled = tenantConfig.Disabled + && string.Equals(tenantConfig.DisabledReason, OffboardingDisabledReason, StringComparison.Ordinal); + if (alreadyDisabled) + { + _tenantConfigService.InvalidateCache(targetTenantId); + return new EnsureDisabledResult(EnsureDisabledOutcome.AlreadyTombstone, tenantConfig.LastUpdated); + } + } + else + { + // Missing-row edge case: create a fresh Disabled=true row so AuthFunction's + // auto-create-default cannot bring the tenant back enabled. + tenantConfig = TenantConfiguration.CreateDefault(targetTenantId); + _logger.LogWarning( + "TenantOffboard: no TenantConfiguration row found for {TenantId} — creating fresh Disabled=true tombstone", + targetTenantId); + } + + tenantConfig.Disabled = true; + tenantConfig.DisabledReason = OffboardingDisabledReason; + tenantConfig.DisabledUntil = null; + tenantConfig.UpdatedBy = upn; + // Explicit LastUpdated stamp — we bypass TenantConfigurationService.SaveConfigurationAsync + // (which would set this) and write to the repo directly, so we own the bookkeeping. + // Critical for the resume-path drain-deadline math: this timestamp is the + // authoritative "when did the cache-drain barrier start" anchor. + var writtenAt = DateTime.UtcNow; + tenantConfig.LastUpdated = writtenAt; + + var saved = await _configRepo.SaveTenantConfigurationAsync(tenantConfig); + if (!saved) + { + // TableConfigRepository.SaveTenantConfigurationAsync returns false on transient + // storage failure instead of throwing. Without throwing here, the caller would + // proceed to enqueue without the auth-gate committed — defeating the whole + // cache-drain barrier. Surface as a hard failure. + throw new InvalidOperationException( + $"SaveTenantConfigurationAsync returned false for tenant {targetTenantId}; Disabled-gate NOT committed"); + } + _tenantConfigService.InvalidateCache(targetTenantId); + return new EnsureDisabledResult(EnsureDisabledOutcome.WroteNewTombstone, writtenAt); + } + + /// + /// Enqueues the offboarding envelope with the given visibility delay (queue-side + /// invisibility window). First-click: full . Resume-click: + /// remaining delay computed from EarliestProcessingAt so a re-enqueue does not + /// reset the drain barrier (the message would otherwise become visible too early). + /// + internal Task EnqueueEnvelopeAsync( + string normalizedTenantId, string historyRowKey, string upn, DateTime initiatedAt, + TimeSpan? visibilityDelay) + { + var envelope = new TenantOffboardingEnvelope + { + EnvelopeVersion = "1", + TenantId = normalizedTenantId, + HistoryPartitionKey = Constants.OffboardingPartitionKeys.History, + HistoryRowKey = historyRowKey, + InitiatedBy = upn, + InitiatedAt = initiatedAt, + EnqueuedAt = DateTime.UtcNow, + DrainPollCount = 0, + }; + return _offboardingEnqueuer.EnqueueAsync(envelope, visibilityDelay); + } + + // Plan §4.4 read-modify-write CAS-loop. Bounded retry; throws on exhaustion so the + // caller turns it into a 500. The pointer is the O(1) re-onboarding index — failure + // here is recoverable (graceful degradation in PR4) but we still surface it so the + // operator can investigate. + internal async Task UpsertPointerWithCasAsync( + string normalizedTenantId, string historyRowKey, DateTime now) + { + for (var attempt = 1; attempt <= PointerCasMaxAttempts; attempt++) + { + var (existing, etag) = await _offboardingRepo.TryGetByTenantPointerAsync(normalizedTenantId); + + if (existing == null) + { + var fresh = new OffboardingByTenantPointer + { + PartitionKey = Constants.OffboardingPartitionKeys.ByTenant, + RowKey = normalizedTenantId, + TenantId = normalizedTenantId, + LatestHistoryRowKey = historyRowKey, + LatestStatus = "Initiated", + LatestUpdatedAt = now, + OffboardCount = 1, + }; + + try + { + await _offboardingRepo.InsertByTenantPointerAsync(fresh); + return; + } + catch (RequestFailedException ex) when (ex.Status == 409) + { + _logger.LogInformation( + "OffboardingByTenant pointer raced on insert for {TenantId} (attempt {Attempt}); falling back to ETag update", + normalizedTenantId, attempt); + continue; + } + } + + existing.LatestHistoryRowKey = historyRowKey; + existing.LatestStatus = "Initiated"; + existing.LatestUpdatedAt = now; + existing.OffboardCount = existing.OffboardCount + 1; + + try + { + await _offboardingRepo.UpdateByTenantPointerWithEtagAsync(existing, etag!); + return; + } + catch (RequestFailedException ex) when (ex.Status == 412) + { + _logger.LogInformation( + "OffboardingByTenant pointer ETag mismatch for {TenantId} (attempt {Attempt}); retrying", + normalizedTenantId, attempt); + } + } + + throw new InvalidOperationException( + $"OffboardingByTenant pointer CAS exhausted for tenant {normalizedTenantId} after {PointerCasMaxAttempts} attempts"); + } + + private static async Task Build500Async(HttpRequestData req, string message) + { + var response = req.CreateResponse(HttpStatusCode.InternalServerError); + await response.WriteAsJsonAsync(new { success = false, error = message }); + return response; + } } -public class OffboardResult +/// +/// 202/200 response body for the offboarding endpoint. Fields point the caller at the +/// History row so subsequent reporting / status polling can resolve back to the audit +/// trail. drives the "data deletion starts in mm ss" +/// countdown in the Web UI's drain-barrier state. +/// +public class OffboardResponse { public string TenantId { get; set; } = string.Empty; - public string InitiatedBy { get; set; } = string.Empty; - public DateTime InitiatedAt { get; set; } - public bool Success { get; set; } - public string? Error { get; set; } - public Dictionary DeletedCounts { get; set; } = new(); + public string Status { get; set; } = string.Empty; + public string HistoryPartitionKey { get; set; } = string.Empty; + public string HistoryRowKey { get; set; } = string.Empty; + public string Message { get; set; } = string.Empty; + + /// UTC timestamp before which the worker MUST NOT start Phase 2. Drives the + /// cache-drain-barrier countdown UI. Null on the idempotent-Completed/Failed branches. + public DateTime? EarliestProcessingAt { get; set; } +} + +/// +/// Decision the resume-path produced; consumed by +/// to shape the HTTP response. Test-visible so unit tests can assert the re-enqueue decision +/// without faking HttpRequestData. +/// +/// is the History row's drain-barrier deadline for the +/// in-flight path (so the UI can keep counting down through a re-click); null on the +/// terminal idempotent branches. +/// +/// +internal sealed record ResumeOutcome( + ResumeOutcomeKind Kind, + System.Net.HttpStatusCode StatusCode, + string Message, + bool ReEnqueued, + DateTime? EarliestProcessingAt = null); + +internal enum ResumeOutcomeKind +{ + /// Marker is Completed; 15-min grace blocker is held, no re-enqueue. + IdempotentCompleted, + /// Marker is Failed; operator action required, no re-enqueue. + IdempotentFailed, + /// Marker is Initiated/InProgress; defensive re-enqueue was sent. + ReEnqueuedInFlight, + /// Marker classified as in-flight but re-enqueue threw; surface 500. + ReEnqueueFailed, } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/OffboardingMarkerCleanupFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/OffboardingMarkerCleanupFunction.cs new file mode 100644 index 00000000..79a55624 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/OffboardingMarkerCleanupFunction.cs @@ -0,0 +1,230 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Functions.Admin; +using AutopilotMonitor.Functions.Services.Offboarding; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Maintenance +{ + /// + /// Marker housekeeping + defense-in-depth cleanup for the tenant-offboarding cascade. + /// Plan §7.5 (PR3.B-revised). + /// + /// Tenant-offboarding markers ( under + /// OffboardingAudit/OffboardingMarker) live past the Completed transition by + /// design — they are the anchor that lets this function find lingering side-effects + /// (Expectations blob, possibly-stale TenantConfiguration tombstone) and clean + /// them up if the handler's Phase 2.F-final/2.G ordering hit a crash. Failed markers + /// are never auto-removed; they require operator action. + /// + /// + /// Note: in the PR3-revised architecture the marker is NO LONGER read in the + /// agent/web auth hotpath. The active gate is TenantConfiguration.Disabled=true; + /// the cache-drain barrier (6 min visibility delay on the worker envelope) plus + /// together absorb the + /// TenantConfigurationService cache TTL (5 min). The marker's lifetime here is + /// purely about cleanup ordering and crash recovery, not auth. + /// + /// + /// Cleanup order per Completed marker, each step's failure keeps the marker for retry: + /// (1) Expectations blob delete, (2) probe TenantConfiguration → conditional sweep if + /// still the offboarding tombstone (skipped when the user has self-service-re-onboarded + /// and the row carries a non-offboarding state), (3) marker delete. + /// + /// + public class OffboardingMarkerCleanupFunction + { + /// + /// 3× TenantConfigurationService.CacheDuration (5min) — safety margin that + /// absorbs clock-skew + GC pauses + cache-read-after-save anomalies on warm + /// function instances. Originally sized to guarantee the marker outlives every + /// cached TenantConfiguration on every host; retained at 15min because the + /// defense-in-depth sweeps (Expectations blob + TenantConfiguration tombstone) + /// still benefit from waiting for the dust to settle. + /// + public static readonly TimeSpan MinCompletedAgeForCleanup = TimeSpan.FromMinutes(15); + + /// Failed markers older than this trigger a warn-log every run (operator nudge). + internal static readonly TimeSpan FailedMarkerWarnAge = TimeSpan.FromHours(1); + + // Every 2h — matches the broader Maintenance cadence and gives at most a 2h lag between + // 15min-eligible and actual cleanup. Plan §7.5: "Function läuft im selben Maintenance- + // Schedule wie SessionDeletionMaintenance". + private const string Cron = "0 0 */2 * * *"; + + private readonly IOffboardingAuditRepository _auditRepo; + private readonly IOffboardingExpectationsStore _expectations; + private readonly SafeWipeService _safeWipe; + private readonly IConfigRepository _configRepo; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public OffboardingMarkerCleanupFunction( + IOffboardingAuditRepository auditRepo, + IOffboardingExpectationsStore expectations, + SafeWipeService safeWipe, + IConfigRepository configRepo, + ILogger logger) + : this(auditRepo, expectations, safeWipe, configRepo, logger, TimeProvider.System) + { + } + + /// Test seam — inject a fake so age comparisons are deterministic. + internal OffboardingMarkerCleanupFunction( + IOffboardingAuditRepository auditRepo, + IOffboardingExpectationsStore expectations, + SafeWipeService safeWipe, + IConfigRepository configRepo, + ILogger logger, + TimeProvider timeProvider) + { + _auditRepo = auditRepo; + _expectations = expectations; + _safeWipe = safeWipe; + _configRepo = configRepo; + _logger = logger; + _timeProvider = timeProvider; + } + + [Function("OffboardingMarkerCleanup")] + public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken) + { + await RunCoreAsync(cancellationToken); + } + + /// Testable core — bypasses TimerInfo / FunctionContext. + internal async Task RunCoreAsync(CancellationToken ct) + { + var now = _timeProvider.GetUtcNow().UtcDateTime; + var result = new CleanupResult(); + + await foreach (var marker in _auditRepo.QueryMarkersAsync(ct)) + { + ct.ThrowIfCancellationRequested(); + result.Scanned++; + + if (marker.Status == "Completed" + && marker.CompletedAt is { } completedAt + && completedAt < now - MinCompletedAgeForCleanup) + { + // Cleanup order — each step's failure must keep the marker for the next run: + // 1. Expectations blob (Rev-9 hygiene) + // 2. TenantConfiguration sweep (PR3.B-revised Finding 2 defense-in-depth) + // 3. Marker delete (last anchor) + try + { + await _expectations.DeleteAsync(marker.TenantId, marker.OffboardingHistoryRowKey, ct); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "OffboardingMarkerCleanup: Expectations blob delete failed for tenant={Tenant} history={History} — keeping marker for next-run retry", + marker.TenantId, marker.OffboardingHistoryRowKey); + result.BlobDeleteRetries++; + continue; + } + + // PR3.B Defense-in-depth: if Phase 2.F-final failed in the handler + // (crash between 2.G commit and 2.F-final, or transient storage), the + // TenantConfiguration row would still carry Disabled=true and block the + // tenant from self-service re-onboarding indefinitely. Sweep it here. + // + // CRITICAL: must be CONDITIONAL on the row still being the offboarding + // tombstone. The handler intentionally deletes the row at the end of + // Phase 2.F-final precisely so the next /api/auth/me triggers a fresh + // Default-Disabled=false config (self-service re-onboarding). If the + // user has re-onboarded between Phase 2.F-final and now, blind wiping + // would destroy their fresh tenant. So: read first, only wipe when + // (Disabled=true AND DisabledReason matches OffboardingDisabledReason). + try + { + var existingConfig = await _configRepo.GetTenantConfigurationAsync(marker.TenantId); + if (existingConfig == null) + { + // Normal happy path: Phase 2.F-final succeeded, row already gone. + } + else if (existingConfig.Disabled + && string.Equals(existingConfig.DisabledReason, TenantOffboardFunction.OffboardingDisabledReason, StringComparison.Ordinal)) + { + // Still the offboarding tombstone → Phase 2.F-final must have failed. + var swept = await _safeWipe.WipeByExactPartitionAsync( + Constants.TableNames.TenantConfiguration, marker.TenantId, ct); + if (swept > 0) + { + result.TenantConfigsSwept++; + _logger.LogWarning( + "OffboardingMarkerCleanup: swept lingering TenantConfiguration tombstone for tenant={Tenant} (Phase 2.F-final must have failed in the handler)", + marker.TenantId); + } + } + else + { + // User has self-service-re-onboarded between Phase 2.F-final and + // now. Leave their fresh config alone. + result.TenantConfigsSpared++; + _logger.LogInformation( + "OffboardingMarkerCleanup: tenant={Tenant} has a non-tombstone TenantConfiguration row (Disabled={Disabled}, Reason={Reason}) — user re-onboarded, skip sweep", + marker.TenantId, existingConfig.Disabled, existingConfig.DisabledReason ?? ""); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "OffboardingMarkerCleanup: TenantConfiguration probe/sweep failed for tenant={Tenant} — keeping marker for next-run retry", + marker.TenantId); + result.TenantConfigSweepRetries++; + continue; + } + + await _auditRepo.DeleteMarkerAsync(marker.RowKey, ct); + result.MarkersDeleted++; + _logger.LogInformation( + "OffboardingMarkerCleanup: deleted Completed marker for tenant={Tenant} (age={Age})", + marker.TenantId, now - completedAt); + } + else if (marker.Status == "Failed") + { + result.FailedMarkersSeen++; + var failedAt = marker.FailedAt ?? marker.InitiatedAt; + var age = now - failedAt; + if (age > FailedMarkerWarnAge) + { + _logger.LogWarning( + "OffboardingMarkerCleanup: stale Failed marker tenant={Tenant} age={Age} phase={Phase} — operator action required", + marker.TenantId, age, marker.FailedPhase ?? "unknown"); + } + } + // Initiated / InProgress → still working, ignore. + } + + _logger.LogInformation( + "OffboardingMarkerCleanup completed: scanned={Scanned} deleted={Deleted} blobRetries={BlobRetries} tenantConfigSwept={ConfigSwept} tenantConfigSpared={ConfigSpared} tenantConfigRetries={ConfigRetries} failed={Failed}", + result.Scanned, result.MarkersDeleted, result.BlobDeleteRetries, + result.TenantConfigsSwept, result.TenantConfigsSpared, result.TenantConfigSweepRetries, result.FailedMarkersSeen); + + return result; + } + + /// Summary counters returned from so tests can assert outcomes. + internal sealed class CleanupResult + { + public int Scanned { get; set; } + public int MarkersDeleted { get; set; } + public int BlobDeleteRetries { get; set; } + public int FailedMarkersSeen { get; set; } + /// Number of stale TenantConfiguration tombstones the defense-in-depth sweep removed. + public int TenantConfigsSwept { get; set; } + /// Number of TenantConfiguration rows the sweep deliberately left alone because the + /// tenant had re-onboarded (DisabledReason is no longer the offboarding tombstone string). + public int TenantConfigsSpared { get; set; } + /// Number of markers kept alive because the TenantConfiguration probe/sweep failed transiently. + public int TenantConfigSweepRetries { get; set; } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index 86013444..d3bfde74 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -155,6 +155,27 @@ builder.Services.AddHostedService(); + +// Tenant-offboarding cascade worker (Plan Rev 9). Consumes per-tenant envelopes off +// `tenant-offboarding`, drives the §3 Phase 2 sequence (cascade enqueue → drain → SafeWipe → +// blob wipe → tenant config delete), and lands the audit row under AuditGlobalTenantId. +// OffboardingFilters is a static helper — no DI registration needed. +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton< + AutopilotMonitor.Functions.Services.Offboarding.IOffboardingExpectationsStore, + AutopilotMonitor.Functions.Services.Offboarding.BlobOffboardingExpectationsStore>(); +builder.Services.AddSingleton< + AutopilotMonitor.Functions.Services.Offboarding.IDeletionProgressDrainProbe, + AutopilotMonitor.Functions.Services.Offboarding.DeletionProgressDrainProbe>(); +builder.Services.AddSingleton< + AutopilotMonitor.Functions.Services.Offboarding.ITenantOffboardingEnqueuer, + AutopilotMonitor.Functions.Services.Offboarding.AzureQueueTenantOffboardingEnqueuer>(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); +// OffboardingMarkerCleanupFunction is TimerTrigger — auto-registered via the Functions host; +// no AddHostedService call needed. DI for its dependencies comes from the singletons above. + builder.Services.AddHostedService(); // Initialize all tables at startup // Data Access Layer — repository interfaces backed by Table Storage. diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index dea05f26..21664f95 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -275,6 +275,18 @@ public static class EndpointAccessPolicyCatalog new("POST", "versions/block", EndpointPolicy.GlobalAdminOnly), new("DELETE", "versions/block/{encodedPattern}", EndpointPolicy.GlobalAdminOnly), new("POST", "maintenance/trigger", EndpointPolicy.GlobalAdminOnly), + + // ── Tenant Offboarding Customs Archive (PR3.B) ───────────────────────── + // Snapshot of each tenant's custom GatherRules / AnalyzeRules / ImeLogPatterns + // rows from prior offboarding runs. Operator-driven cleanup: GA reviews + deletes. + // The {tenantId} route param is the OFFBOARDED tenant whose data was archived; + // the operator (GA) does not belong to that tenant. RouteParam scoping declares + // that the route's tenantId is supplied from the URL, not from the caller's JWT. + new("GET", "global/customs-archive", EndpointPolicy.GlobalAdminOnly), + new("GET", "global/customs-archive/{tenantId}/{historyRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam), + new("GET", "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam), + new("DELETE", "global/customs-archive/{tenantId}/{historyRowKey}/{archiveRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam), + new("DELETE", "global/customs-archive/{tenantId}/{historyRowKey}", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam), new("GET", "health/detailed", EndpointPolicy.AuthenticatedUser), new("POST", "rules/reseed-from-github", EndpointPolicy.GlobalAdminOnly), new("GET", "vulnerability/unmatched-software", EndpointPolicy.GlobalAdminOnly), diff --git a/src/Backend/AutopilotMonitor.Functions/Services/BlobStorageService.cs b/src/Backend/AutopilotMonitor.Functions/Services/BlobStorageService.cs index 91768695..1a939e12 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/BlobStorageService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/BlobStorageService.cs @@ -22,7 +22,8 @@ namespace AutopilotMonitor.Functions.Services { // Container that holds cascade-deletion manifests (snapshot + progress blobs). // Plan §3 + §10: 30d Lifecycle delete + 3d soft-delete = effective max 33-day retention. - private const string DeletionManifestsContainer = "deletion-manifests"; + // Single source of truth: AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests. + private const string DeletionManifestsContainer = AutopilotMonitor.Shared.Constants.BlobContainers.DeletionManifests; private readonly BlobServiceClient _blobServiceClient; private readonly ILogger _logger; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs index f99099ed..5553d6ab 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs @@ -259,18 +259,17 @@ namespace AutopilotMonitor.Functions.Services.Deletion await ExecuteTombstoneAsync(tenantId, sessionId, manifestId, tombstone, cancellationToken); } - (progress, etag) = await UpdateProgressWithRetryAsync( - tenantId, sessionId, manifestId, progress, etag, - mutate: p => - { - p.CompletedSteps.Add(tombstone.Order); - p.CompletedAt = DateTime.UtcNow; - }, - isAlreadyApplied: p => p.CompletedAt != null, - cancellationToken); - sw.Stop(); + // PR1.5 (Rev-5-F1) — Audit + SignalR notify BEFORE we stamp CompletedAt on the + // progress blob. CompletedAt is the drain predicate the tenant-offboarding + // worker uses to decide "all side effects through for this cascade"; if we + // set it before the audit row lands, an in-flight cascade can write + // `deletion_completed` into AuditLogs *after* the offboard worker has wiped + // that table → orphan audit row. Both calls are fail-soft today + // (LogAuditEntryAsync returns false on failure, NotifySessionDeletedAsync + // catches everything internally) so reordering does not change end-to-end + // failure semantics, only the observability window. await _maintenanceRepo.LogAuditEntryAsync( tenantId, action: "deletion_completed", @@ -282,6 +281,16 @@ namespace AutopilotMonitor.Functions.Services.Deletion await _signalR.NotifySessionDeletedAsync(tenantId, sessionId); + (progress, etag) = await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, etag, + mutate: p => + { + p.CompletedSteps.Add(tombstone.Order); + p.CompletedAt = DateTime.UtcNow; + }, + isAlreadyApplied: p => p.CompletedAt != null, + cancellationToken); + _logger.LogInformation( "SessionDeletionHandler completed: tenant={TenantId} session={SessionId} manifestId={ManifestId} durationMs={DurationMs}", tenantId, sessionId, manifestId, sw.ElapsedMilliseconds); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/AzureQueueTenantOffboardingEnqueuer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/AzureQueueTenantOffboardingEnqueuer.cs new file mode 100644 index 00000000..1b24231c --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/AzureQueueTenantOffboardingEnqueuer.cs @@ -0,0 +1,103 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Identity; +using Azure.Storage.Queues; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Azure Storage Queue producer for tenant-offboarding. Mirrors the existing + /// session-deletion / analyze producers but **fail-loud** (plan §7 + memory: + /// feedback_storage_helpers_fail_soft). Caller (TenantOffboardFunction) catches + /// and translates to HTTP 500; the History + Pointer + Marker rows already exist by then + /// so the operator can retry by re-issuing the offboard click (or via maintenance tooling). + /// + public sealed class AzureQueueTenantOffboardingEnqueuer : ITenantOffboardingEnqueuer + { + private readonly QueueClient _queueClient; + private readonly ILogger _logger; + + private int _queueEnsured; + + public AzureQueueTenantOffboardingEnqueuer( + IConfiguration configuration, + ILogger logger) + { + _logger = logger; + + var options = new QueueClientOptions + { + MessageEncoding = QueueMessageEncoding.Base64, + }; + + var storageAccountName = configuration["AzureStorageAccountName"]; + var connectionString = configuration["AzureTableStorageConnectionString"]; + + if (!string.IsNullOrEmpty(storageAccountName)) + { + var queueUri = new Uri( + $"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboarding}"); + _queueClient = new QueueClient(queueUri, new DefaultAzureCredential(), options); + _logger.LogInformation( + "TenantOffboarding enqueuer initialized with Managed Identity (account: {Account})", + storageAccountName); + } + else if (!string.IsNullOrEmpty(connectionString)) + { + _queueClient = new QueueClient( + connectionString, Constants.QueueNames.TenantOffboarding, options); + _logger.LogInformation("TenantOffboarding enqueuer initialized with connection string"); + } + else + { + throw new InvalidOperationException( + "Queue Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureTableStorageConnectionString'."); + } + } + + /// Test seam: bind directly to a (possibly mocked) . + internal AzureQueueTenantOffboardingEnqueuer( + QueueClient queueClient, + ILogger logger) + { + _queueClient = queueClient ?? throw new ArgumentNullException(nameof(queueClient)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task EnqueueAsync( + TenantOffboardingEnvelope envelope, + TimeSpan? visibilityDelay = null, + CancellationToken ct = default) + { + if (envelope == null) throw new ArgumentNullException(nameof(envelope)); + if (string.IsNullOrEmpty(envelope.TenantId)) throw new ArgumentException("envelope.TenantId required", nameof(envelope)); + if (string.IsNullOrEmpty(envelope.HistoryRowKey)) throw new ArgumentException("envelope.HistoryRowKey required", nameof(envelope)); + + await EnsureQueueExistsAsync(ct).ConfigureAwait(false); + + var body = JsonConvert.SerializeObject(envelope); + await _queueClient.SendMessageAsync( + body, + visibilityTimeout: visibilityDelay, + timeToLive: null, + cancellationToken: ct).ConfigureAwait(false); + + _logger.LogInformation( + "TenantOffboarding enqueued tenant={Tenant} history={History} drainPollCount={Polls} visibilityDelay={Delay}", + envelope.TenantId, envelope.HistoryRowKey, envelope.DrainPollCount, visibilityDelay); + } + + private async Task EnsureQueueExistsAsync(CancellationToken ct) + { + if (_queueEnsured == 1) return; + await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false); + Interlocked.Exchange(ref _queueEnsured, 1); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/BlobOffboardingExpectationsStore.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/BlobOffboardingExpectationsStore.cs new file mode 100644 index 00000000..7a4de931 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/BlobOffboardingExpectationsStore.cs @@ -0,0 +1,215 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Blob-backed implementation of . Pins the + /// blob path and container so callers cannot accidentally drop the expectations into + /// deletion-manifests (which gets wiped in Phase 2.E and would orphan crash recovery). + /// + /// The actual blob IO calls are routed through four protected virtual seams so + /// tests can replace them with an in-memory fake without standing up Azurite. Production + /// overrides hit ; tests subclass and override. + /// + /// + public class BlobOffboardingExpectationsStore : IOffboardingExpectationsStore + { + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }; + + private readonly BlobStorageService _blobs; + private readonly ILogger _logger; + private int _containerEnsured; + + public BlobOffboardingExpectationsStore( + BlobStorageService blobs, ILogger logger) + { + _blobs = blobs; + _logger = logger; + } + + // ── Public contract ───────────────────────────────────────────────────── + + public async Task TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default) + { + if (payload == null) throw new ArgumentNullException(nameof(payload)); + ValidatePayload(payload); + + var blobName = BuildBlobName(payload.TenantId, payload.HistoryRowKey); + var bytes = JsonSerializer.SerializeToUtf8Bytes(payload, JsonOptions); + + var (inserted, _) = await WriteIfNoneMatchAsync(blobName, bytes, ct); + + if (inserted) + { + _logger.LogInformation( + "Expectations blob uploaded tenant={Tenant} history={History} count={Count}", + payload.TenantId, payload.HistoryRowKey, payload.Expectations.Count); + } + else + { + _logger.LogInformation( + "Expectations blob already exists tenant={Tenant} history={History} — entering resume path", + payload.TenantId, payload.HistoryRowKey); + } + return inserted; + } + + public async Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync( + string tenantId, string historyRowKey, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey)); + + var blobName = BuildBlobName(tenantId, historyRowKey); + var (rawPayload, etag) = await ReadBlobAsync(blobName, ct); + if (rawPayload == null) return (null, null); + + var payload = JsonSerializer.Deserialize(rawPayload, JsonOptions); + return (payload, etag); + } + + public async Task UpdateWithEtagCasAsync( + OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default) + { + if (payload == null) throw new ArgumentNullException(nameof(payload)); + if (string.IsNullOrEmpty(ifMatchEtag)) throw new ArgumentException("ifMatchEtag required", nameof(ifMatchEtag)); + ValidatePayload(payload); + + var blobName = BuildBlobName(payload.TenantId, payload.HistoryRowKey); + var bytes = JsonSerializer.SerializeToUtf8Bytes(payload, JsonOptions); + return await WriteWithIfMatchAsync(blobName, bytes, ifMatchEtag, ct); + } + + public async Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + if (string.IsNullOrEmpty(historyRowKey)) throw new ArgumentException("historyRowKey required", nameof(historyRowKey)); + + var blobName = BuildBlobName(tenantId, historyRowKey); + await DeleteBlobIfExistsAsync(blobName, ct); + } + + // ── Protected virtual seams (production override = Azure SDK; tests = in-memory) ── + + /// + /// Production path: BlobClient.UploadAsync with If-None-Match=*. Azure + /// can answer with either 409 BlobAlreadyExists OR 412 ConditionNotMet + /// depending on storage tier / SDK version — both mean "exists already" and must + /// translate to (false, null) so the worker's resume path triggers. Other + /// failures propagate. + /// + protected virtual async Task<(bool Inserted, string? ETag)> WriteIfNoneMatchAsync( + string blobName, byte[] payload, CancellationToken ct) + { + var blobClient = await GetBlobClientAsync(blobName, ct); + + var options = new BlobUploadOptions + { + HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" }, + Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All }, + }; + + try + { + using var stream = new MemoryStream(payload); + var response = await blobClient.UploadAsync(stream, options, ct); + return (true, response.Value.ETag.ToString()); + } + catch (RequestFailedException ex) when (IsAlreadyExists(ex)) + { + return (false, null); + } + } + + protected virtual async Task<(byte[]? Payload, string? ETag)> ReadBlobAsync( + string blobName, CancellationToken ct) + { + var blobClient = await GetBlobClientAsync(blobName, ct); + try + { + using var stream = new MemoryStream(); + await blobClient.DownloadToAsync(stream, ct); + var properties = await blobClient.GetPropertiesAsync(cancellationToken: ct); + return (stream.ToArray(), properties.Value.ETag.ToString()); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return (null, null); + } + } + + protected virtual async Task WriteWithIfMatchAsync( + string blobName, byte[] payload, string ifMatchEtag, CancellationToken ct) + { + var blobClient = await GetBlobClientAsync(blobName, ct); + var options = new BlobUploadOptions + { + HttpHeaders = new BlobHttpHeaders { ContentType = "application/json" }, + Conditions = new BlobRequestConditions { IfMatch = new ETag(ifMatchEtag) }, + }; + using var stream = new MemoryStream(payload); + var response = await blobClient.UploadAsync(stream, options, ct); + return response.Value.ETag.ToString(); + } + + protected virtual async Task DeleteBlobIfExistsAsync(string blobName, CancellationToken ct) + { + var blobClient = await GetBlobClientAsync(blobName, ct); + await blobClient.DeleteIfExistsAsync(cancellationToken: ct); + } + + /// + /// Both 409 BlobAlreadyExists and 412 ConditionNotMet are emitted by Azure Blob + /// Storage for the same "blob exists, If-None-Match=* refused" condition, depending on + /// API version / tier. Either is a successful "no-op" for the resume path. + /// + protected static bool IsAlreadyExists(RequestFailedException ex) + => ex.Status == 409 + || ex.Status == 412 + || ex.ErrorCode == BlobErrorCode.BlobAlreadyExists + || ex.ErrorCode == BlobErrorCode.ConditionNotMet; + + // ── Internals ─────────────────────────────────────────────────────────── + + internal static string BuildBlobName(string tenantId, string historyRowKey) + => $"{tenantId}/{historyRowKey}.expectations.json"; + + private static void ValidatePayload(OffboardingExpectations payload) + { + SecurityValidator.EnsureValidGuid(payload.TenantId, $"{nameof(payload)}.TenantId"); + if (string.IsNullOrEmpty(payload.HistoryRowKey)) + throw new ArgumentException("payload.HistoryRowKey required", nameof(payload)); + if (payload.SchemaVersion < 1) + throw new ArgumentException("payload.SchemaVersion must be ≥ 1", nameof(payload)); + } + + private async Task GetBlobClientAsync(string blobName, CancellationToken ct) + { + var container = _blobs.GetContainerClient(Constants.BlobContainers.OffboardingState); + await EnsureContainerExistsAsync(container, ct); + return container.GetBlobClient(blobName); + } + + private async Task EnsureContainerExistsAsync(BlobContainerClient container, CancellationToken ct) + { + if (_containerEnsured == 1) return; + await container.CreateIfNotExistsAsync(cancellationToken: ct); + Interlocked.Exchange(ref _containerEnsured, 1); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/DeletionProgressDrainProbe.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/DeletionProgressDrainProbe.cs new file mode 100644 index 00000000..085a653b --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/DeletionProgressDrainProbe.cs @@ -0,0 +1,45 @@ +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Default backed by + /// . Plan §7.4 step 5: + /// drain is satisfied for a session expectation when its progress blob has + /// CompletedAt != null AND TombstoneStarted == true. + /// + public class DeletionProgressDrainProbe : IDeletionProgressDrainProbe + { + private readonly BlobStorageService _blobs; + private readonly ILogger _logger; + + public DeletionProgressDrainProbe(BlobStorageService blobs, ILogger logger) + { + _blobs = blobs; + _logger = logger; + } + + public async Task IsCascadeCompletedAsync( + string tenantId, string sessionId, string manifestId, CancellationToken ct = default) + { + try + { + var (progress, _) = await _blobs.DownloadDeletionProgressAsync(tenantId, sessionId, manifestId, ct); + return progress.CompletedAt != null && progress.TombstoneStarted; + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + // Progress blob not yet written — cascade still running. NOT a failure: the + // producer creates the progress blob before the queue message lands, but the + // tombstone step happens at the end of the cascade. + _logger.LogDebug( + "Drain probe: progress blob missing for tenant={Tenant} session={Session} manifest={Manifest} — cascade not done", + tenantId, sessionId, manifestId); + return false; + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/IDeletionProgressDrainProbe.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/IDeletionProgressDrainProbe.cs new file mode 100644 index 00000000..db85e795 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/IDeletionProgressDrainProbe.cs @@ -0,0 +1,21 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Per-session predicate over a cascade-deletion progress blob. Used by the + /// tenant-offboarding handler to drain against each + /// that ended up in an Enqueued | AlreadyInFlight outcome. + /// + public interface IDeletionProgressDrainProbe + { + /// + /// Returns true iff the progress blob exists AND + /// CompletedAt != null AND TombstoneStarted == true. 404 is treated as "not yet + /// done" (false). Other RequestFailedExceptions propagate so the worker can retry. + /// + Task IsCascadeCompletedAsync( + string tenantId, string sessionId, string manifestId, CancellationToken ct = default); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/IOffboardingExpectationsStore.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/IOffboardingExpectationsStore.cs new file mode 100644 index 00000000..f9027214 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/IOffboardingExpectationsStore.cs @@ -0,0 +1,45 @@ +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.Models.Offboarding; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// CRUD over the per-tenant Expectations blob. Lives under + /// Constants.BlobContainers.OffboardingState/{tenantId}/{historyRowKey}.expectations.json + /// — separate container from deletion-manifests so Phase 2.E's blob wipe cannot + /// orphan crash-recovery state (plan Rev-8-F1). + /// + public interface IOffboardingExpectationsStore + { + /// + /// Idempotent initial upload via If-None-Match=*. Returns true on a fresh + /// insert, false when a blob already exists for the same + /// (tenantId, historyRowKey) — caller treats false as the resume path and re-reads. + /// Any other Azure error propagates fail-loud. + /// + Task TryUploadInitialAsync(OffboardingExpectations payload, CancellationToken ct = default); + + /// + /// Downloads the Expectations blob; returns (null, null) on 404. Caller's + /// drain probe fails closed on 404 (FailedPhase="expectations_missing"). + /// + Task<(OffboardingExpectations? Payload, string? ETag)> TryDownloadAsync( + string tenantId, string historyRowKey, CancellationToken ct = default); + + /// + /// ETag-CAS update used by the drain loop to re-record retry counters for CasExhausted + /// expectations. Throws on 412 so the caller + /// can re-read and retry. + /// + Task UpdateWithEtagCasAsync( + OffboardingExpectations payload, string ifMatchEtag, CancellationToken ct = default); + + /// + /// Idempotent DeleteIfExistsAsync. Called as Phase 2.G's last step after the + /// history row has been marked Completed, and again as defence-in-depth by + /// OffboardingMarkerCleanupFunction when it removes a Completed marker. + /// + Task DeleteAsync(string tenantId, string historyRowKey, CancellationToken ct = default); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/ITenantOffboardingEnqueuer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/ITenantOffboardingEnqueuer.cs new file mode 100644 index 00000000..fb56961b --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/ITenantOffboardingEnqueuer.cs @@ -0,0 +1,27 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.Models.Offboarding; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Producer side of the tenant-offboarding queue. Implementations MUST be fail-loud: + /// the caller (TenantOffboardFunction) only returns 202 Accepted after a successful + /// enqueue, otherwise the admin's offboard click would silently strand without a worker + /// pickup (memory: feedback_storage_helpers_fail_soft). + /// + public interface ITenantOffboardingEnqueuer + { + /// + /// Enqueue a tenant-offboarding envelope. is used by + /// the handler's drain self-re-enqueue path (Plan §7.4 D2) to delay the next pickup by + /// ~2 minutes; producers in TenantOffboardFunction leave it null for immediate + /// visibility. + /// + Task EnqueueAsync( + TenantOffboardingEnvelope envelope, + TimeSpan? visibilityDelay = null, + CancellationToken ct = default); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/OffboardingFilters.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/OffboardingFilters.cs new file mode 100644 index 00000000..30abd9d8 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/OffboardingFilters.cs @@ -0,0 +1,70 @@ +using System; +using AutopilotMonitor.Functions.Security; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// OData filter builders for tenant-offboarding queries. All inputs are validated and + /// escaped before interpolation. Centralizing the patterns here keeps SafeWipe-callers + /// honest and gives the test suite a single surface to attack with injection payloads. + /// + /// + /// Rules (memory: feedback_storage_helpers_fail_soft, plan §5.5): + /// + /// Every tenant id is run through before use. + /// Every interpolated value passes through . + /// Composite-PK range uses '{tenantId}_~' upper bound — the underscore is the + /// anchor; without it the range bleeds into the next ASCII bucket. + /// + /// + public static class OffboardingFilters + { + /// + /// PartitionKey eq '{tenantId}' — for Variant A SafeWipe of tenant-PK tables. + /// + public static string ExactPartition(string normalizedTenantId) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + return $"PartitionKey eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'"; + } + + /// + /// PartitionKey ge '{tenantId}_' and PartitionKey lt '{tenantId}_~' — for + /// Variant A SafeWipe of composite-PK tables (events, signals, decision transitions, + /// discriminator-PK index tables, …). The underscore-tilde upper bound traps the + /// range strictly to {tenantId}_*; switching to '{tenantId}~' would + /// match keys outside the tenant prefix. + /// + public static string CompositePartitionRange(string normalizedTenantId) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + var safe = ODataSanitizer.EscapeValue(normalizedTenantId); + return $"PartitionKey ge '{safe}_' and PartitionKey lt '{safe}_~'"; + } + + /// + /// PartitionKey eq '{discriminator}' and TenantId eq '{tenantId}' — for + /// Variant B SafeWipe of discriminator-PK tables with a TenantId property as the + /// only tenant anchor. Verify-step MUST re-check the property server-side because + /// the PK is non-anchored. + /// + public static string DiscriminatorWithTenantProp(string discriminator, string normalizedTenantId) + { + if (string.IsNullOrEmpty(discriminator)) throw new ArgumentException("Discriminator required", nameof(discriminator)); + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + return $"PartitionKey eq '{ODataSanitizer.EscapeValue(discriminator)}' " + + $"and TenantId eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'"; + } + + /// + /// TenantId eq '{tenantId}' — Variant C SafeWipe (Property-only full-table + /// filter). Reserve for low-volume tables; Group-by-PartitionKey before batching + /// because rows can span many PKs. + /// + public static string TenantIdProperty(string normalizedTenantId) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + return $"TenantId eq '{ODataSanitizer.EscapeValue(normalizedTenantId)}'"; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/OffboardingSessionEnumerator.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/OffboardingSessionEnumerator.cs new file mode 100644 index 00000000..84f961f4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/OffboardingSessionEnumerator.cs @@ -0,0 +1,42 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared.DataAccess; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Thin wrapper around + /// so the offboarding handler can take an injectable dependency that is trivially mockable + /// (the repo interface has 20+ unrelated members; mocking it for every test is painful). + /// + /// Fail-loud contract: any storage exception from the underlying enumerator + /// propagates unchanged. Plan §7.4 + memory feedback_storage_helpers_fail_soft: + /// silently returning an empty list during enumeration would let the handler proceed + /// straight to wipe without cascade-backup. Tests in + /// OffboardingSessionEnumeratorTests pin this behaviour. + /// + /// + public class OffboardingSessionEnumerator + { + private readonly IMaintenanceRepository _maintenance; + + public OffboardingSessionEnumerator(IMaintenanceRepository maintenance) + { + _maintenance = maintenance; + } + + public virtual async IAsyncEnumerable EnumerateAsync( + string tenantId, + [EnumeratorCancellation] CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + + await foreach (var sessionId in _maintenance.EnumerateSessionsForOffboardingAsync(tenantId, ct)) + { + yield return sessionId; + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/SafeWipeService.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/SafeWipeService.cs new file mode 100644 index 00000000..08e585b1 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/SafeWipeService.cs @@ -0,0 +1,279 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Shared; +using Azure; +using Azure.Data.Tables; +using Azure.Storage.Blobs; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Four-step verified wipe pattern used by the tenant-offboarding worker. + /// + /// Fetch rows matching a tenant-anchored filter (PK or TenantId-Property). + /// Verify each fetched row matches the expected anchor server-side AND client-side. + /// Delete via batched SubmitTransactionAsync with ETag.All; per-row + /// 404-fallback so the helper stays idempotent across crash/resume. + /// Audit the deleted/already-missing counts to the caller's logger. + /// + /// Verify mismatch → before any delete fires. + /// + public class SafeWipeService + { + private const int BatchActionLimit = 100; + + private readonly TableStorageService _tables; + private readonly BlobStorageService _blobs; + private readonly ILogger _logger; + + public SafeWipeService(TableStorageService tables, BlobStorageService blobs, ILogger logger) + { + _tables = tables; + _blobs = blobs; + _logger = logger; + } + + /// + /// Variant A (exact tenant PartitionKey). Used for tables where rows live under + /// PartitionKey == normalizedTenantId (AuditLogs, UsageMetrics, …). + /// + public virtual Task WipeByExactPartitionAsync( + string tableName, string normalizedTenantId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + var filter = OffboardingFilters.ExactPartition(normalizedTenantId); + + return RunFetchVerifyDeleteAsync( + tableName, + filter, + expectedAnchor: normalizedTenantId, + verifyRow: e => string.Equals(e.PartitionKey, normalizedTenantId, StringComparison.Ordinal), + ct); + } + + /// + /// Variant A range (composite tenant PartitionKey). Used for tables where rows live + /// under {tenantId}_{sessionId|discriminator}. + /// + public virtual Task WipeByCompositePartitionRangeAsync( + string tableName, string normalizedTenantId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + var prefix = normalizedTenantId + "_"; + var filter = OffboardingFilters.CompositePartitionRange(normalizedTenantId); + + return RunFetchVerifyDeleteAsync( + tableName, + filter, + expectedAnchor: prefix, + verifyRow: e => e.PartitionKey.StartsWith(prefix, StringComparison.Ordinal), + ct); + } + + /// + /// Variant B (discriminator-PK + TenantId-Property). Used for tables where rows share + /// a fixed string PartitionKey (e.g. "CodeLookup", "reports", "Feedback") and only the + /// TenantId property identifies tenant ownership. Verify-step MUST check the + /// property because the PK is non-anchored. + /// + public virtual Task WipeByDiscriminatorAndTenantPropertyAsync( + string tableName, string discriminator, string normalizedTenantId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + if (string.IsNullOrEmpty(discriminator)) throw new ArgumentException("Discriminator required", nameof(discriminator)); + + var filter = OffboardingFilters.DiscriminatorWithTenantProp(discriminator, normalizedTenantId); + var expectedAnchor = $"{discriminator}+TenantId={normalizedTenantId}"; + + return RunFetchVerifyDeleteAsync( + tableName, + filter, + expectedAnchor: expectedAnchor, + verifyRow: e => string.Equals(e.PartitionKey, discriminator, StringComparison.Ordinal) + && string.Equals(e.GetString("TenantId"), normalizedTenantId, StringComparison.Ordinal), + ct); + } + + /// + /// Variant C (Property-only full-table filter). Reserved for low-volume tables where + /// PartitionKey is not tenant-anchored (e.g. UserUsageLog keyed by userOid). + /// Full-table scan — DO NOT use on high-volume tables without consideration. + /// + public virtual Task WipeByTenantIdPropertyAsync( + string tableName, string normalizedTenantId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + var filter = OffboardingFilters.TenantIdProperty(normalizedTenantId); + + return RunFetchVerifyDeleteAsync( + tableName, + filter, + expectedAnchor: $"TenantId={normalizedTenantId}", + verifyRow: e => string.Equals(e.GetString("TenantId"), normalizedTenantId, StringComparison.Ordinal), + ct); + } + + /// + /// Blob-prefix wipe. Lists blobs under {normalizedTenantId}/ in the named + /// container, verifies each returned name starts with the prefix, then deletes + /// per-blob (DeleteIfExistsAsync) so the helper stays idempotent across crash/resume. + /// Mismatch → abort before any delete. Returns the number of blobs deleted. + /// + public virtual async Task WipeBlobsByTenantPrefixAsync( + string containerName, string normalizedTenantId, CancellationToken ct = default) + { + SecurityValidator.EnsureValidGuid(normalizedTenantId, nameof(normalizedTenantId)); + if (string.IsNullOrEmpty(containerName)) throw new ArgumentException("Container required", nameof(containerName)); + + var prefix = normalizedTenantId + "/"; + var container = _blobs.GetContainerClient(containerName); + + var collected = new List(); + await foreach (var blob in container.GetBlobsAsync( + Azure.Storage.Blobs.Models.BlobTraits.None, + Azure.Storage.Blobs.Models.BlobStates.None, + prefix: prefix, + cancellationToken: ct)) + { + collected.Add(blob.Name); + } + + if (collected.Count == 0) + { + _logger.LogInformation( + "SafeWipe blobs '{Container}' tenant={Tenant}: no blobs found under '{Prefix}'", + containerName, normalizedTenantId, prefix); + return 0; + } + + var mismatches = collected + .Where(name => !name.StartsWith(prefix, StringComparison.Ordinal)) + .ToList(); + if (mismatches.Count > 0) + { + _logger.LogCritical( + "SafeWipe blobs '{Container}' tenant={Tenant}: aborting — {Total} blob(s) listed but {Bad} do not start with '{Prefix}'. First bad: '{Sample}'", + containerName, normalizedTenantId, collected.Count, mismatches.Count, prefix, mismatches[0]); + throw new SafeWipeVerificationException(containerName, prefix, mismatches.Count); + } + + int deleted = 0, missing = 0; + foreach (var name in collected) + { + ct.ThrowIfCancellationRequested(); + var blobClient = container.GetBlobClient(name); + var resp = await blobClient.DeleteIfExistsAsync(cancellationToken: ct); + if (resp.Value) deleted++; + else missing++; + } + + _logger.LogInformation( + "SafeWipe blobs '{Container}' tenant={Tenant}: deleted {Deleted}, already-missing {Missing}", + containerName, normalizedTenantId, deleted, missing); + return deleted; + } + + // ── Internal pipeline ─────────────────────────────────────────────────── + + private async Task RunFetchVerifyDeleteAsync( + string tableName, + string filter, + string expectedAnchor, + Func verifyRow, + CancellationToken ct) + { + if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName required", nameof(tableName)); + + var tableClient = _tables.GetTableClient(tableName); + + // STEP 1 — Fetch keys + the TenantId property if any verify-rule needs it. + // We pull "PartitionKey", "RowKey", "TenantId" so Variant B/C verify works. + var fetched = new List(); + await foreach (var e in tableClient.QueryAsync( + filter, + select: new[] { "PartitionKey", "RowKey", "TenantId" }, + cancellationToken: ct)) + { + fetched.Add(e); + } + + if (fetched.Count == 0) + { + _logger.LogDebug( + "SafeWipe '{Table}' anchor={Anchor}: 0 rows matched filter", + tableName, expectedAnchor); + return 0; + } + + // STEP 2 — Verify every row matches the expected anchor. + var mismatches = fetched.Where(e => !verifyRow(e)).ToList(); + if (mismatches.Count > 0) + { + var sample = mismatches.First(); + _logger.LogCritical( + "SafeWipe '{Table}' anchor={Anchor}: aborting — {Total} fetched, {Bad} fail verify. " + + "Sample bad row: PK='{Pk}' RK='{Rk}' TenantId='{Tid}'", + tableName, expectedAnchor, fetched.Count, mismatches.Count, + sample.PartitionKey, sample.RowKey, sample.GetString("TenantId") ?? ""); + throw new SafeWipeVerificationException(tableName, expectedAnchor, mismatches.Count); + } + + // STEP 3 — Delete in batches, grouped by PartitionKey (transaction requirement). + int deleted = 0, alreadyMissing = 0; + var groupedByPk = fetched + .GroupBy(e => e.PartitionKey, StringComparer.Ordinal) + .ToList(); + + foreach (var group in groupedByPk) + { + var rows = group.ToList(); + for (int i = 0; i < rows.Count; i += BatchActionLimit) + { + ct.ThrowIfCancellationRequested(); + var chunk = rows.Skip(i).Take(BatchActionLimit).ToList(); + var actions = chunk + .Select(e => new TableTransactionAction( + TableTransactionActionType.Delete, + new TableEntity(e.PartitionKey, e.RowKey) { ETag = ETag.All })) + .ToList(); + + try + { + await tableClient.SubmitTransactionAsync(actions, ct); + deleted += chunk.Count; + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + // Azure rolls the whole transaction back on a single 404; degrade to per-row. + foreach (var entity in chunk) + { + ct.ThrowIfCancellationRequested(); + try + { + await tableClient.DeleteEntityAsync( + entity.PartitionKey, entity.RowKey, ETag.All, ct); + deleted++; + } + catch (RequestFailedException rfe) when (rfe.Status == 404) + { + alreadyMissing++; + } + } + } + } + } + + // STEP 4 — Audit. + _logger.LogInformation( + "SafeWipe '{Table}' anchor={Anchor}: deleted {Deleted}, already-missing {Missing}", + tableName, expectedAnchor, deleted, alreadyMissing); + + return deleted; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/SafeWipeVerificationException.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/SafeWipeVerificationException.cs new file mode 100644 index 00000000..25e39c3e --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/SafeWipeVerificationException.cs @@ -0,0 +1,28 @@ +using System; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Thrown by when the verify step finds rows in the fetched + /// set that do not match the expected tenant anchor (PartitionKey, TenantId-Property, or + /// blob-name prefix depending on variant). NO DATA IS DELETED at this point — the wipe is + /// aborted before any delete call so a misbehaving filter cannot cause cross-tenant data + /// loss. Operator must inspect the unexpected rows manually. + /// + public class SafeWipeVerificationException : Exception + { + public string TableOrContainer { get; } + public string ExpectedAnchor { get; } + public int MismatchCount { get; } + + public SafeWipeVerificationException(string tableOrContainer, string expectedAnchor, int mismatchCount) + : base($"SafeWipe verify aborted for '{tableOrContainer}': {mismatchCount} row(s) " + + $"did not match expected anchor '{expectedAnchor}'. No deletes were issued. " + + $"Manual review required — possible filter bug or schema drift.") + { + TableOrContainer = tableOrContainer; + ExpectedAnchor = expectedAnchor; + MismatchCount = mismatchCount; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs new file mode 100644 index 00000000..d549aae4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs @@ -0,0 +1,958 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.DataAccess.TableStorage; +using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models.Deletion; +using AutopilotMonitor.Shared.Models.Offboarding; +using Azure; +using Azure.Data.Tables; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Phase 2 of the tenant-offboarding cascade plan (Rev 9). Drives a queued + /// through: + /// + /// 2.A — History/Pointer/Marker → InProgress; Rev-9-F1 Drain-Skip-Gate when DrainCompletedAt is set. + /// 2.B — fail-loud session enumeration + per-session cascade enqueue + Expectations-Blob upload. + /// 2.C — drain predicate against Expectations; D2 self-re-enqueue with 2-min delay up to 60 polls. + /// 2.D — retain-counter + SafeWipe over all tenant-scoped tables (variants A/B/C). + /// 2.E — blob-prefix cleanup of deletion-manifests/{tenantId}/. + /// 2.D-pass-2 — re-SafeWipe AuditLogs as defence-in-depth. + /// 2.F — delete the TenantConfiguration row. + /// 2.G — Completed state on History/Pointer/Marker; fail-soft Expectations-Blob delete. + /// 2.H — global-tenant audit row. + /// 2.I — TenantOffboarded OpsEvent. + /// + /// All Phase 2.D-G operations are idempotent (ETag.All + 404-fallback in SafeWipe, fail-soft + /// blob delete in 2.G, last-writer-wins on history) so the Rev-9-F1 resume from a crash + /// between 2.E and 2.G is safe even though the progress-blob predicate cannot run any more. + /// + public class TenantOffboardingHandler + { + /// Cap on drain self-re-enqueue iterations (~2h at 2-min visibility delay). + public const int MaxDrainPolls = 60; + + /// Delay between drain self-polls when expectations are not yet satisfied. + internal static readonly TimeSpan DrainPollDelay = TimeSpan.FromMinutes(2); + + /// Maximum CasExhausted retries per session before fail-closing the offboarding. + public const int MaxCasRetriesPerSession = 3; + + /// Throttle between per-session cascade enqueues so a 10k-session tenant doesn't burst. + internal static readonly TimeSpan PerSessionEnqueueThrottle = TimeSpan.FromMilliseconds(50); + + private static readonly Dictionary EmptyDetails = new(StringComparer.Ordinal); + + // Plan §6.2 — exact tenant-PK wipes (Variant A). + private static readonly string[] TenantPartitionTables = + { + Constants.TableNames.AuditLogs, + Constants.TableNames.UsageMetrics, + Constants.TableNames.UserActivity, + Constants.TableNames.AppInstallSummaries, + Constants.TableNames.TenantAdmins, + Constants.TableNames.BootstrapSessions, // main rows; CodeLookup discriminator below + Constants.TableNames.BlockedDevices, + Constants.TableNames.RuleStates, // runtime state — NOT customs (kept; §6.7) + Constants.TableNames.TenantNotifications, + Constants.TableNames.HardwareRejectionNotificationTracker, + Constants.TableNames.SlaTenantStatus, + Constants.TableNames.DeviceSnapshot, + Constants.TableNames.EventSessionIndex, + Constants.TableNames.DistressReports, + Constants.TableNames.SoftwareInventory, + Constants.TableNames.SessionInventoryContributions, + Constants.TableNames.SessionTombstones, + Constants.TableNames.PreviewWhitelist, + }; + + // Plan §6.4 — composite-PK "{tenantId}_..." wipes (Variant A range). + private static readonly string[] CompositePartitionTables = + { + Constants.TableNames.Events, + Constants.TableNames.RuleResults, + Constants.TableNames.VulnerabilityReports, + Constants.TableNames.Signals, + Constants.TableNames.DecisionTransitions, + Constants.TableNames.EventTypeIndex, + Constants.TableNames.CveIndex, + Constants.TableNames.SessionsByTerminal, + Constants.TableNames.SessionsByStage, + Constants.TableNames.DeadEndsByReason, + Constants.TableNames.ClassifierVerdictsByIdLevel, + Constants.TableNames.SignalsByKind, + }; + + // Plan §6.3.1 — Variant B (Discriminator + TenantId property). + // (table, discriminator) + private static readonly (string Table, string Discriminator)[] DiscriminatorTables = + { + (Constants.TableNames.BootstrapSessions, "CodeLookup"), + (Constants.TableNames.SessionReports, "reports"), + (Constants.TableNames.PreviewConfig, "Feedback"), + }; + + // Plan §6.3.2 — Variant C (TenantId property only). + private static readonly string[] PropertyOnlyTables = + { + Constants.TableNames.UserUsageLog, + }; + + // PR3.B plan §3 — Customs rules tables: archive each row to + // TenantOffboardingCustomsArchive (per-run partition) THEN wipe the source so the + // tenant's originals are gone. Counters in OffboardingHistory are recomputed from + // the archive table per run + table to be crash-resume safe. + private static readonly (string Table, string Field)[] ArchivedRuleTables = + { + (Constants.TableNames.GatherRules, nameof(OffboardingHistoryEntry.CustomGatherRulesArchived)), + (Constants.TableNames.AnalyzeRules, nameof(OffboardingHistoryEntry.CustomAnalyzeRulesArchived)), + (Constants.TableNames.ImeLogPatterns, nameof(OffboardingHistoryEntry.ImeLogPatternOverridesArchived)), + }; + + /// + /// Maximum number of archive-then-wipe iterations per rules table before the + /// post-wipe verify re-find triggers fail-closed with customs_arrival_race. + /// First pass handles the normal case; passes 2 + 3 absorb writes that landed + /// during the drain barrier (a warm function-host with a stale TenantConfig + /// cache could still write between Phase 1 commit and Phase 2 pickup). + /// + internal const int ArchiveIterationCap = 3; + + private readonly IOffboardingAuditRepository _auditRepo; + private readonly OffboardingSessionEnumerator _enumerator; + private readonly ISessionDeletionEnqueuer _cascadeEnqueuer; + private readonly IOffboardingExpectationsStore _expectations; + private readonly IDeletionProgressDrainProbe _drainProbe; + private readonly SafeWipeService _safeWipe; + private readonly TableStorageService _storage; + private readonly IMaintenanceRepository _maintenance; + private readonly ITenantOffboardingEnqueuer _reEnqueuer; + private readonly OpsEventService _opsEvents; + private readonly ITenantCustomsArchiveRepository _customsArchive; + private readonly ILogger _logger; + + public TenantOffboardingHandler( + IOffboardingAuditRepository auditRepo, + OffboardingSessionEnumerator enumerator, + ISessionDeletionEnqueuer cascadeEnqueuer, + IOffboardingExpectationsStore expectations, + IDeletionProgressDrainProbe drainProbe, + SafeWipeService safeWipe, + TableStorageService storage, + IMaintenanceRepository maintenance, + ITenantOffboardingEnqueuer reEnqueuer, + OpsEventService opsEvents, + ITenantCustomsArchiveRepository customsArchive, + ILogger logger) + { + _auditRepo = auditRepo; + _enumerator = enumerator; + _cascadeEnqueuer = cascadeEnqueuer; + _expectations = expectations; + _drainProbe = drainProbe; + _safeWipe = safeWipe; + _storage = storage; + _maintenance = maintenance; + _reEnqueuer = reEnqueuer; + _opsEvents = opsEvents; + _customsArchive = customsArchive; + _logger = logger; + } + + public async Task HandleAsync(TenantOffboardingEnvelope envelope, CancellationToken ct = default) + { + if (envelope == null) throw new ArgumentNullException(nameof(envelope)); + var tenantId = envelope.TenantId.ToLowerInvariant(); + + // 2.A — Load history. Idempotent re-pickup: Completed/Failed → return early. + var history = await _auditRepo.TryGetHistoryAsync(envelope.HistoryRowKey, ct) + ?? throw new InvalidOperationException( + $"OffboardingHistory row missing for envelope tenant={tenantId} history={envelope.HistoryRowKey}"); + + if (history.Status == "Completed") + { + _logger.LogInformation("Tenant offboarding already Completed — re-pickup is a no-op. tenant={Tenant}", tenantId); + return; + } + if (history.Status == "Failed") + { + _logger.LogWarning( + "Tenant offboarding History is Failed (phase={Phase}) — operator action required, worker returns. tenant={Tenant}", + history.ErrorMessage ?? "unknown", tenantId); + return; + } + + // 2.A — first pickup transitions History/Pointer/Marker to InProgress. + if (history.Status == "Initiated") + { + history.Status = "InProgress"; + await _auditRepo.UpsertHistoryAsync(history, ct); + await TransitionStatusAsync(tenantId, history.RowKey, "InProgress", ct); + } + + // 2.A.5 — Rev-9-F1 Drain-Skip-Gate. After the first successful drain, DrainCompletedAt + // is stamped on the history row and all subsequent phases (2.D-G) are idempotent. + // A crash between 2.E (blob wipe) and 2.G (status flip) would otherwise leave the + // re-pickup looking for progress blobs that 2.E just deleted — fail-closed with + // expectations_missing, which is wrong (the offboard actually succeeded in spirit). + if (history.DrainCompletedAt != null) + { + _logger.LogInformation( + "Drain-Skip-Gate: History.DrainCompletedAt={DrainedAt} set; running post-drain phases idempotently. tenant={Tenant}", + history.DrainCompletedAt, tenantId); + await RunPostDrainPhasesAsync(history, tenantId, ct); + return; + } + + // 2.B — Ensure Expectations blob exists. The two history markers EnumerationStartedAt + // (ESA) and EnumerationCompletedBeforeUpload (ECBU) disambiguate three resume modes + // when the blob is missing: + // • ESA null, ECBU null → first try → stamp ESA, enumerate, stamp ECBU, upload. + // • ESA set, ECBU null → mid-enumeration crash → fail-closed expectations_missing + // (re-enumerating against a mid-mutation tenant is unsafe). + // • ESA set, ECBU set → upload failure → retry enumerate+upload (idempotent, + // SessionDeletionProducer returns AlreadyInFlight). + // Full decision table lives on EnsureExpectationsBlobAsync. Plan §7.4 step 3. + var ensureFail = await EnsureExpectationsBlobAsync(history, tenantId, ct); + if (ensureFail != null) + { + await FailAsync(history, tenantId, ensureFail, + $"Drain fail-closed: {ensureFail}", ct); + throw new InvalidOperationException( + $"TenantOffboarding fail-closed for tenant={tenantId} phase={ensureFail}"); + } + + // 2.C — Drain probe. Three outcomes: drain OK (proceed), drain not yet (re-enqueue + // with 2-min delay up to MaxDrainPolls), fail-closed (throw → worker poisons). + var (drainOk, failReason) = await EvaluateDrainAsync(history, tenantId, ct); + + if (failReason != null) + { + await FailAsync(history, tenantId, failReason, $"Drain fail-closed: {failReason}", ct); + throw new InvalidOperationException( + $"TenantOffboarding fail-closed for tenant={tenantId} phase={failReason}"); + } + + if (!drainOk) + { + if (envelope.DrainPollCount + 1 >= MaxDrainPolls) + { + await FailAsync(history, tenantId, "drain_timeout", + $"Drain did not settle within {MaxDrainPolls} polls (~{MaxDrainPolls * DrainPollDelay.TotalMinutes:0}min)", ct); + throw new InvalidOperationException( + $"TenantOffboarding drain_timeout for tenant={tenantId}"); + } + + var next = new TenantOffboardingEnvelope + { + EnvelopeVersion = envelope.EnvelopeVersion, + TenantId = envelope.TenantId, + HistoryPartitionKey = envelope.HistoryPartitionKey, + HistoryRowKey = envelope.HistoryRowKey, + InitiatedBy = envelope.InitiatedBy, + InitiatedAt = envelope.InitiatedAt, + EnqueuedAt = DateTime.UtcNow, + DrainPollCount = envelope.DrainPollCount + 1, + }; + await _reEnqueuer.EnqueueAsync(next, DrainPollDelay, ct); + _logger.LogInformation( + "Drain not yet settled — re-enqueued tenant={Tenant} poll={Poll}/{Max} delay={Delay}", + tenantId, next.DrainPollCount, MaxDrainPolls, DrainPollDelay); + return; + } + + // Drain OK — Rev-9-F1: stamp DrainCompletedAt BEFORE Phase 2.D begins. + history.DrainCompletedAt = DateTime.UtcNow; + await _auditRepo.UpsertHistoryAsync(history, ct); + + await RunPostDrainPhasesAsync(history, tenantId, ct); + } + + // ── Phase 2.A status transitions ──────────────────────────────────────── + + private async Task TransitionStatusAsync(string tenantId, string historyRowKey, string newStatus, CancellationToken ct) + { + // Marker — read existing fields so we don't clobber InitiatedAt/InitiatedBy. + var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct); + if (marker != null) + { + marker.Status = newStatus; + await _auditRepo.UpsertMarkerAsync(marker, ct); + } + + // Pointer — read-modify-write with ETag-CAS retry (Plan §4.4). + await UpdatePointerWithRetryAsync(tenantId, historyRowKey, newStatus, ct); + } + + private async Task UpdatePointerWithRetryAsync(string tenantId, string historyRowKey, string newStatus, CancellationToken ct) + { + const int maxRetries = 5; + for (int attempt = 0; attempt < maxRetries; attempt++) + { + ct.ThrowIfCancellationRequested(); + var (pointer, etag) = await _auditRepo.TryGetByTenantPointerAsync(tenantId, ct); + if (pointer == null || etag == null) + { + _logger.LogWarning( + "Pointer missing for tenant={Tenant} while transitioning to {Status} — skip (graceful degradation, Plan §4.4)", + tenantId, newStatus); + return; + } + pointer.LatestStatus = newStatus; + pointer.LatestUpdatedAt = DateTime.UtcNow; + if (!string.Equals(pointer.LatestHistoryRowKey, historyRowKey, StringComparison.Ordinal)) + { + // Pointer points at a different attempt — concurrent re-offboard. Skip. + _logger.LogWarning( + "Pointer.LatestHistoryRowKey={PointerHistory} != envelope.HistoryRowKey={Envelope} — skipping pointer transition", + pointer.LatestHistoryRowKey, historyRowKey); + return; + } + try + { + await _auditRepo.UpdateByTenantPointerWithEtagAsync(pointer, etag, ct); + return; + } + catch (RequestFailedException ex) when (ex.Status == 412) + { + // Concurrent writer beat us — re-read + retry. + } + } + _logger.LogWarning( + "Pointer ETag-CAS exhausted for tenant={Tenant} status={Status} — giving up (Plan §4.4 graceful degradation)", + tenantId, newStatus); + } + + // ── Phase 2.B — initial enqueue + Expectations persistence ────────────── + + /// + /// Returns null on success (blob exists / blob just created). Returns a + /// FailedPhase string when the handler must fail-closed without enumerating. + /// + /// Decision table (Plan §7.4 step 3 + Review-Fix Finding 2): + /// + /// + /// blobESAECBUAction + /// exists——Resume path; return null. + /// missingnullnullFirst try; stamp ESA, enumerate, stamp ECBU, upload. + /// missingsetnullMid-enumeration crash → fail-closed expectations_missing. + /// missingsetsetUpload failure → retry enumerate (idempotent — producer returns AlreadyInFlight) + re-upload. + /// + /// + private async Task EnsureExpectationsBlobAsync( + OffboardingHistoryEntry history, string tenantId, CancellationToken ct) + { + var (existing, _) = await _expectations.TryDownloadAsync(tenantId, history.RowKey, ct); + if (existing != null) + { + _logger.LogInformation( + "Expectations blob already exists for tenant={Tenant} (count={Count}, enumComplete={EnumComplete}) — resume path", + tenantId, existing.Expectations.Count, existing.EnumerationCompleted); + return null; + } + + // Mid-enumeration crash: prior pickup stamped ESA but never reached ECBU. Refuse + // to re-enumerate against a possibly mid-mutation tenant. + if (history.EnumerationStartedAt != null + && history.EnumerationCompletedBeforeUpload == null) + { + _logger.LogError( + "Expectations blob missing, EnumerationStartedAt={StartedAt}, ECBU null — fail-closed (mid-enumeration crash). tenant={Tenant}", + history.EnumerationStartedAt, tenantId); + return "expectations_missing"; + } + + // Either first try (ESA null) or upload-retry (ESA + ECBU set, blob missing). + // First-try only: stamp ESA before touching the enumerator so a crash here lands + // in the fail-closed branch above on the next pickup. + if (history.EnumerationStartedAt == null) + { + history.EnumerationStartedAt = DateTime.UtcNow; + await _auditRepo.UpsertHistoryAsync(history, ct); + } + else + { + _logger.LogInformation( + "Upload-retry path for tenant={Tenant} (ESA={Esa}, ECBU={Ecbu}) — re-running enumerate+upload", + tenantId, history.EnumerationStartedAt, history.EnumerationCompletedBeforeUpload); + } + + // Fail-loud enumeration. Storage exceptions propagate; the queue worker poisons or + // retries via visibility timeout. NEVER catch here — that would re-introduce the + // Rev-5-F3 silent-empty bug. + var collected = new List(); + var actor = new DeletionActor { Type = "system", Actor = "System.TenantOffboarding" }; + + await foreach (var sessionId in _enumerator.EnumerateAsync(tenantId, ct)) + { + ct.ThrowIfCancellationRequested(); + var result = await _cascadeEnqueuer.EnqueueAsync( + tenantId, sessionId, reason: "tenant_offboard", actor: actor, + retentionContext: null, cancellationToken: ct); + + collected.Add(new OffboardingExpectation + { + SessionId = sessionId, + ManifestId = result.ManifestId, + Outcome = result.Outcome.ToString(), + RetryCount = 0, + }); + + if (PerSessionEnqueueThrottle > TimeSpan.Zero) + { + try { await Task.Delay(PerSessionEnqueueThrottle, ct); } + catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } + } + } + + // Enumeration finished cleanly. Persist ECBU BEFORE the upload so a subsequent + // upload failure is recognized as upload-retry-eligible on the next pickup. + // On retry pickups ECBU may already be set — UpsertHistoryAsync is idempotent. + if (history.EnumerationCompletedBeforeUpload == null) + { + history.EnumerationCompletedBeforeUpload = DateTime.UtcNow; + await _auditRepo.UpsertHistoryAsync(history, ct); + } + + // Plan Rev-7-F2: EnumerationCompleted on the payload = the in-blob flag the drain + // probe checks. We can only set it true here because the iterator above ran to + // completion without throwing (the throw-path skipped this whole block). + var payload = new OffboardingExpectations + { + SchemaVersion = 1, + TenantId = tenantId, + HistoryRowKey = history.RowKey, + CreatedAt = DateTime.UtcNow, + EnumerationCompleted = true, + EnumeratedSessionCount = collected.Count, + Expectations = collected, + }; + + var uploaded = await _expectations.TryUploadInitialAsync(payload, ct); + if (!uploaded) + { + // Race: a parallel worker beat us. Their blob is the source of truth. + _logger.LogInformation( + "Expectations blob race lost for tenant={Tenant} history={History} — using existing blob", + tenantId, history.RowKey); + } + else + { + history.CascadeSessionsEnqueued = collected.Count; + await _auditRepo.UpsertHistoryAsync(history, ct); + } + return null; + } + + // ── Phase 2.C — drain predicate ───────────────────────────────────────── + + private async Task<(bool DrainOk, string? FailReason)> EvaluateDrainAsync( + OffboardingHistoryEntry history, string tenantId, CancellationToken ct) + { + var (payload, etag) = await _expectations.TryDownloadAsync(tenantId, history.RowKey, ct); + if (payload == null) return (false, "expectations_missing"); + if (etag == null) return (false, "expectations_missing"); + if (payload.SchemaVersion != 1) return (false, "expectations_corrupt"); + if (!payload.EnumerationCompleted) return (false, "enumeration_incomplete"); + if (payload.Expectations.Count != payload.EnumeratedSessionCount) + return (false, "expectations_size_mismatch"); + + var allSatisfied = true; + var dirty = false; + var actor = new DeletionActor { Type = "system", Actor = "System.TenantOffboarding" }; + + for (int i = 0; i < payload.Expectations.Count; i++) + { + ct.ThrowIfCancellationRequested(); + var expectation = payload.Expectations[i]; + + switch (expectation.Outcome) + { + case nameof(SessionDeletionEnqueueOutcome.KillSwitchActive): + return (false, "killswitch"); + + case nameof(SessionDeletionEnqueueOutcome.Poisoned): + return (false, "poisoned"); + + case nameof(SessionDeletionEnqueueOutcome.SessionNotFound): + // Session was already gone when we tried to enqueue — nothing to drain. + continue; + + case nameof(SessionDeletionEnqueueOutcome.CasExhausted): + if (expectation.RetryCount >= MaxCasRetriesPerSession) + return (false, "cas_exhausted"); + + // Retry the per-session enqueue. Update outcome + counter in-place. + var retry = await _cascadeEnqueuer.EnqueueAsync( + tenantId, expectation.SessionId, reason: "tenant_offboard", actor: actor, + retentionContext: null, cancellationToken: ct); + expectation.Outcome = retry.Outcome.ToString(); + expectation.ManifestId = retry.ManifestId; + expectation.RetryCount++; + dirty = true; + allSatisfied = false; // re-evaluate next drain cycle + break; + + case nameof(SessionDeletionEnqueueOutcome.Enqueued): + case nameof(SessionDeletionEnqueueOutcome.AlreadyInFlight): + if (string.IsNullOrEmpty(expectation.ManifestId)) + { + // Rev-8-F3 — AlreadyInFlight with null ManifestId is a Preparing-without- + // snapshot state. There's no progress blob to drain against; fail-closed. + return (false, "alreadyinflight_no_manifest"); + } + var done = await _drainProbe.IsCascadeCompletedAsync( + tenantId, expectation.SessionId, expectation.ManifestId, ct); + if (!done) allSatisfied = false; + break; + + default: + _logger.LogWarning( + "Unknown Expectation.Outcome '{Outcome}' for tenant={Tenant} session={Session}", + expectation.Outcome, tenantId, expectation.SessionId); + allSatisfied = false; + break; + } + } + + if (dirty) + { + try + { + await _expectations.UpdateWithEtagCasAsync(payload, etag, ct); + } + catch (RequestFailedException ex) when (ex.Status == 412) + { + // A parallel worker re-evaluated and wrote first. Their version wins; the + // next drain cycle will pick up the latest state. We still report + // allSatisfied=false so this cycle re-enqueues. + _logger.LogInformation( + "Expectations CAS 412 for tenant={Tenant} — parallel worker won, will re-read next cycle", + tenantId); + return (false, null); + } + } + + return (allSatisfied, null); + } + + // ── Phases 2.D / 2.E / 2.D-pass-2 / 2.F / 2.G / 2.H / 2.I ─────────────── + + private async Task RunPostDrainPhasesAsync( + OffboardingHistoryEntry history, string tenantId, CancellationToken ct) + { + // 2.D-archive (PR3.B §3) — Archive then wipe GatherRules / AnalyzeRules / + // ImeLogPatterns. Fail-loud: a row that cannot be archived must NOT be wiped. + // After the third iteration if the source still has rows, throw with + // FailedPhase="customs_arrival_race" — marker stays Failed for operator action. + try + { + await ArchiveAndWipeCustomsRulesAsync(history, tenantId, ct); + } + catch (InvalidOperationException ex) when (ex.Message.StartsWith("customs_arrival_race", StringComparison.Ordinal)) + { + await FailAsync(history, tenantId, "customs_arrival_race", ex.Message, ct); + throw; // worker dequeues + eventually poisons + } + + // 2.D-pass-1 — SafeWipe all tenant-scoped tables, including AuditLogs. + var deletedCounts = new Dictionary(StringComparer.Ordinal); + await SafeWipeAllTenantTablesAsync(deletedCounts, tenantId, ct); + + // 2.E — Blob-prefix wipe (deletion-manifests only; offboarding-state stays). + int deletedBlobs = 0; + try + { + deletedBlobs = await _safeWipe.WipeBlobsByTenantPrefixAsync( + Constants.BlobContainers.DeletionManifests, tenantId, ct); + } + catch (SafeWipeVerificationException) + { + throw; // unrecoverable — let worker poison + } + + // 2.D-pass-2 — Re-wipe AuditLogs as defence in depth against late audits that may + // have landed between Phase 2.D-pass-1 and now (the cascade-handler reorder from + // PR1.5 closes the main window, but a struggling SignalR/Audit retry could still + // bring a row in late). + var auditPass2 = await _safeWipe.WipeByExactPartitionAsync(Constants.TableNames.AuditLogs, tenantId, ct); + if (auditPass2 > 0) + { + _logger.LogInformation( + "Audit pass-2 swept {Count} late row(s) for tenant={Tenant}", auditPass2, tenantId); + deletedCounts[Constants.TableNames.AuditLogs] = + deletedCounts.TryGetValue(Constants.TableNames.AuditLogs, out var existing) ? existing + auditPass2 : auditPass2; + } + + // ── 2.G — Order: side-effects FIRST, then History terminal write LAST ──── + // + // Note (PR3.B review): the legacy 2.F TenantConfiguration-delete used to sit + // BEFORE 2.G. That left a tiny window where a /api/auth/me race could re-create + // a fresh Default-Disabled=false TenantConfiguration before the Completed + // state landed — letting an offboarded tenant slip back through. The delete + // has moved to AFTER 2.I (see "2.F-final" below) so the entire Phase 2 sequence + // runs against an existing Disabled=true row, and only once everything is + // committed do we drop the row. Self-service re-onboarding still works because + // the user's NEXT /api/auth/me after 2.F-final sees no row → auto-create-default. + // Plan §3 Phase 2.G + Review-Fix Finding 3: a crash between History=Completed and + // the marker/pointer/blob/audit/ops writes would otherwise make the next pickup + // return early (terminal-status check) and leave the side-effects dangling. By + // flipping History last we keep DrainCompletedAt as the resume anchor — re-pickup + // skips drain and re-runs the (idempotent) side-effect chain. + + // Pre-populate the in-memory history with the final values so the LAST write below + // commits the whole snapshot atomically. + var completedAt = DateTime.UtcNow; + history.CompletedAt = completedAt; + history.DeletedRowCountsJson = JsonSerializer.Serialize(deletedCounts); + history.TotalRowsDeleted = deletedCounts.Values.Sum(); + history.DeletedBlobCount = deletedBlobs; + + // Side-effect 1: Marker → Completed (must happen FIRST so a crash here leaves + // History=InProgress; the next pickup re-runs the idempotent post-drain chain). + var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct); + if (marker != null) + { + marker.Status = "Completed"; + marker.CompletedAt = completedAt; + await _auditRepo.UpsertMarkerAsync(marker, ct); + } + + // Side-effect 2: Pointer → Completed (ETag-CAS retry). + await UpdatePointerWithRetryAsync(tenantId, history.RowKey, "Completed", ct); + + // Side-effect 3: Expectations blob delete (fail-soft; MarkerCleanupFunction retries). + try + { + await _expectations.DeleteAsync(tenantId, history.RowKey, ct); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Expectations blob delete failed for tenant={Tenant} history={History} — MarkerCleanupFunction will retry", + tenantId, history.RowKey); + } + + // Side-effect 4: Global-tenant audit row (persists across the wiped tenant audits). + await _maintenance.LogAuditEntryAsync( + Constants.AuditGlobalTenantId, + action: "DELETE", + entityType: "Tenant", + entityId: tenantId, + performedBy: history.InitiatedBy, + details: new Dictionary(StringComparer.Ordinal) + { + ["Action"] = "Offboard", + ["Phase"] = "Completed", + ["DomainName"] = history.DomainName ?? string.Empty, + ["TotalRowsDeleted"] = (history.TotalRowsDeleted ?? 0).ToString(), + ["DeletedBlobCount"] = (history.DeletedBlobCount ?? 0).ToString(), + ["CascadeSessionsEnqueued"] = (history.CascadeSessionsEnqueued ?? 0).ToString(), + ["HistoryRowKey"] = history.RowKey, + }); + + // Side-effect 5: OpsEvent. Reuses the existing TenantOffboarded helper so the + // dashboard category cell stays "Tenant" / event "TenantOffboarded". + await _opsEvents.RecordTenantOffboardedAsync( + tenantId, history.InitiatedBy, deletedCounts, history.DomainName); + + // COMMIT: History → Completed LAST. Until this write lands the next pickup will + // see Status=InProgress + DrainCompletedAt set and re-run all of the above + // idempotently (duplicate audit/OpsEvent emit is preferable to dangling state). + history.Status = "Completed"; + await _auditRepo.UpsertHistoryAsync(history, ct); + + // ── 2.F-final — TenantConfiguration delete (PR3.B Codex Finding 2 reorder) ── + // + // This is the LAST step in Phase 2. By now Marker / Pointer / History all say + // "Completed" and the Disabled=true auth-gate has carried the tenant through the + // entire async wipe. Deleting the row here means the next /api/auth/me for this + // tenant will see no row → TenantConfigurationService.GetConfigurationAsync's + // auto-create-default path kicks in, giving the same user a fresh, clean tenant + // (the legitimate self-service re-onboarding case). + // + // Fail-soft: if the delete fails, Marker stays Completed and the + // OffboardingMarkerCleanupFunction's TenantConfig-sweep (defense-in-depth) runs + // every 2h and finishes the cleanup. The tenant cannot re-onboard until that + // sweep lands — acceptable, because Disabled=true still applies meanwhile. + try + { + var tenantConfigDeleted = await _safeWipe.WipeByExactPartitionAsync( + Constants.TableNames.TenantConfiguration, tenantId, ct); + if (tenantConfigDeleted > 0) + { + _logger.LogInformation( + "Phase 2.F-final: deleted TenantConfiguration row(s)={Count} for tenant={Tenant}", + tenantConfigDeleted, tenantId); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Phase 2.F-final TenantConfiguration delete failed for tenant={Tenant} — MarkerCleanupFunction's TenantConfig-sweep will retry", + tenantId); + } + + _logger.LogInformation( + "Tenant offboarding Completed: tenant={Tenant} totalRows={Rows} blobs={Blobs} sessions={Sessions} archiveGather={G} archiveAnalyze={A} archivePatterns={P}", + tenantId, + history.TotalRowsDeleted, history.DeletedBlobCount, history.CascadeSessionsEnqueued, + history.CustomGatherRulesArchived, history.CustomAnalyzeRulesArchived, history.ImeLogPatternOverridesArchived); + } + + // ── Helpers ───────────────────────────────────────────────────────────── + + /// + /// PR3.B §3 — Phase 2.D-archive. For each rules table, archive every row to + /// THEN delete that exact (PK, RK) + /// from the source. Up to iterations to absorb + /// late writes that may have landed during the drain barrier; if the source still + /// has rows after the cap, throw so the + /// caller fail-closes the offboarding with FailedPhase="customs_arrival_race". + /// + /// Crash-safety: the archive upsert is idempotent (Replace), so a worker that + /// crashed between archive-insert and source-delete will re-enumerate the same + /// row on re-pickup, re-write the (identical) archive entry, and delete the + /// source row that survived the crash. Counters are recomputed from the archive + /// table at the end so a transient crash mid-loop does not produce a stale 0. + /// + /// + private async Task ArchiveAndWipeCustomsRulesAsync( + OffboardingHistoryEntry history, string tenantId, CancellationToken ct) + { + foreach (var (tableName, fieldName) in ArchivedRuleTables) + { + await ArchiveAndWipeRulesTableAsync(tableName, tenantId, history.RowKey, ct); + + // Recompute the counter from the archive table (per run + table) rather than + // from in-loop accumulation. A crash between this assignment and the final + // UpsertHistoryAsync below would otherwise leave a stale 0. + var counted = await _customsArchive.CountByRunAndTableAsync(tenantId, history.RowKey, tableName, ct); + AssignArchivedCounter(history, fieldName, counted); + } + + await _auditRepo.UpsertHistoryAsync(history, ct); + } + + /// + /// Archive-then-wipe loop for a single rules table. Up to + /// iterations; final-iteration verify-read triggers customs_arrival_race when a + /// writer is still racing us. Marked internal virtual so test harnesses can + /// override it to a no-op without faking the underlying TableServiceClient. + /// + internal virtual async Task ArchiveAndWipeRulesTableAsync( + string tableName, string tenantId, string historyRowKey, CancellationToken ct) + { + var sourceClient = _storage.GetTableClient(tableName); + + for (var attempt = 1; attempt <= ArchiveIterationCap; attempt++) + { + var archivedThisAttempt = 0; + var filter = OffboardingFilters.ExactPartition(tenantId); + await foreach (var entity in sourceClient.QueryAsync(filter, cancellationToken: ct)) + { + await _customsArchive.UpsertAsync(BuildArchiveEntry(entity, tenantId, tableName, historyRowKey), ct); + // Only AFTER the archive entry is persisted do we delete the source. + try + { + await sourceClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey, Azure.ETag.All, ct); + } + catch (Azure.RequestFailedException ex) when (ex.Status == 404) + { + // Already gone (parallel writer / re-pickup) — archive already + // captured the state we observed, so this is idempotent-safe. + } + archivedThisAttempt++; + } + + if (archivedThisAttempt == 0) + { + // Source partition empty → converged. + return; + } + + if (attempt == ArchiveIterationCap) + { + // Final iteration still found rows. One last verify-read; if STILL + // non-empty, a writer is racing us and we cannot guarantee complete + // archive coverage → fail-closed. + var leftover = 0; + await foreach (var _ in sourceClient.QueryAsync( + filter, select: new[] { "PartitionKey", "RowKey" }, cancellationToken: ct)) + { + leftover++; + break; // existence is enough + } + if (leftover > 0) + { + _logger.LogError( + "Customs archive iteration cap exhausted for {Table} tenant={Tenant} run={History}: a writer is still producing rows after {Cap} archive-then-wipe passes — fail-closing", + tableName, tenantId, historyRowKey, ArchiveIterationCap); + throw new InvalidOperationException( + $"customs_arrival_race: table '{tableName}' still has rows after {ArchiveIterationCap} archive-then-wipe iterations"); + } + } + } + } + + internal TenantOffboardingCustomsArchiveEntry BuildArchiveEntry( + TableEntity source, string tenantId, string originalTable, string historyRowKey) + { + // Serialize every non-system property the source row exposes. TableEntity is + // IDictionary internally, so we can iterate. + var snapshot = new Dictionary(StringComparer.Ordinal); + foreach (var kv in source) + { + // Skip pseudo-properties Azure layers add (ETag/Timestamp/odata.*). + if (kv.Key is "odata.etag" or "Timestamp") continue; + snapshot[kv.Key] = kv.Value; + } + var entityJson = JsonSerializer.Serialize(snapshot); + + return new TenantOffboardingCustomsArchiveEntry + { + PartitionKey = TableTenantCustomsArchiveRepository.BuildPartitionKey(tenantId, historyRowKey), + RowKey = TableTenantCustomsArchiveRepository.BuildRowKey(originalTable, source.RowKey), + TenantId = tenantId, + OriginalTable = originalTable, + OriginalPartitionKey = source.PartitionKey, + OriginalRowKey = source.RowKey, + EntityJson = entityJson, + HistoryRowKey = historyRowKey, + ArchivedAt = DateTime.UtcNow, + ArchivedBy = "TenantOffboardingHandler", + }; + } + + private static void AssignArchivedCounter(OffboardingHistoryEntry history, string fieldName, int value) + { + switch (fieldName) + { + case nameof(OffboardingHistoryEntry.CustomGatherRulesArchived): + history.CustomGatherRulesArchived = value; + break; + case nameof(OffboardingHistoryEntry.CustomAnalyzeRulesArchived): + history.CustomAnalyzeRulesArchived = value; + break; + case nameof(OffboardingHistoryEntry.ImeLogPatternOverridesArchived): + history.ImeLogPatternOverridesArchived = value; + break; + default: + throw new InvalidOperationException($"Unknown archived counter field '{fieldName}'"); + } + } + + private async Task SafeWipeAllTenantTablesAsync( + Dictionary counts, string tenantId, CancellationToken ct) + { + foreach (var table in TenantPartitionTables) + { + ct.ThrowIfCancellationRequested(); + counts[table] = await _safeWipe.WipeByExactPartitionAsync(table, tenantId, ct); + } + foreach (var table in CompositePartitionTables) + { + ct.ThrowIfCancellationRequested(); + counts[table] = await _safeWipe.WipeByCompositePartitionRangeAsync(table, tenantId, ct); + } + foreach (var (table, discriminator) in DiscriminatorTables) + { + ct.ThrowIfCancellationRequested(); + var key = $"{table}/{discriminator}"; + counts[key] = await _safeWipe.WipeByDiscriminatorAndTenantPropertyAsync(table, discriminator, tenantId, ct); + } + foreach (var table in PropertyOnlyTables) + { + ct.ThrowIfCancellationRequested(); + counts[table] = await _safeWipe.WipeByTenantIdPropertyAsync(table, tenantId, ct); + } + } + + // ── Failure path ──────────────────────────────────────────────────────── + // + // Same Review-Fix Finding 3 invariant as the Completed path: write Marker/Pointer/Ops + // FIRST, then History terminal LAST. A crash between the side-effects and the History + // commit makes the next pickup re-run the side-effects idempotently — which is exactly + // what we want for the Failed path, because the marker/pointer/ops outcome is the + // operator-facing signal, not History.Status by itself. + + private async Task FailAsync( + OffboardingHistoryEntry history, string tenantId, + string failedPhase, string errorMessage, CancellationToken ct) + { + // Pre-populate in-memory history with the final values so the LAST write below + // commits the snapshot. + history.ErrorMessage = errorMessage; + + // Side-effect 1: Marker → Failed. + var marker = await _auditRepo.TryGetMarkerAsync(tenantId, ct); + if (marker != null) + { + marker.Status = "Failed"; + marker.FailedAt = DateTime.UtcNow; + marker.FailedPhase = failedPhase; + await _auditRepo.UpsertMarkerAsync(marker, ct); + } + + // Side-effect 2: Pointer → Failed. + await UpdatePointerWithRetryAsync(tenantId, history.RowKey, "Failed", ct); + + // Side-effect 3: OpsEvent (Telegram-routable). + await _opsEvents.RecordTenantOffboardingFailedAsync( + tenantId, history.InitiatedBy, failedPhase, errorMessage, history.RetryCount, history.DomainName); + + // COMMIT: History → Failed LAST. + history.Status = "Failed"; + await _auditRepo.UpsertHistoryAsync(history, ct); + + _logger.LogError( + "Tenant offboarding FAILED: tenant={Tenant} phase={Phase} message={Message}", + tenantId, failedPhase, errorMessage); + } + + // ── Worker-side poison transition (Review-Fix Finding 2) ──────────────── + + /// + /// Called by when an envelope hits max-dequeue and + /// moves to the poison queue. Idempotently transitions History/Pointer/Marker to Failed + /// so the operator dashboard reflects the dead-letter rather than leaving the tenant + /// hanging in InProgress. No-op when the row is already Completed/Failed. + /// + public async Task MarkEnvelopeFailedFromPoisonAsync( + TenantOffboardingEnvelope envelope, int dequeueCount, CancellationToken ct = default) + { + if (envelope == null) throw new ArgumentNullException(nameof(envelope)); + var tenantId = envelope.TenantId.ToLowerInvariant(); + + var history = await _auditRepo.TryGetHistoryAsync(envelope.HistoryRowKey, ct); + if (history == null) + { + _logger.LogWarning( + "MarkEnvelopeFailedFromPoison: history row missing for tenant={Tenant} history={History} — nothing to transition", + tenantId, envelope.HistoryRowKey); + return; + } + if (history.Status is "Completed" or "Failed") + { + _logger.LogInformation( + "MarkEnvelopeFailedFromPoison: history already terminal ({Status}) for tenant={Tenant} — skipping", + history.Status, tenantId); + return; + } + + history.RetryCount = Math.Max(history.RetryCount, dequeueCount); + await FailAsync(history, tenantId, + failedPhase: "max_dequeue", + errorMessage: $"Envelope poisoned after {dequeueCount} failed handler attempts", + ct); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingWorker.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingWorker.cs new file mode 100644 index 00000000..48e2cb55 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingWorker.cs @@ -0,0 +1,345 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Identity; +using Azure.Storage.Queues; +using Azure.Storage.Queues.Models; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models.Offboarding; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Functions.Services.Offboarding +{ + /// + /// Background poll-loop for the tenant-offboarding queue (Plan §7.3). Modeled on + /// : + /// + /// = 1 — one tenant cascade per receive bounds memory + /// (per-tenant enumerate + per-session enqueue is long-running) and limits the + /// blast radius of a poison. + /// Heartbeat extends visibility every while + /// the handler runs so concurrent re-delivery can't spawn a parallel offboarding + /// for the same tenant. + /// Max-dequeue → poison queue + the handler's own + /// Failed-state transition (the handler emits TenantOffboardingFailed when + /// it fails closed; the worker emits one too on dequeue exhaustion so an envelope + /// that never reaches the handler still surfaces). + /// + /// + public sealed class TenantOffboardingWorker : BackgroundService + { + internal const int BatchSize = 1; + + internal static readonly TimeSpan VisibilityTimeout = TimeSpan.FromMinutes(5); + internal static readonly TimeSpan DefaultHeartbeatInterval = TimeSpan.FromSeconds(60); + internal static readonly TimeSpan HeartbeatExtendBy = TimeSpan.FromMinutes(5); + internal static readonly TimeSpan DefaultPollInterval = TimeSpan.FromSeconds(10); + internal static readonly TimeSpan ErrorBackoff = TimeSpan.FromSeconds(30); + + internal const int MaxDequeueCount = 5; + private const string PoisonQueueSuffix = "-poison"; + + private readonly QueueClient _mainQueue; + private readonly QueueClient _poisonQueue; + private readonly TenantOffboardingHandler _handler; + private readonly OpsEventService _opsEvents; + private readonly ILogger _logger; + + private readonly TimeSpan _heartbeatInterval; + private readonly TimeSpan _pollInterval; + + public TenantOffboardingWorker( + IConfiguration configuration, + TenantOffboardingHandler handler, + OpsEventService opsEvents, + ILogger logger) + { + _handler = handler ?? throw new ArgumentNullException(nameof(handler)); + _opsEvents = opsEvents ?? throw new ArgumentNullException(nameof(opsEvents)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _heartbeatInterval = DefaultHeartbeatInterval; + _pollInterval = DefaultPollInterval; + + var options = new QueueClientOptions { MessageEncoding = QueueMessageEncoding.Base64 }; + var storageAccountName = configuration["AzureStorageAccountName"]; + var connectionString = configuration["AzureTableStorageConnectionString"]; + + if (!string.IsNullOrEmpty(storageAccountName)) + { + var mainUri = new Uri( + $"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboarding}"); + var poisonUri = new Uri( + $"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.TenantOffboardingPoison}"); + var credential = new DefaultAzureCredential(); + _mainQueue = new QueueClient(mainUri, credential, options); + _poisonQueue = new QueueClient(poisonUri, credential, options); + _logger.LogInformation( + "TenantOffboardingWorker initialized with Managed Identity (account: {Account})", + storageAccountName); + } + else if (!string.IsNullOrEmpty(connectionString)) + { + _mainQueue = new QueueClient(connectionString, Constants.QueueNames.TenantOffboarding, options); + _poisonQueue = new QueueClient(connectionString, Constants.QueueNames.TenantOffboardingPoison, options); + _logger.LogInformation("TenantOffboardingWorker initialized with connection string"); + } + else + { + throw new InvalidOperationException( + "Queue Storage not configured. Set either 'AzureStorageAccountName' (for Managed Identity) or 'AzureTableStorageConnectionString'."); + } + } + + /// Test seam — inject mocked queues + shorter intervals for unit tests. + internal TenantOffboardingWorker( + QueueClient mainQueue, + QueueClient poisonQueue, + TenantOffboardingHandler handler, + OpsEventService opsEvents, + ILogger logger, + TimeSpan? heartbeatInterval = null, + TimeSpan? pollInterval = null) + { + _mainQueue = mainQueue ?? throw new ArgumentNullException(nameof(mainQueue)); + _poisonQueue = poisonQueue ?? throw new ArgumentNullException(nameof(poisonQueue)); + _handler = handler ?? throw new ArgumentNullException(nameof(handler)); + _opsEvents = opsEvents ?? throw new ArgumentNullException(nameof(opsEvents)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _heartbeatInterval = heartbeatInterval ?? DefaultHeartbeatInterval; + _pollInterval = pollInterval ?? DefaultPollInterval; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await TryCreateQueueAsync(_mainQueue, "main", stoppingToken).ConfigureAwait(false); + await TryCreateQueueAsync(_poisonQueue, "poison", stoppingToken).ConfigureAwait(false); + + _logger.LogInformation("TenantOffboardingWorker: poll loop started (queue {Name})", _mainQueue.Name); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var batch = await _mainQueue + .ReceiveMessagesAsync(BatchSize, VisibilityTimeout, stoppingToken) + .ConfigureAwait(false); + + if (batch?.Value is null || batch.Value.Length == 0) + { + await Task.Delay(_pollInterval, stoppingToken).ConfigureAwait(false); + continue; + } + + foreach (var msg in batch.Value) + { + if (stoppingToken.IsCancellationRequested) break; + await ProcessOneAsync(msg, stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, + "TenantOffboardingWorker: poll-loop error — backing off {Backoff}", + ErrorBackoff); + try { await Task.Delay(ErrorBackoff, stoppingToken).ConfigureAwait(false); } + catch (OperationCanceledException) { break; } + } + } + + _logger.LogInformation("TenantOffboardingWorker: poll loop stopped"); + } + + private async Task ProcessOneAsync(QueueMessage msg, CancellationToken ct) + { + if (msg.DequeueCount > MaxDequeueCount) + { + await MoveToPoisonAsync(msg, ct).ConfigureAwait(false); + return; + } + + TenantOffboardingEnvelope? envelope; + try + { + envelope = JsonConvert.DeserializeObject(msg.Body.ToString()); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, + "TenantOffboardingWorker: malformed envelope JSON — dropping (msg {Id})", + msg.MessageId); + await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false); + return; + } + + if (envelope is null + || string.IsNullOrEmpty(envelope.TenantId) + || string.IsNullOrEmpty(envelope.HistoryRowKey)) + { + _logger.LogWarning( + "TenantOffboardingWorker: envelope missing required fields — dropping (msg {Id})", + msg.MessageId); + await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false); + return; + } + + using var heartbeatCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + var heartbeatState = new HeartbeatState(msg.MessageId, msg.PopReceipt); + var heartbeatTask = Task.Run(() => HeartbeatLoopAsync(heartbeatState, heartbeatCts.Token), heartbeatCts.Token); + + try + { + await _handler.HandleAsync(envelope, ct).ConfigureAwait(false); + + heartbeatCts.Cancel(); + try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { } + + await SafeDeleteAsync(msg.MessageId, heartbeatState.PopReceipt, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + heartbeatCts.Cancel(); + throw; + } + catch (Exception ex) + { + heartbeatCts.Cancel(); + try { await heartbeatTask.ConfigureAwait(false); } catch (OperationCanceledException) { } + + _logger.LogWarning(ex, + "TenantOffboardingWorker: handler failed (tenant={Tenant} history={History} dequeue={N}) — visibility-timeout retry", + envelope.TenantId, envelope.HistoryRowKey, msg.DequeueCount); + } + } + + private async Task HeartbeatLoopAsync(HeartbeatState state, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try { await Task.Delay(_heartbeatInterval, ct).ConfigureAwait(false); } + catch (OperationCanceledException) { return; } + + try + { + var updated = await _mainQueue.UpdateMessageAsync( + state.MessageId, state.PopReceipt, + visibilityTimeout: HeartbeatExtendBy, + cancellationToken: ct).ConfigureAwait(false); + state.PopReceipt = updated.Value.PopReceipt; + _logger.LogDebug( + "TenantOffboardingWorker: heartbeat extended visibility for message {Id} by {Extend}", + state.MessageId, HeartbeatExtendBy); + } + catch (OperationCanceledException) { return; } + catch (Exception ex) + { + _logger.LogWarning(ex, + "TenantOffboardingWorker: heartbeat UpdateMessageAsync failed for message {Id} — exiting heartbeat", + state.MessageId); + return; + } + } + } + + private async Task MoveToPoisonAsync(QueueMessage msg, CancellationToken ct) + { + TenantOffboardingEnvelope? envelope = null; + try { envelope = JsonConvert.DeserializeObject(msg.Body.ToString()); } + catch (JsonException) { /* still poison the malformed envelope below */ } + + // Review-Fix (second-pass) Finding 1: the durable Failed-state transition MUST land + // BEFORE the poison-queue send + main-queue delete. Otherwise a transient storage / + // OpsEvent / pointer-CAS failure on the transition would leave the message in poison + // but the Tenant stuck at InProgress. By doing the transition first we keep the + // message visible on the main queue when the transition fails — the worker will + // retry on the next dequeue. The transition itself is idempotent + // (MarkEnvelopeFailedFromPoisonAsync no-ops on terminal History.Status) so retries + // are safe. + if (envelope != null + && !string.IsNullOrEmpty(envelope.TenantId) + && !string.IsNullOrEmpty(envelope.HistoryRowKey)) + { + try + { + await _handler.MarkEnvelopeFailedFromPoisonAsync( + envelope, (int)(msg.DequeueCount - 1), ct); + } + catch (Exception ex) + { + // Critical: leave the message visible so the next dequeue can retry the + // transition. We do NOT proceed to the poison send / SafeDelete here — + // otherwise the operator would see a queue with nothing in it and a tenant + // hanging on InProgress. + _logger.LogError(ex, + "TenantOffboardingWorker: MarkEnvelopeFailedFromPoisonAsync failed for tenant={Tenant} history={History} — message left visible for retry (durable Failed state has higher priority than poison move)", + envelope.TenantId, envelope.HistoryRowKey); + return; + } + } + + // Failed state is now durable (or envelope was malformed and couldn't transition). + // Best-effort poison move + SafeDelete; transient failure here leaves the message + // visible but the next pickup will see History.Status=Failed and SafeDelete cleanly. + try + { + await _poisonQueue.SendMessageAsync(msg.Body.ToString(), ct).ConfigureAwait(false); + await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false); + _logger.LogWarning( + "TenantOffboardingWorker: moved message {Id} to poison queue after {N} failed attempts", + msg.MessageId, msg.DequeueCount - 1); + } + catch (Exception ex) + { + _logger.LogError(ex, + "TenantOffboardingWorker: poison move failed for message {Id} (will retry; durable Failed state already written)", + msg.MessageId); + } + } + + private async Task SafeDeleteAsync(string messageId, string popReceipt, CancellationToken ct) + { + try + { + await _mainQueue.DeleteMessageAsync(messageId, popReceipt, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "TenantOffboardingWorker: DeleteMessageAsync failed for message {Id} — message will reappear after visibility expires", + messageId); + } + } + + private async Task TryCreateQueueAsync(QueueClient queue, string label, CancellationToken ct) + { + try + { + await queue.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "TenantOffboardingWorker: CreateIfNotExists failed on {Label} queue {Name}", + label, queue.Name); + } + } + + private sealed class HeartbeatState + { + public string MessageId { get; } + public string PopReceipt { get; set; } + + public HeartbeatState(string messageId, string popReceipt) + { + MessageId = messageId; + PopReceipt = popReceipt; + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 114e3273..93386dd8 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -275,6 +275,26 @@ namespace AutopilotMonitor.Functions.Services tenantId, performedBy, details); } + /// + /// Fired by the tenant-offboarding worker when the cascade fails closed (kill-switch + /// active mid-enqueue, drain timeout, expectations blob corruption, ETag/CAS exhaustion, + /// SafeWipe verify abort, …). Marker stays Failed until operator action; this event is + /// the Telegram-routable signal that something needs human attention. Plan Rev-4 Q2. + /// + public Task RecordTenantOffboardingFailedAsync( + string tenantId, string performedBy, string failedPhase, string errorMessage, + int retryCount, string? domainName = null) + { + var tenantLabel = string.IsNullOrWhiteSpace(domainName) + ? tenantId + : $"{domainName} ({tenantId})"; + + return WriteAsync(OpsEventCategory.Tenant, "TenantOffboardingFailed", OpsEventSeverity.Error, + $"Tenant {tenantLabel} offboarding failed at phase '{failedPhase}': {errorMessage}", + tenantId, performedBy, + new { domainName, failedPhase, errorMessage, retryCount }); + } + // ── Agent ────────────────────────────────────────────────────────────── public Task RecordSessionTimeoutsAsync(string tenantId, int sessionCount, int timeoutHours) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TenantConfigurationService.cs b/src/Backend/AutopilotMonitor.Functions/Services/TenantConfigurationService.cs index a9f985f9..4e033400 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TenantConfigurationService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TenantConfigurationService.cs @@ -124,7 +124,7 @@ namespace AutopilotMonitor.Functions.Services /// Returns (config, exists). exists=false when no row was found — does NOT auto-create. /// Use for agent security gates where unknown tenants must be rejected. /// - public async Task<(TenantConfiguration config, bool exists)> TryGetConfigurationAsync(string tenantId) + public virtual async Task<(TenantConfiguration config, bool exists)> TryGetConfigurationAsync(string tenantId) { if (string.IsNullOrEmpty(tenantId)) return (TenantConfiguration.CreateDefault("unknown"), false); diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 65200f06..cb16fa2a 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -493,6 +493,25 @@ namespace AutopilotMonitor.Shared public const string ClassifierVerdictsByIdLevel = "ClassifierVerdictsByIdLevel"; public const string SignalsByKind = "SignalsByKind"; + // Tenant offboarding audit (cascade-worker plan, Rev 9). Houses three + // PartitionKey patterns side-by-side: + // - "OffboardingMarker" (1 row/tenant, RK=tenantId) active 403-gate marker + // - "OffboardingHistory" (N rows/tenant, RK="{yyyyMMddHHmmssfff}_{tenantId}") audit trail + // - "OffboardingByTenant" (1 row/tenant, RK=tenantId) O(1) re-onboarding pointer + // Marker survives the Completed transition for at least 15min so that warm + // function-instance caches of TenantConfiguration cannot leak auth after wipe. + // See OffboardingPartitionKeys for the canonical PK string literals. + public const string OffboardingAudit = "OffboardingAudit"; + + // Tenant offboarding customs archive (PR3.B). Phase 2.D-archive writes a + // snapshot of every GatherRules / AnalyzeRules / ImeLogPatterns row keyed by + // PK=tenantId before the originals are safe-wiped, so a Global Admin can + // review them post-offboarding from the /admin/customs-archive page. + // - PartitionKey = "{normalizedTenantId}_{historyRowKey}" (one partition per offboarding run) + // - RowKey = "{originalTable}_{base64url(originalRowKey)}" + // Lives forever; cleanup is operator-driven via the admin UI. + public const string TenantOffboardingCustomsArchive = "TenantOffboardingCustomsArchive"; + /// /// Returns all table names for initialization /// @@ -546,10 +565,54 @@ namespace AutopilotMonitor.Shared SessionsByStage, DeadEndsByReason, ClassifierVerdictsByIdLevel, - SignalsByKind + SignalsByKind, + OffboardingAudit, + TenantOffboardingCustomsArchive }; } + /// + /// Canonical PartitionKey strings stored in . + /// All three patterns coexist in the same table; consumers must always read by + /// (PartitionKey, RowKey) point-lookup, never by table scan. + /// + public static class OffboardingPartitionKeys + { + /// Active 403 sperrmarker. RowKey = normalized tenantId. 1 row per active offboarding. + public const string Marker = "OffboardingMarker"; + + /// Chronological audit history. RowKey = "{yyyyMMddHHmmssfff}_{normalizedTenantId}". N rows per tenant. + public const string History = "OffboardingHistory"; + + /// O(1) pointer index for re-onboarding lookup. RowKey = normalized tenantId. 1 row per tenant. + public const string ByTenant = "OffboardingByTenant"; + } + + /// + /// Azure Blob Storage container names. Single source of truth for container strings + /// so that storage helpers, cascade workers, and offboarding workers all agree. + /// + public static class BlobContainers + { + /// + /// Houses cascade-deletion manifests (snapshot + progress blobs) under prefix + /// {tenantId}/{sessionId}/{manifestId}.{snapshot.json.gz | progress.json}. + /// 30d lifecycle delete + 3d soft delete = ~33d effective retention. + /// Phase 2.E of tenant offboarding wipes this container by {tenantId}/ prefix. + /// + public const string DeletionManifests = "deletion-manifests"; + + /// + /// Houses tenant-offboarding state blobs that MUST survive the deletion-manifests wipe. + /// Currently: per-tenant Expectations-Blob written at the first handler pickup + /// ({tenantId}/{historyRowKey}.expectations.json). Phase 2.E intentionally + /// skips this container; Phase 2.G deletes the blob explicitly as its last step, + /// with as a + /// defense-in-depth second-pass. + /// + public const string OffboardingState = "offboarding-state"; + } + /// /// Azure Storage Queue names. Queues feed async fan-out / reconciliation paths /// that would otherwise block the hot ingest loop. @@ -593,6 +656,20 @@ namespace AutopilotMonitor.Shared /// the Sessions row. Poison suffix -poison, max-dequeue 5. /// public const string SessionDeletion = "session-deletion"; + + /// + /// Tenant offboarding cascade-worker queue. Producer (TenantOffboardFunction) writes + /// one envelope per tenant after inserting the History/Pointer/Marker rows in + /// . Worker (TenantOffboardingWorker) + /// enumerates sessions, enqueues per-session cascades, drains against an + /// Expectations-Blob in , then safe-wipes + /// all remaining tenant-scoped rows. Max-dequeue 5, paired with + /// . + /// + public const string TenantOffboarding = "tenant-offboarding"; + + /// Poison sibling of . + public const string TenantOffboardingPoison = "tenant-offboarding-poison"; } } } diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs index 889b8abf..f64dfe92 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs @@ -69,7 +69,16 @@ namespace AutopilotMonitor.Shared.DataAccess Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId); // --- Tenant Offboarding --- - Task> DeleteAllTenantDataAsync(string tenantId); + + /// + /// Fail-loud iterator over every session belonging to . + /// Used by the tenant-offboarding worker to drive per-session cascade enqueue. + /// Unlike , this method MUST NOT swallow + /// storage exceptions — a transient failure during enumeration must propagate so + /// the queue worker can retry / poison, instead of silently returning zero sessions + /// which would trigger a same-cycle wipe without cascade backup. + /// + IAsyncEnumerable EnumerateSessionsForOffboardingAsync(string tenantId, System.Threading.CancellationToken ct = default); } public class OrphanedEventSession diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IOffboardingAuditRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IOffboardingAuditRepository.cs new file mode 100644 index 00000000..363c3507 --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IOffboardingAuditRepository.cs @@ -0,0 +1,74 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.Models.Offboarding; + +namespace AutopilotMonitor.Shared.DataAccess +{ + /// + /// Access to the OffboardingAudit table. Owns three PartitionKey patterns + /// (Marker / History / ByTenant). All write paths fail-loud — Queue retry / poison + /// owns recovery semantics. + /// + public interface IOffboardingAuditRepository + { + // ── Marker (offboarding lifecycle anchor; idempotency + cleanup) ─────── + + /// + /// Uncached point lookup used by TenantOffboardFunction's idempotency check + /// + OffboardingMarkerCleanupFunction's timer sweep. NOT read in the agent + /// or web auth hotpath any more — the active auth-gate is + /// TenantConfiguration.Disabled=true. Returns null when no marker exists. + /// + Task TryGetMarkerAsync(string normalizedTenantId, CancellationToken ct = default); + + /// Inserts a fresh marker. Throws if a marker for the tenant already exists. + Task InsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default); + + /// Idempotent upsert; preferred path from worker phase transitions. + Task UpsertMarkerAsync(OffboardingMarkerEntry marker, CancellationToken ct = default); + + /// Removes a marker by tenant id. 404 silently swallowed (idempotent). + Task DeleteMarkerAsync(string normalizedTenantId, CancellationToken ct = default); + + /// Enumerates ALL active markers — used by OffboardingMarkerCleanupFunction. + IAsyncEnumerable QueryMarkersAsync(CancellationToken ct = default); + + // ── History (audit trail, one row per offboarding attempt) ────────────── + + Task InsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default); + Task TryGetHistoryAsync(string historyRowKey, CancellationToken ct = default); + Task UpsertHistoryAsync(OffboardingHistoryEntry history, CancellationToken ct = default); + + // ── Pointer (O(1) re-onboarding index) ────────────────────────────────── + // + // Plan §4.4 update protocol is strictly Read-Modify-Write with ETag-CAS: + // 1. TryGet → null + null ETag → Insert with OffboardCount=1 + // 2. TryGet → (pointer, etag) → modify in-memory + UpdateWithEtag(etag), + // 412 on mismatch ⇒ re-read and retry. + // The "blind upsert" path was removed because it cannot satisfy OffboardCount=existing+1 + // under concurrent writers — the read-modify-write window is the whole point of the + // pointer's design. + + /// + /// Reads the pointer row plus its ETag for CAS. Returns (null, null) on 404. + /// + Task<(OffboardingByTenantPointer? Pointer, string? ETag)> TryGetByTenantPointerAsync( + string normalizedTenantId, CancellationToken ct = default); + + /// + /// Inserts a fresh pointer row. Throws with + /// status 409 when a pointer for the tenant already exists — caller is expected to fall + /// back to in that case. + /// + Task InsertByTenantPointerAsync(OffboardingByTenantPointer pointer, CancellationToken ct = default); + + /// + /// Conditional replace: succeeds only when the stored row's ETag still matches + /// . Throws with + /// status 412 on mismatch so the caller can re-read + retry within their bounded loop. + /// + Task UpdateByTenantPointerWithEtagAsync( + OffboardingByTenantPointer pointer, string ifMatchEtag, CancellationToken ct = default); + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/ITenantCustomsArchiveRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/ITenantCustomsArchiveRepository.cs new file mode 100644 index 00000000..c7b5251c --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/ITenantCustomsArchiveRepository.cs @@ -0,0 +1,55 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.Models.Offboarding; + +namespace AutopilotMonitor.Shared.DataAccess +{ + /// + /// Access to TenantOffboardingCustomsArchive. Writes happen during Phase 2.D-archive + /// of the offboarding worker; reads / deletes are driven by the Global Admin + /// /admin/customs-archive UI. + /// + /// All write paths fail-loud — when an Azure storage call throws, exceptions propagate + /// to the caller (queue worker), which owns retry/poison semantics. Reads + deletes + /// from the admin API similarly surface errors to the HTTP caller. + /// + /// + public interface ITenantCustomsArchiveRepository + { + /// + /// Idempotent upsert (Replace). Used during Phase 2.D-archive so a worker that + /// crashed mid-archive can re-pickup without duplicate-key errors. + /// + Task UpsertAsync(TenantOffboardingCustomsArchiveEntry entry, CancellationToken ct = default); + + /// Count entries within a single offboarding run filtered by source table. + Task CountByRunAndTableAsync( + string normalizedTenantId, string historyRowKey, string originalTable, CancellationToken ct = default); + + /// List one offboarding run's archive partition (all 3 source tables interleaved). + IAsyncEnumerable QueryByRunAsync( + string normalizedTenantId, string historyRowKey, CancellationToken ct = default); + + /// List every archive partition for a tenant (one entry per row across all runs). + IAsyncEnumerable QueryByTenantAsync( + string normalizedTenantId, CancellationToken ct = default); + + /// List every archived run as a (tenantId, historyRowKey)-pair summary across all tenants. + IAsyncEnumerable QueryAllAsync(CancellationToken ct = default); + + /// Fetch a single archive entry by (PK, RK). + Task TryGetEntryAsync( + string partitionKey, string rowKey, CancellationToken ct = default); + + /// Delete a single archived entry. 404 silently swallowed (idempotent). + Task DeleteEntryAsync(string partitionKey, string rowKey, CancellationToken ct = default); + + /// + /// Delete every entry in a single run's archive partition. Used by the Global Admin + /// "Delete run" action after they have reviewed the contents. + /// + Task DeleteRunAsync( + string normalizedTenantId, string historyRowKey, CancellationToken ct = default); + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingByTenantPointer.cs b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingByTenantPointer.cs new file mode 100644 index 00000000..8a91dc05 --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingByTenantPointer.cs @@ -0,0 +1,42 @@ +using System; + +namespace AutopilotMonitor.Shared.Models.Offboarding +{ + /// + /// O(1) point-lookup pointer used by the re-onboarding hook to find the latest history + /// row for a given tenant without scanning the OffboardingHistory partition. + /// Azure Tables cannot serve RowKey endswith '_{tenantId}' efficiently + /// server-side (full partition scan + client-side filter), so we maintain a sidecar + /// row keyed by tenantId. + /// + /// Lives in OffboardingAudit under + /// PartitionKey = OffboardingPartitionKeys.ByTenant with + /// RowKey = normalizedTenantId. Upserted with two-step writes (history-first, + /// pointer-second) and updated as the worker progresses through Status transitions. + /// + /// + public sealed class OffboardingByTenantPointer + { + /// Always OffboardingPartitionKeys.ByTenant. + public string PartitionKey { get; set; } = string.Empty; + + /// Normalized (lowercase GUID) tenant id. + public string RowKey { get; set; } = string.Empty; + + public string TenantId { get; set; } = string.Empty; + + /// "{yyyyMMddHHmmssfff}_{tenantId}" of the current/latest history row. + public string LatestHistoryRowKey { get; set; } = string.Empty; + + /// Mirrors the matching history row's Status: + /// Initiated | InProgress | Completed | Failed | Reonboarded. + public string LatestStatus { get; set; } = "Initiated"; + + /// UTC of the most recent pointer update; monotonically increasing. + public DateTime LatestUpdatedAt { get; set; } + + /// How many distinct offboarding rounds this tenant has been through. + /// Incremented on each new Initiate. Useful for "Tenant ging und kam zurück"-Reporting. + public int OffboardCount { get; set; } = 1; + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingExpectations.cs b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingExpectations.cs new file mode 100644 index 00000000..dd067703 --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingExpectations.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; + +namespace AutopilotMonitor.Shared.Models.Offboarding +{ + /// + /// Per-tenant snapshot of the cascade enqueue outcome captured at the first handler pickup. + /// Drives the drain predicate: every entry is matched against either a progress blob + /// (Enqueued/AlreadyInFlight), a no-op (SessionNotFound), a fail-closed condition + /// (KillSwitchActive/Poisoned), or a bounded retry loop (CasExhausted). + /// + /// Stored as a single blob at + /// {BlobContainers.OffboardingState}/{tenantId}/{historyRowKey}.expectations.json. + /// The container is deliberately separate from deletion-manifests because the latter + /// is wiped in Phase 2.E; storing this blob there would make crash-recovery between 2.E + /// and 2.G impossible. + /// + /// + public sealed class OffboardingExpectations + { + /// Bump on breaking-schema changes. Drain-probe fail-closed if mismatch. + public int SchemaVersion { get; set; } = 1; + + public string TenantId { get; set; } = string.Empty; + + /// Matching history row's RowKey, so a blob can be paired back to its run. + public string HistoryRowKey { get; set; } = string.Empty; + + public DateTime CreatedAt { get; set; } + + /// + /// True only after the session enumerator iterated to completion without exception AND + /// the upload is being written. Default false; drain-probe treats false as + /// "this blob was uploaded with incomplete data" → fail-closed with + /// FailedPhase="enumeration_incomplete". Critical disambiguation: lets the + /// probe tell apart "0 sessions, happy path" from "0 entries because enumerator + /// crashed before yielding any". + /// + public bool EnumerationCompleted { get; set; } + + /// + /// Number of sessions the enumerator yielded BEFORE per-session enqueue ran. Must equal + /// .Count on upload; mismatch → fail-closed. + /// + public int EnumeratedSessionCount { get; set; } + + public List Expectations { get; set; } = new(); + } + + /// One per session that the enumerator returned. + public sealed class OffboardingExpectation + { + public string SessionId { get; set; } = string.Empty; + + /// + /// Manifest id returned by SessionDeletionProducer.EnqueueAsync. Null for outcomes + /// that did not produce a cascade (SessionNotFound, KillSwitchActive, CasExhausted) or + /// when the producer returned AlreadyInFlight against a Preparing-without-snapshot + /// state. The drain probe fails closed on Enqueued/AlreadyInFlight with a null/empty + /// manifest id. + /// + public string? ManifestId { get; set; } + + /// + /// String form of SessionDeletionProducer.EnqueueOutcome: + /// Enqueued | AlreadyInFlight | SessionNotFound | KillSwitchActive | Poisoned | CasExhausted. + /// Stored as string so the shared assembly doesn't need to reference the + /// Functions-only enum and so future enum additions don't break already-written blobs. + /// + public string Outcome { get; set; } = string.Empty; + + /// + /// CasExhausted retry counter. Drain-probe re-enqueues this session and increments; + /// after 3 retries the offboarding fails closed with FailedPhase="cas_exhausted". + /// + public int RetryCount { get; set; } + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingHistoryEntry.cs b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingHistoryEntry.cs new file mode 100644 index 00000000..5c2de24e --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingHistoryEntry.cs @@ -0,0 +1,110 @@ +using System; + +namespace AutopilotMonitor.Shared.Models.Offboarding +{ + /// + /// Chronological audit-trail row for one tenant-offboarding attempt. Multiple rows per + /// tenant when the tenant offboarded, re-onboarded, and offboarded again. Lives in + /// OffboardingAudit under PartitionKey = OffboardingPartitionKeys.History + /// with RowKey = "{yyyyMMddHHmmssfff}_{normalizedTenantId}" so rows sort + /// newest-last lexicographically within the partition. + /// + public sealed class OffboardingHistoryEntry + { + /// Always OffboardingPartitionKeys.History. + public string PartitionKey { get; set; } = string.Empty; + + /// "{yyyyMMddHHmmssfff}_{normalizedTenantId}". Unique per offboarding attempt. + public string RowKey { get; set; } = string.Empty; + + public string TenantId { get; set; } = string.Empty; + public string DomainName { get; set; } = string.Empty; + public string InitiatedBy { get; set; } = string.Empty; + + public DateTime OffboardedAt { get; set; } + public DateTime? CompletedAt { get; set; } + + /// Initiated | InProgress | Completed | Failed. + public string Status { get; set; } = "Initiated"; + + /// + /// Cache-Drain-Barrier (plan v2 §2). The worker MUST NOT begin destructive Phase 2 + /// before this UTC timestamp. TenantOffboardFunction sets it to + /// OffboardedAt + DrainBarrier (default 6 min) and enqueues the envelope + /// with the matching visibilityDelay, so the message is invisible to the + /// worker until then. All warm function-host instances will have hit their 5-min + /// TenantConfigurationService cache TTL by the time it expires, so the + /// existing Disabled=true gate carries the 403 instead of the marker. + /// + /// Also rendered by the Web UI as the "data deletion starts in mm ss" countdown + /// during the drain barrier state, and used by the resume-path to compute the + /// remaining delay when an admin re-clicks mid-barrier. + /// + /// + public DateTime? EarliestProcessingAt { get; set; } + + // ── Diagnostics ──────────────────────────────────────────────────────── + + /// JSON: { tableName: deletedRowCount } across SafeWipe + Cascade phases. + public string? DeletedRowCountsJson { get; set; } + + public int? TotalRowsDeleted { get; set; } + public int? DeletedBlobCount { get; set; } + public int? CascadeSessionsEnqueued { get; set; } + public string? ErrorMessage { get; set; } + public int RetryCount { get; set; } + + /// + /// Rev-9-F1 phase marker. Set once by the handler immediately after the drain predicate + /// returns true, BEFORE Phase 2.D begins wiping tables. Re-pickup after crash skips the + /// drain predicate when this is non-null because Phase 2.E may have wiped the cascade + /// progress blobs that the drain predicate reads from; the post-drain phases (2.D-G) + /// are all idempotent so a clean resume requires no progress blobs. + /// + public DateTime? DrainCompletedAt { get; set; } + + /// + /// Stamped by the handler the first time it enters EnsureExpectationsBlobAsync, + /// BEFORE the session enumerator starts iterating. On re-pickup after an enumerator + /// crash this field is non-null while the Expectations blob is still missing — the + /// handler then fail-closes with FailedPhase="expectations_missing" instead of + /// re-enumerating against a tenant whose underlying Sessions table may already be + /// mid-mutation. Plan §7.4 step 3 (Rev-5-F3 + Rev-7-F2 crash-after-enumerate-start). + /// + public DateTime? EnumerationStartedAt { get; set; } + + /// + /// Stamped after the session enumerator loop finishes cleanly, BEFORE the Expectations + /// blob upload. Disambiguates two failure modes that both leave the blob missing with + /// set: + /// + /// Mid-enumeration crash: ECBU is null → fail-closed + /// (FailedPhase="expectations_missing"); the tenant may be mid-mutation + /// and re-enumerating is unsafe. + /// Upload failure: ECBU is non-null → retry the enumerate+upload pair + /// on the next pickup. The session-deletion producer is idempotent + /// (AlreadyInFlight outcome with the existing ManifestId), so re-running + /// the loop produces the same cascade-state set. + /// + /// Plan §7.4 step 3 + Review-Fix (second-pass) Finding 2. + /// + public DateTime? EnumerationCompletedBeforeUpload { get; set; } + + // ── Archived Customs (PR3.B §3) ──────────────────────────────────────── + // + // Counts of rows that were snapshotted into TenantOffboardingCustomsArchive + // during Phase 2.D-archive, immediately before the originals were safe-wiped. + // Recomputed from the archive table by post-archive query so a crash between + // archive-insert and counter-write does not leave a stale 0. + + public int? CustomGatherRulesArchived { get; set; } + public int? CustomAnalyzeRulesArchived { get; set; } + public int? ImeLogPatternOverridesArchived { get; set; } + + // ── Re-onboarding tracking (legacy, kept for audit history continuity) ─ + + public DateTime? ReonboardedAt { get; set; } + public string? ReonboardedBy { get; set; } + public int? CustomsAutoWipedOnReonboard { get; set; } + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingMarkerEntry.cs b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingMarkerEntry.cs new file mode 100644 index 00000000..9d6c2b69 --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/OffboardingMarkerEntry.cs @@ -0,0 +1,67 @@ +using System; + +namespace AutopilotMonitor.Shared.Models.Offboarding +{ + /// + /// Lifecycle anchor + audit row for a tenant currently (or recently) being offboarded. + /// Lives in OffboardingAudit table under + /// PartitionKey = OffboardingPartitionKeys.Marker with RowKey = normalizedTenantId. + /// + /// In the PR3-revised architecture this row is NO LONGER read in the agent or web + /// auth hotpath. The active gate is TenantConfiguration.Disabled=true + the + /// 6-minute cache-drain barrier on the worker envelope. The marker exists to: + /// + /// + /// Provide an idempotency anchor for the TenantOffboardFunction + /// admin endpoint (re-clicks resolve to the existing marker and resume). + /// Track the offboarding lifecycle status () across + /// Phase 1 → Phase 2 → Completed/Failed for ops visibility. + /// Anchor the post-completion cleanup sweeps run by + /// OffboardingMarkerCleanupFunction (Expectations blob delete + + /// defense-in-depth TenantConfiguration sweep if Phase 2.F-final failed). + /// + /// + /// Marker survives the Completed transition: Phase 2.G sets = + /// "Completed" + . OffboardingMarkerCleanupFunction + /// removes Completed markers after 15min (post-cleanup-sweep settle buffer). Failed + /// markers are never auto-deleted — operator action required. + /// + /// + public sealed class OffboardingMarkerEntry + { + /// Always OffboardingPartitionKeys.Marker. + public string PartitionKey { get; set; } = string.Empty; + + /// Normalized (lowercase GUID) tenant id. + public string RowKey { get; set; } = string.Empty; + + public string TenantId { get; set; } = string.Empty; + + /// Cross-reference to the matching OffboardingHistory row's RowKey. + public string OffboardingHistoryRowKey { get; set; } = string.Empty; + + public DateTime InitiatedAt { get; set; } + + /// UPN of the admin (or "System.*") that triggered the offboarding. + public string InitiatedBy { get; set; } = string.Empty; + + /// + /// Lifecycle: Initiated → InProgress → Completed | Failed. + /// Marker remains active in all four states; only the cleanup function removes + /// Completed markers (and never Failed). + /// + public string Status { get; set; } = "Initiated"; + + /// UTC timestamp set by Phase 2.G; gates 15-min cleanup window. + public DateTime? CompletedAt { get; set; } + + /// UTC timestamp set after max-dequeue → Failed; marker stays active until operator action. + public DateTime? FailedAt { get; set; } + + /// Phase that failed: drain, wipe, blobs, tenantconfig, + /// killswitch, poisoned, cas_exhausted, drain_timeout, + /// expectations_missing, expectations_corrupt, enumeration_incomplete, + /// expectations_size_mismatch, alreadyinflight_no_manifest. + public string? FailedPhase { get; set; } + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/TenantOffboardingCustomsArchiveEntry.cs b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/TenantOffboardingCustomsArchiveEntry.cs new file mode 100644 index 00000000..7a0f9b7c --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/TenantOffboardingCustomsArchiveEntry.cs @@ -0,0 +1,58 @@ +using System; + +namespace AutopilotMonitor.Shared.Models.Offboarding +{ + /// + /// Snapshot of a tenant's custom rule row written by the offboarding handler during + /// Phase 2.D-archive, BEFORE the original row is safe-wiped. Survives forever so a + /// Global Admin can review (and selectively delete) the tenant's custom rules from + /// /admin/customs-archive after offboarding completes. + /// + /// Storage layout (see PR3.B plan §3.2): + /// + /// PartitionKey: "{normalizedTenantId}_{historyRowKey}". One + /// partition per offboarding run — Re-Re-Offboarding produces a fresh, immutable + /// partition each time so multiple runs co-exist without RowKey collisions. + /// RowKey: "{originalTable}_{base64url(originalRowKey)}". The + /// table-name prefix disambiguates collisions across the three rules tables; the + /// base64url encoding of avoids the Azure-Tables + /// RowKey-forbidden characters (#, ?, /, \, control chars). + /// + /// + /// + public sealed class TenantOffboardingCustomsArchiveEntry + { + /// "{normalizedTenantId}_{historyRowKey}". + public string PartitionKey { get; set; } = string.Empty; + + /// "{originalTable}_{base64url(originalRowKey)}". + public string RowKey { get; set; } = string.Empty; + + public string TenantId { get; set; } = string.Empty; + + /// One of GatherRules / AnalyzeRules / ImeLogPatterns. + public string OriginalTable { get; set; } = string.Empty; + + public string OriginalPartitionKey { get; set; } = string.Empty; + + /// Original RowKey verbatim — base64url-encoded into 's suffix. + public string OriginalRowKey { get; set; } = string.Empty; + + /// + /// JSON dump of every property on the source row (excluding system properties). + /// The full body is small enough to fit in a single Azure Table string property + /// (64 KB limit; observed rule bodies are well under 4 KB). If a future rule type + /// ever exceeds that, an EntityJsonBlobUrl overflow pointer can be added — not + /// part of this PR. + /// + public string EntityJson { get; set; } = string.Empty; + + /// Cross-reference to the OffboardingHistory row for the run. + public string HistoryRowKey { get; set; } = string.Empty; + + public DateTime ArchivedAt { get; set; } + + /// Constant "TenantOffboardingHandler" in production. Test fakes may override. + public string ArchivedBy { get; set; } = "TenantOffboardingHandler"; + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/TenantOffboardingEnvelope.cs b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/TenantOffboardingEnvelope.cs new file mode 100644 index 00000000..2fbdd8c6 --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/Offboarding/TenantOffboardingEnvelope.cs @@ -0,0 +1,37 @@ +using System; + +namespace AutopilotMonitor.Shared.Models.Offboarding +{ + /// + /// Queue message envelope carried on tenant-offboarding (and its poison sibling). + /// Producer (TenantOffboardFunction) writes one envelope per tenant after the + /// History/Pointer/Marker rows are committed. Worker (TenantOffboardingWorker) + /// dequeues, runs TenantOffboardingHandler.HandleAsync, and either acks on + /// success or re-enqueues itself with incremented when the + /// drain predicate is still in flight. + /// + public sealed class TenantOffboardingEnvelope + { + /// Schema version — bump on breaking envelope changes. + public string EnvelopeVersion { get; set; } = "1"; + + public string TenantId { get; set; } = string.Empty; + + /// Always OffboardingPartitionKeys.History; kept for self-documentation. + public string HistoryPartitionKey { get; set; } = string.Empty; + + /// "{yyyyMMddHHmmssfff}_{tenantId}" pointing at the matching history row. + public string HistoryRowKey { get; set; } = string.Empty; + + public string InitiatedBy { get; set; } = string.Empty; + public DateTime InitiatedAt { get; set; } + public DateTime EnqueuedAt { get; set; } + + /// + /// Drain self-poll counter. Handler re-enqueues with this incremented when the + /// cascade-progress drain predicate has not yet settled. Capped at 60 (~2h) before + /// failing the offboarding with FailedPhase="drain_timeout". + /// + public int DrainPollCount { get; set; } + } +} diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index 5a26ea9d..07e87cb0 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -44,7 +44,11 @@ const OPS_EVENT_TYPES: Record = { "PoisonQueueBacklogHigh", "PoisonQueueBacklogCritical", ], - Tenant: ["TenantOffboarded"], + // TenantOffboardingFailed is dispatched by TenantOffboardingWorker when the async cascade fails + // closed (drain timeout, KillSwitch active, Poisoned session, ETag/CAS exhaustion, SafeWipe verify + // abort, etc.). Marker remains in Failed state until operator action — this event is the + // Telegram-routable signal that something needs human attention. Plan Rev-4 Q2 + Rev-9 §11 PR3. + Tenant: ["TenantOffboarded", "TenantOffboardingFailed"], Agent: ["BlobStorageMissing", "BlobStorageUnreachable", "NewImeVersionDetected", "ExcessiveSessionEvents"], }; diff --git a/src/Web/autopilot-monitor-web/app/admin/customs-archive/[tenantId]/[historyRowKey]/page.tsx b/src/Web/autopilot-monitor-web/app/admin/customs-archive/[tenantId]/[historyRowKey]/page.tsx new file mode 100644 index 00000000..e4b57d1b --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/customs-archive/[tenantId]/[historyRowKey]/page.tsx @@ -0,0 +1,243 @@ +"use client"; + +import { useCallback, useEffect, useState, use as usePromise } from "react"; +import Link from "next/link"; +import { + api, + type CustomsArchiveEntrySummary, + type CustomsArchiveFullEntry, + type CustomsArchiveListEntriesResponse, +} from "@/lib/api"; +import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch"; +import { useAdminConfig } from "../../../AdminConfigContext"; +import { AdminNotifications } from "../../../AdminNotifications"; +import { DeleteConfirmModal } from "../../components/DeleteConfirmModal"; + +interface PageProps { + params: Promise<{ tenantId: string; historyRowKey: string }>; +} + +export default function CustomsArchiveDetailPage({ params }: PageProps) { + // Next.js 15: route params are async. Resolve once. + const resolved = usePromise(params); + const tenantId = decodeURIComponent(resolved.tenantId); + const historyRowKey = decodeURIComponent(resolved.historyRowKey); + + const { getAccessToken, setError, setSuccessMessage } = useAdminConfig(); + const [entries, setEntries] = useState([]); + const [loading, setLoading] = useState(false); + const [expanded, setExpanded] = useState>({}); + const [deletingRk, setDeletingRk] = useState(null); + const [pendingDelete, setPendingDelete] = useState(null); + + const load = useCallback(async () => { + try { + setLoading(true); + const url = api.customsArchive.listEntries(tenantId, historyRowKey); + const res = await authenticatedFetch(url, getAccessToken); + if (!res.ok) { + throw new Error(`List entries failed: ${res.status} ${res.statusText}`); + } + const body = (await res.json()) as CustomsArchiveListEntriesResponse; + setEntries(body.entries ?? []); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError(err.message); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setLoading(false); + } + }, [tenantId, historyRowKey, getAccessToken, setError]); + + useEffect(() => { + void load(); + }, [load]); + + const toggleExpand = useCallback(async (entry: CustomsArchiveEntrySummary) => { + const key = entry.rowKey; + if (expanded[key]) { + // Collapse + setExpanded((prev) => { + const next = { ...prev }; + delete next[key]; + return next; + }); + return; + } + + setExpanded((prev) => ({ ...prev, [key]: "loading" })); + try { + const res = await authenticatedFetch( + api.customsArchive.getEntry(tenantId, historyRowKey, entry.rowKey), + getAccessToken, + ); + if (!res.ok) throw new Error(`Get entry failed: ${res.status}`); + const body = await res.json(); + setExpanded((prev) => ({ ...prev, [key]: body.entry as CustomsArchiveFullEntry })); + } catch (err) { + setExpanded((prev) => { + const next = { ...prev }; + delete next[key]; + return next; + }); + setError(err instanceof Error ? err.message : String(err)); + } + }, [expanded, tenantId, historyRowKey, getAccessToken, setError]); + + const confirmDelete = useCallback(async () => { + const entry = pendingDelete; + if (!entry) return; + try { + setDeletingRk(entry.rowKey); + const res = await authenticatedFetch( + api.customsArchive.deleteEntry(tenantId, historyRowKey, entry.rowKey), + getAccessToken, + { method: "DELETE" }, + ); + if (!res.ok) throw new Error(`Delete failed: ${res.status}`); + setSuccessMessage(`Deleted ${entry.originalTable} / ${entry.originalRowKey}`); + setPendingDelete(null); + await load(); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setDeletingRk(null); + } + }, [pendingDelete, tenantId, historyRowKey, getAccessToken, load, setError, setSuccessMessage]); + + return ( +
+
+
+ + ← Back to runs + +

+ Customs Archive — Run detail +

+

+ Tenant: {tenantId} · Run: {historyRowKey} +

+
+
+ +
+ + +
+ + {entries.length} entry/entries{loading ? " — loading…" : ""} + + +
+ + {entries.length === 0 && !loading && ( +
+ No archived entries for this run. +
+ )} + +
+ {entries.map((entry) => { + const detail = expanded[entry.rowKey]; + const isOpen = !!detail; + const isLoadingFull = detail === "loading"; + const fullEntry = typeof detail === "object" ? (detail as CustomsArchiveFullEntry) : null; + + return ( +
+
+
+
+ + {entry.originalTable} + + + {entry.originalRowKey} + +
+
+ Archived {new Date(entry.archivedAt).toLocaleString()} +
+
+
+ + +
+
+ + {isOpen && ( +
+ {isLoadingFull ? ( +
Loading…
+ ) : fullEntry ? ( + 
+                        {prettyJson(fullEntry.entityJson)}
+
+ ) : null} +
+ )} +
+ ); + })} +
+
+ + +

+ Delete this archived entry from{" "} + {pendingDelete.originalTable}? +

+

+ {pendingDelete.originalRowKey} +

+

This action is irreversible.

+ + )} + busy={deletingRk !== null} + onCancel={() => setPendingDelete(null)} + onConfirm={confirmDelete} + /> +
+ ); +} + +function prettyJson(raw: string): string { + try { + return JSON.stringify(JSON.parse(raw), null, 2); + } catch { + return raw; + } +} diff --git a/src/Web/autopilot-monitor-web/app/admin/customs-archive/components/DeleteConfirmModal.tsx b/src/Web/autopilot-monitor-web/app/admin/customs-archive/components/DeleteConfirmModal.tsx new file mode 100644 index 00000000..fa8b0940 --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/customs-archive/components/DeleteConfirmModal.tsx @@ -0,0 +1,83 @@ +"use client"; + +import { useEffect, useState } from "react"; + +interface DeleteConfirmModalProps { + open: boolean; + title: string; + description: React.ReactNode; + busy?: boolean; + onCancel: () => void; + onConfirm: () => void; +} + +const CONFIRM_WORD = "DELETE"; + +/** + * Lightweight typed-confirmation modal for destructive customs-archive actions. + * Replaces the native window.confirm() so the dialog can render rich context + * (tenant id, table/row, count) and require a typed "DELETE" before the + * primary button activates. + */ +export function DeleteConfirmModal({ + open, + title, + description, + busy = false, + onCancel, + onConfirm, +}: DeleteConfirmModalProps) { + const [text, setText] = useState(""); + + useEffect(() => { + if (!open) setText(""); + }, [open]); + + if (!open) return null; + + const canConfirm = text === CONFIRM_WORD && !busy; + + return ( +
+
+

{title}

+
{description}
+ +
+ + setText(e.target.value)} + placeholder={CONFIRM_WORD} + disabled={busy} + autoFocus + autoComplete="off" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-900 text-gray-900 dark:text-white focus:outline-none focus:ring-2 focus:ring-red-500 focus:border-red-500" + /> +
+ +
+ + +
+
+
+ ); +} diff --git a/src/Web/autopilot-monitor-web/app/admin/customs-archive/page.tsx b/src/Web/autopilot-monitor-web/app/admin/customs-archive/page.tsx new file mode 100644 index 00000000..04f64fb5 --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/customs-archive/page.tsx @@ -0,0 +1,182 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import Link from "next/link"; +import { + api, + type CustomsArchiveListRunsResponse, + type CustomsArchiveRunSummary, +} from "@/lib/api"; +import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch"; +import { useAdminConfig } from "../AdminConfigContext"; +import { AdminNotifications } from "../AdminNotifications"; +import { DeleteConfirmModal } from "./components/DeleteConfirmModal"; + +export default function CustomsArchivePage() { + const { getAccessToken, setError, setSuccessMessage } = useAdminConfig(); + const [filter, setFilter] = useState(""); + const [runs, setRuns] = useState([]); + const [loading, setLoading] = useState(false); + const [deletingPk, setDeletingPk] = useState(null); + const [pendingDelete, setPendingDelete] = useState(null); + + const load = useCallback(async () => { + try { + setLoading(true); + const url = api.customsArchive.listRuns({ tenantId: filter.trim() || undefined }); + const res = await authenticatedFetch(url, getAccessToken); + if (!res.ok) { + throw new Error(`List runs failed: ${res.status} ${res.statusText}`); + } + const body = (await res.json()) as CustomsArchiveListRunsResponse; + setRuns(body.runs ?? []); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError(err.message); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setLoading(false); + } + }, [filter, getAccessToken, setError]); + + useEffect(() => { + void load(); + }, [load]); + + const confirmDelete = useCallback(async () => { + const run = pendingDelete; + if (!run) return; + try { + setDeletingPk(run.partitionKey); + const res = await authenticatedFetch( + api.customsArchive.deleteRun(run.tenantId, run.historyRowKey), + getAccessToken, + { method: "DELETE" } + ); + if (!res.ok) { + throw new Error(`Delete run failed: ${res.status} ${res.statusText}`); + } + const body = await res.json(); + setSuccessMessage(`Deleted ${body.deleted ?? 0} archived rules.`); + setPendingDelete(null); + await load(); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError(err.message); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setDeletingPk(null); + } + }, [pendingDelete, getAccessToken, load, setError, setSuccessMessage]); + + return ( +
+
+
+

Customs Archive

+

+ Snapshots of each tenant's custom GatherRules / AnalyzeRules /{" "} + ImeLogPatterns taken during offboarding (Phase 2.D-archive). Review the content, + then delete entries selectively when no longer needed. No auto-cleanup. +

+
+
+ +
+ + +
+ setFilter(e.target.value)} + placeholder="Filter by tenant ID (optional)" + className="px-3 py-1.5 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm text-gray-900 dark:text-white" + /> + + {runs.length} run(s) +
+ + {runs.length === 0 && !loading && ( +
+ No archived offboarding runs found{filter ? ` for tenant ${filter}.` : "."} +
+ )} + +
+ {runs.map((run) => ( +
+
+
{run.tenantId}
+
+ Run: {run.historyRowKey} · Archived{" "} + {new Date(run.archivedAt).toLocaleString()} +
+
+ Gather: {run.gatherRulesCount} + Analyze: {run.analyzeRulesCount} + ImePatterns: {run.imeLogPatternsCount} +
+
+
+ + Inspect + + +
+
+ ))} +
+
+ + +

+ Delete the entire archive run for tenant{" "} + {pendingDelete.tenantId}? +

+

+ This permanently removes{" "} + + {pendingDelete.gatherRulesCount + + pendingDelete.analyzeRulesCount + + pendingDelete.imeLogPatternsCount} + {" "} + archived rules. This action is irreversible. +

+ + )} + busy={deletingPk !== null} + onCancel={() => setPendingDelete(null)} + onConfirm={confirmDelete} + /> +
+ ); +} diff --git a/src/Web/autopilot-monitor-web/app/settings/TenantConfigContext.tsx b/src/Web/autopilot-monitor-web/app/settings/TenantConfigContext.tsx index d77396b8..b9fa8ae1 100644 --- a/src/Web/autopilot-monitor-web/app/settings/TenantConfigContext.tsx +++ b/src/Web/autopilot-monitor-web/app/settings/TenantConfigContext.tsx @@ -15,6 +15,18 @@ import { type BootstrapSessionItem } from "./components/BootstrapSessionsSection // --------------------------------------------------------------------------- // Context value interface // --------------------------------------------------------------------------- +/** + * State surfaced by handleOffboard once the DELETE returns 202. Drives the post-confirm + * banner in OffboardingSection. EarliestProcessingAt is rendered as a live countdown; + * when it elapses, handleDrainBarrierElapsed() runs and logs the user out. + */ +export interface OffboardingInProgressInfo { + status: string; + historyRowKey: string; + earliestProcessingAt?: string | null; + message: string; +} + interface TenantConfigContextValue { // Core config: TenantConfiguration | null; @@ -200,6 +212,12 @@ interface TenantConfigContextValue { setOffboardError: (v: string | null) => void; handleOffboard: () => Promise; + /** Set after the DELETE returns 202; drives the post-confirm drain-barrier banner. */ + offboardingInProgress: OffboardingInProgressInfo | null; + + /** Called by the banner countdown when the cache-drain barrier elapses → triggers logout. */ + handleDrainBarrierElapsed: () => void; + // Auth helpers user: ReturnType["user"]; getAccessToken: () => Promise; @@ -243,6 +261,7 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode } const [offboardConfirmText, setOffboardConfirmText] = useState(""); const [offboarding, setOffboarding] = useState(false); const [offboardError, setOffboardError] = useState(null); + const [offboardingInProgress, setOffboardingInProgress] = useState(null); // Bootstrap sessions const [bootstrapSessions, setBootstrapSessions] = useState([]); @@ -1163,18 +1182,40 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode } throw new Error(data?.error || `Offboard failed: ${response.statusText}`); } - // Offboard successful – sign out the user as their admin access is gone + // Backend returns 202 (or 200 for idempotent re-clicks) with the History row pointer + // and EarliestProcessingAt (cache-drain barrier deadline). Switch the UI into the + // drain-barrier banner state; the banner's countdown will auto-logout once the + // barrier elapses (by then the worker has started Phase 2 and the auth pipeline + // returns 403 via the existing Disabled-flag gate). + const body = await response.json().catch(() => ({})); trackEvent("tenant_offboarded"); - logout(); + + setOffboardingInProgress({ + status: body?.status ?? "Queued", + historyRowKey: body?.historyRowKey ?? "", + earliestProcessingAt: body?.earliestProcessingAt ?? null, + message: body?.message ?? "Tenant offboarding queued.", + }); + // Dismiss the confirmation dialog now that the banner has taken over. + setShowOffboardDialog(false); } catch (err) { if (err instanceof TokenExpiredError) { addNotification('error', 'Session Expired', err.message, 'session-expired-error'); } else { setOffboardError(err instanceof Error ? err.message : 'Offboard failed'); } + } finally { setOffboarding(false); } - }, [tenantId, getAccessToken, addNotification, logout]); + }, [tenantId, getAccessToken, addNotification]); + + const handleDrainBarrierElapsed = useCallback(() => { + // The cache-drain barrier has expired. The worker is starting Phase 2 right now and + // all function-host instances have refreshed their TenantConfiguration cache to see + // Disabled=true. Sign the user out — any further authenticated call will fail with + // 403 anyway. + logout(); + }, [logout]); // ----------------------------------------------------------------------- // Provider value @@ -1281,6 +1322,7 @@ export function TenantConfigProvider({ children }: { children: React.ReactNode } offboardConfirmText, setOffboardConfirmText, offboarding, offboardError, setOffboardError, handleOffboard, + offboardingInProgress, handleDrainBarrierElapsed, // Auth user, getAccessToken, diff --git a/src/Web/autopilot-monitor-web/app/settings/components/OffboardingSection.tsx b/src/Web/autopilot-monitor-web/app/settings/components/OffboardingSection.tsx index b5d07ebb..97ad270d 100644 --- a/src/Web/autopilot-monitor-web/app/settings/components/OffboardingSection.tsx +++ b/src/Web/autopilot-monitor-web/app/settings/components/OffboardingSection.tsx @@ -1,5 +1,14 @@ "use client"; +import { useEffect, useRef, useState } from "react"; + +interface OffboardingInProgressInfo { + status: string; // "Queued" | "Initiated" | "InProgress" | "Completed" | "Failed" + historyRowKey: string; + earliestProcessingAt?: string | null; // ISO UTC; drives countdown in drain-barrier state + message: string; +} + interface OffboardingSectionProps { showOffboardDialog: boolean; setShowOffboardDialog: (value: boolean) => void; @@ -9,6 +18,58 @@ interface OffboardingSectionProps { offboardError: string | null; setOffboardError: (value: string | null) => void; onOffboard: () => void; + + /** Set after the DELETE returns 202; switches the section into the drain-barrier banner state. */ + offboardingInProgress: OffboardingInProgressInfo | null; + + /** Invoked by the banner's auto-logout timer once the drain barrier has fully elapsed. */ + onDrainBarrierElapsed: () => void; +} + +function formatRemaining(totalSeconds: number): string { + if (totalSeconds <= 0) return "0 min"; + return `~${Math.ceil(totalSeconds / 60)} min`; +} + +/** Live countdown — re-renders every second until EarliestProcessingAt passes, then calls onElapsed once. */ +function useCountdown(targetIso: string | null | undefined, onElapsed: () => void): number { + const [secondsLeft, setSecondsLeft] = useState(() => { + if (!targetIso) return 0; + return Math.max(0, Math.floor((new Date(targetIso).getTime() - Date.now()) / 1000)); + }); + const firedRef = useRef(false); + + useEffect(() => { + if (!targetIso) return; + firedRef.current = false; + + // Anchor the remaining-time once at mount, then drive subsequent ticks off + // performance.now() — a monotonic clock that is NOT affected by NTP-syncs + // or manual clock changes during the ~6 min drain barrier. The previous + // implementation re-read Date.now() on every tick, so a wall-clock jump + // forward would fire onElapsed (auto-logout) prematurely. + const initialRemainingMs = Math.max( + 0, + new Date(targetIso).getTime() - Date.now(), + ); + const monoAtMount = performance.now(); + + const tick = () => { + const elapsedMs = performance.now() - monoAtMount; + const remaining = Math.max(0, Math.floor((initialRemainingMs - elapsedMs) / 1000)); + setSecondsLeft(remaining); + if (remaining === 0 && !firedRef.current) { + firedRef.current = true; + onElapsed(); + } + }; + + tick(); // immediate sync + const interval = setInterval(tick, 1000); + return () => clearInterval(interval); + }, [targetIso, onElapsed]); + + return secondsLeft; } export default function OffboardingSection({ @@ -20,7 +81,20 @@ export default function OffboardingSection({ offboardError, setOffboardError, onOffboard, + offboardingInProgress, + onDrainBarrierElapsed, }: OffboardingSectionProps) { + // When the offboarding has been queued, replace the section entirely with the + // drain-barrier banner. The dialog is dismissed by handleOffboard on success. + if (offboardingInProgress) { + return ( + + ); + } + return ( <> {/* Danger Zone: Tenant Offboarding */} @@ -79,7 +153,10 @@ export default function OffboardingSection({
  • Tenant configuration
  • All admin accounts (including yours)
    • -

    After offboarding you will be signed out and lose all access.

    +

    + Data deletion will start after a ~6 minute preparation window and then run in the background. + You can close this tab once the request has been sent. +

    @@ -118,10 +195,10 @@ export default function OffboardingSection({ {offboarding ? ( <>
    - Offboarding... + Queuing... ) : ( - Permanently Delete All Data + Queue Permanent Deletion )}
    @@ -131,3 +208,95 @@ export default function OffboardingSection({ ); } + +/** + * Banner shown after the DELETE returns 202. Replaces the entire Danger Zone section. + * Renders a live countdown until `earliestProcessingAt`, then auto-logs the user out + * (the worker has by then started Phase 2 and the auth pipeline will return 403 + * anyway via the Disabled-flag gate). + */ +function OffboardingInProgressBanner({ + info, + onDrainBarrierElapsed, +}: { + info: OffboardingInProgressInfo; + onDrainBarrierElapsed: () => void; +}) { + const secondsLeft = useCountdown(info.earliestProcessingAt ?? null, onDrainBarrierElapsed); + const inDrainBarrier = (info.earliestProcessingAt && secondsLeft > 0) === true; + + // Single 4-state visual; status text drives the badge color + label. + // "Preparing" is derived from earliestProcessingAt + countdown rather than from status, + // because the backend status stays "Queued" during the cache-drain window. + const stateLabel = inDrainBarrier + ? "Preparing" + : info.status === "Failed" + ? "Failed" + : info.status === "Completed" + ? "Completed" + : "In Progress"; + + const stateColors: Record = { + "Preparing": "bg-amber-100 text-amber-800 border-amber-300", + "In Progress": "bg-blue-100 text-blue-800 border-blue-300", + "Completed": "bg-green-100 text-green-800 border-green-300", + "Failed": "bg-red-100 text-red-800 border-red-300", + }; + + return ( +
    +
    +
    + + + +
    +
    +

    Tenant Offboarding queued

    +

    + This tenant will be permanently deleted. You do not need to keep this page open. +

    +
    +
    + +
    + + {stateLabel} + + {inDrainBarrier && ( + + Data deletion starts in {formatRemaining(secondsLeft)} + + )} +
    + + {inDrainBarrier ? ( +
    +

    Preparing for deletion

    +

    + Your tenant has been disabled and is being prepared for permanent deletion. The wipe will start + automatically once the countdown reaches zero. +

    +

    + Everything needed for the deletion is already recorded — you can sign out and close this tab. + The deletion continues in the background. +

    +
    + ) : ( +
    +

    Deletion in progress

    +

    + Your tenant is being permanently deleted. You can close this tab safely. +

    +
    + )} + +
    +

    History row: {info.historyRowKey}

    + {info.earliestProcessingAt && ( +

    Earliest processing at: {info.earliestProcessingAt}

    + )} +
    +
    + ); +} diff --git a/src/Web/autopilot-monitor-web/app/settings/management/sections/SectionOffboarding.tsx b/src/Web/autopilot-monitor-web/app/settings/management/sections/SectionOffboarding.tsx index d1eda5e6..07afb667 100644 --- a/src/Web/autopilot-monitor-web/app/settings/management/sections/SectionOffboarding.tsx +++ b/src/Web/autopilot-monitor-web/app/settings/management/sections/SectionOffboarding.tsx @@ -10,6 +10,7 @@ export function SectionOffboarding() { offboardConfirmText, setOffboardConfirmText, offboarding, offboardError, setOffboardError, handleOffboard, + offboardingInProgress, handleDrainBarrierElapsed, } = useTenantConfig(); return ( @@ -24,6 +25,8 @@ export function SectionOffboarding() { offboardError={offboardError} setOffboardError={setOffboardError} onOffboard={handleOffboard} + offboardingInProgress={offboardingInProgress} + onDrainBarrierElapsed={handleDrainBarrierElapsed} /> ); diff --git a/src/Web/autopilot-monitor-web/lib/api.ts b/src/Web/autopilot-monitor-web/lib/api.ts index 05e00819..07fb841d 100644 --- a/src/Web/autopilot-monitor-web/lib/api.ts +++ b/src/Web/autopilot-monitor-web/lib/api.ts @@ -402,6 +402,21 @@ export const api = { })}`, }, + // ── Customs Archive (PR3.B Tenant offboarding archive) ──────────────────── + // Response types defined below (CustomsArchive*). + customsArchive: { + listRuns: (opts?: { tenantId?: string }) => + `${API_BASE_URL}/api/global/customs-archive${qs({ tenantId: opts?.tenantId })}`, + listEntries: (tenantId: string, historyRowKey: string) => + `${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}`, + getEntry: (tenantId: string, historyRowKey: string, archiveRowKey: string) => + `${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}/${encodeURIComponent(archiveRowKey)}`, + deleteEntry: (tenantId: string, historyRowKey: string, archiveRowKey: string) => + `${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}/${encodeURIComponent(archiveRowKey)}`, + deleteRun: (tenantId: string, historyRowKey: string) => + `${API_BASE_URL}/api/global/customs-archive/${encodeURIComponent(tenantId)}/${encodeURIComponent(historyRowKey)}`, + }, + // ── Feedback ────────────────────────────────────────────────────────────── feedback: { status: () => `${API_BASE_URL}/api/feedback/status`, @@ -484,3 +499,45 @@ export const api = { leaveGroup: () => `${API_BASE_URL}/api/realtime/groups/leave`, }, }; + +// ── Customs Archive response types ──────────────────────────────────────── +// Shared by /admin/customs-archive (list runs) and +// /admin/customs-archive/[tenantId]/[historyRowKey] (list/inspect entries). +export interface CustomsArchiveRunSummary { + partitionKey: string; + tenantId: string; + historyRowKey: string; + archivedAt: string; + gatherRulesCount: number; + analyzeRulesCount: number; + imeLogPatternsCount: number; +} + +export interface CustomsArchiveListRunsResponse { + success: boolean; + count: number; + runs: CustomsArchiveRunSummary[]; +} + +export interface CustomsArchiveEntrySummary { + partitionKey: string; + rowKey: string; + originalTable: string; + originalRowKey: string; + archivedAt: string; + entityJsonPreview: string; +} + +export interface CustomsArchiveListEntriesResponse { + success: boolean; + count: number; + entries: CustomsArchiveEntrySummary[]; +} + +export interface CustomsArchiveFullEntry extends CustomsArchiveEntrySummary { + tenantId: string; + originalPartitionKey: string; + entityJson: string; + historyRowKey: string; + archivedBy: string; +} diff --git a/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx b/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx index 6890886e..5eb86e30 100644 --- a/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx +++ b/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx @@ -263,6 +263,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [ items: [ { id: "ga-maintenance", label: "Maintenance", href: "/admin/ops" }, { id: "ga-session-cleanup", label: "Session Cleanup", href: "/admin/ops/session-cleanup" }, + { id: "ga-customs-archive", label: "Customs Archive", href: "/admin/customs-archive" }, ], }, {