Files
AutopilotMonitor/src/Backend/AutopilotMonitor.Functions.Tests/TenantOffboardFunctionTests.cs
T
Oliver Kieselbach e816c92b88 Tenant configuration has one conditional write path; the full-model PUT is gone
A tenant configuration row is never written blind any more (D-290). Three writers saved
from the 5-minute instance cache and four wrote a fresh read without an ETag: auto-re-enable
could lift a suspension a Global Admin had just renewed, the PUT could write the cached
Disabled=false over the offboarding tombstone during the drain barrier, and a concurrent
first login could overwrite the write-once OnboardedBy.

- TenantConfigurationService.CreateOrUpdateAsync next to UpdateAsync (shared core): fresh
  read with ETag, tombstone refused, mutation decides, conditional replace, conditional
  insert for a missing row, re-read and re-decide on a lost race.
- SaveConfigurationAsync and IConfigRepository.SaveTenantConfigurationAsync deleted; new
  TryCreateTenantConfigurationAsync. The default row of GetConfigurationAsync is a
  conditional insert.
- Converted: first-login seed (writes only while the stored row has no domain; only the
  winning login fires the signup side effects), auto-re-enable (decided on the fresh row),
  auth client-id tracking, app-homing flip, conferred-Pro grace stamp, offboarding tombstone.
- PUT/POST config/{tenantId} removed (function, catalog entries, response DTO). The Global
  Admin tenant editor sends only its changed fields to the field PATCH; the Collect Logs
  quick-config PATCHes its two diagnostics fields with ?intent=collect-logs, which keeps
  the CollectLogsQuickConfigEnabled ops event. Shared diff helper lib/tenantConfigSave.ts.
- Tests: create-or-update and default-row races, concurrent first login, renewed
  suspension, tombstone paths; repository and service doubles in ConfigRepoMockExtensions.
  Regenerated shared manifest and wire types (web, MCP).

Deploy order: backend, then web right after (the old web's editor save and quick-config
use the removed PUT).

Bumps internal (D-290, plan, docs).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-27 20:11:56 +02:00

1342 lines
60 KiB
C#

using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Functions.Admin;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Offboarding;
using AutopilotMonitor.Functions.Tests.Offboarding;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Offboarding;
using Azure;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
// Capture-helper for the recording enqueuer used in Finding-1 tests below.
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// PR3-side unit tests for the rewritten <see cref="TenantOffboardFunction"/>. The HTTP
/// entrypoint itself lives on top of <c>HttpRequestData</c> (abstract, hard to fake without
/// a real Functions host) so these tests focus on the function's CAS-loop helper. The
/// end-to-end happy/idempotent/race paths are covered by integration tests against the
/// real Functions host on dev tenants (see §10.3 smoketest in the plan).
/// </summary>
public sealed class TenantOffboardFunctionTests
{
private const string TenantId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee";
private static TenantOffboardFunction Build(IOffboardingAuditRepository repo)
=> Build(repo, Mock.Of<ITenantOffboardingEnqueuer>());
private static TenantOffboardFunction Build(
IOffboardingAuditRepository repo, ITenantOffboardingEnqueuer enqueuer)
{
// Resume-path tests (UpsertPointer / Resume_*) exercise just the CAS-loop +
// enqueuer wiring and don't care about the Disabled-gate. Stub IConfigRepository
// so EnsureTenantDisabledAsync's idempotent no-op path fires (returns an existing
// row already marked Disabled with the offboarding reason → no Save call).
//
// LastUpdated is set FAR in the past so the tombstone-deadline (LastUpdated +
// DrainBarrier) doesn't dominate the max() computation in the resume-path. That
// means the existing history-based-deadline tests still observe history-derived
// delays. Tests that exercise the LastUpdated-dominates path stand up their own
// configRepoMock with a recent LastUpdated.
var configRepoMock = new Mock<IConfigRepository>();
var alreadyDisabled = TenantConfiguration.CreateDefault(TenantId);
alreadyDisabled.Disabled = true;
alreadyDisabled.DisabledReason = "Offboarding in progress";
alreadyDisabled.LastUpdated = DateTime.UtcNow.AddDays(-30);
configRepoMock.BackWithRow(alreadyDisabled);
var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache(
new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions());
var service = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
return new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: service,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer,
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
}
/// <summary>
/// Builds a real <see cref="PreviewWhitelistService"/> wired to the supplied
/// <see cref="IConfigRepository"/> mock. Tests that exercise the Phase 1 notification-
/// email capture set up <c>configRepoMock.Setup(r =&gt; r.GetNotificationEmailAsync(...))</c>
/// to drive the captured value. Resume-path tests don't care — the mock returns null by
/// default which surfaces as a null email on the History row.
/// </summary>
private static PreviewWhitelistService BuildPreviewWhitelistService(Mock<IConfigRepository> configRepoMock)
{
var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache(
new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions());
return new PreviewWhitelistService(
configRepoMock.Object,
cache,
NullLogger<PreviewWhitelistService>.Instance,
new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache));
}
/// <summary>
/// Test-side enqueuer recording every call. Production enqueuer is sealed + needs
/// configuration to ctor, so this fake lets the tests assert "did Phase 1 actually
/// hand off to the worker?" — the question Finding 1 says we previously got wrong.
/// </summary>
private sealed class RecordingEnqueuer : ITenantOffboardingEnqueuer
{
public readonly List<TenantOffboardingEnvelope> Sent = new();
public readonly List<TimeSpan?> VisibilityDelays = new();
public Exception? ThrowOnEnqueue { get; set; }
public Task EnqueueAsync(TenantOffboardingEnvelope envelope, TimeSpan? visibilityDelay = null, CancellationToken ct = default)
{
if (ThrowOnEnqueue is { } ex) throw ex;
Sent.Add(envelope);
VisibilityDelays.Add(visibilityDelay);
return Task.CompletedTask;
}
}
// Build helper exposing IConfigRepository + TenantConfigurationService for the
// EnsureTenantDisabledAsync tests. The repo is a Mock so we can pin Save calls; the
// service uses a real MemoryCache so InvalidateCache(tenantId) is observable via a
// pre-seeded cache entry.
private static (TenantOffboardFunction Sut, Mock<IConfigRepository> ConfigRepo, Microsoft.Extensions.Caching.Memory.IMemoryCache Cache, TenantConfigurationService Service)
BuildWithConfigService()
{
var configRepoMock = new Mock<IConfigRepository>();
var cache = new Microsoft.Extensions.Caching.Memory.MemoryCache(
new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions());
var service = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: service,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: new FakeOffboardingAuditRepository(),
offboardingEnqueuer: new RecordingEnqueuer(),
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
return (sut, configRepoMock, cache, service);
}
// ── CaptureNotificationEmailAsync ────────────────────────────────────────
[Fact]
public async Task CaptureNotificationEmail_ReturnsValue_WhenPreviewEmailSet()
{
// Phase 1 must snapshot the Preview-Notification-Email into OffboardingHistory
// BEFORE Phase 2.D wipes the PreviewWhitelist table. This test pins that contract.
var (sut, configRepoMock, _, _) = BuildWithConfigService();
configRepoMock
.Setup(r => r.GetNotificationEmailAsync(TenantId))
.ReturnsAsync("ops@contoso.invalid");
var captured = await sut.CaptureNotificationEmailAsync(TenantId);
Assert.Equal("ops@contoso.invalid", captured);
}
[Fact]
public async Task CaptureNotificationEmail_ReturnsNull_WhenTenantNeverSetEmail()
{
// The farewell-send skip-gate in TenantOffboardingHandler depends on null-vs-set.
var (sut, configRepoMock, _, _) = BuildWithConfigService();
configRepoMock
.Setup(r => r.GetNotificationEmailAsync(TenantId))
.ReturnsAsync((string?)null);
var captured = await sut.CaptureNotificationEmailAsync(TenantId);
Assert.Null(captured);
}
[Fact]
public async Task CaptureNotificationEmail_FailSoftReturnsNull_OnRepoException()
{
// The farewell email is a courtesy, not a correctness contract. A storage hiccup
// on the email lookup must NOT block the offboarding itself.
var (sut, configRepoMock, _, _) = BuildWithConfigService();
configRepoMock
.Setup(r => r.GetNotificationEmailAsync(TenantId))
.ThrowsAsync(new InvalidOperationException("simulated table-storage outage"));
var captured = await sut.CaptureNotificationEmailAsync(TenantId);
Assert.Null(captured);
}
// ── UpsertPointerWithCasAsync ────────────────────────────────────────────
[Fact]
public async Task UpsertPointer_FreshTenant_Inserts_OffboardCount_1()
{
var repo = new FakeOffboardingAuditRepository();
var sut = Build(repo);
var historyRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await sut.UpsertPointerWithCasAsync(TenantId, historyRowKey, DateTime.UtcNow, trialConsumed: false);
Assert.True(repo.Pointers.TryGetValue(TenantId, out var entry));
Assert.Equal(historyRowKey, entry.Pointer.LatestHistoryRowKey);
Assert.Equal("Initiated", entry.Pointer.LatestStatus);
Assert.Equal(1, entry.Pointer.OffboardCount);
}
[Fact]
public async Task UpsertPointer_ExistingTenant_IncrementsOffboardCount()
{
var repo = new FakeOffboardingAuditRepository();
await repo.InsertByTenantPointerAsync(new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
LatestUpdatedAt = DateTime.UtcNow.AddDays(-30),
OffboardCount = 2,
});
var sut = Build(repo);
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow, trialConsumed: false);
Assert.Equal(newHistoryRowKey, repo.Pointers[TenantId].Pointer.LatestHistoryRowKey);
Assert.Equal("Initiated", repo.Pointers[TenantId].Pointer.LatestStatus);
Assert.Equal(3, repo.Pointers[TenantId].Pointer.OffboardCount);
}
[Theory]
[InlineData(false, false, false)]
[InlineData(false, true, true)]
[InlineData(true, false, true)] // a later round without a trial never clears it
[InlineData(true, true, true)]
public async Task UpsertPointer_TrialConsumed_IsStickyAcrossRounds(bool stored, bool thisRound, bool expected)
{
var repo = new FakeOffboardingAuditRepository();
await repo.InsertByTenantPointerAsync(new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
OffboardCount = 1,
TrialConsumed = stored,
});
await Build(repo).UpsertPointerWithCasAsync(TenantId, "20260201000000000_" + TenantId, DateTime.UtcNow, thisRound);
Assert.Equal(expected, repo.Pointers[TenantId].Pointer.TrialConsumed);
Assert.Equal(2, repo.Pointers[TenantId].Pointer.OffboardCount);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task UpsertPointer_FreshTenant_StampsTrialConsumed(bool trialConsumed)
{
var repo = new FakeOffboardingAuditRepository();
await Build(repo).UpsertPointerWithCasAsync(TenantId, "20260201000000000_" + TenantId, DateTime.UtcNow, trialConsumed);
Assert.Equal(trialConsumed, repo.Pointers[TenantId].Pointer.TrialConsumed);
}
// ── OffboardTenant (Phase 1): the consumed trial must reach the pointer before anything is deleted ──
private static (TenantOffboardFunction Sut, FakeOffboardingAuditRepository Repo, RecordingEnqueuer Enqueuer, Mock<IConfigRepository> ConfigRepo)
BuildForOffboard(TenantConfiguration? liveConfig, Exception? readFailure = null)
{
var configRepo = new Mock<IConfigRepository>();
configRepo.BackWithRow(liveConfig).ThrowOnRead = readFailure;
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepo.Object,
tenantConfigService: new TenantConfigurationService(
configRepo.Object, NullLogger<TenantConfigurationService>.Instance,
new Microsoft.Extensions.Caching.Memory.MemoryCache(new Microsoft.Extensions.Caching.Memory.MemoryCacheOptions())),
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer,
previewWhitelistService: BuildPreviewWhitelistService(configRepo));
return (sut, repo, enqueuer, configRepo);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task OffboardTenant_CarriesTheLiveTrialConsumedOntoThePointer(bool trialConsumed)
{
var live = TenantConfiguration.CreateDefault(TenantId);
live.DomainName = "contoso.com";
live.TrialConsumed = trialConsumed;
var (sut, repo, enqueuer, _) = BuildForOffboard(live);
var (req, context) = EndpointHarness.Request(TenantId);
var response = await sut.OffboardTenant(req, TenantId, context);
Assert.Equal(System.Net.HttpStatusCode.Accepted, response.StatusCode);
Assert.Equal(trialConsumed, repo.Pointers[TenantId].Pointer.TrialConsumed);
Assert.Single(enqueuer.Sent);
}
[Fact]
public async Task OffboardTenant_ConfigReadFails_RefusesBeforeAnyStateIsWritten()
{
// Proceeding without the read would reset the one-trial rule for the returning tenant.
var (sut, repo, enqueuer, configRepo) = BuildForOffboard(null, new RequestFailedException(503, "Server busy"));
var (req, context) = EndpointHarness.Request(TenantId);
var response = await sut.OffboardTenant(req, TenantId, context);
Assert.Equal(System.Net.HttpStatusCode.InternalServerError, response.StatusCode);
Assert.Empty(repo.History);
Assert.Empty(repo.Pointers);
Assert.Empty(repo.Markers);
Assert.Empty(enqueuer.Sent);
configRepo.VerifyNoTenantConfigWrite();
}
[Fact]
public async Task UpsertPointer_409OnInsert_FallsBackToEtagUpdate()
{
// First TryGet returns null, but InsertByTenantPointerAsync throws 409 because
// a parallel writer inserted between our read and write. The CAS-loop must
// re-read and update with the now-existing pointer's ETag.
var repoMock = new Mock<IOffboardingAuditRepository>();
var existingPointer = new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
OffboardCount = 1,
};
var attempt = 0;
repoMock
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
.Returns(() =>
{
attempt++;
return attempt == 1
? Task.FromResult<(OffboardingByTenantPointer?, string?)>((null, null))
: Task.FromResult<(OffboardingByTenantPointer?, string?)>((existingPointer, "\"etag-1\""));
});
repoMock
.Setup(r => r.InsertByTenantPointerAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new RequestFailedException(409, "Conflict", "EntityAlreadyExists", null));
OffboardingByTenantPointer? updatedPointer = null;
repoMock
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), "\"etag-1\"", It.IsAny<CancellationToken>()))
.Returns<OffboardingByTenantPointer, string, CancellationToken>((p, _, _) =>
{
updatedPointer = p;
return Task.CompletedTask;
});
var sut = Build(repoMock.Object);
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow, trialConsumed: false);
Assert.NotNull(updatedPointer);
Assert.Equal(newHistoryRowKey, updatedPointer!.LatestHistoryRowKey);
Assert.Equal(2, updatedPointer.OffboardCount);
repoMock.Verify(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), "\"etag-1\"", It.IsAny<CancellationToken>()), Times.Once);
}
[Fact]
public async Task UpsertPointer_412OnUpdate_RetriesAndSucceeds()
{
// Concurrent writer mutates the pointer between our read and our update. CAS-loop
// must re-read (getting the fresh ETag) and try once more.
var repoMock = new Mock<IOffboardingAuditRepository>();
var freshState = new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
OffboardCount = 1,
};
var readEtags = new[] { "\"etag-old\"", "\"etag-new\"" };
var readIdx = 0;
repoMock
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
.Returns(() =>
{
var etag = readEtags[Math.Min(readIdx, readEtags.Length - 1)];
readIdx++;
return Task.FromResult<(OffboardingByTenantPointer?, string?)>((freshState, etag));
});
var updateCalls = 0;
repoMock
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.Returns<OffboardingByTenantPointer, string, CancellationToken>((_, etag, _) =>
{
updateCalls++;
if (etag == "\"etag-old\"")
{
throw new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null);
}
return Task.CompletedTask;
});
var sut = Build(repoMock.Object);
var newHistoryRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await sut.UpsertPointerWithCasAsync(TenantId, newHistoryRowKey, DateTime.UtcNow, trialConsumed: false);
Assert.Equal(2, updateCalls);
Assert.Equal(2, readIdx);
}
// ── ResumeExistingMarkerAsync (PR3 Review Finding 1) ─────────────────────
//
// The previous implementation returned 200 from the idempotency path UNCONDITIONALLY.
// If the very first enqueue failed (transient queue outage), the marker was committed
// and the queue was empty: every subsequent re-click took the idempotency-200 branch
// and never re-enqueued, leaving the tenant stuck Initiated forever. These tests pin
// the new defensive-re-enqueue behaviour: Initiated/InProgress → re-enqueue; Completed
// → no re-enqueue (worker is done); Failed → no re-enqueue (operator action required).
[Theory]
[InlineData("Initiated")]
[InlineData("InProgress")]
public async Task Resume_InFlightStatus_DefensivelyReEnqueues(string status)
{
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var historyRowKey = "20260518091523123_" + TenantId;
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = DateTime.UtcNow,
InitiatedBy = "alice@contoso.com",
Status = status,
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "bob@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.True(outcome.ReEnqueued);
Assert.Single(enqueuer.Sent);
Assert.Equal(historyRowKey, enqueuer.Sent[0].HistoryRowKey);
Assert.Equal(TenantId, enqueuer.Sent[0].TenantId);
}
[Fact]
public async Task Resume_Completed_NoReEnqueue_ReturnsIdempotentOk()
{
var enqueuer = new RecordingEnqueuer();
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var marker = new OffboardingMarkerEntry
{
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
Status = "Completed",
CompletedAt = DateTime.UtcNow.AddMinutes(-5),
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.IdempotentCompleted, outcome.Kind);
Assert.False(outcome.ReEnqueued);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Resume_Failed_NoReEnqueue_SurfacesPhaseInMessage()
{
var enqueuer = new RecordingEnqueuer();
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var marker = new OffboardingMarkerEntry
{
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
Status = "Failed",
FailedAt = DateTime.UtcNow.AddMinutes(-10),
FailedPhase = "drain_timeout",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.IdempotentFailed, outcome.Kind);
Assert.False(outcome.ReEnqueued);
Assert.Contains("drain_timeout", outcome.Message);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Resume_InFlight_EnqueueThrows_SurfacesReEnqueueFailed()
{
// The defensive re-enqueue must surface its failure (500) so the caller knows
// the resume did NOT take effect. The marker stays Initiated; the next click
// will trigger another resume attempt — that's the operator-visible recovery
// mechanism that previously didn't exist (stuck-tenant bug).
var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") };
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var marker = new OffboardingMarkerEntry
{
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueueFailed, outcome.Kind);
Assert.False(outcome.ReEnqueued);
}
[Fact]
public async Task Resume_409Race_StillReEnqueues_BecauseWinnerMayHaveFailed()
{
// The 409-race path resolves to the winning marker and routes through the same
// resume code. We can't know whether the winner's enqueue succeeded, so we
// must re-enqueue defensively — same Finding-1 contract as the plain idempotency
// re-click path.
var enqueuer = new RecordingEnqueuer();
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var marker = new OffboardingMarkerEntry
{
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = "20260101000000000_" + TenantId,
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: true);
Assert.True(outcome.ReEnqueued);
Assert.Single(enqueuer.Sent);
Assert.Contains("race", outcome.Message);
}
[Fact]
public async Task UpsertPointer_CasExhausted_Throws()
{
// Every UpdateWithEtag returns 412 — the bounded retry-loop must surface this
// rather than retry indefinitely.
var repoMock = new Mock<IOffboardingAuditRepository>();
var freshState = new OffboardingByTenantPointer
{
PartitionKey = Constants.OffboardingPartitionKeys.ByTenant,
RowKey = TenantId,
TenantId = TenantId,
LatestHistoryRowKey = "20260101000000000_" + TenantId,
LatestStatus = "Completed",
OffboardCount = 1,
};
repoMock
.Setup(r => r.TryGetByTenantPointerAsync(TenantId, It.IsAny<CancellationToken>()))
.ReturnsAsync((freshState, "\"some-etag\""));
repoMock
.Setup(r => r.UpdateByTenantPointerWithEtagAsync(It.IsAny<OffboardingByTenantPointer>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new RequestFailedException(412, "ConditionNotMet", "UpdateConditionNotSatisfied", null));
var sut = Build(repoMock.Object);
var historyRowKey = $"{DateTime.UtcNow:yyyyMMddHHmmssfff}_{TenantId}";
await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.UpsertPointerWithCasAsync(TenantId, historyRowKey, DateTime.UtcNow, trialConsumed: false));
}
// ── Cache-Drain-Barrier (plan v2 §2.2 / §2.3) ─────────────────────────────
//
// The DrainBarrier (hardcoded 6min) prevents Phase 2 from starting before all
// function-host instances drain their TenantConfigurationService cache. Phase 1
// enqueues with visibilityDelay=DrainBarrier so the worker only sees the message
// after the barrier elapses. A resume-click reads the stored EarliestProcessingAt
// and re-enqueues with the REMAINING delay so the barrier does not reset.
[Fact]
public void DrainBarrier_HardcodedAt6Minutes()
{
// Plan v2 §2.2: 5min TenantConfigurationService.CacheDuration + 1min safety buffer.
Assert.Equal(TimeSpan.FromMinutes(6), TenantOffboardFunction.DrainBarrier);
}
[Fact]
public async Task Resume_InFlight_UsesRemainingBarrier_NotFullBarrier()
{
// Re-click 2 min into the 6-min barrier → re-enqueue must carry ~4min visibility delay,
// not a fresh 6min. Otherwise each re-click would reset the cache-drain window.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var initiatedAt = DateTime.UtcNow.AddMinutes(-2);
var earliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // ~+4min from now
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = initiatedAt,
EarliestProcessingAt = earliestProcessingAt,
Status = "Initiated",
});
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = initiatedAt,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
var delay = enqueuer.VisibilityDelays[0];
Assert.NotNull(delay);
// Allow a few seconds of test-runtime slack — clock advanced between Setup and Act.
Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.5));
Assert.Equal(earliestProcessingAt, outcome.EarliestProcessingAt);
}
[Fact]
public async Task Resume_InFlight_PastBarrier_UsesZeroDelay()
{
// Re-click 8 min into a 6-min barrier → barrier already elapsed → re-enqueue with
// zero visibility delay so the worker picks up immediately.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var initiatedAt = DateTime.UtcNow.AddMinutes(-8);
var earliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // 2 min ago
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = initiatedAt,
EarliestProcessingAt = earliestProcessingAt,
Status = "InProgress",
});
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = initiatedAt,
InitiatedBy = "alice@contoso.com",
Status = "InProgress",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
Assert.Equal(TimeSpan.Zero, enqueuer.VisibilityDelays[0]);
}
[Fact]
public async Task Resume_InFlight_MissingHistoryRow_UsesTombstoneLastUpdatedAsDeadlineAnchor()
{
// The marker references a History row that does not exist (corruption / manual
// delete). Without history.EarliestProcessingAt to read, the resume falls back
// to tombstone.LastUpdated + DrainBarrier as the authoritative drain-deadline.
// The Build-helper's default tombstone has LastUpdated = 30 days ago → the
// tombstone-derived deadline is far in the past → drain barrier already expired
// → visibility-delay = Zero (worker picks immediately). That is the CORRECT
// outcome: if the tombstone has been sitting for 30 days, every cache has been
// through the 5-min TTL many times already.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = "20260518091523123_" + TenantId,
InitiatedAt = DateTime.UtcNow,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
// 30-day-old tombstone → drain barrier long expired → 0 delay is safe.
Assert.Equal(TimeSpan.Zero, enqueuer.VisibilityDelays[0]);
}
// ── EnsureTenantDisabledAsync (PR3.B Review Finding 1 — fail-loud Disabled-gate) ──
//
// Without Disabled=true, the cache-drain barrier guards nothing: warm function-hosts
// would keep accepting agent traffic for a tenant whose Phase 2 is about to wipe
// their data. Every initial enqueue + every resume re-enqueue MUST commit this gate.
[Fact]
public async Task EnsureTenantDisabled_FlipsDisabled_AndInvalidatesCache()
{
var (sut, configRepo, cache, _) = BuildWithConfigService();
var existing = TenantConfiguration.CreateDefault(TenantId);
existing.Disabled = false;
existing.DomainName = "contoso.com";
var store = configRepo.BackWithRow(existing);
// Seed cache with stale "Disabled=false" entry so we can prove invalidation.
cache.Set($"tenant-config:{TenantId}", existing);
await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
var (saved, source, reason) = Assert.Single(store.Writes);
Assert.True(saved.Disabled);
Assert.Equal("Offboarding in progress", saved.DisabledReason);
Assert.Null(saved.DisabledUntil);
Assert.Equal("alice@contoso.com", saved.UpdatedBy);
Assert.Equal("contoso.com", saved.DomainName); // the rest of the row survives
Assert.Equal(("offboard", "Disabled-gate before cascade"), (source, reason));
Assert.False(cache.TryGetValue($"tenant-config:{TenantId}", out _),
"cache entry must be invalidated so warm reads pick up the new Disabled=true on next access");
}
[Fact]
public async Task EnsureTenantDisabled_DecidesOnTheStoredRow_NotAStaleCachedView()
{
// A cached copy still says "not disabled"; the stored row is already the tombstone.
var (sut, configRepo, cache, _) = BuildWithConfigService();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
var store = configRepo.BackWithRow(tombstone);
cache.Set($"tenant-config:{TenantId}", TenantConfiguration.CreateDefault(TenantId));
var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.AlreadyTombstone, result.Outcome);
Assert.Empty(store.Writes);
}
[Fact]
public async Task EnsureTenantDisabled_AlreadyDisabledWithSameReason_DoesNotRewrite()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
var existing = TenantConfiguration.CreateDefault(TenantId);
existing.Disabled = true;
existing.DisabledReason = "Offboarding in progress";
var store = configRepo.BackWithRow(existing);
await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
// Idempotent: no write thrash on resume-clicks while the marker is already in flight.
Assert.Empty(store.Writes);
}
[Fact]
public async Task EnsureTenantDisabled_StorageThrowsOnRead_PropagatesAsExpected()
{
// Caller (initial enqueue + resume re-enqueue) MUST treat this as fail-loud and
// skip the enqueue. We just assert the exception propagates — the caller's
// try/catch + 500-return is exercised by integration tests.
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.BackWithRow(TenantConfiguration.CreateDefault(TenantId)).ThrowOnRead =
new InvalidOperationException("storage unavailable");
await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
}
[Fact]
public async Task EnsureTenantDisabled_StorageThrowsOnWrite_PropagatesAsExpected()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.BackWithRow(TenantConfiguration.CreateDefault(TenantId)).ThrowOnWrite =
new InvalidOperationException("storage write failed");
await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
}
[Fact]
public async Task EnsureTenantDisabled_NoConfigRow_CreatesFreshTombstone()
{
// PR3.B-revised Finding 3: missing row used to log+continue, leaving AuthFunction's
// auto-create-default free to spawn a Disabled=false row on the next /api/auth/me.
// The fix creates a fresh CreateDefault row with Disabled=true so the auth-gate is
// committed even in this edge case.
var (sut, configRepo, _, _) = BuildWithConfigService();
var store = configRepo.BackWithRow(null);
var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
var saved = Assert.Single(store.Writes).Row;
Assert.Equal(TenantId, saved.TenantId);
Assert.True(saved.Disabled, "fresh tombstone must carry Disabled=true");
Assert.Equal("Offboarding in progress", saved.DisabledReason);
Assert.Equal("alice@contoso.com", saved.UpdatedBy);
Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.WroteNewTombstone, result.Outcome);
}
[Fact]
public async Task EnsureTenantDisabled_EveryWriteLost_ThrowsAsHardFail()
{
// Without Disabled=true the caller would proceed to enqueue without the auth-gate
// committed: every lost conditional write is a hard exception, never a silent skip.
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.BackWithRow(TenantConfiguration.CreateDefault(TenantId)).RejectWrites = true;
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
Assert.Contains("Disabled-gate NOT committed", ex.Message);
}
[Fact]
public async Task EnsureTenantDisabled_NoConfigRow_EveryWriteLost_ThrowsAsHardFail()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
configRepo.BackWithRow(null).RejectWrites = true;
await Assert.ThrowsAsync<InvalidOperationException>(() =>
sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com"));
}
[Fact]
public async Task EnsureTenantDisabled_AlreadyTombstone_ReturnsAlreadyTombstone()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
var store = configRepo.BackWithRow(tombstone);
var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.AlreadyTombstone, result.Outcome);
Assert.Empty(store.Writes);
}
[Fact]
public async Task EnsureTenantDisabled_FlipsDisabled_ReturnsWroteNewTombstone()
{
var (sut, configRepo, _, _) = BuildWithConfigService();
var existing = TenantConfiguration.CreateDefault(TenantId);
existing.Disabled = false;
var store = configRepo.BackWithRow(existing);
var result = await sut.EnsureTenantDisabledAsync(TenantId, "alice@contoso.com");
Assert.Equal(TenantOffboardFunction.EnsureDisabledOutcome.WroteNewTombstone, result.Outcome);
// The anchor of the drain-deadline math is the stored write time.
Assert.Equal(Assert.Single(store.Writes).Row.LastUpdated, result.TombstoneLastUpdated);
}
// ── Resume-path cache-drain bypass fix (Codex review) ────────────────────
//
// Scenario: initial attempt commits Marker/History/Pointer (History.EarliestProcessingAt
// = T+6min) but then EnsureTenantDisabledAsync FAILS → 500 returned, no enqueue. The
// admin re-clicks 10 minutes later. The History row still carries the now-past
// EarliestProcessingAt = T+6min. Without the fix, the resume would naively use the
// remaining-from-history delta and arrive at Zero → enqueue with no visibility-delay
// → worker picks immediately → but Disabled=true was JUST written for the first time
// → warm function-host caches still hold Disabled=false → cache-drain barrier violated.
//
// The fix anchors on tombstone.LastUpdated + DrainBarrier so the deadline is
// authoritative from the same atomic write as the Disabled-flip.
[Fact]
public async Task Resume_FirstAttemptFailedBeforeDisabledWrite_UsesFullBarrierAndPatchesHistory()
{
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var initiatedAt = DateTime.UtcNow.AddMinutes(-10); // first attempt 10min ago
var staleEarliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier; // ~4 min in the past
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = initiatedAt,
EarliestProcessingAt = staleEarliestProcessingAt,
Status = "Initiated",
});
// Build SUT where the first attempt FAILED at EnsureTenantDisabledAsync: Config
// exists but is NOT yet the offboarding tombstone (Disabled=false). The resume's
// EnsureTenantDisabledAsync will write it now → WroteNewTombstone.
var configRepoMock = new Mock<IConfigRepository>();
var notYetDisabled = TenantConfiguration.CreateDefault(TenantId);
notYetDisabled.Disabled = false;
configRepoMock.BackWithRow(notYetDisabled);
var cache = new MemoryCache(new MemoryCacheOptions());
var tenantConfigService = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: tenantConfigService,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer,
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = initiatedAt,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
// CRITICAL: visibility-delay must be the full DrainBarrier (6 min), NOT Zero. The
// History row's stale EarliestProcessingAt must NOT be honoured when Disabled was
// just written for the first time. Allow a few-ms slack because the delay is
// computed as (effectiveDeadline - DateTime.UtcNow), so a moment passes between
// the LastUpdated stamp and the delay computation.
var actualDelay = enqueuer.VisibilityDelays[0]!.Value;
Assert.InRange(actualDelay, TenantOffboardFunction.DrainBarrier - TimeSpan.FromSeconds(1), TenantOffboardFunction.DrainBarrier);
// The History row must have been patched so further re-clicks see the new deadline.
Assert.True(repo.History.ContainsKey(historyRowKey));
var patchedHistory = repo.History[historyRowKey];
Assert.NotNull(patchedHistory.EarliestProcessingAt);
Assert.True(patchedHistory.EarliestProcessingAt > DateTime.UtcNow.AddMinutes(5),
$"EarliestProcessingAt must be roughly now+6min after the patch, got {patchedHistory.EarliestProcessingAt:O}");
// ResumeOutcome's EarliestProcessingAt is the NEW deadline (UI countdown source of truth).
Assert.NotNull(outcome.EarliestProcessingAt);
Assert.True(outcome.EarliestProcessingAt > DateTime.UtcNow.AddMinutes(5));
}
[Fact]
public async Task Resume_HistoryPatchFails_SecondResumeStillRespectsBarrierViaTombstoneLastUpdated()
{
// The exact Codex Finding scenario: first resume writes tombstone successfully
// but the History.EarliestProcessingAt patch fails (transient). A second resume
// arriving moments later reads History with the OLD stale EarliestProcessingAt
// but sees the tombstone (AlreadyTombstone path). The deadline computation must
// still respect the cache-drain barrier via tombstone.LastUpdated + DrainBarrier.
//
// Without the LastUpdated-anchor fix, the second resume would compute
// visibility-delay = max(0, stale-history-deadline - now) = 0 → worker picks
// before the cache-drain barrier from the tombstone-write completes.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var firstAttempt = DateTime.UtcNow.AddMinutes(-10);
var staleHistoryDeadline = firstAttempt + TenantOffboardFunction.DrainBarrier; // ~now-4min
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = firstAttempt,
EarliestProcessingAt = staleHistoryDeadline, // STALE — patch failed before
Status = "Initiated",
});
// Tombstone-already-in-place from the prior (first) resume; LastUpdated reflects
// when Disabled was committed in that first resume = recently (now). The History
// patch had failed, so EarliestProcessingAt is still the stale firstAttempt+6min.
var configRepoMock = new Mock<IConfigRepository>();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
tombstone.LastUpdated = DateTime.UtcNow.AddSeconds(-1); // freshly written ~1s ago
configRepoMock.BackWithRow(tombstone);
var cache = new MemoryCache(new MemoryCacheOptions());
var tenantConfigService = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: tenantConfigService,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer,
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = firstAttempt,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
// SAFETY: the visibility-delay must come from tombstone.LastUpdated + DrainBarrier
// (~6min remaining), NOT from the stale history deadline (which would yield 0).
// Anything below ~5min would mean we relied on the corrupted history value.
var delay = enqueuer.VisibilityDelays[0]!.Value;
Assert.InRange(delay, TimeSpan.FromMinutes(5.5), TenantOffboardFunction.DrainBarrier);
}
[Fact]
public async Task Resume_TombstoneAlreadyInPlace_UsesRemainingBarrier_DoesNotPatchHistory()
{
// Counterpoint: tombstone was already there from the prior successful attempt;
// the original deadline still applies. NO patch of History.EarliestProcessingAt.
var repo = new FakeOffboardingAuditRepository();
var enqueuer = new RecordingEnqueuer();
var initiatedAt = DateTime.UtcNow.AddMinutes(-2); // first attempt 2min ago, mid-barrier
var originalEarliestProcessingAt = initiatedAt + TenantOffboardFunction.DrainBarrier;
var historyRowKey = "20260518091523123_" + TenantId;
await repo.InsertHistoryAsync(new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = historyRowKey,
TenantId = TenantId,
DomainName = "contoso.com",
InitiatedBy = "alice@contoso.com",
OffboardedAt = initiatedAt,
EarliestProcessingAt = originalEarliestProcessingAt,
Status = "Initiated",
});
// Build SUT where the tombstone is already in place from the prior attempt at
// initiatedAt (2min ago). The tombstone.LastUpdated reflects when Disabled was
// committed; the resume's deadline math is max(history.EarliestProcessingAt,
// tombstone.LastUpdated + DrainBarrier). Both terms ≈ initiatedAt + 6min here,
// so remaining ≈ 4min as before.
var configRepoMock = new Mock<IConfigRepository>();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
tombstone.LastUpdated = initiatedAt;
configRepoMock.BackWithRow(tombstone);
var cache = new MemoryCache(new MemoryCacheOptions());
var tenantConfigService = new TenantConfigurationService(
configRepoMock.Object,
NullLogger<TenantConfigurationService>.Instance,
cache);
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: tenantConfigService,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer,
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
var marker = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId,
TenantId = TenantId,
OffboardingHistoryRowKey = historyRowKey,
InitiatedAt = initiatedAt,
InitiatedBy = "alice@contoso.com",
Status = "Initiated",
};
var outcome = await sut.ResumeExistingMarkerAsync(marker, "alice@contoso.com", TenantId, isRace: false);
Assert.Equal(ResumeOutcomeKind.ReEnqueuedInFlight, outcome.Kind);
Assert.Single(enqueuer.VisibilityDelays);
// Remaining delay should be ~4min (6min - 2min). Allow a couple seconds of slack.
var delay = enqueuer.VisibilityDelays[0]!.Value;
Assert.InRange(delay, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.5));
// History.EarliestProcessingAt must NOT have been overwritten — it still carries
// the original deadline from the first attempt.
var historyAfter = repo.History[historyRowKey];
Assert.Equal(originalEarliestProcessingAt, historyAfter.EarliestProcessingAt);
// No write (the tombstone was already correct).
configRepoMock.VerifyNoTenantConfigWrite();
}
// ── RetryFailedOffboardingAsync (operator retry of a Failed offboarding) ──────────
private const string HistoryRowKey = "20260101000000000_" + TenantId;
private static FakeOffboardingAuditRepository SeedFailedOffboarding(
string failedPhase = "max_dequeue", DateTime? drainCompletedAt = null, int retryCount = 3)
{
var repo = new FakeOffboardingAuditRepository();
repo.Markers[TenantId] = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = HistoryRowKey,
InitiatedAt = DateTime.UtcNow.AddHours(-3),
InitiatedBy = "alice@contoso.com",
Status = "Failed",
FailedAt = DateTime.UtcNow.AddHours(-2),
FailedPhase = failedPhase,
};
repo.History[HistoryRowKey] = new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = HistoryRowKey, TenantId = TenantId,
Status = "Failed",
ErrorMessage = "Poison queue: max dequeue count reached",
RetryCount = retryCount,
DrainCompletedAt = drainCompletedAt,
EarliestProcessingAt = DateTime.UtcNow.AddHours(-3),
};
return repo;
}
[Fact]
public async Task Retry_Failed_ResetsHistoryAndMarker_KeepsDrainAnchor_ReEnqueues()
{
var drained = DateTime.UtcNow.AddHours(-2).AddMinutes(-30);
var repo = SeedFailedOffboarding(drainCompletedAt: drained, retryCount: 5);
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind);
Assert.Equal(HistoryRowKey, outcome.HistoryRowKey);
Assert.Contains("max_dequeue", outcome.Message);
var history = repo.History[HistoryRowKey];
Assert.Equal("Initiated", history.Status);
Assert.Equal(6, history.RetryCount);
Assert.Null(history.ErrorMessage);
Assert.Equal(drained, history.DrainCompletedAt); // the worker's resume anchor survives
Assert.Equal(HistoryRowKey, history.RowKey); // same run, no new history row
Assert.Single(repo.History);
var marker = repo.Markers[TenantId];
Assert.Equal("Initiated", marker.Status);
Assert.Null(marker.FailedAt);
Assert.Null(marker.FailedPhase);
// History is written before the marker (worker authority first, HTTP anchor second).
Assert.Equal(new[] { "Initiated" }, repo.HistoryWrites);
Assert.Equal(new[] { "Initiated" }, repo.MarkerWrites);
var envelope = Assert.Single(enqueuer.Sent);
Assert.Equal(TenantId, envelope.TenantId);
Assert.Equal(HistoryRowKey, envelope.HistoryRowKey);
Assert.Equal("ops@contoso.com", envelope.InitiatedBy);
}
[Fact]
public async Task Retry_Completed_Conflict_NoWrites()
{
var repo = SeedFailedOffboarding();
repo.Markers[TenantId].Status = "Completed";
repo.Markers[TenantId].FailedPhase = null;
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind);
Assert.Contains("already completed", outcome.Message);
Assert.Empty(repo.HistoryWrites);
Assert.Empty(repo.MarkerWrites);
Assert.Empty(enqueuer.Sent);
}
[Theory]
[InlineData("Initiated")]
[InlineData("InProgress")]
public async Task Retry_InFlight_Conflict_PointsAtTheOffboardResume(string status)
{
var repo = SeedFailedOffboarding();
repo.Markers[TenantId].Status = status;
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind);
Assert.Contains(status, outcome.Message);
Assert.Equal("Failed", repo.History[HistoryRowKey].Status);
Assert.Empty(repo.MarkerWrites);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Retry_NoMarker_NotFound()
{
var enqueuer = new RecordingEnqueuer();
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.NotFound, outcome.Kind);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Retry_HistoryRowMissing_FailsBeforeTouchingTheMarker()
{
var repo = SeedFailedOffboarding();
repo.History.Clear();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.HistoryMissing, outcome.Kind);
Assert.Equal("Failed", repo.Markers[TenantId].Status);
Assert.Empty(repo.MarkerWrites);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Retry_EnqueueThrows_LeavesMarkerInitiated_ForTheOffboardResume()
{
var repo = SeedFailedOffboarding();
var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") };
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.ReEnqueueFailed, outcome.Kind);
// Rows are reset: the ordinary DELETE offboard re-click now hits the Initiated resume path.
Assert.Equal("Initiated", repo.Markers[TenantId].Status);
Assert.Equal("Initiated", repo.History[HistoryRowKey].Status);
}
[Fact]
public async Task Retry_ResumesThroughTheDisabledGate_WithRemainingBarrierDelay()
{
// Tombstone written 2 minutes ago → the resume path must wait out the remaining
// 4 minutes of the 6-minute drain barrier, not restart it and not skip it.
var repo = SeedFailedOffboarding();
repo.History[HistoryRowKey].EarliestProcessingAt = DateTime.UtcNow.AddHours(-3);
var configRepoMock = new Mock<IConfigRepository>();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
tombstone.LastUpdated = DateTime.UtcNow.AddMinutes(-2);
configRepoMock.BackWithRow(tombstone);
var cache = new MemoryCache(new MemoryCacheOptions());
var service = new TenantConfigurationService(configRepoMock.Object, NullLogger<TenantConfigurationService>.Instance, cache);
var enqueuer = new RecordingEnqueuer();
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: service,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer,
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind);
var delay = Assert.Single(enqueuer.VisibilityDelays);
Assert.NotNull(delay);
Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.1));
configRepoMock.VerifyNoTenantConfigWrite();
}
[Fact]
public void BuildStatusResponse_ProjectsMarkerStatus_AndHistoryDetails()
{
var repo = SeedFailedOffboarding(retryCount: 5);
var marker = repo.Markers[TenantId];
var history = repo.History[HistoryRowKey];
var dto = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, history);
Assert.Equal("Failed", dto.Status);
Assert.Equal("max_dequeue", dto.FailedPhase);
Assert.Equal(marker.FailedAt, dto.FailedAt);
Assert.Equal(5, dto.RetryCount);
Assert.Equal(history.ErrorMessage, dto.ErrorMessage);
Assert.Equal(HistoryRowKey, dto.HistoryRowKey);
// History row gone (should not happen, but the record must still render).
var bare = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, null);
Assert.Equal("Failed", bare.Status);
Assert.Equal(0, bare.RetryCount);
Assert.Null(bare.ErrorMessage);
}
}