This commit is contained in:
Oliver Kieselbach
2026-09-19 14:40:22 +02:00
13 changed files with 896 additions and 36 deletions
@@ -1046,4 +1046,210 @@ public sealed class TenantOffboardFunctionTests
// No Save call (the tombstone was already correct).
configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Never);
}
// ── RetryFailedOffboardingAsync (operator retry of a Failed offboarding) ──────────
private const string HistoryRowKey = "20260101000000000_" + TenantId;
private static FakeOffboardingAuditRepository SeedFailedOffboarding(
string failedPhase = "max_dequeue", DateTime? drainCompletedAt = null, int retryCount = 3)
{
var repo = new FakeOffboardingAuditRepository();
repo.Markers[TenantId] = new OffboardingMarkerEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
RowKey = TenantId, TenantId = TenantId,
OffboardingHistoryRowKey = HistoryRowKey,
InitiatedAt = DateTime.UtcNow.AddHours(-3),
InitiatedBy = "alice@contoso.com",
Status = "Failed",
FailedAt = DateTime.UtcNow.AddHours(-2),
FailedPhase = failedPhase,
};
repo.History[HistoryRowKey] = new OffboardingHistoryEntry
{
PartitionKey = Constants.OffboardingPartitionKeys.History,
RowKey = HistoryRowKey, TenantId = TenantId,
Status = "Failed",
ErrorMessage = "Poison queue: max dequeue count reached",
RetryCount = retryCount,
DrainCompletedAt = drainCompletedAt,
EarliestProcessingAt = DateTime.UtcNow.AddHours(-3),
};
return repo;
}
[Fact]
public async Task Retry_Failed_ResetsHistoryAndMarker_KeepsDrainAnchor_ReEnqueues()
{
var drained = DateTime.UtcNow.AddHours(-2).AddMinutes(-30);
var repo = SeedFailedOffboarding(drainCompletedAt: drained, retryCount: 5);
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind);
Assert.Equal(HistoryRowKey, outcome.HistoryRowKey);
Assert.Contains("max_dequeue", outcome.Message);
var history = repo.History[HistoryRowKey];
Assert.Equal("Initiated", history.Status);
Assert.Equal(6, history.RetryCount);
Assert.Null(history.ErrorMessage);
Assert.Equal(drained, history.DrainCompletedAt); // the worker's resume anchor survives
Assert.Equal(HistoryRowKey, history.RowKey); // same run, no new history row
Assert.Single(repo.History);
var marker = repo.Markers[TenantId];
Assert.Equal("Initiated", marker.Status);
Assert.Null(marker.FailedAt);
Assert.Null(marker.FailedPhase);
// History is written before the marker (worker authority first, HTTP anchor second).
Assert.Equal(new[] { "Initiated" }, repo.HistoryWrites);
Assert.Equal(new[] { "Initiated" }, repo.MarkerWrites);
var envelope = Assert.Single(enqueuer.Sent);
Assert.Equal(TenantId, envelope.TenantId);
Assert.Equal(HistoryRowKey, envelope.HistoryRowKey);
Assert.Equal("ops@contoso.com", envelope.InitiatedBy);
}
[Fact]
public async Task Retry_Completed_Conflict_NoWrites()
{
var repo = SeedFailedOffboarding();
repo.Markers[TenantId].Status = "Completed";
repo.Markers[TenantId].FailedPhase = null;
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind);
Assert.Contains("already completed", outcome.Message);
Assert.Empty(repo.HistoryWrites);
Assert.Empty(repo.MarkerWrites);
Assert.Empty(enqueuer.Sent);
}
[Theory]
[InlineData("Initiated")]
[InlineData("InProgress")]
public async Task Retry_InFlight_Conflict_PointsAtTheOffboardResume(string status)
{
var repo = SeedFailedOffboarding();
repo.Markers[TenantId].Status = status;
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind);
Assert.Contains(status, outcome.Message);
Assert.Equal("Failed", repo.History[HistoryRowKey].Status);
Assert.Empty(repo.MarkerWrites);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Retry_NoMarker_NotFound()
{
var enqueuer = new RecordingEnqueuer();
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.NotFound, outcome.Kind);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Retry_HistoryRowMissing_FailsBeforeTouchingTheMarker()
{
var repo = SeedFailedOffboarding();
repo.History.Clear();
var enqueuer = new RecordingEnqueuer();
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.HistoryMissing, outcome.Kind);
Assert.Equal("Failed", repo.Markers[TenantId].Status);
Assert.Empty(repo.MarkerWrites);
Assert.Empty(enqueuer.Sent);
}
[Fact]
public async Task Retry_EnqueueThrows_LeavesMarkerInitiated_ForTheOffboardResume()
{
var repo = SeedFailedOffboarding();
var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") };
var sut = Build(repo, enqueuer);
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.ReEnqueueFailed, outcome.Kind);
// Rows are reset: the ordinary DELETE offboard re-click now hits the Initiated resume path.
Assert.Equal("Initiated", repo.Markers[TenantId].Status);
Assert.Equal("Initiated", repo.History[HistoryRowKey].Status);
}
[Fact]
public async Task Retry_ResumesThroughTheDisabledGate_WithRemainingBarrierDelay()
{
// Tombstone written 2 minutes ago → the resume path must wait out the remaining
// 4 minutes of the 6-minute drain barrier, not restart it and not skip it.
var repo = SeedFailedOffboarding();
repo.History[HistoryRowKey].EarliestProcessingAt = DateTime.UtcNow.AddHours(-3);
var configRepoMock = new Mock<IConfigRepository>();
var tombstone = TenantConfiguration.CreateDefault(TenantId);
tombstone.Disabled = true;
tombstone.DisabledReason = "Offboarding in progress";
tombstone.LastUpdated = DateTime.UtcNow.AddMinutes(-2);
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>())).ReturnsAsync(tombstone);
var cache = new MemoryCache(new MemoryCacheOptions());
var service = new TenantConfigurationService(configRepoMock.Object, NullLogger<TenantConfigurationService>.Instance, cache);
var enqueuer = new RecordingEnqueuer();
var sut = new TenantOffboardFunction(
logger: NullLogger<TenantOffboardFunction>.Instance,
configRepo: configRepoMock.Object,
tenantConfigService: service,
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
offboardingRepo: repo,
offboardingEnqueuer: enqueuer,
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind);
var delay = Assert.Single(enqueuer.VisibilityDelays);
Assert.NotNull(delay);
Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.1));
configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>(), It.IsAny<string>(), It.IsAny<string>()), Times.Never);
}
[Fact]
public void BuildStatusResponse_ProjectsMarkerStatus_AndHistoryDetails()
{
var repo = SeedFailedOffboarding(retryCount: 5);
var marker = repo.Markers[TenantId];
var history = repo.History[HistoryRowKey];
var dto = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, history);
Assert.Equal("Failed", dto.Status);
Assert.Equal("max_dequeue", dto.FailedPhase);
Assert.Equal(marker.FailedAt, dto.FailedAt);
Assert.Equal(5, dto.RetryCount);
Assert.Equal(history.ErrorMessage, dto.ErrorMessage);
Assert.Equal(HistoryRowKey, dto.HistoryRowKey);
// History row gone (should not happen, but the record must still render).
var bare = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, null);
Assert.Equal("Failed", bare.Status);
Assert.Equal(0, bare.RetryCount);
Assert.Null(bare.ErrorMessage);
}
}
@@ -888,4 +888,75 @@ public class TypedContractCompletionWireParityTests
},
});
}
// ── global/tenants/{tenantId}/offboarding ────────────────────────────────────────────
[Fact]
public void TenantOffboardingStatusResponse_pins_the_failed_record_shape()
{
var initiated = new DateTime(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc);
var failed = initiated.AddMinutes(30);
AssertWireIdentical(
new
{
tenantId = "11111111-1111-1111-1111-111111111111",
status = "Failed",
historyRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
initiatedAt = initiated,
initiatedBy = "alice@contoso.com",
retryCount = 5,
earliestProcessingAt = (DateTime?)initiated.AddMinutes(6),
drainCompletedAt = (DateTime?)initiated.AddMinutes(8),
completedAt = (DateTime?)null,
failedAt = (DateTime?)failed,
failedPhase = (string?)"max_dequeue",
errorMessage = (string?)"Poison queue: max dequeue count reached",
},
new TenantOffboardingStatusResponse
{
TenantId = "11111111-1111-1111-1111-111111111111",
Status = "Failed",
HistoryRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
InitiatedAt = initiated,
InitiatedBy = "alice@contoso.com",
RetryCount = 5,
EarliestProcessingAt = initiated.AddMinutes(6),
DrainCompletedAt = initiated.AddMinutes(8),
CompletedAt = null,
FailedAt = failed,
FailedPhase = "max_dequeue",
ErrorMessage = "Poison queue: max dequeue count reached",
});
}
[Fact]
public void TenantOffboardingStatusResponse_keeps_null_slots_for_an_in_flight_record()
{
var initiated = new DateTime(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc);
AssertWireIdentical(
new
{
tenantId = "11111111-1111-1111-1111-111111111111",
status = "InProgress",
historyRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
initiatedAt = initiated,
initiatedBy = "alice@contoso.com",
retryCount = 0,
earliestProcessingAt = (DateTime?)initiated.AddMinutes(6),
drainCompletedAt = (DateTime?)null,
completedAt = (DateTime?)null,
failedAt = (DateTime?)null,
failedPhase = (string?)null,
errorMessage = (string?)null,
},
new TenantOffboardingStatusResponse
{
TenantId = "11111111-1111-1111-1111-111111111111",
Status = "InProgress",
HistoryRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
InitiatedAt = initiated,
InitiatedBy = "alice@contoso.com",
EarliestProcessingAt = initiated.AddMinutes(6),
});
}
}
@@ -506,6 +506,194 @@ public class TenantOffboardFunction
EarliestProcessingAt: earliestProcessingAtForResponse);
}
// ── Offboarding record + operator retry (Global Admin tooling) ────────────────────
/// <summary>
/// GET /api/global/tenants/{tenantId}/offboarding — the tenant's current offboarding record
/// (marker + history) for the admin tenant editor. 404 without a marker: the tenant was
/// never offboarded, or the Completed marker is already cleaned up
/// (<see cref="OffboardingMarkerCleanupFunction"/>, 15-min grace) — the normal end state,
/// nothing for an operator to do. Global Admin / Global Reader only (middleware-enforced).
/// </summary>
[Function("GetTenantOffboardingStatus")]
[Authorize]
public async Task<HttpResponseData> GetTenantOffboardingStatus(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/tenants/{tenantId}/offboarding")] HttpRequestData req,
string tenantId,
FunctionContext context)
{
var targetTenantId = context.GetRequestContext().TargetTenantId;
if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
{
return await req.BadRequestAsync("tenantId must be a valid GUID");
}
var normalizedTenantId = targetTenantId.ToLowerInvariant();
var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
if (marker == null)
{
return await req.NotFoundAsync("No offboarding on record for this tenant");
}
var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey);
return await req.OkAsync(BuildStatusResponse(normalizedTenantId, marker, history));
}
internal static TenantOffboardingStatusResponse BuildStatusResponse(
string normalizedTenantId, OffboardingMarkerEntry marker, OffboardingHistoryEntry? history)
=> new()
{
TenantId = normalizedTenantId,
Status = marker.Status,
HistoryRowKey = marker.OffboardingHistoryRowKey,
InitiatedAt = marker.InitiatedAt,
InitiatedBy = marker.InitiatedBy,
RetryCount = history?.RetryCount ?? 0,
EarliestProcessingAt = history?.EarliestProcessingAt,
DrainCompletedAt = history?.DrainCompletedAt,
CompletedAt = marker.CompletedAt,
FailedAt = marker.FailedAt,
FailedPhase = marker.FailedPhase,
ErrorMessage = history?.ErrorMessage,
};
/// <summary>
/// POST /api/global/tenants/{tenantId}/offboarding/retry — re-drives an offboarding whose
/// marker is Failed. Failed is fail-closed by design (the worker returns, the offboard
/// re-click refuses to re-enqueue), so this is the explicit operator action that design
/// asks for; nothing retries on its own. History and marker go back to Initiated — the
/// History row keeps its RowKey and <c>DrainCompletedAt</c> (the worker's resume anchor)
/// and counts the retry — and the ordinary resume path re-affirms the Disabled-gate,
/// computes the remaining drain barrier and enqueues the envelope. Global Admin only.
/// </summary>
[Function("RetryTenantOffboarding")]
[Authorize]
public async Task<HttpResponseData> RetryTenantOffboarding(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/tenants/{tenantId}/offboarding/retry")] HttpRequestData req,
string tenantId,
FunctionContext context)
{
var requestCtx = context.GetRequestContext();
var upn = requestCtx.UserPrincipalName;
var targetTenantId = requestCtx.TargetTenantId;
if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
{
return await req.BadRequestAsync("tenantId must be a valid GUID");
}
var normalizedTenantId = targetTenantId.ToLowerInvariant();
var retry = await RetryFailedOffboardingAsync(normalizedTenantId, upn);
switch (retry.Kind)
{
case RetryOutcomeKind.Retried:
{
var response = req.CreateResponse(HttpStatusCode.Accepted);
await response.WriteAsJsonAsync(new OffboardResponse
{
TenantId = normalizedTenantId,
Status = "Queued",
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
HistoryRowKey = retry.HistoryRowKey ?? string.Empty,
EarliestProcessingAt = retry.EarliestProcessingAt,
Message = retry.Message,
});
return response;
}
case RetryOutcomeKind.NotFound:
return await req.NotFoundAsync(retry.Message);
case RetryOutcomeKind.NotFailed:
return await req.ConflictAsync(retry.Message);
default:
return await Build500Async(req, retry.Message);
}
}
/// <summary>
/// Pure retry decision behind <see cref="RetryTenantOffboarding"/>; internal for unit tests.
/// Write order matters: History first (the worker's authority — a Failed history makes
/// it return early), Marker second (the HTTP idempotency anchor). A crash in between
/// leaves History=Initiated + Marker=Failed, and the next retry click resolves it the
/// same way. The worker's first pickup then flips History/Pointer/Marker to InProgress.
/// </summary>
internal async Task<RetryOutcome> RetryFailedOffboardingAsync(string normalizedTenantId, string upn)
{
var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
if (marker == null)
{
return new RetryOutcome(RetryOutcomeKind.NotFound, "No offboarding on record for this tenant");
}
if (!string.Equals(marker.Status, "Failed", StringComparison.OrdinalIgnoreCase))
{
var message = string.Equals(marker.Status, "Completed", StringComparison.OrdinalIgnoreCase)
? "Tenant offboarding already completed — nothing to retry"
: $"Tenant offboarding is {marker.Status} — not Failed; re-run the offboard action to resume it";
_logger.LogInformation(
"TenantOffboard retry for {TenantId} refused: marker status={Status}", normalizedTenantId, marker.Status);
return new RetryOutcome(RetryOutcomeKind.NotFailed, message, marker.OffboardingHistoryRowKey);
}
var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey);
if (history == null)
{
_logger.LogError(
"TenantOffboard retry for {TenantId}: History row {History} is missing — operator inspection required",
normalizedTenantId, marker.OffboardingHistoryRowKey);
return new RetryOutcome(
RetryOutcomeKind.HistoryMissing,
"Offboarding history row is missing — operator inspection required",
marker.OffboardingHistoryRowKey);
}
var failedPhase = marker.FailedPhase ?? "unknown";
history.Status = "Initiated";
history.RetryCount += 1;
history.ErrorMessage = null;
await _offboardingRepo.UpsertHistoryAsync(history);
marker.Status = "Initiated";
marker.FailedAt = null;
marker.FailedPhase = null;
await _offboardingRepo.UpsertMarkerAsync(marker);
_logger.LogWarning(
"TENANT OFFBOARD RETRY for {TenantId} by {Upn}: previous failure phase={Phase}; historyRowKey={History} retryCount={RetryCount}",
normalizedTenantId, upn, failedPhase, history.RowKey, history.RetryCount);
// Global audit row (survives the TenantConfiguration wipe like the initiation's DELETE row).
await _maintenanceRepo.LogAuditEntryAsync(
Constants.AuditGlobalTenantId,
"RETRY",
"TenantOffboarding",
normalizedTenantId,
upn,
new Dictionary<string, string>
{
["failedPhase"] = failedPhase,
["historyRowKey"] = history.RowKey,
["retryCount"] = history.RetryCount.ToString(System.Globalization.CultureInfo.InvariantCulture),
});
var resume = await ResumeExistingMarkerAsync(marker, upn, normalizedTenantId, isRace: false);
if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed)
{
// Marker is Initiated now, so the ordinary offboard re-click resumes it.
return new RetryOutcome(
RetryOutcomeKind.ReEnqueueFailed,
"Failed to re-enqueue tenant offboarding — re-run the offboard action to resume it",
history.RowKey);
}
return new RetryOutcome(
RetryOutcomeKind.Retried,
$"Tenant offboarding retried after the failure in phase '{failedPhase}'",
history.RowKey,
resume.EarliestProcessingAt);
}
/// <summary>
/// Magic-string DisabledReason that identifies a TenantConfiguration row as the
/// offboarding tombstone. Used by <c>EnsureTenantDisabledAsync</c> for the idempotent
@@ -775,3 +963,24 @@ internal enum ResumeOutcomeKind
/// <summary>Marker classified as in-flight but re-enqueue threw; surface 500.</summary>
ReEnqueueFailed,
}
/// <summary>Result of <see cref="TenantOffboardFunction.RetryFailedOffboardingAsync"/>.</summary>
internal sealed record RetryOutcome(
RetryOutcomeKind Kind,
string Message,
string? HistoryRowKey = null,
DateTime? EarliestProcessingAt = null);
internal enum RetryOutcomeKind
{
/// <summary>Marker was Failed; history + marker reset to Initiated and the envelope re-enqueued (202).</summary>
Retried,
/// <summary>No marker on record (404).</summary>
NotFound,
/// <summary>Marker is Completed or in flight — nothing to retry (409).</summary>
NotFailed,
/// <summary>Marker points at a History row that no longer exists (500).</summary>
HistoryMissing,
/// <summary>Rows were reset but the re-enqueue threw (500); the offboard re-click resumes.</summary>
ReEnqueueFailed,
}
@@ -428,6 +428,11 @@ public static class EndpointAccessPolicyCatalog
new("GET", "global/sessions/{sessionId}/deletion-manifest", EndpointPolicy.GlobalReadOrAdmin),
// Platform-operational (cascade-delete restore prep) — GA + read-only Reader only, NOT delegated.
new("GET", "global/tenants/{tenantId}/deletion-manifests", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true),
// Offboarding record (marker + history) behind the tenant editor's status card, and the
// operator retry of a Failed offboarding. Platform-operational: GA + Reader read, GA-only
// retry, never delegated (an MSP manages active tenants, not the platform's offboarding).
new("GET", "global/tenants/{tenantId}/offboarding", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true),
new("POST", "global/tenants/{tenantId}/offboarding/retry", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam, excludeDelegated: true),
new("GET", "global/session-deletions", EndpointPolicy.GlobalReadOrAdmin),
// Manual maintenance-run trigger (GC sweeps + retention fanout) — mutating, GA only.
new("POST", "global/session-deletions/maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
@@ -4994,6 +4994,23 @@ export interface TenantOffboardingCustomsArchiveEntry {
archivedBy: string;
}
/** Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one. */
export interface TenantOffboardingStatusResponse {
tenantId: string;
status: string;
historyRowKey: string;
initiatedAt: string;
initiatedBy: string;
/** Queue dequeue attempts plus operator retries of this run. */
retryCount: number;
earliestProcessingAt?: string;
drainCompletedAt?: string;
completedAt?: string;
failedAt?: string;
failedPhase?: string;
errorMessage?: string;
}
/** Body of POST rules/gather/test-pattern. */
export interface TestLogPatternRequest {
pattern?: string | null;
@@ -23,6 +23,30 @@ namespace AutopilotMonitor.Shared.Models
public DateTime? EarliestProcessingAt { get; set; }
}
/// <summary>
/// Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding
/// record for the admin tenant editor. <see cref="Status"/> is the marker's status (the
/// anchor every HTTP decision keys on); the history fields describe the run behind it.
/// A Failed record carries <see cref="FailedPhase"/> so the operator sees what to expect
/// from a retry before starting one.
/// </summary>
public class TenantOffboardingStatusResponse : IApiResponse
{
public string TenantId { get; set; } = string.Empty;
public string Status { get; set; } = string.Empty;
public string HistoryRowKey { get; set; } = string.Empty;
public DateTime InitiatedAt { get; set; }
public string InitiatedBy { get; set; } = string.Empty;
/// <summary>Queue dequeue attempts plus operator retries of this run.</summary>
public int RetryCount { get; set; }
public DateTime? EarliestProcessingAt { get; set; }
public DateTime? DrainCompletedAt { get; set; }
public DateTime? CompletedAt { get; set; }
public DateTime? FailedAt { get; set; }
public string? FailedPhase { get; set; }
public string? ErrorMessage { get; set; }
}
/// <summary>Body of POST tenants/{tenantId}/offboard/feedback.</summary>
public class SubmitOffboardingFeedbackRequest : IApiRequest
{
@@ -12,6 +12,9 @@ interface OffboardTenantConfirmDialogProps {
error: string | null;
onCancel: () => void;
onConfirm: () => void;
/** Set when the dialog confirms the RETRY of a Failed offboarding instead of a new one:
* the phase the previous run died in, shown so the operator knows what the retry re-runs. */
retryOfFailedPhase?: string | null;
}
/**
@@ -27,26 +30,37 @@ export function OffboardTenantConfirmDialog({
error,
onCancel,
onConfirm,
retryOfFailedPhase,
}: OffboardTenantConfirmDialogProps) {
const [confirmText, setConfirmText] = useState("");
const armed = confirmText === "OFFBOARD";
const retry = retryOfFailedPhase !== undefined && retryOfFailedPhase !== null;
return (
<ModalPortal>
<div className="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-[60] p-4">
<div className="bg-white rounded-lg shadow-xl max-w-lg w-full">
<div className="p-5 rounded-t-lg text-white bg-red-600">
<h2 className="text-lg font-bold">Offboard tenant</h2>
<h2 className="text-lg font-bold">{retry ? "Retry offboarding" : "Offboard tenant"}</h2>
<p className="text-sm opacity-90 mt-0.5">{tenantLabel}</p>
<p className="text-xs opacity-75">{tenantId}</p>
</div>
<div className="p-5 space-y-3 text-sm text-gray-700">
<p>
This queues the offboarding cascade: the tenant is <strong>suspended immediately</strong>,
and after a short drain window <strong>all of its data is permanently deleted</strong> —
sessions, events, rules, admins, and the tenant configuration itself.
</p>
{retry ? (
<p>
The previous run failed in phase <span className="font-mono font-semibold">{retryOfFailedPhase}</span>.
This re-queues the same offboarding: the tenant stays suspended, the cascade re-runs its
remaining phases (every step is idempotent) and <strong>permanently deletes whatever data
is left</strong> — including the tenant configuration itself.
</p>
) : (
<p>
This queues the offboarding cascade: the tenant is <strong>suspended immediately</strong>,
and after a short drain window <strong>all of its data is permanently deleted</strong> —
sessions, events, rules, admins, and the tenant configuration itself.
</p>
)}
<div className="bg-red-50 border border-red-200 rounded-lg p-3 text-red-800">
<strong>This cannot be undone,</strong> and it is <strong>not a ban</strong>: because the
configuration row (including any suspension) is deleted, a new sign-in from this tenant
@@ -69,7 +83,7 @@ export function OffboardTenantConfirmDialog({
</div>
{error && (
<div role="alert" className="bg-red-100 border border-red-300 rounded-lg p-3 text-red-800">
<strong>Offboarding failed:</strong>{" "}{error}
<strong>{retry ? "Retry failed:" : "Offboarding failed:"}</strong>{" "}{error}
</div>
)}
</div>
@@ -90,10 +104,10 @@ export function OffboardTenantConfirmDialog({
{saving ? (
<>
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-white"></div>
<span>Offboarding…</span>
<span>{retry ? "Retrying…" : "Offboarding…"}</span>
</>
) : (
<span>Offboard tenant</span>
<span>{retry ? "Retry offboarding" : "Offboard tenant"}</span>
)}
</button>
</div>
@@ -14,6 +14,7 @@ import type {
SaveNotificationEmailRequest,
SetTenantPlanTierResponse,
TenantConfiguration as WireTenantConfiguration,
TenantOffboardingStatusResponse,
UpdateTenantAppHomingResponse,
UpdateTenantConfigurationResponse,
} from "@/utils/wire-types.generated";
@@ -36,7 +37,9 @@ import {
effectiveEdition,
facetCounts,
facetOption,
isOffboardingTombstone,
matchesTenantFilters,
onWaitlist,
planOverview,
toggleFacetValue,
visibleFacets,
@@ -195,6 +198,37 @@ function TenantManagementSectionInner({
cancelled = true;
};
}, [editingTenantId, getAccessToken]);
// Offboarding record, loaded only for a row the cascade owns (the Disabled-gate tombstone).
// Keyed by the response's tenantId rather than reset on every editor switch: a record is
// only ever rendered for the tenant it names (editingRecord below), so a stale one from a
// previously opened editor can never show. null = no marker (never offboarded, or
// Completed and already cleaned up).
const [offboardingRecord, setOffboardingRecord] = useState<TenantOffboardingStatusResponse | null>(null);
const [retryDialogOpen, setRetryDialogOpen] = useState(false);
const [retrying, setRetrying] = useState(false);
const [retryError, setRetryError] = useState<string | null>(null);
const editingIsOffboarding = !!editingTenant && isOffboardingTombstone(editingTenant);
const editingRecord =
editingIsOffboarding && offboardingRecord && editingTenantId
&& offboardingRecord.tenantId.toLowerCase() === editingTenantId.toLowerCase()
? offboardingRecord
: null;
useEffect(() => {
// A 404 (no marker) and any other failure both leave the card hidden: the Danger Zone then
// shows the ordinary Offboard action, whose own resume/refusal messages still apply.
if (!editingTenantId || !editingIsOffboarding) return;
let cancelled = false;
fetchJson<TenantOffboardingStatusResponse>(api.tenants.offboardingStatus(editingTenantId), getAccessToken)
.then((record) => {
if (!cancelled) setOffboardingRecord(record);
})
.catch(() => {
if (!cancelled) setOffboardingRecord(null);
});
return () => {
cancelled = true;
};
}, [editingTenantId, editingIsOffboarding, getAccessToken]);
const [homingDialogTarget, setHomingDialogTarget] = useState<"primary" | "legacy" | null>(null);
const [savingHoming, setSavingHoming] = useState(false);
// Failures of the homing/offboard calls go into these instead of the page-level
@@ -231,7 +265,7 @@ function TenantManagementSectionInner({
// Statistics (always over all tenants, not filtered)
const readyCount = tenants.filter(t => t.validateAutopilotDevice).length;
const waitlistCount = tenants.filter(t => !previewApproved.has(t.tenantId)).length;
const waitlistCount = tenants.filter(t => onWaitlist(t, filterCtx)).length;
const totalCount = tenants.length;
const plans = planOverview(tenants, nowMs);
@@ -411,6 +445,34 @@ function TenantManagementSectionInner({
}
};
// Operator retry of a Failed offboarding (POST global/tenants/{id}/offboarding/retry). Failed
// is fail-closed on the backend: the worker returns and the DELETE re-click refuses, so this
// is the one path that re-drives the cascade. Same confirm dialog, retry variant.
const handleRetryOffboarding = async (tenant: TenantConfiguration) => {
if (!canMutate) return; // read-only Global Reader
try {
setRetrying(true);
setRetryError(null);
setSuccessMessage(null);
const data = await fetchJson<OffboardResponse>(
api.tenants.offboardingRetry(tenant.tenantId), getAccessToken, { method: "POST" });
trackEvent("admin_tenant_offboard_retried", {
tenantId: tenant.tenantId,
failedPhase: editingRecord?.failedPhase ?? "unknown",
});
setRetryDialogOpen(false);
setEditingTenant(null);
fetchTenants();
setSuccessMessage(data.message || `Offboarding retried for tenant ${tenant.tenantId}`);
setTimeout(() => setSuccessMessage(null), 8000);
} catch (err) {
console.error("Error retrying tenant offboarding:", err);
setRetryError(apiErrorText(err, "Failed to retry the offboarding"));
} finally {
setRetrying(false);
}
};
const handleSendWelcomeEmail = async (tenantId: string, email?: string) => {
if (!canMutate) return; // read-only Global Reader
try {
@@ -593,7 +655,14 @@ function TenantManagementSectionInner({
</div>
<div className="flex flex-wrap items-center gap-2">
<div className="flex flex-wrap items-center gap-2">
{tenant.disabled && (
{isOffboardingTombstone(tenant) ? (
<span
className={`inline-flex items-center px-2 py-1 rounded-full text-xs font-medium ${facetOption("status", "offboarding")?.badgeClass}`}
title="The offboarding cascade owns this row — open the editor for its status"
>
Offboarding
</span>
) : tenant.disabled && (
<span className="inline-flex items-center px-2 py-1 rounded-full text-xs font-medium bg-red-100 text-red-800">
Suspended
</span>
@@ -606,7 +675,7 @@ function TenantManagementSectionInner({
MCP off
</span>
)}
{!previewApproved.has(tenant.tenantId) && (
{onWaitlist(tenant, filterCtx) && (
<span className="inline-flex items-center px-2 py-1 rounded-full text-xs font-medium bg-amber-100 text-amber-800">
Waitlist
</span>
@@ -1265,28 +1334,103 @@ function TenantManagementSectionInner({
</div>
</div>
{/* Danger Zone — offboarding cascade (own path: DELETE tenants/{id}/offboard;
deliberately NOT part of the modal's Save button) */}
<div className="bg-red-50 border-2 border-red-300 rounded-lg p-4">
<h3 className="font-semibold text-red-900 mb-1">Offboard Tenant</h3>
<p className="text-xs text-gray-600 mb-2">
Suspends the tenant immediately and permanently deletes all of its data
(sessions, events, rules, admins, configuration) after a short drain window.
Same cascade as the tenant&apos;s self-service offboarding.
</p>
<p className="text-xs text-red-700 font-medium mb-3">
Not a ban: the deletion includes the suspension, so once the cascade completes a
new sign-in re-onboards (and auto-activates) the tenant. To lock a tenant out,
suspend it above and leave its data in place.
</p>
<button
onClick={() => { setOffboardError(null); setOffboardDialogOpen(true); }}
disabled={!canMutate || offboarding}
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 disabled:opacity-50 text-sm font-medium"
>
Offboard Tenant…
</button>
</div>
{/* Offboarding record — shown instead of the Offboard action while the cascade owns
the row. Failed is a dead end on the backend until an operator retries here. */}
{editingRecord ? (
<div className="bg-red-50 border-2 border-red-300 rounded-lg p-4">
<div className="flex items-center justify-between mb-2">
<h3 className="font-semibold text-red-900">Offboarding</h3>
<span
className={`inline-flex items-center px-2 py-1 rounded-full text-xs font-medium ${
editingRecord.status === "Failed"
? "bg-red-100 text-red-800"
: editingRecord.status === "Completed"
? "bg-green-100 text-green-800"
: "bg-amber-100 text-amber-800"
}`}
>
{editingRecord.status}
</span>
</div>
<dl className="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1 text-xs text-gray-700 mb-3">
<dt className="text-gray-500">Started</dt>
<dd>{new Date(editingRecord.initiatedAt).toLocaleString()} by {editingRecord.initiatedBy}</dd>
<dt className="text-gray-500">Attempts</dt>
<dd>{editingRecord.retryCount}</dd>
{editingRecord.drainCompletedAt && (
<>
<dt className="text-gray-500">Drain done</dt>
<dd>{new Date(editingRecord.drainCompletedAt).toLocaleString()}</dd>
</>
)}
{editingRecord.status === "Failed" && (
<>
<dt className="text-gray-500">Failed</dt>
<dd>
{editingRecord.failedAt ? new Date(editingRecord.failedAt).toLocaleString() : "—"}
{" "}in phase <span className="font-mono">{editingRecord.failedPhase || "unknown"}</span>
</dd>
{editingRecord.errorMessage && (
<>
<dt className="text-gray-500">Error</dt>
<dd className="break-words">{editingRecord.errorMessage}</dd>
</>
)}
</>
)}
</dl>
{editingRecord.status === "Failed" ? (
<>
<p className="text-xs text-red-700 font-medium mb-3">
The cascade stopped and will not resume on its own. The tenant stays suspended and
its remaining data stays in place until the offboarding is retried. Fix the cause
first if the phase points at one (the backend log carries the details).
</p>
<button
onClick={() => { setRetryError(null); setRetryDialogOpen(true); }}
disabled={!canMutate || retrying}
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 disabled:opacity-50 text-sm font-medium"
>
Retry offboarding…
</button>
</>
) : editingRecord.status === "Completed" ? (
<p className="text-xs text-gray-600">
The cascade has finished; the tenant row disappears once the marker&apos;s grace window
has passed.
</p>
) : (
<p className="text-xs text-gray-600">
The cascade is running. Re-running the Offboard action would only re-queue it; nothing to do here.
</p>
)}
</div>
) : (
<>
{/* Danger Zone — offboarding cascade (own path: DELETE tenants/{id}/offboard;
deliberately NOT part of the modal's Save button) */}
<div className="bg-red-50 border-2 border-red-300 rounded-lg p-4">
<h3 className="font-semibold text-red-900 mb-1">Offboard Tenant</h3>
<p className="text-xs text-gray-600 mb-2">
Suspends the tenant immediately and permanently deletes all of its data
(sessions, events, rules, admins, configuration) after a short drain window.
Same cascade as the tenant&apos;s self-service offboarding.
</p>
<p className="text-xs text-red-700 font-medium mb-3">
Not a ban: the deletion includes the suspension, so once the cascade completes a
new sign-in re-onboards (and auto-activates) the tenant. To lock a tenant out,
suspend it above and leave its data in place.
</p>
<button
onClick={() => { setOffboardError(null); setOffboardDialogOpen(true); }}
disabled={!canMutate || offboarding}
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 disabled:opacity-50 text-sm font-medium"
>
Offboard Tenant…
</button>
</div>
</>
)}
</div>
@@ -1331,6 +1475,19 @@ function TenantManagementSectionInner({
/>
)}
{/* Retry confirmation for a Failed offboarding (same dialog, retry variant) */}
{editingTenant && retryDialogOpen && editingRecord && (
<OffboardTenantConfirmDialog
tenantLabel={editingTenant.domainName || editingTenant.tenantId}
tenantId={editingTenant.tenantId}
saving={retrying}
error={retryError}
retryOfFailedPhase={editingRecord.failedPhase || "unknown"}
onCancel={() => setRetryDialogOpen(false)}
onConfirm={() => handleRetryOffboarding(editingTenant)}
/>
)}
{/* App-homing flip confirmation (renders above the editor modal) */}
{editingTenant && homingDialogTarget && (
<AppHomingConfirmDialog
@@ -6,7 +6,9 @@ import {
effectiveEdition,
facetCounts,
matchesTenantFilters,
onWaitlist,
planOverview,
tenantFacetValues,
toggleFacetValue,
trialState,
visibleFacets,
@@ -117,6 +119,21 @@ describe("matchesTenantFilters", () => {
expect(matchesTenantFilters(t, filters({ status: ["suspended"] }), waitlisted)).toBe(false);
});
it("an offboarding tombstone is Offboarding — neither Waitlist nor Suspended", () => {
const t = tenant({ tenantId: "w", disabled: true, disabledReason: "Offboarding in progress" });
const waitlisted = ctx({ isWaitlisted: (id) => id === "w" });
expect(tenantFacetValues(t, waitlisted).status).toEqual(["offboarding"]);
expect(onWaitlist(t, waitlisted)).toBe(false);
expect(matchesTenantFilters(t, filters({ status: ["offboarding"] }), waitlisted)).toBe(true);
expect(matchesTenantFilters(t, filters({ status: ["waitlist"] }), waitlisted)).toBe(false);
expect(matchesTenantFilters(t, filters({ status: ["suspended"] }), waitlisted)).toBe(false);
// An admin suspension with any other reason stays Suspended (+ Waitlist while unactivated).
const suspended = tenant({ tenantId: "w", disabled: true, disabledReason: "abuse" });
expect(tenantFacetValues(suspended, waitlisted).status).toEqual(["waitlist", "suspended"]);
expect(onWaitlist(suspended, waitlisted)).toBe(true);
});
it("classifies the app registration through the context", () => {
const newApp = tenant({ homedAppClientId: PRIMARY });
const legacyApp = tenant({ homedAppClientId: null });
@@ -28,6 +28,7 @@ export interface TenantFilterFields {
payingCustomer: boolean;
homedAppClientId?: string | null;
disabled: boolean;
disabledReason?: string | null;
mcpDisabled: boolean;
validateAutopilotDevice: boolean;
}
@@ -111,6 +112,7 @@ export const FACETS: readonly FacetDefinition[] = [
options: [
{ value: "ready", label: "Ready", badgeClass: "bg-blue-100 text-blue-800" },
{ value: "waitlist", label: "Waitlist", badgeClass: AMBER },
{ value: "offboarding", label: "Offboarding", badgeClass: "bg-orange-100 text-orange-800" },
{ value: "suspended", label: "Suspended", badgeClass: "bg-red-100 text-red-800" },
{ value: "mcp-off", label: "MCP off", badgeClass: AMBER },
],
@@ -155,12 +157,31 @@ export function trialState(t: TenantFilterFields, nowMs: number): TrialState {
return "never";
}
/**
* DisabledReason the offboarding cascade writes on the tenant row (Phase 1 Disabled-gate,
* `TenantOffboardFunction.OffboardingDisabledReason`). While it is set the row is the
* cascade's tombstone: the tenant is neither suspended by an admin nor waiting for activation
* (its whitelist row is wiped early in the cascade), it is being deleted.
*/
export const OFFBOARDING_TOMBSTONE_REASON = "Offboarding in progress";
export function isOffboardingTombstone(t: Pick<TenantFilterFields, "disabled" | "disabledReason">): boolean {
return t.disabled && t.disabledReason === OFFBOARDING_TOMBSTONE_REASON;
}
/** Waitlist = not activated yet, and not a row the offboarding cascade is deleting. */
export function onWaitlist(t: TenantFilterFields, ctx: TenantFilterContext): boolean {
return !isOffboardingTombstone(t) && ctx.isWaitlisted(t.tenantId);
}
/** Every facet value a tenant carries; Status is the only multi-valued facet. */
export function tenantFacetValues(t: TenantFilterFields, ctx: TenantFilterContext): Record<FacetKey, readonly string[]> {
const status: string[] = [];
const offboarding = isOffboardingTombstone(t);
if (t.validateAutopilotDevice) status.push("ready");
if (ctx.isWaitlisted(t.tenantId)) status.push("waitlist");
if (t.disabled) status.push("suspended");
if (onWaitlist(t, ctx)) status.push("waitlist");
if (offboarding) status.push("offboarding");
if (t.disabled && !offboarding) status.push("suspended");
if (t.mcpDisabled) status.push("mcp-off");
return {
plan: [effectiveEdition(t, ctx.nowMs)],
+5
View File
@@ -213,6 +213,11 @@ export const api = {
`${API_BASE_URL}/api/tenants/${encodeURIComponent(tenantId)}/offboard`,
offboardFeedback: (tenantId: string) =>
`${API_BASE_URL}/api/tenants/${encodeURIComponent(tenantId)}/offboard/feedback`,
// Offboarding record + operator retry of a Failed run (Global Admin tooling, /api/global/*).
offboardingStatus: (tenantId: string) =>
`${API_BASE_URL}/api/global/tenants/${encodeURIComponent(tenantId)}/offboarding`,
offboardingRetry: (tenantId: string) =>
`${API_BASE_URL}/api/global/tenants/${encodeURIComponent(tenantId)}/offboarding/retry`,
},
// ── Devices ───────────────────────────────────────────────────────────────
@@ -22816,6 +22816,103 @@
}
]
},
"TenantOffboardingStatusResponse": {
"kind": "object",
"doc": "Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one.",
"fields": [
{
"name": "tenantId",
"type": {
"kind": "primitive",
"name": "string"
}
},
{
"name": "status",
"type": {
"kind": "primitive",
"name": "string"
}
},
{
"name": "historyRowKey",
"type": {
"kind": "primitive",
"name": "string"
}
},
{
"name": "initiatedAt",
"type": {
"kind": "primitive",
"name": "string"
}
},
{
"name": "initiatedBy",
"type": {
"kind": "primitive",
"name": "string"
}
},
{
"name": "retryCount",
"type": {
"kind": "primitive",
"name": "number"
},
"doc": "Queue dequeue attempts plus operator retries of this run."
},
{
"name": "earliestProcessingAt",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true
},
{
"name": "drainCompletedAt",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true
},
{
"name": "completedAt",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true
},
{
"name": "failedAt",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true
},
{
"name": "failedPhase",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true
},
{
"name": "errorMessage",
"type": {
"kind": "primitive",
"name": "string"
},
"optional": true
}
]
},
"TestLogPatternRequest": {
"kind": "object",
"doc": "Body of POST rules/gather/test-pattern.",
@@ -4994,6 +4994,23 @@ export interface TenantOffboardingCustomsArchiveEntry {
archivedBy: string;
}
/** Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one. */
export interface TenantOffboardingStatusResponse {
tenantId: string;
status: string;
historyRowKey: string;
initiatedAt: string;
initiatedBy: string;
/** Queue dequeue attempts plus operator retries of this run. */
retryCount: number;
earliestProcessingAt?: string;
drainCompletedAt?: string;
completedAt?: string;
failedAt?: string;
failedPhase?: string;
errorMessage?: string;
}
/** Body of POST rules/gather/test-pattern. */
export interface TestLogPatternRequest {
pattern?: string | null;