From c81591b498869d1427a8074bb7bedb0883b51c75 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 10:30:06 +0000 Subject: [PATCH] feat(offboarding): operator retry of a Failed tenant offboarding from Tenant Management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Failed offboarding was a dead end: the worker returns early on a Failed history, the offboard re-click refuses to re-enqueue on a Failed marker, and the marker cleanup only warns. Recovery meant deleting the marker row in Table Storage by hand, while the tenant sat in the list as a tombstone counted on the Waitlist (its whitelist row is wiped early in the cascade). - GET global/tenants/{tenantId}/offboarding returns the offboarding record (marker status, failed phase, attempts, history details) as the typed TenantOffboardingStatusResponse; GA + Reader, never delegated. - POST global/tenants/{tenantId}/offboarding/retry re-drives a Failed run: the History row keeps its RowKey and DrainCompletedAt (the worker's resume anchor) and counts the retry, marker and history go back to Initiated, the ordinary resume path re-affirms the Disabled-gate, waits out the remaining drain barrier and enqueues the envelope. Completed / in-flight answer 409, no marker 404, a missing history row 500 before any write. Global audit row RETRY/TenantOffboarding. Nothing retries on its own. - Tenant editor: an "Offboarding" card replaces the Offboard action while the cascade owns the row (status, started by, attempts, failed phase and error); Failed offers "Retry offboarding…" behind the type-to-confirm dialog in its retry variant. - Tenant list: tombstone rows carry an Offboarding badge (own Status facet) instead of Suspended, and no longer count as Waitlist. - Tests: retry decision (reset order, drain anchor, conflicts, not found, enqueue failure, remaining-barrier delay), status projection, wire parity facts, filter facet. Shared manifest and generated wire types regenerated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XPG5WCmSwfNAv4xkK62n5r --- .../TenantOffboardFunctionTests.cs | 206 +++++++++++++++++ .../TypedContractCompletionWireParityTests.cs | 71 ++++++ .../Functions/Admin/TenantOffboardFunction.cs | 209 ++++++++++++++++++ .../Security/EndpointAccessPolicyCatalog.cs | 5 + .../src/generated/wire-types.generated.ts | 17 ++ .../Models/OffboardApiModels.cs | 24 ++ .../OffboardTenantConfirmDialog.tsx | 32 ++- .../components/TenantManagementSection.tsx | 207 ++++++++++++++--- .../__tests__/tenantFilters.test.ts | 17 ++ .../app/admin/components/tenantFilters.ts | 25 ++- src/Web/autopilot-monitor-web/lib/api.ts | 5 + .../utils/shared-manifests.json | 97 ++++++++ .../utils/wire-types.generated.ts | 17 ++ 13 files changed, 896 insertions(+), 36 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TenantOffboardFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TenantOffboardFunctionTests.cs index 6c9b877d..c5b5253a 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/TenantOffboardFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TenantOffboardFunctionTests.cs @@ -1046,4 +1046,210 @@ public sealed class TenantOffboardFunctionTests // No Save call (the tombstone was already correct). configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); } + + // ── RetryFailedOffboardingAsync (operator retry of a Failed offboarding) ────────── + + private const string HistoryRowKey = "20260101000000000_" + TenantId; + + private static FakeOffboardingAuditRepository SeedFailedOffboarding( + string failedPhase = "max_dequeue", DateTime? drainCompletedAt = null, int retryCount = 3) + { + var repo = new FakeOffboardingAuditRepository(); + repo.Markers[TenantId] = new OffboardingMarkerEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.Marker, + RowKey = TenantId, TenantId = TenantId, + OffboardingHistoryRowKey = HistoryRowKey, + InitiatedAt = DateTime.UtcNow.AddHours(-3), + InitiatedBy = "alice@contoso.com", + Status = "Failed", + FailedAt = DateTime.UtcNow.AddHours(-2), + FailedPhase = failedPhase, + }; + repo.History[HistoryRowKey] = new OffboardingHistoryEntry + { + PartitionKey = Constants.OffboardingPartitionKeys.History, + RowKey = HistoryRowKey, TenantId = TenantId, + Status = "Failed", + ErrorMessage = "Poison queue: max dequeue count reached", + RetryCount = retryCount, + DrainCompletedAt = drainCompletedAt, + EarliestProcessingAt = DateTime.UtcNow.AddHours(-3), + }; + return repo; + } + + [Fact] + public async Task Retry_Failed_ResetsHistoryAndMarker_KeepsDrainAnchor_ReEnqueues() + { + var drained = DateTime.UtcNow.AddHours(-2).AddMinutes(-30); + var repo = SeedFailedOffboarding(drainCompletedAt: drained, retryCount: 5); + var enqueuer = new RecordingEnqueuer(); + var sut = Build(repo, enqueuer); + + var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com"); + + Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind); + Assert.Equal(HistoryRowKey, outcome.HistoryRowKey); + Assert.Contains("max_dequeue", outcome.Message); + + var history = repo.History[HistoryRowKey]; + Assert.Equal("Initiated", history.Status); + Assert.Equal(6, history.RetryCount); + Assert.Null(history.ErrorMessage); + Assert.Equal(drained, history.DrainCompletedAt); // the worker's resume anchor survives + Assert.Equal(HistoryRowKey, history.RowKey); // same run, no new history row + Assert.Single(repo.History); + + var marker = repo.Markers[TenantId]; + Assert.Equal("Initiated", marker.Status); + Assert.Null(marker.FailedAt); + Assert.Null(marker.FailedPhase); + + // History is written before the marker (worker authority first, HTTP anchor second). + Assert.Equal(new[] { "Initiated" }, repo.HistoryWrites); + Assert.Equal(new[] { "Initiated" }, repo.MarkerWrites); + + var envelope = Assert.Single(enqueuer.Sent); + Assert.Equal(TenantId, envelope.TenantId); + Assert.Equal(HistoryRowKey, envelope.HistoryRowKey); + Assert.Equal("ops@contoso.com", envelope.InitiatedBy); + } + + [Fact] + public async Task Retry_Completed_Conflict_NoWrites() + { + var repo = SeedFailedOffboarding(); + repo.Markers[TenantId].Status = "Completed"; + repo.Markers[TenantId].FailedPhase = null; + var enqueuer = new RecordingEnqueuer(); + var sut = Build(repo, enqueuer); + + var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com"); + + Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind); + Assert.Contains("already completed", outcome.Message); + Assert.Empty(repo.HistoryWrites); + Assert.Empty(repo.MarkerWrites); + Assert.Empty(enqueuer.Sent); + } + + [Theory] + [InlineData("Initiated")] + [InlineData("InProgress")] + public async Task Retry_InFlight_Conflict_PointsAtTheOffboardResume(string status) + { + var repo = SeedFailedOffboarding(); + repo.Markers[TenantId].Status = status; + var enqueuer = new RecordingEnqueuer(); + var sut = Build(repo, enqueuer); + + var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com"); + + Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind); + Assert.Contains(status, outcome.Message); + Assert.Equal("Failed", repo.History[HistoryRowKey].Status); + Assert.Empty(repo.MarkerWrites); + Assert.Empty(enqueuer.Sent); + } + + [Fact] + public async Task Retry_NoMarker_NotFound() + { + var enqueuer = new RecordingEnqueuer(); + var sut = Build(new FakeOffboardingAuditRepository(), enqueuer); + + var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com"); + + Assert.Equal(RetryOutcomeKind.NotFound, outcome.Kind); + Assert.Empty(enqueuer.Sent); + } + + [Fact] + public async Task Retry_HistoryRowMissing_FailsBeforeTouchingTheMarker() + { + var repo = SeedFailedOffboarding(); + repo.History.Clear(); + var enqueuer = new RecordingEnqueuer(); + var sut = Build(repo, enqueuer); + + var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com"); + + Assert.Equal(RetryOutcomeKind.HistoryMissing, outcome.Kind); + Assert.Equal("Failed", repo.Markers[TenantId].Status); + Assert.Empty(repo.MarkerWrites); + Assert.Empty(enqueuer.Sent); + } + + [Fact] + public async Task Retry_EnqueueThrows_LeavesMarkerInitiated_ForTheOffboardResume() + { + var repo = SeedFailedOffboarding(); + var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") }; + var sut = Build(repo, enqueuer); + + var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com"); + + Assert.Equal(RetryOutcomeKind.ReEnqueueFailed, outcome.Kind); + // Rows are reset: the ordinary DELETE offboard re-click now hits the Initiated resume path. + Assert.Equal("Initiated", repo.Markers[TenantId].Status); + Assert.Equal("Initiated", repo.History[HistoryRowKey].Status); + } + + [Fact] + public async Task Retry_ResumesThroughTheDisabledGate_WithRemainingBarrierDelay() + { + // Tombstone written 2 minutes ago → the resume path must wait out the remaining + // 4 minutes of the 6-minute drain barrier, not restart it and not skip it. + var repo = SeedFailedOffboarding(); + repo.History[HistoryRowKey].EarliestProcessingAt = DateTime.UtcNow.AddHours(-3); + var configRepoMock = new Mock(); + var tombstone = TenantConfiguration.CreateDefault(TenantId); + tombstone.Disabled = true; + tombstone.DisabledReason = "Offboarding in progress"; + tombstone.LastUpdated = DateTime.UtcNow.AddMinutes(-2); + configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny())).ReturnsAsync(tombstone); + var cache = new MemoryCache(new MemoryCacheOptions()); + var service = new TenantConfigurationService(configRepoMock.Object, NullLogger.Instance, cache); + var enqueuer = new RecordingEnqueuer(); + var sut = new TenantOffboardFunction( + logger: NullLogger.Instance, + configRepo: configRepoMock.Object, + tenantConfigService: service, + maintenanceRepo: Mock.Of(), + offboardingRepo: repo, + offboardingEnqueuer: enqueuer, + previewWhitelistService: BuildPreviewWhitelistService(configRepoMock)); + + var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com"); + + Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind); + var delay = Assert.Single(enqueuer.VisibilityDelays); + Assert.NotNull(delay); + Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.1)); + configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public void BuildStatusResponse_ProjectsMarkerStatus_AndHistoryDetails() + { + var repo = SeedFailedOffboarding(retryCount: 5); + var marker = repo.Markers[TenantId]; + var history = repo.History[HistoryRowKey]; + + var dto = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, history); + + Assert.Equal("Failed", dto.Status); + Assert.Equal("max_dequeue", dto.FailedPhase); + Assert.Equal(marker.FailedAt, dto.FailedAt); + Assert.Equal(5, dto.RetryCount); + Assert.Equal(history.ErrorMessage, dto.ErrorMessage); + Assert.Equal(HistoryRowKey, dto.HistoryRowKey); + + // History row gone (should not happen, but the record must still render). + var bare = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, null); + Assert.Equal("Failed", bare.Status); + Assert.Equal(0, bare.RetryCount); + Assert.Null(bare.ErrorMessage); + } } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TypedContractCompletionWireParityTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TypedContractCompletionWireParityTests.cs index 598e228a..fc31daaf 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/TypedContractCompletionWireParityTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TypedContractCompletionWireParityTests.cs @@ -888,4 +888,75 @@ public class TypedContractCompletionWireParityTests }, }); } + + // ── global/tenants/{tenantId}/offboarding ──────────────────────────────────────────── + + [Fact] + public void TenantOffboardingStatusResponse_pins_the_failed_record_shape() + { + var initiated = new DateTime(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc); + var failed = initiated.AddMinutes(30); + AssertWireIdentical( + new + { + tenantId = "11111111-1111-1111-1111-111111111111", + status = "Failed", + historyRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111", + initiatedAt = initiated, + initiatedBy = "alice@contoso.com", + retryCount = 5, + earliestProcessingAt = (DateTime?)initiated.AddMinutes(6), + drainCompletedAt = (DateTime?)initiated.AddMinutes(8), + completedAt = (DateTime?)null, + failedAt = (DateTime?)failed, + failedPhase = (string?)"max_dequeue", + errorMessage = (string?)"Poison queue: max dequeue count reached", + }, + new TenantOffboardingStatusResponse + { + TenantId = "11111111-1111-1111-1111-111111111111", + Status = "Failed", + HistoryRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111", + InitiatedAt = initiated, + InitiatedBy = "alice@contoso.com", + RetryCount = 5, + EarliestProcessingAt = initiated.AddMinutes(6), + DrainCompletedAt = initiated.AddMinutes(8), + CompletedAt = null, + FailedAt = failed, + FailedPhase = "max_dequeue", + ErrorMessage = "Poison queue: max dequeue count reached", + }); + } + + [Fact] + public void TenantOffboardingStatusResponse_keeps_null_slots_for_an_in_flight_record() + { + var initiated = new DateTime(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc); + AssertWireIdentical( + new + { + tenantId = "11111111-1111-1111-1111-111111111111", + status = "InProgress", + historyRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111", + initiatedAt = initiated, + initiatedBy = "alice@contoso.com", + retryCount = 0, + earliestProcessingAt = (DateTime?)initiated.AddMinutes(6), + drainCompletedAt = (DateTime?)null, + completedAt = (DateTime?)null, + failedAt = (DateTime?)null, + failedPhase = (string?)null, + errorMessage = (string?)null, + }, + new TenantOffboardingStatusResponse + { + TenantId = "11111111-1111-1111-1111-111111111111", + Status = "InProgress", + HistoryRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111", + InitiatedAt = initiated, + InitiatedBy = "alice@contoso.com", + EarliestProcessingAt = initiated.AddMinutes(6), + }); + } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs index 2934beab..f3e371d1 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/TenantOffboardFunction.cs @@ -506,6 +506,194 @@ public class TenantOffboardFunction EarliestProcessingAt: earliestProcessingAtForResponse); } + // ── Offboarding record + operator retry (Global Admin tooling) ──────────────────── + + /// + /// GET /api/global/tenants/{tenantId}/offboarding — the tenant's current offboarding record + /// (marker + history) for the admin tenant editor. 404 without a marker: the tenant was + /// never offboarded, or the Completed marker is already cleaned up + /// (, 15-min grace) — the normal end state, + /// nothing for an operator to do. Global Admin / Global Reader only (middleware-enforced). + /// + [Function("GetTenantOffboardingStatus")] + [Authorize] + public async Task GetTenantOffboardingStatus( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/tenants/{tenantId}/offboarding")] HttpRequestData req, + string tenantId, + FunctionContext context) + { + var targetTenantId = context.GetRequestContext().TargetTenantId; + if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId)) + { + return await req.BadRequestAsync("tenantId must be a valid GUID"); + } + + var normalizedTenantId = targetTenantId.ToLowerInvariant(); + var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId); + if (marker == null) + { + return await req.NotFoundAsync("No offboarding on record for this tenant"); + } + + var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey); + return await req.OkAsync(BuildStatusResponse(normalizedTenantId, marker, history)); + } + + internal static TenantOffboardingStatusResponse BuildStatusResponse( + string normalizedTenantId, OffboardingMarkerEntry marker, OffboardingHistoryEntry? history) + => new() + { + TenantId = normalizedTenantId, + Status = marker.Status, + HistoryRowKey = marker.OffboardingHistoryRowKey, + InitiatedAt = marker.InitiatedAt, + InitiatedBy = marker.InitiatedBy, + RetryCount = history?.RetryCount ?? 0, + EarliestProcessingAt = history?.EarliestProcessingAt, + DrainCompletedAt = history?.DrainCompletedAt, + CompletedAt = marker.CompletedAt, + FailedAt = marker.FailedAt, + FailedPhase = marker.FailedPhase, + ErrorMessage = history?.ErrorMessage, + }; + + /// + /// POST /api/global/tenants/{tenantId}/offboarding/retry — re-drives an offboarding whose + /// marker is Failed. Failed is fail-closed by design (the worker returns, the offboard + /// re-click refuses to re-enqueue), so this is the explicit operator action that design + /// asks for; nothing retries on its own. History and marker go back to Initiated — the + /// History row keeps its RowKey and DrainCompletedAt (the worker's resume anchor) + /// and counts the retry — and the ordinary resume path re-affirms the Disabled-gate, + /// computes the remaining drain barrier and enqueues the envelope. Global Admin only. + /// + [Function("RetryTenantOffboarding")] + [Authorize] + public async Task RetryTenantOffboarding( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/tenants/{tenantId}/offboarding/retry")] HttpRequestData req, + string tenantId, + FunctionContext context) + { + var requestCtx = context.GetRequestContext(); + var upn = requestCtx.UserPrincipalName; + var targetTenantId = requestCtx.TargetTenantId; + + if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId)) + { + return await req.BadRequestAsync("tenantId must be a valid GUID"); + } + + var normalizedTenantId = targetTenantId.ToLowerInvariant(); + var retry = await RetryFailedOffboardingAsync(normalizedTenantId, upn); + + switch (retry.Kind) + { + case RetryOutcomeKind.Retried: + { + var response = req.CreateResponse(HttpStatusCode.Accepted); + await response.WriteAsJsonAsync(new OffboardResponse + { + TenantId = normalizedTenantId, + Status = "Queued", + HistoryPartitionKey = Constants.OffboardingPartitionKeys.History, + HistoryRowKey = retry.HistoryRowKey ?? string.Empty, + EarliestProcessingAt = retry.EarliestProcessingAt, + Message = retry.Message, + }); + return response; + } + case RetryOutcomeKind.NotFound: + return await req.NotFoundAsync(retry.Message); + case RetryOutcomeKind.NotFailed: + return await req.ConflictAsync(retry.Message); + default: + return await Build500Async(req, retry.Message); + } + } + + /// + /// Pure retry decision behind ; internal for unit tests. + /// Write order matters: History first (the worker's authority — a Failed history makes + /// it return early), Marker second (the HTTP idempotency anchor). A crash in between + /// leaves History=Initiated + Marker=Failed, and the next retry click resolves it the + /// same way. The worker's first pickup then flips History/Pointer/Marker to InProgress. + /// + internal async Task RetryFailedOffboardingAsync(string normalizedTenantId, string upn) + { + var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId); + if (marker == null) + { + return new RetryOutcome(RetryOutcomeKind.NotFound, "No offboarding on record for this tenant"); + } + + if (!string.Equals(marker.Status, "Failed", StringComparison.OrdinalIgnoreCase)) + { + var message = string.Equals(marker.Status, "Completed", StringComparison.OrdinalIgnoreCase) + ? "Tenant offboarding already completed — nothing to retry" + : $"Tenant offboarding is {marker.Status} — not Failed; re-run the offboard action to resume it"; + _logger.LogInformation( + "TenantOffboard retry for {TenantId} refused: marker status={Status}", normalizedTenantId, marker.Status); + return new RetryOutcome(RetryOutcomeKind.NotFailed, message, marker.OffboardingHistoryRowKey); + } + + var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey); + if (history == null) + { + _logger.LogError( + "TenantOffboard retry for {TenantId}: History row {History} is missing — operator inspection required", + normalizedTenantId, marker.OffboardingHistoryRowKey); + return new RetryOutcome( + RetryOutcomeKind.HistoryMissing, + "Offboarding history row is missing — operator inspection required", + marker.OffboardingHistoryRowKey); + } + + var failedPhase = marker.FailedPhase ?? "unknown"; + + history.Status = "Initiated"; + history.RetryCount += 1; + history.ErrorMessage = null; + await _offboardingRepo.UpsertHistoryAsync(history); + + marker.Status = "Initiated"; + marker.FailedAt = null; + marker.FailedPhase = null; + await _offboardingRepo.UpsertMarkerAsync(marker); + + _logger.LogWarning( + "TENANT OFFBOARD RETRY for {TenantId} by {Upn}: previous failure phase={Phase}; historyRowKey={History} retryCount={RetryCount}", + normalizedTenantId, upn, failedPhase, history.RowKey, history.RetryCount); + + // Global audit row (survives the TenantConfiguration wipe like the initiation's DELETE row). + await _maintenanceRepo.LogAuditEntryAsync( + Constants.AuditGlobalTenantId, + "RETRY", + "TenantOffboarding", + normalizedTenantId, + upn, + new Dictionary + { + ["failedPhase"] = failedPhase, + ["historyRowKey"] = history.RowKey, + ["retryCount"] = history.RetryCount.ToString(System.Globalization.CultureInfo.InvariantCulture), + }); + + var resume = await ResumeExistingMarkerAsync(marker, upn, normalizedTenantId, isRace: false); + if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed) + { + // Marker is Initiated now, so the ordinary offboard re-click resumes it. + return new RetryOutcome( + RetryOutcomeKind.ReEnqueueFailed, + "Failed to re-enqueue tenant offboarding — re-run the offboard action to resume it", + history.RowKey); + } + + return new RetryOutcome( + RetryOutcomeKind.Retried, + $"Tenant offboarding retried after the failure in phase '{failedPhase}'", + history.RowKey, + resume.EarliestProcessingAt); + } + /// /// Magic-string DisabledReason that identifies a TenantConfiguration row as the /// offboarding tombstone. Used by EnsureTenantDisabledAsync for the idempotent @@ -775,3 +963,24 @@ internal enum ResumeOutcomeKind /// Marker classified as in-flight but re-enqueue threw; surface 500. ReEnqueueFailed, } + +/// Result of . +internal sealed record RetryOutcome( + RetryOutcomeKind Kind, + string Message, + string? HistoryRowKey = null, + DateTime? EarliestProcessingAt = null); + +internal enum RetryOutcomeKind +{ + /// Marker was Failed; history + marker reset to Initiated and the envelope re-enqueued (202). + Retried, + /// No marker on record (404). + NotFound, + /// Marker is Completed or in flight — nothing to retry (409). + NotFailed, + /// Marker points at a History row that no longer exists (500). + HistoryMissing, + /// Rows were reset but the re-enqueue threw (500); the offboard re-click resumes. + ReEnqueueFailed, +} diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index 0dd041df..e96bbcbf 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -428,6 +428,11 @@ public static class EndpointAccessPolicyCatalog new("GET", "global/sessions/{sessionId}/deletion-manifest", EndpointPolicy.GlobalReadOrAdmin), // Platform-operational (cascade-delete restore prep) — GA + read-only Reader only, NOT delegated. new("GET", "global/tenants/{tenantId}/deletion-manifests", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true), + // Offboarding record (marker + history) behind the tenant editor's status card, and the + // operator retry of a Failed offboarding. Platform-operational: GA + Reader read, GA-only + // retry, never delegated (an MSP manages active tenants, not the platform's offboarding). + new("GET", "global/tenants/{tenantId}/offboarding", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true), + new("POST", "global/tenants/{tenantId}/offboarding/retry", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam, excludeDelegated: true), new("GET", "global/session-deletions", EndpointPolicy.GlobalReadOrAdmin), // Manual maintenance-run trigger (GC sweeps + retention fanout) — mutating, GA only. new("POST", "global/session-deletions/maintenance/trigger", EndpointPolicy.GlobalAdminOnly), diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index 760083d8..70dceb97 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -5010,6 +5010,23 @@ export interface TenantOffboardingCustomsArchiveEntry { archivedBy: string; } +/** Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one. */ +export interface TenantOffboardingStatusResponse { + tenantId: string; + status: string; + historyRowKey: string; + initiatedAt: string; + initiatedBy: string; + /** Queue dequeue attempts plus operator retries of this run. */ + retryCount: number; + earliestProcessingAt?: string; + drainCompletedAt?: string; + completedAt?: string; + failedAt?: string; + failedPhase?: string; + errorMessage?: string; +} + /** Body of POST rules/gather/test-pattern. */ export interface TestLogPatternRequest { pattern?: string | null; diff --git a/src/Shared/AutopilotMonitor.Shared/Models/OffboardApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/OffboardApiModels.cs index 3b2d6296..f09af999 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/OffboardApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/OffboardApiModels.cs @@ -23,6 +23,30 @@ namespace AutopilotMonitor.Shared.Models public DateTime? EarliestProcessingAt { get; set; } } + /// + /// Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding + /// record for the admin tenant editor. is the marker's status (the + /// anchor every HTTP decision keys on); the history fields describe the run behind it. + /// A Failed record carries so the operator sees what to expect + /// from a retry before starting one. + /// + public class TenantOffboardingStatusResponse : IApiResponse + { + public string TenantId { get; set; } = string.Empty; + public string Status { get; set; } = string.Empty; + public string HistoryRowKey { get; set; } = string.Empty; + public DateTime InitiatedAt { get; set; } + public string InitiatedBy { get; set; } = string.Empty; + /// Queue dequeue attempts plus operator retries of this run. + public int RetryCount { get; set; } + public DateTime? EarliestProcessingAt { get; set; } + public DateTime? DrainCompletedAt { get; set; } + public DateTime? CompletedAt { get; set; } + public DateTime? FailedAt { get; set; } + public string? FailedPhase { get; set; } + public string? ErrorMessage { get; set; } + } + /// Body of POST tenants/{tenantId}/offboard/feedback. public class SubmitOffboardingFeedbackRequest : IApiRequest { diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OffboardTenantConfirmDialog.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OffboardTenantConfirmDialog.tsx index 235d03bc..4194b228 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OffboardTenantConfirmDialog.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OffboardTenantConfirmDialog.tsx @@ -12,6 +12,9 @@ interface OffboardTenantConfirmDialogProps { error: string | null; onCancel: () => void; onConfirm: () => void; + /** Set when the dialog confirms the RETRY of a Failed offboarding instead of a new one: + * the phase the previous run died in, shown so the operator knows what the retry re-runs. */ + retryOfFailedPhase?: string | null; } /** @@ -27,26 +30,37 @@ export function OffboardTenantConfirmDialog({ error, onCancel, onConfirm, + retryOfFailedPhase, }: OffboardTenantConfirmDialogProps) { const [confirmText, setConfirmText] = useState(""); const armed = confirmText === "OFFBOARD"; + const retry = retryOfFailedPhase !== undefined && retryOfFailedPhase !== null; return (
-

Offboard tenant

+

{retry ? "Retry offboarding" : "Offboard tenant"}

{tenantLabel}

{tenantId}

-

- This queues the offboarding cascade: the tenant is suspended immediately, - and after a short drain window all of its data is permanently deleted — - sessions, events, rules, admins, and the tenant configuration itself. -

+ {retry ? ( +

+ The previous run failed in phase {retryOfFailedPhase}. + This re-queues the same offboarding: the tenant stays suspended, the cascade re-runs its + remaining phases (every step is idempotent) and permanently deletes whatever data + is left — including the tenant configuration itself. +

+ ) : ( +

+ This queues the offboarding cascade: the tenant is suspended immediately, + and after a short drain window all of its data is permanently deleted — + sessions, events, rules, admins, and the tenant configuration itself. +

+ )}
This cannot be undone, and it is not a ban: because the configuration row (including any suspension) is deleted, a new sign-in from this tenant @@ -69,7 +83,7 @@ export function OffboardTenantConfirmDialog({
{error && (
- Offboarding failed:{" "}{error} + {retry ? "Retry failed:" : "Offboarding failed:"}{" "}{error}
)}
@@ -90,10 +104,10 @@ export function OffboardTenantConfirmDialog({ {saving ? ( <>
- Offboarding… + {retry ? "Retrying…" : "Offboarding…"} ) : ( - Offboard tenant + {retry ? "Retry offboarding" : "Offboard tenant"} )}
diff --git a/src/Web/autopilot-monitor-web/app/admin/components/TenantManagementSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/TenantManagementSection.tsx index b827e735..53f96e1a 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/TenantManagementSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/TenantManagementSection.tsx @@ -14,6 +14,7 @@ import type { SaveNotificationEmailRequest, SetTenantPlanTierResponse, TenantConfiguration as WireTenantConfiguration, + TenantOffboardingStatusResponse, UpdateTenantAppHomingResponse, UpdateTenantConfigurationResponse, } from "@/utils/wire-types.generated"; @@ -36,7 +37,9 @@ import { effectiveEdition, facetCounts, facetOption, + isOffboardingTombstone, matchesTenantFilters, + onWaitlist, planOverview, toggleFacetValue, visibleFacets, @@ -195,6 +198,37 @@ function TenantManagementSectionInner({ cancelled = true; }; }, [editingTenantId, getAccessToken]); + // Offboarding record, loaded only for a row the cascade owns (the Disabled-gate tombstone). + // Keyed by the response's tenantId rather than reset on every editor switch: a record is + // only ever rendered for the tenant it names (editingRecord below), so a stale one from a + // previously opened editor can never show. null = no marker (never offboarded, or + // Completed and already cleaned up). + const [offboardingRecord, setOffboardingRecord] = useState(null); + const [retryDialogOpen, setRetryDialogOpen] = useState(false); + const [retrying, setRetrying] = useState(false); + const [retryError, setRetryError] = useState(null); + const editingIsOffboarding = !!editingTenant && isOffboardingTombstone(editingTenant); + const editingRecord = + editingIsOffboarding && offboardingRecord && editingTenantId + && offboardingRecord.tenantId.toLowerCase() === editingTenantId.toLowerCase() + ? offboardingRecord + : null; + useEffect(() => { + // A 404 (no marker) and any other failure both leave the card hidden: the Danger Zone then + // shows the ordinary Offboard action, whose own resume/refusal messages still apply. + if (!editingTenantId || !editingIsOffboarding) return; + let cancelled = false; + fetchJson(api.tenants.offboardingStatus(editingTenantId), getAccessToken) + .then((record) => { + if (!cancelled) setOffboardingRecord(record); + }) + .catch(() => { + if (!cancelled) setOffboardingRecord(null); + }); + return () => { + cancelled = true; + }; + }, [editingTenantId, editingIsOffboarding, getAccessToken]); const [homingDialogTarget, setHomingDialogTarget] = useState<"primary" | "legacy" | null>(null); const [savingHoming, setSavingHoming] = useState(false); // Failures of the homing/offboard calls go into these instead of the page-level @@ -231,7 +265,7 @@ function TenantManagementSectionInner({ // Statistics (always over all tenants, not filtered) const readyCount = tenants.filter(t => t.validateAutopilotDevice).length; - const waitlistCount = tenants.filter(t => !previewApproved.has(t.tenantId)).length; + const waitlistCount = tenants.filter(t => onWaitlist(t, filterCtx)).length; const totalCount = tenants.length; const plans = planOverview(tenants, nowMs); @@ -411,6 +445,34 @@ function TenantManagementSectionInner({ } }; + // Operator retry of a Failed offboarding (POST global/tenants/{id}/offboarding/retry). Failed + // is fail-closed on the backend: the worker returns and the DELETE re-click refuses, so this + // is the one path that re-drives the cascade. Same confirm dialog, retry variant. + const handleRetryOffboarding = async (tenant: TenantConfiguration) => { + if (!canMutate) return; // read-only Global Reader + try { + setRetrying(true); + setRetryError(null); + setSuccessMessage(null); + const data = await fetchJson( + api.tenants.offboardingRetry(tenant.tenantId), getAccessToken, { method: "POST" }); + trackEvent("admin_tenant_offboard_retried", { + tenantId: tenant.tenantId, + failedPhase: editingRecord?.failedPhase ?? "unknown", + }); + setRetryDialogOpen(false); + setEditingTenant(null); + fetchTenants(); + setSuccessMessage(data.message || `Offboarding retried for tenant ${tenant.tenantId}`); + setTimeout(() => setSuccessMessage(null), 8000); + } catch (err) { + console.error("Error retrying tenant offboarding:", err); + setRetryError(apiErrorText(err, "Failed to retry the offboarding")); + } finally { + setRetrying(false); + } + }; + const handleSendWelcomeEmail = async (tenantId: string, email?: string) => { if (!canMutate) return; // read-only Global Reader try { @@ -593,7 +655,14 @@ function TenantManagementSectionInner({
- {tenant.disabled && ( + {isOffboardingTombstone(tenant) ? ( + + Offboarding + + ) : tenant.disabled && ( Suspended @@ -606,7 +675,7 @@ function TenantManagementSectionInner({ MCP off )} - {!previewApproved.has(tenant.tenantId) && ( + {onWaitlist(tenant, filterCtx) && ( Waitlist @@ -1265,28 +1334,103 @@ function TenantManagementSectionInner({
- {/* Danger Zone — offboarding cascade (own path: DELETE tenants/{id}/offboard; - deliberately NOT part of the modal's Save button) */} -
-

Offboard Tenant

-

- Suspends the tenant immediately and permanently deletes all of its data - (sessions, events, rules, admins, configuration) after a short drain window. - Same cascade as the tenant's self-service offboarding. -

-

- Not a ban: the deletion includes the suspension, so once the cascade completes a - new sign-in re-onboards (and auto-activates) the tenant. To lock a tenant out, - suspend it above and leave its data in place. -

- -
+ {/* Offboarding record — shown instead of the Offboard action while the cascade owns + the row. Failed is a dead end on the backend until an operator retries here. */} + {editingRecord ? ( +
+
+

Offboarding

+ + {editingRecord.status} + +
+
+
Started
+
{new Date(editingRecord.initiatedAt).toLocaleString()} by {editingRecord.initiatedBy}
+
Attempts
+
{editingRecord.retryCount}
+ {editingRecord.drainCompletedAt && ( + <> +
Drain done
+
{new Date(editingRecord.drainCompletedAt).toLocaleString()}
+ + )} + {editingRecord.status === "Failed" && ( + <> +
Failed
+
+ {editingRecord.failedAt ? new Date(editingRecord.failedAt).toLocaleString() : "—"} + {" "}in phase {editingRecord.failedPhase || "unknown"} +
+ {editingRecord.errorMessage && ( + <> +
Error
+
{editingRecord.errorMessage}
+ + )} + + )} +
+ {editingRecord.status === "Failed" ? ( + <> +

+ The cascade stopped and will not resume on its own. The tenant stays suspended and + its remaining data stays in place until the offboarding is retried. Fix the cause + first if the phase points at one (the backend log carries the details). +

+ + + ) : editingRecord.status === "Completed" ? ( +

+ The cascade has finished; the tenant row disappears once the marker's grace window + has passed. +

+ ) : ( +

+ The cascade is running. Re-running the Offboard action would only re-queue it; nothing to do here. +

+ )} +
+ ) : ( + <> + {/* Danger Zone — offboarding cascade (own path: DELETE tenants/{id}/offboard; + deliberately NOT part of the modal's Save button) */} +
+

Offboard Tenant

+

+ Suspends the tenant immediately and permanently deletes all of its data + (sessions, events, rules, admins, configuration) after a short drain window. + Same cascade as the tenant's self-service offboarding. +

+

+ Not a ban: the deletion includes the suspension, so once the cascade completes a + new sign-in re-onboards (and auto-activates) the tenant. To lock a tenant out, + suspend it above and leave its data in place. +

+ +
+ + )} @@ -1331,6 +1475,19 @@ function TenantManagementSectionInner({ /> )} + {/* Retry confirmation for a Failed offboarding (same dialog, retry variant) */} + {editingTenant && retryDialogOpen && editingRecord && ( + setRetryDialogOpen(false)} + onConfirm={() => handleRetryOffboarding(editingTenant)} + /> + )} + {/* App-homing flip confirmation (renders above the editor modal) */} {editingTenant && homingDialogTarget && ( { expect(matchesTenantFilters(t, filters({ status: ["suspended"] }), waitlisted)).toBe(false); }); + it("an offboarding tombstone is Offboarding — neither Waitlist nor Suspended", () => { + const t = tenant({ tenantId: "w", disabled: true, disabledReason: "Offboarding in progress" }); + const waitlisted = ctx({ isWaitlisted: (id) => id === "w" }); + expect(tenantFacetValues(t, waitlisted).status).toEqual(["offboarding"]); + expect(onWaitlist(t, waitlisted)).toBe(false); + expect(matchesTenantFilters(t, filters({ status: ["offboarding"] }), waitlisted)).toBe(true); + expect(matchesTenantFilters(t, filters({ status: ["waitlist"] }), waitlisted)).toBe(false); + expect(matchesTenantFilters(t, filters({ status: ["suspended"] }), waitlisted)).toBe(false); + + // An admin suspension with any other reason stays Suspended (+ Waitlist while unactivated). + const suspended = tenant({ tenantId: "w", disabled: true, disabledReason: "abuse" }); + expect(tenantFacetValues(suspended, waitlisted).status).toEqual(["waitlist", "suspended"]); + expect(onWaitlist(suspended, waitlisted)).toBe(true); + }); + it("classifies the app registration through the context", () => { const newApp = tenant({ homedAppClientId: PRIMARY }); const legacyApp = tenant({ homedAppClientId: null }); diff --git a/src/Web/autopilot-monitor-web/app/admin/components/tenantFilters.ts b/src/Web/autopilot-monitor-web/app/admin/components/tenantFilters.ts index 002f6947..5c078fb3 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/tenantFilters.ts +++ b/src/Web/autopilot-monitor-web/app/admin/components/tenantFilters.ts @@ -28,6 +28,7 @@ export interface TenantFilterFields { payingCustomer: boolean; homedAppClientId?: string | null; disabled: boolean; + disabledReason?: string | null; mcpDisabled: boolean; validateAutopilotDevice: boolean; } @@ -111,6 +112,7 @@ export const FACETS: readonly FacetDefinition[] = [ options: [ { value: "ready", label: "Ready", badgeClass: "bg-blue-100 text-blue-800" }, { value: "waitlist", label: "Waitlist", badgeClass: AMBER }, + { value: "offboarding", label: "Offboarding", badgeClass: "bg-orange-100 text-orange-800" }, { value: "suspended", label: "Suspended", badgeClass: "bg-red-100 text-red-800" }, { value: "mcp-off", label: "MCP off", badgeClass: AMBER }, ], @@ -155,12 +157,31 @@ export function trialState(t: TenantFilterFields, nowMs: number): TrialState { return "never"; } +/** + * DisabledReason the offboarding cascade writes on the tenant row (Phase 1 Disabled-gate, + * `TenantOffboardFunction.OffboardingDisabledReason`). While it is set the row is the + * cascade's tombstone: the tenant is neither suspended by an admin nor waiting for activation + * (its whitelist row is wiped early in the cascade), it is being deleted. + */ +export const OFFBOARDING_TOMBSTONE_REASON = "Offboarding in progress"; + +export function isOffboardingTombstone(t: Pick): boolean { + return t.disabled && t.disabledReason === OFFBOARDING_TOMBSTONE_REASON; +} + +/** Waitlist = not activated yet, and not a row the offboarding cascade is deleting. */ +export function onWaitlist(t: TenantFilterFields, ctx: TenantFilterContext): boolean { + return !isOffboardingTombstone(t) && ctx.isWaitlisted(t.tenantId); +} + /** Every facet value a tenant carries; Status is the only multi-valued facet. */ export function tenantFacetValues(t: TenantFilterFields, ctx: TenantFilterContext): Record { const status: string[] = []; + const offboarding = isOffboardingTombstone(t); if (t.validateAutopilotDevice) status.push("ready"); - if (ctx.isWaitlisted(t.tenantId)) status.push("waitlist"); - if (t.disabled) status.push("suspended"); + if (onWaitlist(t, ctx)) status.push("waitlist"); + if (offboarding) status.push("offboarding"); + if (t.disabled && !offboarding) status.push("suspended"); if (t.mcpDisabled) status.push("mcp-off"); return { plan: [effectiveEdition(t, ctx.nowMs)], diff --git a/src/Web/autopilot-monitor-web/lib/api.ts b/src/Web/autopilot-monitor-web/lib/api.ts index 5d0e3eab..85cb282e 100644 --- a/src/Web/autopilot-monitor-web/lib/api.ts +++ b/src/Web/autopilot-monitor-web/lib/api.ts @@ -213,6 +213,11 @@ export const api = { `${API_BASE_URL}/api/tenants/${encodeURIComponent(tenantId)}/offboard`, offboardFeedback: (tenantId: string) => `${API_BASE_URL}/api/tenants/${encodeURIComponent(tenantId)}/offboard/feedback`, + // Offboarding record + operator retry of a Failed run (Global Admin tooling, /api/global/*). + offboardingStatus: (tenantId: string) => + `${API_BASE_URL}/api/global/tenants/${encodeURIComponent(tenantId)}/offboarding`, + offboardingRetry: (tenantId: string) => + `${API_BASE_URL}/api/global/tenants/${encodeURIComponent(tenantId)}/offboarding/retry`, }, // ── Devices ─────────────────────────────────────────────────────────────── diff --git a/src/Web/autopilot-monitor-web/utils/shared-manifests.json b/src/Web/autopilot-monitor-web/utils/shared-manifests.json index 00890b5a..0e97c6ef 100644 --- a/src/Web/autopilot-monitor-web/utils/shared-manifests.json +++ b/src/Web/autopilot-monitor-web/utils/shared-manifests.json @@ -22906,6 +22906,103 @@ } ] }, + "TenantOffboardingStatusResponse": { + "kind": "object", + "doc": "Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one.", + "fields": [ + { + "name": "tenantId", + "type": { + "kind": "primitive", + "name": "string" + } + }, + { + "name": "status", + "type": { + "kind": "primitive", + "name": "string" + } + }, + { + "name": "historyRowKey", + "type": { + "kind": "primitive", + "name": "string" + } + }, + { + "name": "initiatedAt", + "type": { + "kind": "primitive", + "name": "string" + } + }, + { + "name": "initiatedBy", + "type": { + "kind": "primitive", + "name": "string" + } + }, + { + "name": "retryCount", + "type": { + "kind": "primitive", + "name": "number" + }, + "doc": "Queue dequeue attempts plus operator retries of this run." + }, + { + "name": "earliestProcessingAt", + "type": { + "kind": "primitive", + "name": "string" + }, + "optional": true + }, + { + "name": "drainCompletedAt", + "type": { + "kind": "primitive", + "name": "string" + }, + "optional": true + }, + { + "name": "completedAt", + "type": { + "kind": "primitive", + "name": "string" + }, + "optional": true + }, + { + "name": "failedAt", + "type": { + "kind": "primitive", + "name": "string" + }, + "optional": true + }, + { + "name": "failedPhase", + "type": { + "kind": "primitive", + "name": "string" + }, + "optional": true + }, + { + "name": "errorMessage", + "type": { + "kind": "primitive", + "name": "string" + }, + "optional": true + } + ] + }, "TestLogPatternRequest": { "kind": "object", "doc": "Body of POST rules/gather/test-pattern.", diff --git a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts index 0153f129..1765c462 100644 --- a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts @@ -5010,6 +5010,23 @@ export interface TenantOffboardingCustomsArchiveEntry { archivedBy: string; } +/** Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one. */ +export interface TenantOffboardingStatusResponse { + tenantId: string; + status: string; + historyRowKey: string; + initiatedAt: string; + initiatedBy: string; + /** Queue dequeue attempts plus operator retries of this run. */ + retryCount: number; + earliestProcessingAt?: string; + drainCompletedAt?: string; + completedAt?: string; + failedAt?: string; + failedPhase?: string; + errorMessage?: string; +} + /** Body of POST rules/gather/test-pattern. */ export interface TestLogPatternRequest { pattern?: string | null;