mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
Merge branch 'main' of https://github.com/okieselbach/AutopilotMonitor
This commit is contained in:
@@ -1046,4 +1046,210 @@ public sealed class TenantOffboardFunctionTests
|
||||
// No Save call (the tombstone was already correct).
|
||||
configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>(), It.IsAny<string?>(), It.IsAny<string?>()), Times.Never);
|
||||
}
|
||||
|
||||
// ── RetryFailedOffboardingAsync (operator retry of a Failed offboarding) ──────────
|
||||
|
||||
private const string HistoryRowKey = "20260101000000000_" + TenantId;
|
||||
|
||||
private static FakeOffboardingAuditRepository SeedFailedOffboarding(
|
||||
string failedPhase = "max_dequeue", DateTime? drainCompletedAt = null, int retryCount = 3)
|
||||
{
|
||||
var repo = new FakeOffboardingAuditRepository();
|
||||
repo.Markers[TenantId] = new OffboardingMarkerEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.Marker,
|
||||
RowKey = TenantId, TenantId = TenantId,
|
||||
OffboardingHistoryRowKey = HistoryRowKey,
|
||||
InitiatedAt = DateTime.UtcNow.AddHours(-3),
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
Status = "Failed",
|
||||
FailedAt = DateTime.UtcNow.AddHours(-2),
|
||||
FailedPhase = failedPhase,
|
||||
};
|
||||
repo.History[HistoryRowKey] = new OffboardingHistoryEntry
|
||||
{
|
||||
PartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
RowKey = HistoryRowKey, TenantId = TenantId,
|
||||
Status = "Failed",
|
||||
ErrorMessage = "Poison queue: max dequeue count reached",
|
||||
RetryCount = retryCount,
|
||||
DrainCompletedAt = drainCompletedAt,
|
||||
EarliestProcessingAt = DateTime.UtcNow.AddHours(-3),
|
||||
};
|
||||
return repo;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_Failed_ResetsHistoryAndMarker_KeepsDrainAnchor_ReEnqueues()
|
||||
{
|
||||
var drained = DateTime.UtcNow.AddHours(-2).AddMinutes(-30);
|
||||
var repo = SeedFailedOffboarding(drainCompletedAt: drained, retryCount: 5);
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind);
|
||||
Assert.Equal(HistoryRowKey, outcome.HistoryRowKey);
|
||||
Assert.Contains("max_dequeue", outcome.Message);
|
||||
|
||||
var history = repo.History[HistoryRowKey];
|
||||
Assert.Equal("Initiated", history.Status);
|
||||
Assert.Equal(6, history.RetryCount);
|
||||
Assert.Null(history.ErrorMessage);
|
||||
Assert.Equal(drained, history.DrainCompletedAt); // the worker's resume anchor survives
|
||||
Assert.Equal(HistoryRowKey, history.RowKey); // same run, no new history row
|
||||
Assert.Single(repo.History);
|
||||
|
||||
var marker = repo.Markers[TenantId];
|
||||
Assert.Equal("Initiated", marker.Status);
|
||||
Assert.Null(marker.FailedAt);
|
||||
Assert.Null(marker.FailedPhase);
|
||||
|
||||
// History is written before the marker (worker authority first, HTTP anchor second).
|
||||
Assert.Equal(new[] { "Initiated" }, repo.HistoryWrites);
|
||||
Assert.Equal(new[] { "Initiated" }, repo.MarkerWrites);
|
||||
|
||||
var envelope = Assert.Single(enqueuer.Sent);
|
||||
Assert.Equal(TenantId, envelope.TenantId);
|
||||
Assert.Equal(HistoryRowKey, envelope.HistoryRowKey);
|
||||
Assert.Equal("ops@contoso.com", envelope.InitiatedBy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_Completed_Conflict_NoWrites()
|
||||
{
|
||||
var repo = SeedFailedOffboarding();
|
||||
repo.Markers[TenantId].Status = "Completed";
|
||||
repo.Markers[TenantId].FailedPhase = null;
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind);
|
||||
Assert.Contains("already completed", outcome.Message);
|
||||
Assert.Empty(repo.HistoryWrites);
|
||||
Assert.Empty(repo.MarkerWrites);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Initiated")]
|
||||
[InlineData("InProgress")]
|
||||
public async Task Retry_InFlight_Conflict_PointsAtTheOffboardResume(string status)
|
||||
{
|
||||
var repo = SeedFailedOffboarding();
|
||||
repo.Markers[TenantId].Status = status;
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.NotFailed, outcome.Kind);
|
||||
Assert.Contains(status, outcome.Message);
|
||||
Assert.Equal("Failed", repo.History[HistoryRowKey].Status);
|
||||
Assert.Empty(repo.MarkerWrites);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_NoMarker_NotFound()
|
||||
{
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(new FakeOffboardingAuditRepository(), enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.NotFound, outcome.Kind);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_HistoryRowMissing_FailsBeforeTouchingTheMarker()
|
||||
{
|
||||
var repo = SeedFailedOffboarding();
|
||||
repo.History.Clear();
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.HistoryMissing, outcome.Kind);
|
||||
Assert.Equal("Failed", repo.Markers[TenantId].Status);
|
||||
Assert.Empty(repo.MarkerWrites);
|
||||
Assert.Empty(enqueuer.Sent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_EnqueueThrows_LeavesMarkerInitiated_ForTheOffboardResume()
|
||||
{
|
||||
var repo = SeedFailedOffboarding();
|
||||
var enqueuer = new RecordingEnqueuer { ThrowOnEnqueue = new InvalidOperationException("queue down") };
|
||||
var sut = Build(repo, enqueuer);
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.ReEnqueueFailed, outcome.Kind);
|
||||
// Rows are reset: the ordinary DELETE offboard re-click now hits the Initiated resume path.
|
||||
Assert.Equal("Initiated", repo.Markers[TenantId].Status);
|
||||
Assert.Equal("Initiated", repo.History[HistoryRowKey].Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retry_ResumesThroughTheDisabledGate_WithRemainingBarrierDelay()
|
||||
{
|
||||
// Tombstone written 2 minutes ago → the resume path must wait out the remaining
|
||||
// 4 minutes of the 6-minute drain barrier, not restart it and not skip it.
|
||||
var repo = SeedFailedOffboarding();
|
||||
repo.History[HistoryRowKey].EarliestProcessingAt = DateTime.UtcNow.AddHours(-3);
|
||||
var configRepoMock = new Mock<IConfigRepository>();
|
||||
var tombstone = TenantConfiguration.CreateDefault(TenantId);
|
||||
tombstone.Disabled = true;
|
||||
tombstone.DisabledReason = "Offboarding in progress";
|
||||
tombstone.LastUpdated = DateTime.UtcNow.AddMinutes(-2);
|
||||
configRepoMock.Setup(r => r.GetTenantConfigurationAsync(It.IsAny<string>())).ReturnsAsync(tombstone);
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var service = new TenantConfigurationService(configRepoMock.Object, NullLogger<TenantConfigurationService>.Instance, cache);
|
||||
var enqueuer = new RecordingEnqueuer();
|
||||
var sut = new TenantOffboardFunction(
|
||||
logger: NullLogger<TenantOffboardFunction>.Instance,
|
||||
configRepo: configRepoMock.Object,
|
||||
tenantConfigService: service,
|
||||
maintenanceRepo: Mock.Of<IMaintenanceRepository>(),
|
||||
offboardingRepo: repo,
|
||||
offboardingEnqueuer: enqueuer,
|
||||
previewWhitelistService: BuildPreviewWhitelistService(configRepoMock));
|
||||
|
||||
var outcome = await sut.RetryFailedOffboardingAsync(TenantId, "ops@contoso.com");
|
||||
|
||||
Assert.Equal(RetryOutcomeKind.Retried, outcome.Kind);
|
||||
var delay = Assert.Single(enqueuer.VisibilityDelays);
|
||||
Assert.NotNull(delay);
|
||||
Assert.InRange(delay!.Value, TimeSpan.FromMinutes(3.5), TimeSpan.FromMinutes(4.1));
|
||||
configRepoMock.Verify(r => r.SaveTenantConfigurationAsync(It.IsAny<TenantConfiguration>(), It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildStatusResponse_ProjectsMarkerStatus_AndHistoryDetails()
|
||||
{
|
||||
var repo = SeedFailedOffboarding(retryCount: 5);
|
||||
var marker = repo.Markers[TenantId];
|
||||
var history = repo.History[HistoryRowKey];
|
||||
|
||||
var dto = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, history);
|
||||
|
||||
Assert.Equal("Failed", dto.Status);
|
||||
Assert.Equal("max_dequeue", dto.FailedPhase);
|
||||
Assert.Equal(marker.FailedAt, dto.FailedAt);
|
||||
Assert.Equal(5, dto.RetryCount);
|
||||
Assert.Equal(history.ErrorMessage, dto.ErrorMessage);
|
||||
Assert.Equal(HistoryRowKey, dto.HistoryRowKey);
|
||||
|
||||
// History row gone (should not happen, but the record must still render).
|
||||
var bare = TenantOffboardFunction.BuildStatusResponse(TenantId, marker, null);
|
||||
Assert.Equal("Failed", bare.Status);
|
||||
Assert.Equal(0, bare.RetryCount);
|
||||
Assert.Null(bare.ErrorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
+71
@@ -888,4 +888,75 @@ public class TypedContractCompletionWireParityTests
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ── global/tenants/{tenantId}/offboarding ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void TenantOffboardingStatusResponse_pins_the_failed_record_shape()
|
||||
{
|
||||
var initiated = new DateTime(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc);
|
||||
var failed = initiated.AddMinutes(30);
|
||||
AssertWireIdentical(
|
||||
new
|
||||
{
|
||||
tenantId = "11111111-1111-1111-1111-111111111111",
|
||||
status = "Failed",
|
||||
historyRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
|
||||
initiatedAt = initiated,
|
||||
initiatedBy = "alice@contoso.com",
|
||||
retryCount = 5,
|
||||
earliestProcessingAt = (DateTime?)initiated.AddMinutes(6),
|
||||
drainCompletedAt = (DateTime?)initiated.AddMinutes(8),
|
||||
completedAt = (DateTime?)null,
|
||||
failedAt = (DateTime?)failed,
|
||||
failedPhase = (string?)"max_dequeue",
|
||||
errorMessage = (string?)"Poison queue: max dequeue count reached",
|
||||
},
|
||||
new TenantOffboardingStatusResponse
|
||||
{
|
||||
TenantId = "11111111-1111-1111-1111-111111111111",
|
||||
Status = "Failed",
|
||||
HistoryRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
|
||||
InitiatedAt = initiated,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
RetryCount = 5,
|
||||
EarliestProcessingAt = initiated.AddMinutes(6),
|
||||
DrainCompletedAt = initiated.AddMinutes(8),
|
||||
CompletedAt = null,
|
||||
FailedAt = failed,
|
||||
FailedPhase = "max_dequeue",
|
||||
ErrorMessage = "Poison queue: max dequeue count reached",
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TenantOffboardingStatusResponse_keeps_null_slots_for_an_in_flight_record()
|
||||
{
|
||||
var initiated = new DateTime(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc);
|
||||
AssertWireIdentical(
|
||||
new
|
||||
{
|
||||
tenantId = "11111111-1111-1111-1111-111111111111",
|
||||
status = "InProgress",
|
||||
historyRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
|
||||
initiatedAt = initiated,
|
||||
initiatedBy = "alice@contoso.com",
|
||||
retryCount = 0,
|
||||
earliestProcessingAt = (DateTime?)initiated.AddMinutes(6),
|
||||
drainCompletedAt = (DateTime?)null,
|
||||
completedAt = (DateTime?)null,
|
||||
failedAt = (DateTime?)null,
|
||||
failedPhase = (string?)null,
|
||||
errorMessage = (string?)null,
|
||||
},
|
||||
new TenantOffboardingStatusResponse
|
||||
{
|
||||
TenantId = "11111111-1111-1111-1111-111111111111",
|
||||
Status = "InProgress",
|
||||
HistoryRowKey = "20260918060000000_11111111-1111-1111-1111-111111111111",
|
||||
InitiatedAt = initiated,
|
||||
InitiatedBy = "alice@contoso.com",
|
||||
EarliestProcessingAt = initiated.AddMinutes(6),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,6 +506,194 @@ public class TenantOffboardFunction
|
||||
EarliestProcessingAt: earliestProcessingAtForResponse);
|
||||
}
|
||||
|
||||
// ── Offboarding record + operator retry (Global Admin tooling) ────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// GET /api/global/tenants/{tenantId}/offboarding — the tenant's current offboarding record
|
||||
/// (marker + history) for the admin tenant editor. 404 without a marker: the tenant was
|
||||
/// never offboarded, or the Completed marker is already cleaned up
|
||||
/// (<see cref="OffboardingMarkerCleanupFunction"/>, 15-min grace) — the normal end state,
|
||||
/// nothing for an operator to do. Global Admin / Global Reader only (middleware-enforced).
|
||||
/// </summary>
|
||||
[Function("GetTenantOffboardingStatus")]
|
||||
[Authorize]
|
||||
public async Task<HttpResponseData> GetTenantOffboardingStatus(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/tenants/{tenantId}/offboarding")] HttpRequestData req,
|
||||
string tenantId,
|
||||
FunctionContext context)
|
||||
{
|
||||
var targetTenantId = context.GetRequestContext().TargetTenantId;
|
||||
if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
|
||||
{
|
||||
return await req.BadRequestAsync("tenantId must be a valid GUID");
|
||||
}
|
||||
|
||||
var normalizedTenantId = targetTenantId.ToLowerInvariant();
|
||||
var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
|
||||
if (marker == null)
|
||||
{
|
||||
return await req.NotFoundAsync("No offboarding on record for this tenant");
|
||||
}
|
||||
|
||||
var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey);
|
||||
return await req.OkAsync(BuildStatusResponse(normalizedTenantId, marker, history));
|
||||
}
|
||||
|
||||
internal static TenantOffboardingStatusResponse BuildStatusResponse(
|
||||
string normalizedTenantId, OffboardingMarkerEntry marker, OffboardingHistoryEntry? history)
|
||||
=> new()
|
||||
{
|
||||
TenantId = normalizedTenantId,
|
||||
Status = marker.Status,
|
||||
HistoryRowKey = marker.OffboardingHistoryRowKey,
|
||||
InitiatedAt = marker.InitiatedAt,
|
||||
InitiatedBy = marker.InitiatedBy,
|
||||
RetryCount = history?.RetryCount ?? 0,
|
||||
EarliestProcessingAt = history?.EarliestProcessingAt,
|
||||
DrainCompletedAt = history?.DrainCompletedAt,
|
||||
CompletedAt = marker.CompletedAt,
|
||||
FailedAt = marker.FailedAt,
|
||||
FailedPhase = marker.FailedPhase,
|
||||
ErrorMessage = history?.ErrorMessage,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// POST /api/global/tenants/{tenantId}/offboarding/retry — re-drives an offboarding whose
|
||||
/// marker is Failed. Failed is fail-closed by design (the worker returns, the offboard
|
||||
/// re-click refuses to re-enqueue), so this is the explicit operator action that design
|
||||
/// asks for; nothing retries on its own. History and marker go back to Initiated — the
|
||||
/// History row keeps its RowKey and <c>DrainCompletedAt</c> (the worker's resume anchor)
|
||||
/// and counts the retry — and the ordinary resume path re-affirms the Disabled-gate,
|
||||
/// computes the remaining drain barrier and enqueues the envelope. Global Admin only.
|
||||
/// </summary>
|
||||
[Function("RetryTenantOffboarding")]
|
||||
[Authorize]
|
||||
public async Task<HttpResponseData> RetryTenantOffboarding(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/tenants/{tenantId}/offboarding/retry")] HttpRequestData req,
|
||||
string tenantId,
|
||||
FunctionContext context)
|
||||
{
|
||||
var requestCtx = context.GetRequestContext();
|
||||
var upn = requestCtx.UserPrincipalName;
|
||||
var targetTenantId = requestCtx.TargetTenantId;
|
||||
|
||||
if (string.IsNullOrEmpty(targetTenantId) || !SecurityValidator.IsValidGuid(targetTenantId))
|
||||
{
|
||||
return await req.BadRequestAsync("tenantId must be a valid GUID");
|
||||
}
|
||||
|
||||
var normalizedTenantId = targetTenantId.ToLowerInvariant();
|
||||
var retry = await RetryFailedOffboardingAsync(normalizedTenantId, upn);
|
||||
|
||||
switch (retry.Kind)
|
||||
{
|
||||
case RetryOutcomeKind.Retried:
|
||||
{
|
||||
var response = req.CreateResponse(HttpStatusCode.Accepted);
|
||||
await response.WriteAsJsonAsync(new OffboardResponse
|
||||
{
|
||||
TenantId = normalizedTenantId,
|
||||
Status = "Queued",
|
||||
HistoryPartitionKey = Constants.OffboardingPartitionKeys.History,
|
||||
HistoryRowKey = retry.HistoryRowKey ?? string.Empty,
|
||||
EarliestProcessingAt = retry.EarliestProcessingAt,
|
||||
Message = retry.Message,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
case RetryOutcomeKind.NotFound:
|
||||
return await req.NotFoundAsync(retry.Message);
|
||||
case RetryOutcomeKind.NotFailed:
|
||||
return await req.ConflictAsync(retry.Message);
|
||||
default:
|
||||
return await Build500Async(req, retry.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure retry decision behind <see cref="RetryTenantOffboarding"/>; internal for unit tests.
|
||||
/// Write order matters: History first (the worker's authority — a Failed history makes
|
||||
/// it return early), Marker second (the HTTP idempotency anchor). A crash in between
|
||||
/// leaves History=Initiated + Marker=Failed, and the next retry click resolves it the
|
||||
/// same way. The worker's first pickup then flips History/Pointer/Marker to InProgress.
|
||||
/// </summary>
|
||||
internal async Task<RetryOutcome> RetryFailedOffboardingAsync(string normalizedTenantId, string upn)
|
||||
{
|
||||
var marker = await _offboardingRepo.TryGetMarkerAsync(normalizedTenantId);
|
||||
if (marker == null)
|
||||
{
|
||||
return new RetryOutcome(RetryOutcomeKind.NotFound, "No offboarding on record for this tenant");
|
||||
}
|
||||
|
||||
if (!string.Equals(marker.Status, "Failed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var message = string.Equals(marker.Status, "Completed", StringComparison.OrdinalIgnoreCase)
|
||||
? "Tenant offboarding already completed — nothing to retry"
|
||||
: $"Tenant offboarding is {marker.Status} — not Failed; re-run the offboard action to resume it";
|
||||
_logger.LogInformation(
|
||||
"TenantOffboard retry for {TenantId} refused: marker status={Status}", normalizedTenantId, marker.Status);
|
||||
return new RetryOutcome(RetryOutcomeKind.NotFailed, message, marker.OffboardingHistoryRowKey);
|
||||
}
|
||||
|
||||
var history = await _offboardingRepo.TryGetHistoryAsync(marker.OffboardingHistoryRowKey);
|
||||
if (history == null)
|
||||
{
|
||||
_logger.LogError(
|
||||
"TenantOffboard retry for {TenantId}: History row {History} is missing — operator inspection required",
|
||||
normalizedTenantId, marker.OffboardingHistoryRowKey);
|
||||
return new RetryOutcome(
|
||||
RetryOutcomeKind.HistoryMissing,
|
||||
"Offboarding history row is missing — operator inspection required",
|
||||
marker.OffboardingHistoryRowKey);
|
||||
}
|
||||
|
||||
var failedPhase = marker.FailedPhase ?? "unknown";
|
||||
|
||||
history.Status = "Initiated";
|
||||
history.RetryCount += 1;
|
||||
history.ErrorMessage = null;
|
||||
await _offboardingRepo.UpsertHistoryAsync(history);
|
||||
|
||||
marker.Status = "Initiated";
|
||||
marker.FailedAt = null;
|
||||
marker.FailedPhase = null;
|
||||
await _offboardingRepo.UpsertMarkerAsync(marker);
|
||||
|
||||
_logger.LogWarning(
|
||||
"TENANT OFFBOARD RETRY for {TenantId} by {Upn}: previous failure phase={Phase}; historyRowKey={History} retryCount={RetryCount}",
|
||||
normalizedTenantId, upn, failedPhase, history.RowKey, history.RetryCount);
|
||||
|
||||
// Global audit row (survives the TenantConfiguration wipe like the initiation's DELETE row).
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
Constants.AuditGlobalTenantId,
|
||||
"RETRY",
|
||||
"TenantOffboarding",
|
||||
normalizedTenantId,
|
||||
upn,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failedPhase"] = failedPhase,
|
||||
["historyRowKey"] = history.RowKey,
|
||||
["retryCount"] = history.RetryCount.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
});
|
||||
|
||||
var resume = await ResumeExistingMarkerAsync(marker, upn, normalizedTenantId, isRace: false);
|
||||
if (resume.Kind == ResumeOutcomeKind.ReEnqueueFailed)
|
||||
{
|
||||
// Marker is Initiated now, so the ordinary offboard re-click resumes it.
|
||||
return new RetryOutcome(
|
||||
RetryOutcomeKind.ReEnqueueFailed,
|
||||
"Failed to re-enqueue tenant offboarding — re-run the offboard action to resume it",
|
||||
history.RowKey);
|
||||
}
|
||||
|
||||
return new RetryOutcome(
|
||||
RetryOutcomeKind.Retried,
|
||||
$"Tenant offboarding retried after the failure in phase '{failedPhase}'",
|
||||
history.RowKey,
|
||||
resume.EarliestProcessingAt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Magic-string DisabledReason that identifies a TenantConfiguration row as the
|
||||
/// offboarding tombstone. Used by <c>EnsureTenantDisabledAsync</c> for the idempotent
|
||||
@@ -775,3 +963,24 @@ internal enum ResumeOutcomeKind
|
||||
/// <summary>Marker classified as in-flight but re-enqueue threw; surface 500.</summary>
|
||||
ReEnqueueFailed,
|
||||
}
|
||||
|
||||
/// <summary>Result of <see cref="TenantOffboardFunction.RetryFailedOffboardingAsync"/>.</summary>
|
||||
internal sealed record RetryOutcome(
|
||||
RetryOutcomeKind Kind,
|
||||
string Message,
|
||||
string? HistoryRowKey = null,
|
||||
DateTime? EarliestProcessingAt = null);
|
||||
|
||||
internal enum RetryOutcomeKind
|
||||
{
|
||||
/// <summary>Marker was Failed; history + marker reset to Initiated and the envelope re-enqueued (202).</summary>
|
||||
Retried,
|
||||
/// <summary>No marker on record (404).</summary>
|
||||
NotFound,
|
||||
/// <summary>Marker is Completed or in flight — nothing to retry (409).</summary>
|
||||
NotFailed,
|
||||
/// <summary>Marker points at a History row that no longer exists (500).</summary>
|
||||
HistoryMissing,
|
||||
/// <summary>Rows were reset but the re-enqueue threw (500); the offboard re-click resumes.</summary>
|
||||
ReEnqueueFailed,
|
||||
}
|
||||
|
||||
@@ -428,6 +428,11 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("GET", "global/sessions/{sessionId}/deletion-manifest", EndpointPolicy.GlobalReadOrAdmin),
|
||||
// Platform-operational (cascade-delete restore prep) — GA + read-only Reader only, NOT delegated.
|
||||
new("GET", "global/tenants/{tenantId}/deletion-manifests", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true),
|
||||
// Offboarding record (marker + history) behind the tenant editor's status card, and the
|
||||
// operator retry of a Failed offboarding. Platform-operational: GA + Reader read, GA-only
|
||||
// retry, never delegated (an MSP manages active tenants, not the platform's offboarding).
|
||||
new("GET", "global/tenants/{tenantId}/offboarding", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true),
|
||||
new("POST", "global/tenants/{tenantId}/offboarding/retry", EndpointPolicy.GlobalAdminOnly, TenantScoping.RouteParam, excludeDelegated: true),
|
||||
new("GET", "global/session-deletions", EndpointPolicy.GlobalReadOrAdmin),
|
||||
// Manual maintenance-run trigger (GC sweeps + retention fanout) — mutating, GA only.
|
||||
new("POST", "global/session-deletions/maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
@@ -4994,6 +4994,23 @@ export interface TenantOffboardingCustomsArchiveEntry {
|
||||
archivedBy: string;
|
||||
}
|
||||
|
||||
/** Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one. */
|
||||
export interface TenantOffboardingStatusResponse {
|
||||
tenantId: string;
|
||||
status: string;
|
||||
historyRowKey: string;
|
||||
initiatedAt: string;
|
||||
initiatedBy: string;
|
||||
/** Queue dequeue attempts plus operator retries of this run. */
|
||||
retryCount: number;
|
||||
earliestProcessingAt?: string;
|
||||
drainCompletedAt?: string;
|
||||
completedAt?: string;
|
||||
failedAt?: string;
|
||||
failedPhase?: string;
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
/** Body of POST rules/gather/test-pattern. */
|
||||
export interface TestLogPatternRequest {
|
||||
pattern?: string | null;
|
||||
|
||||
@@ -23,6 +23,30 @@ namespace AutopilotMonitor.Shared.Models
|
||||
public DateTime? EarliestProcessingAt { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding
|
||||
/// record for the admin tenant editor. <see cref="Status"/> is the marker's status (the
|
||||
/// anchor every HTTP decision keys on); the history fields describe the run behind it.
|
||||
/// A Failed record carries <see cref="FailedPhase"/> so the operator sees what to expect
|
||||
/// from a retry before starting one.
|
||||
/// </summary>
|
||||
public class TenantOffboardingStatusResponse : IApiResponse
|
||||
{
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
public string Status { get; set; } = string.Empty;
|
||||
public string HistoryRowKey { get; set; } = string.Empty;
|
||||
public DateTime InitiatedAt { get; set; }
|
||||
public string InitiatedBy { get; set; } = string.Empty;
|
||||
/// <summary>Queue dequeue attempts plus operator retries of this run.</summary>
|
||||
public int RetryCount { get; set; }
|
||||
public DateTime? EarliestProcessingAt { get; set; }
|
||||
public DateTime? DrainCompletedAt { get; set; }
|
||||
public DateTime? CompletedAt { get; set; }
|
||||
public DateTime? FailedAt { get; set; }
|
||||
public string? FailedPhase { get; set; }
|
||||
public string? ErrorMessage { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Body of POST tenants/{tenantId}/offboard/feedback.</summary>
|
||||
public class SubmitOffboardingFeedbackRequest : IApiRequest
|
||||
{
|
||||
|
||||
@@ -12,6 +12,9 @@ interface OffboardTenantConfirmDialogProps {
|
||||
error: string | null;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
/** Set when the dialog confirms the RETRY of a Failed offboarding instead of a new one:
|
||||
* the phase the previous run died in, shown so the operator knows what the retry re-runs. */
|
||||
retryOfFailedPhase?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -27,26 +30,37 @@ export function OffboardTenantConfirmDialog({
|
||||
error,
|
||||
onCancel,
|
||||
onConfirm,
|
||||
retryOfFailedPhase,
|
||||
}: OffboardTenantConfirmDialogProps) {
|
||||
const [confirmText, setConfirmText] = useState("");
|
||||
const armed = confirmText === "OFFBOARD";
|
||||
const retry = retryOfFailedPhase !== undefined && retryOfFailedPhase !== null;
|
||||
|
||||
return (
|
||||
<ModalPortal>
|
||||
<div className="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-[60] p-4">
|
||||
<div className="bg-white rounded-lg shadow-xl max-w-lg w-full">
|
||||
<div className="p-5 rounded-t-lg text-white bg-red-600">
|
||||
<h2 className="text-lg font-bold">Offboard tenant</h2>
|
||||
<h2 className="text-lg font-bold">{retry ? "Retry offboarding" : "Offboard tenant"}</h2>
|
||||
<p className="text-sm opacity-90 mt-0.5">{tenantLabel}</p>
|
||||
<p className="text-xs opacity-75">{tenantId}</p>
|
||||
</div>
|
||||
|
||||
<div className="p-5 space-y-3 text-sm text-gray-700">
|
||||
<p>
|
||||
This queues the offboarding cascade: the tenant is <strong>suspended immediately</strong>,
|
||||
and after a short drain window <strong>all of its data is permanently deleted</strong> —
|
||||
sessions, events, rules, admins, and the tenant configuration itself.
|
||||
</p>
|
||||
{retry ? (
|
||||
<p>
|
||||
The previous run failed in phase <span className="font-mono font-semibold">{retryOfFailedPhase}</span>.
|
||||
This re-queues the same offboarding: the tenant stays suspended, the cascade re-runs its
|
||||
remaining phases (every step is idempotent) and <strong>permanently deletes whatever data
|
||||
is left</strong> — including the tenant configuration itself.
|
||||
</p>
|
||||
) : (
|
||||
<p>
|
||||
This queues the offboarding cascade: the tenant is <strong>suspended immediately</strong>,
|
||||
and after a short drain window <strong>all of its data is permanently deleted</strong> —
|
||||
sessions, events, rules, admins, and the tenant configuration itself.
|
||||
</p>
|
||||
)}
|
||||
<div className="bg-red-50 border border-red-200 rounded-lg p-3 text-red-800">
|
||||
<strong>This cannot be undone,</strong> and it is <strong>not a ban</strong>: because the
|
||||
configuration row (including any suspension) is deleted, a new sign-in from this tenant
|
||||
@@ -69,7 +83,7 @@ export function OffboardTenantConfirmDialog({
|
||||
</div>
|
||||
{error && (
|
||||
<div role="alert" className="bg-red-100 border border-red-300 rounded-lg p-3 text-red-800">
|
||||
<strong>Offboarding failed:</strong>{" "}{error}
|
||||
<strong>{retry ? "Retry failed:" : "Offboarding failed:"}</strong>{" "}{error}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -90,10 +104,10 @@ export function OffboardTenantConfirmDialog({
|
||||
{saving ? (
|
||||
<>
|
||||
<div className="animate-spin rounded-full h-4 w-4 border-b-2 border-white"></div>
|
||||
<span>Offboarding…</span>
|
||||
<span>{retry ? "Retrying…" : "Offboarding…"}</span>
|
||||
</>
|
||||
) : (
|
||||
<span>Offboard tenant</span>
|
||||
<span>{retry ? "Retry offboarding" : "Offboard tenant"}</span>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@@ -14,6 +14,7 @@ import type {
|
||||
SaveNotificationEmailRequest,
|
||||
SetTenantPlanTierResponse,
|
||||
TenantConfiguration as WireTenantConfiguration,
|
||||
TenantOffboardingStatusResponse,
|
||||
UpdateTenantAppHomingResponse,
|
||||
UpdateTenantConfigurationResponse,
|
||||
} from "@/utils/wire-types.generated";
|
||||
@@ -36,7 +37,9 @@ import {
|
||||
effectiveEdition,
|
||||
facetCounts,
|
||||
facetOption,
|
||||
isOffboardingTombstone,
|
||||
matchesTenantFilters,
|
||||
onWaitlist,
|
||||
planOverview,
|
||||
toggleFacetValue,
|
||||
visibleFacets,
|
||||
@@ -195,6 +198,37 @@ function TenantManagementSectionInner({
|
||||
cancelled = true;
|
||||
};
|
||||
}, [editingTenantId, getAccessToken]);
|
||||
// Offboarding record, loaded only for a row the cascade owns (the Disabled-gate tombstone).
|
||||
// Keyed by the response's tenantId rather than reset on every editor switch: a record is
|
||||
// only ever rendered for the tenant it names (editingRecord below), so a stale one from a
|
||||
// previously opened editor can never show. null = no marker (never offboarded, or
|
||||
// Completed and already cleaned up).
|
||||
const [offboardingRecord, setOffboardingRecord] = useState<TenantOffboardingStatusResponse | null>(null);
|
||||
const [retryDialogOpen, setRetryDialogOpen] = useState(false);
|
||||
const [retrying, setRetrying] = useState(false);
|
||||
const [retryError, setRetryError] = useState<string | null>(null);
|
||||
const editingIsOffboarding = !!editingTenant && isOffboardingTombstone(editingTenant);
|
||||
const editingRecord =
|
||||
editingIsOffboarding && offboardingRecord && editingTenantId
|
||||
&& offboardingRecord.tenantId.toLowerCase() === editingTenantId.toLowerCase()
|
||||
? offboardingRecord
|
||||
: null;
|
||||
useEffect(() => {
|
||||
// A 404 (no marker) and any other failure both leave the card hidden: the Danger Zone then
|
||||
// shows the ordinary Offboard action, whose own resume/refusal messages still apply.
|
||||
if (!editingTenantId || !editingIsOffboarding) return;
|
||||
let cancelled = false;
|
||||
fetchJson<TenantOffboardingStatusResponse>(api.tenants.offboardingStatus(editingTenantId), getAccessToken)
|
||||
.then((record) => {
|
||||
if (!cancelled) setOffboardingRecord(record);
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) setOffboardingRecord(null);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [editingTenantId, editingIsOffboarding, getAccessToken]);
|
||||
const [homingDialogTarget, setHomingDialogTarget] = useState<"primary" | "legacy" | null>(null);
|
||||
const [savingHoming, setSavingHoming] = useState(false);
|
||||
// Failures of the homing/offboard calls go into these instead of the page-level
|
||||
@@ -231,7 +265,7 @@ function TenantManagementSectionInner({
|
||||
|
||||
// Statistics (always over all tenants, not filtered)
|
||||
const readyCount = tenants.filter(t => t.validateAutopilotDevice).length;
|
||||
const waitlistCount = tenants.filter(t => !previewApproved.has(t.tenantId)).length;
|
||||
const waitlistCount = tenants.filter(t => onWaitlist(t, filterCtx)).length;
|
||||
const totalCount = tenants.length;
|
||||
const plans = planOverview(tenants, nowMs);
|
||||
|
||||
@@ -411,6 +445,34 @@ function TenantManagementSectionInner({
|
||||
}
|
||||
};
|
||||
|
||||
// Operator retry of a Failed offboarding (POST global/tenants/{id}/offboarding/retry). Failed
|
||||
// is fail-closed on the backend: the worker returns and the DELETE re-click refuses, so this
|
||||
// is the one path that re-drives the cascade. Same confirm dialog, retry variant.
|
||||
const handleRetryOffboarding = async (tenant: TenantConfiguration) => {
|
||||
if (!canMutate) return; // read-only Global Reader
|
||||
try {
|
||||
setRetrying(true);
|
||||
setRetryError(null);
|
||||
setSuccessMessage(null);
|
||||
const data = await fetchJson<OffboardResponse>(
|
||||
api.tenants.offboardingRetry(tenant.tenantId), getAccessToken, { method: "POST" });
|
||||
trackEvent("admin_tenant_offboard_retried", {
|
||||
tenantId: tenant.tenantId,
|
||||
failedPhase: editingRecord?.failedPhase ?? "unknown",
|
||||
});
|
||||
setRetryDialogOpen(false);
|
||||
setEditingTenant(null);
|
||||
fetchTenants();
|
||||
setSuccessMessage(data.message || `Offboarding retried for tenant ${tenant.tenantId}`);
|
||||
setTimeout(() => setSuccessMessage(null), 8000);
|
||||
} catch (err) {
|
||||
console.error("Error retrying tenant offboarding:", err);
|
||||
setRetryError(apiErrorText(err, "Failed to retry the offboarding"));
|
||||
} finally {
|
||||
setRetrying(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleSendWelcomeEmail = async (tenantId: string, email?: string) => {
|
||||
if (!canMutate) return; // read-only Global Reader
|
||||
try {
|
||||
@@ -593,7 +655,14 @@ function TenantManagementSectionInner({
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
{tenant.disabled && (
|
||||
{isOffboardingTombstone(tenant) ? (
|
||||
<span
|
||||
className={`inline-flex items-center px-2 py-1 rounded-full text-xs font-medium ${facetOption("status", "offboarding")?.badgeClass}`}
|
||||
title="The offboarding cascade owns this row — open the editor for its status"
|
||||
>
|
||||
Offboarding
|
||||
</span>
|
||||
) : tenant.disabled && (
|
||||
<span className="inline-flex items-center px-2 py-1 rounded-full text-xs font-medium bg-red-100 text-red-800">
|
||||
Suspended
|
||||
</span>
|
||||
@@ -606,7 +675,7 @@ function TenantManagementSectionInner({
|
||||
MCP off
|
||||
</span>
|
||||
)}
|
||||
{!previewApproved.has(tenant.tenantId) && (
|
||||
{onWaitlist(tenant, filterCtx) && (
|
||||
<span className="inline-flex items-center px-2 py-1 rounded-full text-xs font-medium bg-amber-100 text-amber-800">
|
||||
Waitlist
|
||||
</span>
|
||||
@@ -1265,28 +1334,103 @@ function TenantManagementSectionInner({
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Danger Zone — offboarding cascade (own path: DELETE tenants/{id}/offboard;
|
||||
deliberately NOT part of the modal's Save button) */}
|
||||
<div className="bg-red-50 border-2 border-red-300 rounded-lg p-4">
|
||||
<h3 className="font-semibold text-red-900 mb-1">Offboard Tenant</h3>
|
||||
<p className="text-xs text-gray-600 mb-2">
|
||||
Suspends the tenant immediately and permanently deletes all of its data
|
||||
(sessions, events, rules, admins, configuration) after a short drain window.
|
||||
Same cascade as the tenant's self-service offboarding.
|
||||
</p>
|
||||
<p className="text-xs text-red-700 font-medium mb-3">
|
||||
Not a ban: the deletion includes the suspension, so once the cascade completes a
|
||||
new sign-in re-onboards (and auto-activates) the tenant. To lock a tenant out,
|
||||
suspend it above and leave its data in place.
|
||||
</p>
|
||||
<button
|
||||
onClick={() => { setOffboardError(null); setOffboardDialogOpen(true); }}
|
||||
disabled={!canMutate || offboarding}
|
||||
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 disabled:opacity-50 text-sm font-medium"
|
||||
>
|
||||
Offboard Tenant…
|
||||
</button>
|
||||
</div>
|
||||
{/* Offboarding record — shown instead of the Offboard action while the cascade owns
|
||||
the row. Failed is a dead end on the backend until an operator retries here. */}
|
||||
{editingRecord ? (
|
||||
<div className="bg-red-50 border-2 border-red-300 rounded-lg p-4">
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<h3 className="font-semibold text-red-900">Offboarding</h3>
|
||||
<span
|
||||
className={`inline-flex items-center px-2 py-1 rounded-full text-xs font-medium ${
|
||||
editingRecord.status === "Failed"
|
||||
? "bg-red-100 text-red-800"
|
||||
: editingRecord.status === "Completed"
|
||||
? "bg-green-100 text-green-800"
|
||||
: "bg-amber-100 text-amber-800"
|
||||
}`}
|
||||
>
|
||||
{editingRecord.status}
|
||||
</span>
|
||||
</div>
|
||||
<dl className="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1 text-xs text-gray-700 mb-3">
|
||||
<dt className="text-gray-500">Started</dt>
|
||||
<dd>{new Date(editingRecord.initiatedAt).toLocaleString()} by {editingRecord.initiatedBy}</dd>
|
||||
<dt className="text-gray-500">Attempts</dt>
|
||||
<dd>{editingRecord.retryCount}</dd>
|
||||
{editingRecord.drainCompletedAt && (
|
||||
<>
|
||||
<dt className="text-gray-500">Drain done</dt>
|
||||
<dd>{new Date(editingRecord.drainCompletedAt).toLocaleString()}</dd>
|
||||
</>
|
||||
)}
|
||||
{editingRecord.status === "Failed" && (
|
||||
<>
|
||||
<dt className="text-gray-500">Failed</dt>
|
||||
<dd>
|
||||
{editingRecord.failedAt ? new Date(editingRecord.failedAt).toLocaleString() : "—"}
|
||||
{" "}in phase <span className="font-mono">{editingRecord.failedPhase || "unknown"}</span>
|
||||
</dd>
|
||||
{editingRecord.errorMessage && (
|
||||
<>
|
||||
<dt className="text-gray-500">Error</dt>
|
||||
<dd className="break-words">{editingRecord.errorMessage}</dd>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</dl>
|
||||
{editingRecord.status === "Failed" ? (
|
||||
<>
|
||||
<p className="text-xs text-red-700 font-medium mb-3">
|
||||
The cascade stopped and will not resume on its own. The tenant stays suspended and
|
||||
its remaining data stays in place until the offboarding is retried. Fix the cause
|
||||
first if the phase points at one (the backend log carries the details).
|
||||
</p>
|
||||
<button
|
||||
onClick={() => { setRetryError(null); setRetryDialogOpen(true); }}
|
||||
disabled={!canMutate || retrying}
|
||||
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 disabled:opacity-50 text-sm font-medium"
|
||||
>
|
||||
Retry offboarding…
|
||||
</button>
|
||||
</>
|
||||
) : editingRecord.status === "Completed" ? (
|
||||
<p className="text-xs text-gray-600">
|
||||
The cascade has finished; the tenant row disappears once the marker's grace window
|
||||
has passed.
|
||||
</p>
|
||||
) : (
|
||||
<p className="text-xs text-gray-600">
|
||||
The cascade is running. Re-running the Offboard action would only re-queue it; nothing to do here.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
{/* Danger Zone — offboarding cascade (own path: DELETE tenants/{id}/offboard;
|
||||
deliberately NOT part of the modal's Save button) */}
|
||||
<div className="bg-red-50 border-2 border-red-300 rounded-lg p-4">
|
||||
<h3 className="font-semibold text-red-900 mb-1">Offboard Tenant</h3>
|
||||
<p className="text-xs text-gray-600 mb-2">
|
||||
Suspends the tenant immediately and permanently deletes all of its data
|
||||
(sessions, events, rules, admins, configuration) after a short drain window.
|
||||
Same cascade as the tenant's self-service offboarding.
|
||||
</p>
|
||||
<p className="text-xs text-red-700 font-medium mb-3">
|
||||
Not a ban: the deletion includes the suspension, so once the cascade completes a
|
||||
new sign-in re-onboards (and auto-activates) the tenant. To lock a tenant out,
|
||||
suspend it above and leave its data in place.
|
||||
</p>
|
||||
<button
|
||||
onClick={() => { setOffboardError(null); setOffboardDialogOpen(true); }}
|
||||
disabled={!canMutate || offboarding}
|
||||
className="px-4 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 disabled:opacity-50 text-sm font-medium"
|
||||
>
|
||||
Offboard Tenant…
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
</div>
|
||||
|
||||
@@ -1331,6 +1475,19 @@ function TenantManagementSectionInner({
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Retry confirmation for a Failed offboarding (same dialog, retry variant) */}
|
||||
{editingTenant && retryDialogOpen && editingRecord && (
|
||||
<OffboardTenantConfirmDialog
|
||||
tenantLabel={editingTenant.domainName || editingTenant.tenantId}
|
||||
tenantId={editingTenant.tenantId}
|
||||
saving={retrying}
|
||||
error={retryError}
|
||||
retryOfFailedPhase={editingRecord.failedPhase || "unknown"}
|
||||
onCancel={() => setRetryDialogOpen(false)}
|
||||
onConfirm={() => handleRetryOffboarding(editingTenant)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* App-homing flip confirmation (renders above the editor modal) */}
|
||||
{editingTenant && homingDialogTarget && (
|
||||
<AppHomingConfirmDialog
|
||||
|
||||
@@ -6,7 +6,9 @@ import {
|
||||
effectiveEdition,
|
||||
facetCounts,
|
||||
matchesTenantFilters,
|
||||
onWaitlist,
|
||||
planOverview,
|
||||
tenantFacetValues,
|
||||
toggleFacetValue,
|
||||
trialState,
|
||||
visibleFacets,
|
||||
@@ -117,6 +119,21 @@ describe("matchesTenantFilters", () => {
|
||||
expect(matchesTenantFilters(t, filters({ status: ["suspended"] }), waitlisted)).toBe(false);
|
||||
});
|
||||
|
||||
it("an offboarding tombstone is Offboarding — neither Waitlist nor Suspended", () => {
|
||||
const t = tenant({ tenantId: "w", disabled: true, disabledReason: "Offboarding in progress" });
|
||||
const waitlisted = ctx({ isWaitlisted: (id) => id === "w" });
|
||||
expect(tenantFacetValues(t, waitlisted).status).toEqual(["offboarding"]);
|
||||
expect(onWaitlist(t, waitlisted)).toBe(false);
|
||||
expect(matchesTenantFilters(t, filters({ status: ["offboarding"] }), waitlisted)).toBe(true);
|
||||
expect(matchesTenantFilters(t, filters({ status: ["waitlist"] }), waitlisted)).toBe(false);
|
||||
expect(matchesTenantFilters(t, filters({ status: ["suspended"] }), waitlisted)).toBe(false);
|
||||
|
||||
// An admin suspension with any other reason stays Suspended (+ Waitlist while unactivated).
|
||||
const suspended = tenant({ tenantId: "w", disabled: true, disabledReason: "abuse" });
|
||||
expect(tenantFacetValues(suspended, waitlisted).status).toEqual(["waitlist", "suspended"]);
|
||||
expect(onWaitlist(suspended, waitlisted)).toBe(true);
|
||||
});
|
||||
|
||||
it("classifies the app registration through the context", () => {
|
||||
const newApp = tenant({ homedAppClientId: PRIMARY });
|
||||
const legacyApp = tenant({ homedAppClientId: null });
|
||||
|
||||
@@ -28,6 +28,7 @@ export interface TenantFilterFields {
|
||||
payingCustomer: boolean;
|
||||
homedAppClientId?: string | null;
|
||||
disabled: boolean;
|
||||
disabledReason?: string | null;
|
||||
mcpDisabled: boolean;
|
||||
validateAutopilotDevice: boolean;
|
||||
}
|
||||
@@ -111,6 +112,7 @@ export const FACETS: readonly FacetDefinition[] = [
|
||||
options: [
|
||||
{ value: "ready", label: "Ready", badgeClass: "bg-blue-100 text-blue-800" },
|
||||
{ value: "waitlist", label: "Waitlist", badgeClass: AMBER },
|
||||
{ value: "offboarding", label: "Offboarding", badgeClass: "bg-orange-100 text-orange-800" },
|
||||
{ value: "suspended", label: "Suspended", badgeClass: "bg-red-100 text-red-800" },
|
||||
{ value: "mcp-off", label: "MCP off", badgeClass: AMBER },
|
||||
],
|
||||
@@ -155,12 +157,31 @@ export function trialState(t: TenantFilterFields, nowMs: number): TrialState {
|
||||
return "never";
|
||||
}
|
||||
|
||||
/**
|
||||
* DisabledReason the offboarding cascade writes on the tenant row (Phase 1 Disabled-gate,
|
||||
* `TenantOffboardFunction.OffboardingDisabledReason`). While it is set the row is the
|
||||
* cascade's tombstone: the tenant is neither suspended by an admin nor waiting for activation
|
||||
* (its whitelist row is wiped early in the cascade), it is being deleted.
|
||||
*/
|
||||
export const OFFBOARDING_TOMBSTONE_REASON = "Offboarding in progress";
|
||||
|
||||
export function isOffboardingTombstone(t: Pick<TenantFilterFields, "disabled" | "disabledReason">): boolean {
|
||||
return t.disabled && t.disabledReason === OFFBOARDING_TOMBSTONE_REASON;
|
||||
}
|
||||
|
||||
/** Waitlist = not activated yet, and not a row the offboarding cascade is deleting. */
|
||||
export function onWaitlist(t: TenantFilterFields, ctx: TenantFilterContext): boolean {
|
||||
return !isOffboardingTombstone(t) && ctx.isWaitlisted(t.tenantId);
|
||||
}
|
||||
|
||||
/** Every facet value a tenant carries; Status is the only multi-valued facet. */
|
||||
export function tenantFacetValues(t: TenantFilterFields, ctx: TenantFilterContext): Record<FacetKey, readonly string[]> {
|
||||
const status: string[] = [];
|
||||
const offboarding = isOffboardingTombstone(t);
|
||||
if (t.validateAutopilotDevice) status.push("ready");
|
||||
if (ctx.isWaitlisted(t.tenantId)) status.push("waitlist");
|
||||
if (t.disabled) status.push("suspended");
|
||||
if (onWaitlist(t, ctx)) status.push("waitlist");
|
||||
if (offboarding) status.push("offboarding");
|
||||
if (t.disabled && !offboarding) status.push("suspended");
|
||||
if (t.mcpDisabled) status.push("mcp-off");
|
||||
return {
|
||||
plan: [effectiveEdition(t, ctx.nowMs)],
|
||||
|
||||
@@ -213,6 +213,11 @@ export const api = {
|
||||
`${API_BASE_URL}/api/tenants/${encodeURIComponent(tenantId)}/offboard`,
|
||||
offboardFeedback: (tenantId: string) =>
|
||||
`${API_BASE_URL}/api/tenants/${encodeURIComponent(tenantId)}/offboard/feedback`,
|
||||
// Offboarding record + operator retry of a Failed run (Global Admin tooling, /api/global/*).
|
||||
offboardingStatus: (tenantId: string) =>
|
||||
`${API_BASE_URL}/api/global/tenants/${encodeURIComponent(tenantId)}/offboarding`,
|
||||
offboardingRetry: (tenantId: string) =>
|
||||
`${API_BASE_URL}/api/global/tenants/${encodeURIComponent(tenantId)}/offboarding/retry`,
|
||||
},
|
||||
|
||||
// ── Devices ───────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -22816,6 +22816,103 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"TenantOffboardingStatusResponse": {
|
||||
"kind": "object",
|
||||
"doc": "Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one.",
|
||||
"fields": [
|
||||
{
|
||||
"name": "tenantId",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "historyRowKey",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "initiatedAt",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "initiatedBy",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "retryCount",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "number"
|
||||
},
|
||||
"doc": "Queue dequeue attempts plus operator retries of this run."
|
||||
},
|
||||
{
|
||||
"name": "earliestProcessingAt",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "drainCompletedAt",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "completedAt",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "failedAt",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "failedPhase",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "errorMessage",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"TestLogPatternRequest": {
|
||||
"kind": "object",
|
||||
"doc": "Body of POST rules/gather/test-pattern.",
|
||||
|
||||
@@ -4994,6 +4994,23 @@ export interface TenantOffboardingCustomsArchiveEntry {
|
||||
archivedBy: string;
|
||||
}
|
||||
|
||||
/** Body of GET global/tenants/{tenantId}/offboarding — the tenant's current offboarding record for the admin tenant editor. Status is the marker's status (the anchor every HTTP decision keys on); the history fields describe the run behind it. A Failed record carries FailedPhase so the operator sees what to expect from a retry before starting one. */
|
||||
export interface TenantOffboardingStatusResponse {
|
||||
tenantId: string;
|
||||
status: string;
|
||||
historyRowKey: string;
|
||||
initiatedAt: string;
|
||||
initiatedBy: string;
|
||||
/** Queue dequeue attempts plus operator retries of this run. */
|
||||
retryCount: number;
|
||||
earliestProcessingAt?: string;
|
||||
drainCompletedAt?: string;
|
||||
completedAt?: string;
|
||||
failedAt?: string;
|
||||
failedPhase?: string;
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
/** Body of POST rules/gather/test-pattern. */
|
||||
export interface TestLogPatternRequest {
|
||||
pattern?: string | null;
|
||||
|
||||
Reference in New Issue
Block a user