mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 04:32:32 +00:00
feat(maintenance): session-deletion run budget, live watchdog counters, manual trigger
- Watchdog fix: FanoutResult is caller-allocated and mutated live per tenant/session, so 30/60min LongRunning warnings report real progress instead of the always-zero counters (today's warning showed 0/0 while 5 tenants / 54 sessions were already done). - 50min run budget: the retention fanout stops cleanly at the deadline (tenant boundary + per-session check) and reports AbortedByBudget — no more risk of the 60min Flex functionTimeout hard-aborting mid-run. New Warning OpsEvent SessionDeletionMaintenanceBudgetExceeded; Completed gains abortedByBudget. - Blob-lease serialization: dedicated sentinel _lock/session-deletion-maintenance.lock (deletion-manifests container) via SessionDeletionMaintenanceLockStore + reused MaintenanceLeaseHolder; contended runs emit SessionDeletionMaintenanceSkippedLocked and return. - Manual trigger: POST /api/global/session-deletions/maintenance/trigger (GlobalAdminOnly, policy catalog) -> fail-hard queue producer -> SessionDeletionMaintenanceQueueWorker runs the same RunCoreAsync body with triggeredBy = requesting admin; lease probe returns 409 while a run is active. New Started OpsEvent carries triggeredBy. - Web: MaintenanceStatusBanner on the Session Cleanup page derives active / budget-exceeded / failed / idle from the lifecycle ops events and offers a Run-now button (202/409 handling). New event types dual-registered in OPS_EVENT_TYPES. - Tests: 7 new (budget abort immediate/mid-tenant, live counter increments, watchdog live-progress regression, BudgetExceeded + Completed flag, SkippedLocked, Started triggeredBy); suites green (backend 2935, web 557, tsc clean). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+109
-24
@@ -46,10 +46,10 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
harness.SeedSessionsByState(SessionDeletionState.Preparing); // no rows aged out in this scenario
|
||||
harness.SeedSessionsByState(SessionDeletionState.Queued);
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
harness.BlobMock.Verify(b => b.DeleteDeletionManifestPairAsync("t", "s", "m", It.IsAny<CancellationToken>()), Times.Once);
|
||||
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<CancellationToken>()), Times.Once);
|
||||
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()), Times.Once);
|
||||
// PR6 follow-up F3: lifecycle status now lives on OpsEvents, not AuditLogs (the latter
|
||||
// would silently fail because PartitionKey requires a non-null tenantId).
|
||||
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted");
|
||||
@@ -62,12 +62,12 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
harness.SetKillSwitch(true);
|
||||
harness.SeedTtlBlobs(("t", "s", "m", DateTime.UtcNow.AddDays(-31)));
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
// GCs still ran — TTL sweep emitted the DeleteDeletionManifestPairAsync call.
|
||||
harness.BlobMock.Verify(b => b.DeleteDeletionManifestPairAsync("t", "s", "m", It.IsAny<CancellationToken>()), Times.Once);
|
||||
// Fanout was skipped.
|
||||
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<CancellationToken>()), Times.Never);
|
||||
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
// PR6 follow-up F3: fanout-skip + completion now both live on OpsEvents.
|
||||
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceFanoutSkipped");
|
||||
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted");
|
||||
@@ -82,32 +82,55 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
// but not severe.
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(500));
|
||||
harness.SetKillSwitch(false);
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<CancellationToken>()))
|
||||
.Returns(async (CancellationToken ct) =>
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) =>
|
||||
{
|
||||
r.TenantsProcessed = 1;
|
||||
await Task.Delay(150, ct);
|
||||
return new SessionRetentionFanoutService.FanoutResult { TenantsProcessed = 1, SessionsEnqueued = 0 };
|
||||
});
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
|
||||
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunningSevere");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunCore_watchdog_reports_live_fanout_progress()
|
||||
{
|
||||
// Regression for the "tenants=0, enqueued=0" watchdog bug: the counters used to be
|
||||
// assigned only AFTER the fanout returned, so every LongRunning warning showed zeros.
|
||||
// The fanout now mutates the caller-supplied result incrementally; the watchdog snapshot
|
||||
// must reflect that progress mid-run.
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromSeconds(5));
|
||||
harness.SetKillSwitch(false);
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) =>
|
||||
{
|
||||
r.TenantsProcessed = 3;
|
||||
r.SessionsEnqueued = 7;
|
||||
await Task.Delay(200, ct); // long enough for the 50ms watchdog to fire mid-run
|
||||
});
|
||||
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
var warning = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
|
||||
Assert.Contains("tenants=3", warning.Message);
|
||||
Assert.Contains("enqueued=7", warning.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunCore_emits_LongRunningSevere_when_body_runs_past_severe_threshold()
|
||||
{
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(150));
|
||||
harness.SetKillSwitch(false);
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<CancellationToken>()))
|
||||
.Returns(async (CancellationToken ct) =>
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(async (SessionRetentionFanoutService.FanoutResult _, DateTime _, CancellationToken ct) =>
|
||||
{
|
||||
await Task.Delay(300, ct);
|
||||
return new SessionRetentionFanoutService.FanoutResult();
|
||||
});
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
// Both watchdogs fire: warning at 50ms, severe at 150ms.
|
||||
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
|
||||
@@ -126,7 +149,7 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
("tenant-a", "session-y", "MANIFEST-2"),
|
||||
("tenant-b", "session-z", "MANIFEST-3"));
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-x", It.IsAny<CancellationToken>()), Times.Once);
|
||||
harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-y", It.IsAny<CancellationToken>()), Times.Once);
|
||||
@@ -143,7 +166,7 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
harness.SetKillSwitch(true);
|
||||
harness.SeedExpiredTombstones(("tenant-a", "session-x", "M1"));
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-x", It.IsAny<CancellationToken>()),
|
||||
Times.Once);
|
||||
@@ -155,12 +178,63 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromSeconds(5), watchdogSevere: TimeSpan.FromSeconds(10));
|
||||
harness.SetKillSwitch(false);
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
|
||||
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunningSevere");
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────── Run budget + lease + lifecycle ─
|
||||
|
||||
[Fact]
|
||||
public async Task RunCore_emits_BudgetExceeded_and_flags_Completed_when_fanout_hits_the_budget()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetKillSwitch(false);
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.Returns((SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken _) =>
|
||||
{
|
||||
r.TenantsProcessed = 12;
|
||||
r.SessionsEnqueued = 40;
|
||||
r.AbortedByBudget = true;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
var budget = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceBudgetExceeded");
|
||||
Assert.Contains("tenants=12", budget.Message);
|
||||
var completed = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted");
|
||||
Assert.Contains("abortedByBudget=True", completed.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunCore_emits_SkippedLocked_and_skips_everything_when_lease_is_held()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new AutopilotMonitor.Functions.Services.Backup.LeaseHeldException("held", new Exception("409")));
|
||||
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceSkippedLocked");
|
||||
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceStarted");
|
||||
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted");
|
||||
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunCore_emits_Started_event_with_triggeredBy()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.SetKillSwitch(false);
|
||||
|
||||
await harness.Sut.RunCoreAsync("admin@contoso.com", CancellationToken.None);
|
||||
|
||||
var started = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceStarted");
|
||||
Assert.Contains("admin@contoso.com", started.Message);
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────── Exception path ─
|
||||
|
||||
[Fact]
|
||||
@@ -169,10 +243,10 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
var harness = new Harness();
|
||||
harness.SetKillSwitch(false);
|
||||
// Make the fanout throw.
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<CancellationToken>()))
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new InvalidOperationException("fanout exploded"));
|
||||
|
||||
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.RunCoreAsync(CancellationToken.None));
|
||||
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.RunCoreAsync("Test", CancellationToken.None));
|
||||
Assert.Equal("fanout exploded", ex.Message);
|
||||
|
||||
// PR6 follow-up F3: the SessionDeletionMaintenanceFailed OpsEvent IS the audit. The prior
|
||||
@@ -192,7 +266,7 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
(TenantA, SessionA1, Manifest2, stuckSince),
|
||||
(TenantB, SessionB1, Manifest2, stuckSince));
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
var stranded = harness.OpsEvents.Where(e => e.EventName == "SessionDeletionStrandedQueued").ToList();
|
||||
Assert.Equal(2, stranded.Count);
|
||||
@@ -206,7 +280,7 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
var freshly = DateTime.UtcNow.AddMinutes(-5); // well under 30min
|
||||
harness.SeedSessionsByState(SessionDeletionState.Queued, (TenantA, SessionA1, Manifest2, freshly));
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionStrandedQueued");
|
||||
}
|
||||
@@ -234,7 +308,7 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
.Setup(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA1, Manifest1, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(true);
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA1, Manifest1, It.IsAny<CancellationToken>()), Times.Once);
|
||||
harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA2, Manifest1 + "X", It.IsAny<CancellationToken>()), Times.Never);
|
||||
@@ -250,7 +324,7 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
var recent = DateTime.UtcNow.AddMinutes(-5);
|
||||
harness.SeedSessionsByState(SessionDeletionState.Preparing, (TenantA, SessionA1, Manifest1, recent));
|
||||
|
||||
await harness.Sut.RunCoreAsync(CancellationToken.None);
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
@@ -263,6 +337,7 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
public Mock<MaintenanceBlobStub> BlobMock { get; }
|
||||
public Mock<AdminConfigurationService> AdminConfig { get; }
|
||||
public Mock<SessionRetentionFanoutService> Fanout { get; }
|
||||
public Mock<SessionDeletionMaintenanceLockStore> LockStore { get; }
|
||||
public List<CapturedOpsEvent> OpsEvents { get; } = new();
|
||||
public List<AuditEntry> AuditCalls { get; } = new();
|
||||
public SessionDeletionMaintenanceFunction Sut { get; }
|
||||
@@ -328,14 +403,24 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
Mock.Of<ISessionDeletionEnqueuer>(),
|
||||
AdminConfig.Object,
|
||||
NullLogger<SessionRetentionFanoutService>.Instance);
|
||||
Fanout.Setup(f => f.RunAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new SessionRetentionFanoutService.FanoutResult { TenantsProcessed = 0, SessionsEnqueued = 0 });
|
||||
Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
// Lease store: default acquire succeeds with a mocked lease client. The renewal loop
|
||||
// never ticks inside these millisecond-scale tests; DisposeAsync's release failure on
|
||||
// the bare mock is swallowed by the holder's try/catch.
|
||||
LockStore = new Mock<SessionDeletionMaintenanceLockStore>(
|
||||
BlobMock.Object, NullLogger<SessionDeletionMaintenanceLockStore>.Instance);
|
||||
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new Mock<Azure.Storage.Blobs.Specialized.BlobLeaseClient>().Object);
|
||||
|
||||
Sut = new SessionDeletionMaintenanceFunction(
|
||||
StorageMock.Object, BlobMock.Object, AdminConfig.Object, opsService, maintRepo.Object, Fanout.Object,
|
||||
LockStore.Object,
|
||||
NullLogger<SessionDeletionMaintenanceFunction>.Instance,
|
||||
watchdogWarning: watchdogWarn ?? TimeSpan.FromSeconds(30),
|
||||
watchdogSevere: watchdogSevere ?? TimeSpan.FromMinutes(1));
|
||||
watchdogSevere: watchdogSevere ?? TimeSpan.FromMinutes(1),
|
||||
runBudget: TimeSpan.FromMinutes(50));
|
||||
}
|
||||
|
||||
public void SetKillSwitch(bool active) =>
|
||||
|
||||
+94
-15
@@ -29,7 +29,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
harness.WithTenant(TenantA, retentionDays: 30,
|
||||
sessions: new[] { Old("s1", 60), Old("s2", 45) });
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.Equal(2, result.SessionsEnqueued);
|
||||
harness.Enqueuer.Verify(
|
||||
@@ -56,7 +56,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
harness.MaintenanceRepo.Setup(m => m.GetSessionsOlderThanAsync(TenantB, It.Is<DateTime>(d => d <= DateTime.UtcNow.AddDays(-120)), It.IsAny<int>(), It.IsAny<bool>()))
|
||||
.ReturnsAsync(new List<SessionSummary> { Summary(TenantB, "b2") });
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.Equal(2, result.TenantsProcessed);
|
||||
Assert.Equal(2, result.SessionsEnqueued);
|
||||
@@ -75,7 +75,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
many.Add(Summary(TenantA, $"s{i:000}"));
|
||||
harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: many);
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.Equal(SessionRetentionFanoutService.MaxEnqueuesPerTenantPerRun, result.SessionsEnqueued);
|
||||
Assert.Equal(1, result.RateLimitedTenants);
|
||||
@@ -94,7 +94,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
var harness = new Harness();
|
||||
harness.WithTenant(TenantA, retentionDays: 30, sessions: new[] { Old("s1", 60) });
|
||||
|
||||
await harness.Sut.RunAsync(CancellationToken.None);
|
||||
await harness.RunAsync();
|
||||
|
||||
harness.MaintenanceRepo.Verify(
|
||||
m => m.GetSessionsOlderThanAsync(
|
||||
@@ -113,7 +113,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
var harness = new Harness();
|
||||
harness.WithTenant(TenantA, retentionDays: 0, sessions: new[] { Old("s1", 365) });
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.Equal(0, result.SessionsEnqueued);
|
||||
// GetSessionsOlderThanAsync must not have been called for that tenant.
|
||||
@@ -129,7 +129,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
var harness = new Harness();
|
||||
harness.WithTenant(TenantA, retentionDays: 180, sessions: new[] { Old("s1", 100) });
|
||||
|
||||
await harness.Sut.RunAsync(CancellationToken.None);
|
||||
await harness.RunAsync();
|
||||
|
||||
harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(
|
||||
TenantA,
|
||||
@@ -143,7 +143,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
var harness = new Harness();
|
||||
harness.WithTenant(TenantA, retentionDays: 180, sessions: new[] { Old("s1", 200) }, planTier: "enterprise");
|
||||
|
||||
await harness.Sut.RunAsync(CancellationToken.None);
|
||||
await harness.RunAsync();
|
||||
|
||||
harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(
|
||||
TenantA,
|
||||
@@ -174,7 +174,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
return Task.FromResult(calls >= 4);
|
||||
});
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.True(result.AbortedByKillSwitch);
|
||||
Assert.Equal(2, result.SessionsEnqueued); // only sessions 0 and 1 made it through
|
||||
@@ -203,7 +203,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
return Task.FromResult(calls >= 3);
|
||||
});
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.True(result.AbortedByKillSwitch);
|
||||
Assert.Equal(1, result.TenantsProcessed);
|
||||
@@ -235,7 +235,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
: new SessionDeletionEnqueueResult { Outcome = SessionDeletionEnqueueOutcome.KillSwitchActive };
|
||||
});
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.True(result.AbortedByKillSwitch);
|
||||
Assert.Equal(1, result.SessionsEnqueued);
|
||||
@@ -272,7 +272,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
: new SessionDeletionEnqueueResult { Outcome = SessionDeletionEnqueueOutcome.KillSwitchActive };
|
||||
});
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.True(result.AbortedByKillSwitch);
|
||||
Assert.Equal(1, result.TenantsProcessed); // tenant A completed its RunForTenantAsync; B never started
|
||||
@@ -301,12 +301,79 @@ public class SessionRetentionFanoutServiceTests
|
||||
ExistingState = SessionDeletionState.Running,
|
||||
});
|
||||
|
||||
var result = await harness.Sut.RunAsync(CancellationToken.None);
|
||||
var result = await harness.RunAsync();
|
||||
|
||||
Assert.Equal(0, result.SessionsEnqueued);
|
||||
Assert.Equal(1, result.SessionsSkipped);
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────── Run budget ─
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_aborts_immediately_when_deadline_already_passed()
|
||||
{
|
||||
var harness = new Harness();
|
||||
harness.WithTenant(TenantA, retentionDays: 30, sessions: new[] { Old("s1", 60) });
|
||||
|
||||
var result = await harness.RunAsync(deadlineUtc: DateTime.UtcNow.AddMinutes(-1));
|
||||
|
||||
Assert.True(result.AbortedByBudget);
|
||||
Assert.Equal(0, result.TenantsProcessed);
|
||||
Assert.Equal(0, result.SessionsEnqueued);
|
||||
harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(TenantA, It.IsAny<DateTime>(), It.IsAny<int>(), It.IsAny<bool>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_aborts_mid_tenant_when_budget_deadline_crosses()
|
||||
{
|
||||
// Scripted clock: the deadline crosses between session 1 and session 2 of tenant A —
|
||||
// the remaining sessions of A and all of tenant B must be left untouched.
|
||||
var baseTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var deadline = baseTime.AddMinutes(50);
|
||||
var clockCalls = 0;
|
||||
// Call sequence: tenant-A boundary (1), session s0 (2), session s1 (3), session s2 (4 → past deadline).
|
||||
var harness = new Harness(utcNow: () => ++clockCalls >= 4 ? deadline.AddMinutes(1) : baseTime);
|
||||
|
||||
var sessions = new List<SessionSummary>();
|
||||
for (int i = 0; i < 4; i++) sessions.Add(Summary(TenantA, $"s{i}"));
|
||||
harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: sessions);
|
||||
harness.WithTenant(TenantB, retentionDays: 30, sessions: new[] { Old("b1", 60) });
|
||||
|
||||
var result = await harness.RunAsync(deadlineUtc: deadline);
|
||||
|
||||
Assert.True(result.AbortedByBudget);
|
||||
Assert.Equal(2, result.SessionsEnqueued); // only s0 and s1 made it through
|
||||
Assert.Equal(1, result.TenantsProcessed); // tenant A returned normally, B never started
|
||||
harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantA, "s2", It.IsAny<string>(), It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantB, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(TenantB, It.IsAny<DateTime>(), It.IsAny<int>(), It.IsAny<bool>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_mutates_the_shared_result_incrementally_during_the_run()
|
||||
{
|
||||
// Watchdog-progress contract: the caller-supplied FanoutResult must be bumped per
|
||||
// session (not once at the end), so mid-run snapshots show real progress. The injected
|
||||
// throttle runs between sessions — snapshot the counter there.
|
||||
var snapshots = new List<int>();
|
||||
var result = new SessionRetentionFanoutService.FanoutResult();
|
||||
var harness = new Harness(throttle: (_, _) =>
|
||||
{
|
||||
snapshots.Add(result.SessionsEnqueued);
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
var sessions = new List<SessionSummary>();
|
||||
for (int i = 0; i < 3; i++) sessions.Add(Summary(TenantA, $"s{i}"));
|
||||
harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: sessions);
|
||||
|
||||
await harness.Sut.RunAsync(result, DateTime.UtcNow.AddMinutes(50), CancellationToken.None);
|
||||
|
||||
Assert.Equal(3, result.SessionsEnqueued);
|
||||
// Throttle fires after s0 and s1 (not after the last session) — each snapshot must
|
||||
// already reflect the enqueues so far.
|
||||
Assert.Equal(new[] { 1, 2 }, snapshots);
|
||||
}
|
||||
|
||||
// ============================================================ Helpers ====
|
||||
|
||||
private static SessionSummary Summary(string tenantId, string sessionId) =>
|
||||
@@ -333,7 +400,7 @@ public class SessionRetentionFanoutServiceTests
|
||||
|
||||
private readonly List<string> _tenantIds = new();
|
||||
|
||||
public Harness()
|
||||
public Harness(Func<TimeSpan, CancellationToken, Task>? throttle = null, Func<DateTime>? utcNow = null)
|
||||
{
|
||||
MaintenanceRepo = new Mock<IMaintenanceRepository>();
|
||||
var memCache = new MemoryCache(new MemoryCacheOptions());
|
||||
@@ -363,12 +430,24 @@ public class SessionRetentionFanoutServiceTests
|
||||
.ReturnsAsync(true);
|
||||
|
||||
// Use internal ctor to inject a no-op throttle so the rate-limit test runs in real-time
|
||||
// (50ms × 100 = 5s otherwise).
|
||||
// (50ms × 100 = 5s otherwise) and an optional scripted clock for the budget tests.
|
||||
Sut = new SessionRetentionFanoutService(
|
||||
MaintenanceRepo.Object, TenantConfig.Object, Enqueuer.Object,
|
||||
AdminConfig.Object,
|
||||
NullLogger<SessionRetentionFanoutService>.Instance,
|
||||
throttle: (_, _) => Task.CompletedTask);
|
||||
throttle: throttle ?? ((_, _) => Task.CompletedTask),
|
||||
utcNow: utcNow);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drives <see cref="SessionRetentionFanoutService.RunAsync"/> with a fresh result object
|
||||
/// and a far-away default deadline (the budget is exercised by dedicated tests).
|
||||
/// </summary>
|
||||
public async Task<SessionRetentionFanoutService.FanoutResult> RunAsync(DateTime? deadlineUtc = null)
|
||||
{
|
||||
var result = new SessionRetentionFanoutService.FanoutResult();
|
||||
await Sut.RunAsync(result, deadlineUtc ?? DateTime.UtcNow.AddMinutes(50), CancellationToken.None);
|
||||
return result;
|
||||
}
|
||||
|
||||
public void WithTenant(string tenantId, int retentionDays, SessionSummary[] sessions, string planTier = "free")
|
||||
|
||||
+98
-35
@@ -4,10 +4,12 @@ using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
@@ -32,9 +34,18 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
/// <para>
|
||||
/// 30-minute watchdog emits a Warning OpsEvent; 60-minute watchdog escalates to Error. The
|
||||
/// 60-minute mark matches the Flex Consumption hard <c>functionTimeout</c> (host.json) so the
|
||||
/// operator gets a Telegram alert before the Azure runtime abort. Cron <c>0 0 */12 * * *</c>
|
||||
/// (every 12h at 00:00 / 12:00 UTC) — const per repo convention; change-and-redeploy if you
|
||||
/// need a different cadence.
|
||||
/// operator gets a Telegram alert before the Azure runtime abort. The retention fanout stops
|
||||
/// itself at the <see cref="RunBudget"/> (50min) deadline so a growing backlog degrades into
|
||||
/// a clean <c>BudgetExceeded</c> + partial sweep instead of a host abort. Cron
|
||||
/// <c>0 0 */12 * * *</c> (every 12h at 00:00 / 12:00 UTC) — const per repo convention;
|
||||
/// change-and-redeploy if you need a different cadence.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Runs are serialized by a blob lease (<see cref="SessionDeletionMaintenanceLockStore"/>):
|
||||
/// the timer and the manual trigger (<c>POST /api/global/session-deletions/maintenance/trigger</c>
|
||||
/// → <see cref="SessionDeletionMaintenanceQueueWorker"/>) both enter through
|
||||
/// <see cref="RunCoreAsync"/>, which acquires the lease first and emits
|
||||
/// <c>SkippedLocked</c> when another run is active.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class SessionDeletionMaintenanceFunction
|
||||
@@ -58,15 +69,21 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
internal static readonly TimeSpan WatchdogWarning = TimeSpan.FromMinutes(30);
|
||||
internal static readonly TimeSpan WatchdogSevere = TimeSpan.FromMinutes(60);
|
||||
|
||||
// Run budget for the retention fanout (matches the critical-table backup convention):
|
||||
// stop cleanly at 50min so the 60min host functionTimeout never aborts us mid-tenant.
|
||||
internal static readonly TimeSpan RunBudget = TimeSpan.FromMinutes(50);
|
||||
|
||||
private readonly TableStorageService _storage;
|
||||
private readonly BlobStorageService _blob;
|
||||
private readonly AdminConfigurationService _adminConfig;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly SessionRetentionFanoutService _fanout;
|
||||
private readonly SessionDeletionMaintenanceLockStore _lockStore;
|
||||
private readonly ILogger<SessionDeletionMaintenanceFunction> _logger;
|
||||
private readonly TimeSpan _watchdogWarning;
|
||||
private readonly TimeSpan _watchdogSevere;
|
||||
private readonly TimeSpan _runBudget;
|
||||
|
||||
public SessionDeletionMaintenanceFunction(
|
||||
TableStorageService storage,
|
||||
@@ -75,14 +92,16 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
OpsEventService opsEvents,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
SessionRetentionFanoutService fanout,
|
||||
SessionDeletionMaintenanceLockStore lockStore,
|
||||
ILogger<SessionDeletionMaintenanceFunction> logger)
|
||||
: this(storage, blob, adminConfig, opsEvents, maintenanceRepo, fanout, logger, WatchdogWarning, WatchdogSevere)
|
||||
: this(storage, blob, adminConfig, opsEvents, maintenanceRepo, fanout, lockStore, logger, WatchdogWarning, WatchdogSevere, RunBudget)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test seam: lets unit tests inject sub-second watchdog thresholds so the function can be
|
||||
/// driven end-to-end in milliseconds. Production code uses the static defaults (30min / 60min).
|
||||
/// Test seam: lets unit tests inject sub-second watchdog thresholds and run budget so the
|
||||
/// function can be driven end-to-end in milliseconds. Production code uses the static
|
||||
/// defaults (30min / 60min / 50min).
|
||||
/// </summary>
|
||||
internal SessionDeletionMaintenanceFunction(
|
||||
TableStorageService storage,
|
||||
@@ -91,9 +110,11 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
OpsEventService opsEvents,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
SessionRetentionFanoutService fanout,
|
||||
SessionDeletionMaintenanceLockStore lockStore,
|
||||
ILogger<SessionDeletionMaintenanceFunction> logger,
|
||||
TimeSpan watchdogWarning,
|
||||
TimeSpan watchdogSevere)
|
||||
TimeSpan watchdogSevere,
|
||||
TimeSpan runBudget)
|
||||
{
|
||||
_storage = storage;
|
||||
_blob = blob;
|
||||
@@ -101,63 +122,95 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
_opsEvents = opsEvents;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_fanout = fanout;
|
||||
_lockStore = lockStore;
|
||||
_logger = logger;
|
||||
_watchdogWarning = watchdogWarning;
|
||||
_watchdogSevere = watchdogSevere;
|
||||
_runBudget = runBudget;
|
||||
}
|
||||
|
||||
[Function("SessionDeletionMaintenance")]
|
||||
public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken)
|
||||
{
|
||||
await RunCoreAsync(cancellationToken);
|
||||
await RunCoreAsync("Timer", cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Testable core. Separated from the TimerTrigger entry so unit tests can drive it
|
||||
/// directly without a TimerInfo / FunctionContext.
|
||||
/// directly without a TimerInfo / FunctionContext, and shared with the manual-trigger
|
||||
/// queue worker. <paramref name="triggeredBy"/> is <c>"Timer"</c> for the cron path and
|
||||
/// the requesting Global Admin's identifier for manual runs.
|
||||
/// </summary>
|
||||
internal async Task RunCoreAsync(CancellationToken cancellationToken)
|
||||
internal async Task RunCoreAsync(string triggeredBy, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("SessionDeletionMaintenance: tick started at {Now:o}", DateTime.UtcNow);
|
||||
var sw = Stopwatch.StartNew();
|
||||
_logger.LogInformation("SessionDeletionMaintenance: tick started at {Now:o} (triggeredBy={TriggeredBy})", DateTime.UtcNow, triggeredBy);
|
||||
|
||||
using var watchdogCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
int sessionsEnqueuedSoFar = 0;
|
||||
int tenantsProcessedSoFar = 0;
|
||||
// Serialize timer vs manual trigger vs second host instance. Acquired BEFORE the
|
||||
// Started OpsEvent so a lease-skip never masquerades as an active run to the UI.
|
||||
BlobLeaseClient lease;
|
||||
try
|
||||
{
|
||||
lease = await _lockStore.AcquireLeaseAsync(ct: cancellationToken);
|
||||
}
|
||||
catch (LeaseHeldException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SessionDeletionMaintenance: lease held by another run — skipping (triggeredBy={TriggeredBy})",
|
||||
triggeredBy);
|
||||
await _opsEvents.RecordSessionDeletionMaintenanceSkippedLockedAsync(triggeredBy);
|
||||
return;
|
||||
}
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var deadlineUtc = DateTime.UtcNow + _runBudget;
|
||||
|
||||
// handlerCts: cancelled by the lease holder on renewal failure so the run never
|
||||
// continues rogue under an expired lease. The body + watchdogs run off this token.
|
||||
using var handlerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
var leaseHolder = new MaintenanceLeaseHolder(lease, handlerCts, _logger);
|
||||
using var watchdogCts = CancellationTokenSource.CreateLinkedTokenSource(handlerCts.Token);
|
||||
|
||||
// Live progress object: the fanout mutates it per tenant / per session, so the
|
||||
// watchdog snapshots below report real progress mid-run (previously the counters
|
||||
// were only assigned after the fanout returned — every warning showed 0/0).
|
||||
var fanoutResult = new SessionRetentionFanoutService.FanoutResult();
|
||||
|
||||
// Watchdogs: each one auto-cancels when the body completes (we cancel watchdogCts in
|
||||
// the finally). The await Task.WhenAny inside RunWatchdogAsync only emits if the body
|
||||
// is still running at the threshold.
|
||||
var watchdogWarn = RunWatchdogAsync(_watchdogWarning, severe: false, () => (tenantsProcessedSoFar, sessionsEnqueuedSoFar), watchdogCts.Token);
|
||||
var watchdogSevere = RunWatchdogAsync(_watchdogSevere, severe: true, () => (tenantsProcessedSoFar, sessionsEnqueuedSoFar), watchdogCts.Token);
|
||||
var watchdogWarn = RunWatchdogAsync(_watchdogWarning, severe: false, () => (fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued), watchdogCts.Token);
|
||||
var watchdogSevere = RunWatchdogAsync(_watchdogSevere, severe: true, () => (fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued), watchdogCts.Token);
|
||||
|
||||
try
|
||||
{
|
||||
await _opsEvents.RecordSessionDeletionMaintenanceStartedAsync(triggeredBy);
|
||||
|
||||
var ct = handlerCts.Token;
|
||||
|
||||
// PR5 finding 1: uncached, fail-closed read so a flip-ON is honored within seconds
|
||||
// across all Function-host instances.
|
||||
var killSwitchActive = await _adminConfig.IsSessionDeletionKillSwitchActiveAsync();
|
||||
|
||||
// (1) Manifest-Blob-TTL sweep — runs even when kill-switch=true (Plan PR6 step 3).
|
||||
var blobsTtlGced = await SweepManifestBlobsAsync(cancellationToken);
|
||||
var blobsTtlGced = await SweepManifestBlobsAsync(ct);
|
||||
|
||||
// (2) Stale-Preparing GC — runs even when kill-switch=true.
|
||||
var preparingRowsCleared = await GcStalePreparingAsync(cancellationToken);
|
||||
var preparingRowsCleared = await GcStalePreparingAsync(ct);
|
||||
|
||||
// (3) Stranded-Queued detection (alert-only) — runs even when kill-switch=true.
|
||||
var strandedQueuedDetected = await DetectStrandedQueuedAsync(cancellationToken);
|
||||
var strandedQueuedDetected = await DetectStrandedQueuedAsync(ct);
|
||||
|
||||
// (3b) Tombstone-marker pruning — physically remove rows past their ExpiresAt
|
||||
// (Codex F3). Runs even when kill-switch=true: these markers are short-lived
|
||||
// race-shields, not policy-bound deletion artefacts. Failures are non-fatal
|
||||
// because the in-flight Guard already treats expired markers as absent.
|
||||
var tombstonesPruned = await PruneExpiredTombstonesAsync(cancellationToken);
|
||||
var tombstonesPruned = await PruneExpiredTombstonesAsync(ct);
|
||||
|
||||
// (4) Retention fanout — skipped when kill-switch=true. Emit a Fanout-Skipped
|
||||
// OpsEvent so the operator can fold the skip into the OpsEvents dashboard alongside
|
||||
// the 30/60min watchdogs. PR6 follow-up F3: previously a LogAuditEntryAsync(null!)
|
||||
// call that silently failed because the AuditLogs schema requires a non-null
|
||||
// PartitionKey — OpsEvents tolerates null TenantId, so global-scope events live there.
|
||||
SessionRetentionFanoutService.FanoutResult? fanoutResult = null;
|
||||
if (killSwitchActive)
|
||||
{
|
||||
await _opsEvents.RecordSessionDeletionMaintenanceFanoutSkippedAsync(
|
||||
@@ -165,32 +218,38 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
}
|
||||
else
|
||||
{
|
||||
fanoutResult = await _fanout.RunAsync(cancellationToken);
|
||||
tenantsProcessedSoFar = fanoutResult.TenantsProcessed;
|
||||
sessionsEnqueuedSoFar = fanoutResult.SessionsEnqueued;
|
||||
await _fanout.RunAsync(fanoutResult, deadlineUtc, ct);
|
||||
|
||||
if (fanoutResult.AbortedByBudget)
|
||||
{
|
||||
await _opsEvents.RecordSessionDeletionMaintenanceBudgetExceededAsync(
|
||||
(int)_runBudget.TotalMinutes, fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued);
|
||||
}
|
||||
}
|
||||
|
||||
// (5) Completion OpsEvent — same shape regardless of kill-switch state, so dashboards
|
||||
// can fold both paths together. PR6 follow-up F3.
|
||||
await _opsEvents.RecordSessionDeletionMaintenanceCompletedAsync(
|
||||
killSwitchActive: killSwitchActive,
|
||||
tenantsProcessed: fanoutResult?.TenantsProcessed ?? 0,
|
||||
sessionsEnqueued: fanoutResult?.SessionsEnqueued ?? 0,
|
||||
sessionsSkipped: fanoutResult?.SessionsSkipped ?? 0,
|
||||
rateLimitedTenants: fanoutResult?.RateLimitedTenants ?? 0,
|
||||
tenantsProcessed: fanoutResult.TenantsProcessed,
|
||||
sessionsEnqueued: fanoutResult.SessionsEnqueued,
|
||||
sessionsSkipped: fanoutResult.SessionsSkipped,
|
||||
rateLimitedTenants: fanoutResult.RateLimitedTenants,
|
||||
blobsTtlGced: blobsTtlGced,
|
||||
preparingRowsCleared: preparingRowsCleared,
|
||||
strandedQueuedDetected: strandedQueuedDetected,
|
||||
durationMs: (int)sw.ElapsedMilliseconds,
|
||||
abortedByKillSwitch: fanoutResult?.AbortedByKillSwitch ?? false);
|
||||
abortedByKillSwitch: fanoutResult.AbortedByKillSwitch,
|
||||
abortedByBudget: fanoutResult.AbortedByBudget);
|
||||
|
||||
_logger.LogInformation(
|
||||
"SessionDeletionMaintenance: completed in {Ms}ms — killSwitch={KillSwitch} tenants={Tenants} enqueued={Enqueued} skipped={Skipped} blobs={Blobs} preparing={Preparing} stranded={Stranded} tombstones={Tombstones}",
|
||||
"SessionDeletionMaintenance: completed in {Ms}ms — killSwitch={KillSwitch} tenants={Tenants} enqueued={Enqueued} skipped={Skipped} blobs={Blobs} preparing={Preparing} stranded={Stranded} tombstones={Tombstones} abortedByBudget={Budget}",
|
||||
sw.ElapsedMilliseconds, killSwitchActive,
|
||||
fanoutResult?.TenantsProcessed ?? 0,
|
||||
fanoutResult?.SessionsEnqueued ?? 0,
|
||||
fanoutResult?.SessionsSkipped ?? 0,
|
||||
blobsTtlGced, preparingRowsCleared, strandedQueuedDetected, tombstonesPruned);
|
||||
fanoutResult.TenantsProcessed,
|
||||
fanoutResult.SessionsEnqueued,
|
||||
fanoutResult.SessionsSkipped,
|
||||
blobsTtlGced, preparingRowsCleared, strandedQueuedDetected, tombstonesPruned,
|
||||
fanoutResult.AbortedByBudget);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -209,6 +268,10 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
// Observe the watchdog tasks so the OperationCanceledException doesn't get
|
||||
// surfaced as an unhandled exception in the host log.
|
||||
try { await Task.WhenAll(watchdogWarn, watchdogSevere); } catch (OperationCanceledException) { /* expected */ }
|
||||
|
||||
// Release the lease LAST — the Failed/Completed OpsEvents above are written while
|
||||
// the run is still exclusive, matching the backup worker's ordering.
|
||||
await leaseHolder.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// <c>POST /api/global/session-deletions/maintenance/trigger</c> — manual GA trigger for a
|
||||
/// one-off session-deletion maintenance run (GC sweeps + retention fanout), the on-demand
|
||||
/// counterpart to the 12h <see cref="SessionDeletionMaintenanceFunction"/> timer. GA-only via
|
||||
/// <c>EndpointAccessPolicyCatalog</c>; authentication + authorization are enforced by
|
||||
/// <c>PolicyEnforcementMiddleware</c>, the body does not re-check.
|
||||
/// <para>
|
||||
/// Semantics (mirrors <c>TriggerCriticalTableBackupFunction</c>'s fail-hard producer): a
|
||||
/// lease probe first rejects with <c>409 RunAlreadyActive</c> while a run is in flight; on
|
||||
/// successful enqueue we return <c>202 Accepted</c>; on enqueue failure <c>500</c> so the
|
||||
/// operator never sees a hollow success. Run progress surfaces via the
|
||||
/// <c>SessionDeletionMaintenance*</c> OpsEvents (Session Cleanup banner + Ops Events page).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class TriggerSessionDeletionMaintenanceFunction
|
||||
{
|
||||
private readonly ISessionDeletionMaintenanceTriggerProducer _producer;
|
||||
private readonly SessionDeletionMaintenanceLockStore _lockStore;
|
||||
private readonly ILogger<TriggerSessionDeletionMaintenanceFunction> _logger;
|
||||
|
||||
public TriggerSessionDeletionMaintenanceFunction(
|
||||
ISessionDeletionMaintenanceTriggerProducer producer,
|
||||
SessionDeletionMaintenanceLockStore lockStore,
|
||||
ILogger<TriggerSessionDeletionMaintenanceFunction> logger)
|
||||
{
|
||||
_producer = producer;
|
||||
_lockStore = lockStore;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[Function("TriggerSessionDeletionMaintenance")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/session-deletions/maintenance/trigger")] HttpRequestData req)
|
||||
{
|
||||
var actor = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin";
|
||||
var ct = req.FunctionContext.CancellationToken;
|
||||
|
||||
// Acquire+release probe: a held lease means a run is active RIGHT NOW — tell the
|
||||
// operator instead of queuing a message that would only produce a SkippedLocked
|
||||
// event. Racing the probe (run starts between probe and worker pickup) is benign:
|
||||
// the worker's own lease acquire lands on the SkippedLocked path.
|
||||
try
|
||||
{
|
||||
var lease = await _lockStore.AcquireLeaseAsync(ct: ct);
|
||||
try { await lease.ReleaseAsync(cancellationToken: ct); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Release failure is harmless — the 60s lease auto-expires; the worker's
|
||||
// acquire retries land within the queue's poll cadence.
|
||||
_logger.LogWarning(ex, "TriggerSessionDeletionMaintenance: probe-lease release failed (auto-expires in 60s)");
|
||||
}
|
||||
}
|
||||
catch (LeaseHeldException)
|
||||
{
|
||||
return await WriteJsonAsync(req, HttpStatusCode.Conflict, new
|
||||
{
|
||||
error = "RunAlreadyActive",
|
||||
message = "A session-deletion maintenance run is already active — check the Session Cleanup page for progress.",
|
||||
});
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _producer.EnqueueAsync(new SessionDeletionMaintenanceTriggerEnvelope { TriggeredBy = actor }, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "TriggerSessionDeletionMaintenance: enqueue failed (triggeredBy={TriggeredBy})", actor);
|
||||
return await WriteJsonAsync(req, HttpStatusCode.InternalServerError, new
|
||||
{
|
||||
error = "EnqueueFailed",
|
||||
message = "Failed to enqueue the maintenance run — please retry.",
|
||||
});
|
||||
}
|
||||
|
||||
_logger.LogInformation("TriggerSessionDeletionMaintenance: run queued (triggeredBy={TriggeredBy})", actor);
|
||||
return await WriteJsonAsync(req, HttpStatusCode.Accepted, new
|
||||
{
|
||||
message = "Maintenance run queued — progress surfaces as SessionDeletionMaintenance* ops events.",
|
||||
triggeredBy = actor,
|
||||
});
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseData> WriteJsonAsync(HttpRequestData req, HttpStatusCode status, object body)
|
||||
{
|
||||
var response = req.CreateResponse(status);
|
||||
response.Headers.Add("Content-Type", "application/json; charset=utf-8");
|
||||
await response.WriteStringAsync(JsonSerializer.Serialize(body));
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -189,6 +189,17 @@ builder.Services.AddHostedService<AutopilotMonitor.Functions.Services.Deletion.S
|
||||
// retention fanout (V2 + legacy), manifest-blob TTL sweep, stale-Preparing GC, and stranded-
|
||||
// Queued detection. Independent cadence + kill-switch from the generic 2h Maintenance timer.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionRetentionFanoutService>();
|
||||
// Manual-trigger path: blob-lease lock store (serializes timer vs manual runs), fail-hard
|
||||
// queue producer for POST /api/global/session-deletions/maintenance/trigger, and the
|
||||
// BackgroundService worker that consumes trigger envelopes. The maintenance Function class is
|
||||
// registered explicitly so the worker can invoke the same RunCoreAsync body the timer uses.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionDeletionMaintenanceLockStore>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Services.Deletion.ISessionDeletionMaintenanceTriggerProducer,
|
||||
AutopilotMonitor.Functions.Services.Deletion.AzureQueueSessionDeletionMaintenanceTriggerProducer>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Functions.Maintenance.SessionDeletionMaintenanceFunction>();
|
||||
builder.Services.AddHostedService<
|
||||
AutopilotMonitor.Functions.Services.Deletion.SessionDeletionMaintenanceQueueWorker>();
|
||||
|
||||
// Tenant-offboarding cascade worker (Plan Rev 9). Consumes per-tenant envelopes off
|
||||
// `tenant-offboarding`, drives the §3 Phase 2 sequence (cascade enqueue → drain → SafeWipe →
|
||||
|
||||
@@ -328,6 +328,8 @@ public static class EndpointAccessPolicyCatalog
|
||||
// Platform-operational (cascade-delete restore prep) — GA + read-only Reader only, NOT delegated.
|
||||
new("GET", "global/tenants/{tenantId}/deletion-manifests", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true),
|
||||
new("GET", "global/session-deletions", EndpointPolicy.GlobalReadOrAdmin),
|
||||
// Manual maintenance-run trigger (GC sweeps + retention fanout) — mutating, GA only.
|
||||
new("POST", "global/session-deletions/maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "global/raw/sessions", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
new("GET", "global/raw/events", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
|
||||
// Raw table/log access stays GlobalAdminOnly: query_table can dump arbitrary tables
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Queueing;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure.Storage.Queues;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>
|
||||
/// Azure-Storage-Queue producer for the manual session-deletion-maintenance trigger.
|
||||
/// Mirrors <c>AzureQueueCriticalTableBackupProducer</c>: Managed Identity preferred via
|
||||
/// <see cref="QueueClientFactory"/>, fail-hard semantics (SendMessageAsync errors propagate
|
||||
/// so the HTTP trigger returns 5xx instead of a hollow 202).
|
||||
/// </summary>
|
||||
public sealed class AzureQueueSessionDeletionMaintenanceTriggerProducer : ISessionDeletionMaintenanceTriggerProducer
|
||||
{
|
||||
private readonly QueueClient _queueClient;
|
||||
private readonly ILogger<AzureQueueSessionDeletionMaintenanceTriggerProducer> _logger;
|
||||
private int _queueEnsured;
|
||||
|
||||
public AzureQueueSessionDeletionMaintenanceTriggerProducer(
|
||||
QueueClientFactory queueFactory,
|
||||
ILogger<AzureQueueSessionDeletionMaintenanceTriggerProducer> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
_queueClient = queueFactory.Create(Constants.QueueNames.SessionDeletionMaintenance);
|
||||
}
|
||||
|
||||
public async Task EnqueueAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct = default)
|
||||
{
|
||||
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
|
||||
if (string.IsNullOrEmpty(envelope.TriggeredBy)) throw new ArgumentException("TriggeredBy required", nameof(envelope));
|
||||
|
||||
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
|
||||
|
||||
// Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and returns 500.
|
||||
var body = JsonConvert.SerializeObject(envelope);
|
||||
await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false);
|
||||
_logger.LogInformation("SessionDeletionMaintenance trigger enqueued (triggeredBy={TriggeredBy})", envelope.TriggeredBy);
|
||||
}
|
||||
|
||||
private async Task EnsureQueueExistsAsync(CancellationToken ct)
|
||||
{
|
||||
if (_queueEnsured == 1) return;
|
||||
try
|
||||
{
|
||||
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _queueEnsured, 1);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Ensure failure during enqueue is itself a fail-hard signal — surface to the caller.
|
||||
_logger.LogError(ex, "SessionDeletionMaintenance queue CreateIfNotExists failed");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>
|
||||
/// Envelope for a single manual session-deletion-maintenance trigger message. Tiny on
|
||||
/// purpose — the run has no per-job state row; the OpsEvents lifecycle
|
||||
/// (Started / BudgetExceeded / SkippedLocked / Completed / Failed) is the status surface.
|
||||
/// </summary>
|
||||
public sealed class SessionDeletionMaintenanceTriggerEnvelope
|
||||
{
|
||||
/// <summary>Identifier of the requesting Global Admin (UPN) — flows into the Started OpsEvent.</summary>
|
||||
public string TriggeredBy { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Producer for the <c>session-deletion-maintenance</c> queue. <b>Fail-hard</b>:
|
||||
/// SendMessageAsync exceptions propagate so the HTTP trigger can return 5xx — silent
|
||||
/// enqueue loss would leave the operator with a 202 and no run ever starting.
|
||||
/// </summary>
|
||||
public interface ISessionDeletionMaintenanceTriggerProducer
|
||||
{
|
||||
Task EnqueueAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct = default);
|
||||
}
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Azure;
|
||||
using Azure.Storage.Blobs.Models;
|
||||
using Azure.Storage.Blobs.Specialized;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>
|
||||
/// Blob-lease lock for the session-deletion maintenance run. Serializes the 12h timer
|
||||
/// against the manual HTTP trigger (and against a second host instance) — without it a
|
||||
/// manual run racing the timer would double-run the GC sweeps and the retention fanout.
|
||||
/// <para>
|
||||
/// Deliberately a DEDICATED sentinel (<see cref="LockBlobName"/> in the
|
||||
/// <c>deletion-manifests</c> container) rather than the backup sentinel in
|
||||
/// <see cref="BlobBackupStore"/> — sharing that lease would serialize backups against the
|
||||
/// retention fanout, which have no reason to exclude each other. Lease mechanics
|
||||
/// (60s lease + <see cref="MaintenanceLeaseHolder"/> 45s renewal, <see cref="LeaseHeldException"/>
|
||||
/// on contention) are identical to the backup path.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The <c>deletion-manifests</c> container has a 30-day lifecycle policy that may delete an
|
||||
/// idle sentinel; <see cref="AcquireLeaseAsync"/> re-ensures it on every acquire, so the
|
||||
/// lock self-heals. The manifest-TTL sweep skips the sentinel because <c>_lock/…</c> does
|
||||
/// not parse as a manifest blob name.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class SessionDeletionMaintenanceLockStore
|
||||
{
|
||||
/// <summary>Sentinel blob path — leased by the timer / manual-trigger worker to serialize runs.</summary>
|
||||
public const string LockBlobName = "_lock/session-deletion-maintenance.lock";
|
||||
|
||||
/// <summary>Lease duration (Azure spec allows 15..60s for renewable leases).</summary>
|
||||
public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
|
||||
|
||||
private readonly BlobStorageService _blobs;
|
||||
private readonly ILogger<SessionDeletionMaintenanceLockStore> _logger;
|
||||
private int _containerEnsured;
|
||||
|
||||
public SessionDeletionMaintenanceLockStore(
|
||||
BlobStorageService blobs,
|
||||
ILogger<SessionDeletionMaintenanceLockStore> logger)
|
||||
{
|
||||
_blobs = blobs;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to acquire the session-deletion maintenance lease. Throws
|
||||
/// <see cref="LeaseHeldException"/> on 409 (another holder); other errors propagate.
|
||||
/// Caller is responsible for renewing (via <see cref="MaintenanceLeaseHolder"/>) and
|
||||
/// releasing. <c>virtual</c> so tests can stub it.
|
||||
/// </summary>
|
||||
public virtual async Task<BlobLeaseClient> AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
|
||||
{
|
||||
await EnsureSentinelAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var blob = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests).GetBlobClient(LockBlobName);
|
||||
var leaseClient = blob.GetBlobLeaseClient();
|
||||
try
|
||||
{
|
||||
await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
|
||||
return leaseClient;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409)
|
||||
{
|
||||
throw new LeaseHeldException("session-deletion maintenance lease is held by another run", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
|
||||
/// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
|
||||
/// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
|
||||
/// </summary>
|
||||
private async Task EnsureSentinelAsync(CancellationToken ct)
|
||||
{
|
||||
var container = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests);
|
||||
if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
|
||||
{
|
||||
await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
||||
Interlocked.Exchange(ref _containerEnsured, 1);
|
||||
}
|
||||
|
||||
var blob = container.GetBlobClient(LockBlobName);
|
||||
var options = new BlobUploadOptions
|
||||
{
|
||||
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
|
||||
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
|
||||
};
|
||||
try
|
||||
{
|
||||
using var empty = new MemoryStream(Array.Empty<byte>());
|
||||
await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
|
||||
{
|
||||
// Sentinel exists already — steady state.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Maintenance;
|
||||
using AutopilotMonitor.Functions.Services.Queueing;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{
|
||||
/// <summary>
|
||||
/// Background worker for the <c>session-deletion-maintenance</c> queue — the manual-trigger
|
||||
/// counterpart to the 12h timer. Each message runs the full
|
||||
/// <see cref="SessionDeletionMaintenanceFunction.RunCoreAsync"/> body with the requesting
|
||||
/// admin as <c>triggeredBy</c>.
|
||||
/// <para>
|
||||
/// Concurrency against the timer (and a second host instance) is handled INSIDE
|
||||
/// <c>RunCoreAsync</c> via the session-deletion maintenance blob lease: a lease-held run
|
||||
/// returns normally (SkippedLocked OpsEvent), so the message is deleted rather than
|
||||
/// retried — the operator asked for "a run", and one is already active.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// BatchSize=1 + VisibilityTimeout=60min (run budget is 50min plus cushion — same rationale
|
||||
/// as <c>CriticalTableBackupQueueWorker</c>). Real exceptions rethrow → visibility-timeout
|
||||
/// retry → poison after <see cref="QueuePollingWorkerBase.MaxDequeueCount"/>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class SessionDeletionMaintenanceQueueWorker : QueuePollingWorker<SessionDeletionMaintenanceTriggerEnvelope>
|
||||
{
|
||||
private readonly SessionDeletionMaintenanceFunction _maintenance;
|
||||
|
||||
public SessionDeletionMaintenanceQueueWorker(
|
||||
QueueClientFactory queueFactory,
|
||||
SessionDeletionMaintenanceFunction maintenance,
|
||||
ILogger<SessionDeletionMaintenanceQueueWorker> logger)
|
||||
: base(queueFactory, Constants.QueueNames.SessionDeletionMaintenance, logger, Constants.QueueNames.SessionDeletionMaintenancePoison)
|
||||
{
|
||||
_maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance));
|
||||
}
|
||||
|
||||
protected override int BatchSize => 1;
|
||||
protected override TimeSpan VisibilityTimeout => TimeSpan.FromMinutes(60);
|
||||
|
||||
protected override bool TryValidate(SessionDeletionMaintenanceTriggerEnvelope envelope)
|
||||
=> !string.IsNullOrEmpty(envelope.TriggeredBy);
|
||||
|
||||
protected override string DescribeForLog(SessionDeletionMaintenanceTriggerEnvelope envelope)
|
||||
=> $"triggeredBy={envelope.TriggeredBy}";
|
||||
|
||||
protected override Task HandleAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct)
|
||||
=> _maintenance.RunCoreAsync(envelope.TriggeredBy, ct);
|
||||
}
|
||||
}
|
||||
+56
-21
@@ -34,6 +34,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
private readonly AdminConfigurationService _adminConfig;
|
||||
private readonly ILogger<SessionRetentionFanoutService> _logger;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _throttle;
|
||||
private readonly Func<DateTime> _utcNow;
|
||||
|
||||
public SessionRetentionFanoutService(
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
@@ -41,13 +42,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
ISessionDeletionEnqueuer enqueuer,
|
||||
AdminConfigurationService adminConfig,
|
||||
ILogger<SessionRetentionFanoutService> logger)
|
||||
: this(maintenanceRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay)
|
||||
: this(maintenanceRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay, utcNow: () => DateTime.UtcNow)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test seam — tests inject a no-op throttle to make the rate-limit loop exercise its
|
||||
/// counter without waiting 50ms × N in real time.
|
||||
/// counter without waiting 50ms × N in real time, and a scripted clock so the run-budget
|
||||
/// deadline can be crossed deterministically.
|
||||
/// </summary>
|
||||
internal SessionRetentionFanoutService(
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
@@ -55,7 +57,8 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
ISessionDeletionEnqueuer enqueuer,
|
||||
AdminConfigurationService adminConfig,
|
||||
ILogger<SessionRetentionFanoutService> logger,
|
||||
Func<TimeSpan, CancellationToken, Task> throttle)
|
||||
Func<TimeSpan, CancellationToken, Task> throttle,
|
||||
Func<DateTime>? utcNow = null)
|
||||
{
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_tenantConfig = tenantConfig;
|
||||
@@ -63,11 +66,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
_adminConfig = adminConfig;
|
||||
_logger = logger;
|
||||
_throttle = throttle;
|
||||
_utcNow = utcNow ?? (() => DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Aggregate result of a single fanout invocation; used for the per-tenant audit + the
|
||||
/// completion summary OpsEvent.
|
||||
/// completion summary OpsEvent. The caller allocates it and <see cref="RunAsync"/> mutates
|
||||
/// it incrementally (per tenant / per session), so watchdog snapshots taken while the
|
||||
/// fanout is still running read real progress instead of zeros.
|
||||
/// </summary>
|
||||
public sealed class FanoutResult
|
||||
{
|
||||
@@ -76,6 +82,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
public int SessionsSkipped { get; set; } // already locked / poisoned / kill-switch / etc.
|
||||
public int RateLimitedTenants { get; set; } // tenants that hit MaxEnqueuesPerTenantPerRun
|
||||
public bool AbortedByKillSwitch { get; set; } // kill-switch flipped mid-run
|
||||
public bool AbortedByBudget { get; set; } // run-budget deadline crossed mid-run
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -83,16 +90,32 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
/// Each tenant is processed independently; an exception on tenant A is logged and the loop
|
||||
/// continues with tenant B (matches the existing maintenance behaviour: per-tenant
|
||||
/// failures must not cascade).
|
||||
/// <para>
|
||||
/// <paramref name="deadlineUtc"/> is the run-budget cutoff: once crossed, the fanout stops
|
||||
/// cleanly at the next tenant/session boundary and sets <see cref="FanoutResult.AbortedByBudget"/>
|
||||
/// — the remaining backlog is picked up by the next run. This keeps the maintenance
|
||||
/// function comfortably inside the Flex Consumption 60min <c>functionTimeout</c> instead of
|
||||
/// being hard-aborted by the host mid-tenant.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public virtual async Task<FanoutResult> RunAsync(CancellationToken cancellationToken)
|
||||
public virtual async Task RunAsync(FanoutResult result, DateTime deadlineUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
var result = new FanoutResult();
|
||||
var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync().ConfigureAwait(false);
|
||||
|
||||
foreach (var tenantId in tenantIds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
// Run-budget check at the tenant boundary — cheapest clean stopping point.
|
||||
if (_utcNow() >= deadlineUtc)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SessionRetentionFanout: run budget exhausted — halting before tenant {TenantId} (deadline {Deadline:o})",
|
||||
tenantId, deadlineUtc);
|
||||
result.AbortedByBudget = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Per-tenant kill-switch check so a flip-ON mid-run halts the remaining tenants.
|
||||
// The maintenance function gates entry; this gates iteration.
|
||||
if (await _adminConfig.IsSessionDeletionKillSwitchActiveAsync().ConfigureAwait(false))
|
||||
@@ -106,7 +129,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
|
||||
try
|
||||
{
|
||||
await RunForTenantAsync(tenantId, result, cancellationToken).ConfigureAwait(false);
|
||||
await RunForTenantAsync(tenantId, deadlineUtc, result, cancellationToken).ConfigureAwait(false);
|
||||
result.TenantsProcessed++;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
@@ -118,22 +141,20 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
_logger.LogError(ex, "SessionRetentionFanout failed for tenant {TenantId} — continuing with next tenant", tenantId);
|
||||
}
|
||||
|
||||
// The inner loop sets this when the producer's Step 0 reports KillSwitchActive
|
||||
// mid-tenant. That's the authoritative fail-closed outcome — stop touching the
|
||||
// remaining tenants instead of relying on the next pre-check to catch the flip.
|
||||
if (result.AbortedByKillSwitch)
|
||||
// The inner loop sets these when the producer's Step 0 reports KillSwitchActive
|
||||
// mid-tenant, or when the budget deadline is crossed mid-tenant. Both are
|
||||
// authoritative stop signals — don't touch the remaining tenants.
|
||||
if (result.AbortedByKillSwitch || result.AbortedByBudget)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SessionRetentionFanout: aborting outer loop after tenant {TenantId} reported KillSwitchActive from producer",
|
||||
tenantId);
|
||||
"SessionRetentionFanout: aborting outer loop after tenant {TenantId} (killSwitch={KillSwitch}, budget={Budget})",
|
||||
tenantId, result.AbortedByKillSwitch, result.AbortedByBudget);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task RunForTenantAsync(string tenantId, FanoutResult result, CancellationToken cancellationToken)
|
||||
private async Task RunForTenantAsync(string tenantId, DateTime deadlineUtc, FanoutResult result, CancellationToken cancellationToken)
|
||||
{
|
||||
var config = await _tenantConfig.GetConfigurationAsync(tenantId).ConfigureAwait(false);
|
||||
// Edition retention cap (fail-closed backstop): the stored value is clamped to the
|
||||
@@ -200,6 +221,18 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
break;
|
||||
}
|
||||
|
||||
// Run-budget check per session — a single tenant's batch can take tens of
|
||||
// minutes (manifest build ~7-25s per session), so the tenant-boundary check
|
||||
// alone would not stop a run drifting into the host's hard functionTimeout.
|
||||
if (_utcNow() >= deadlineUtc)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"SessionRetentionFanout: run budget exhausted mid-tenant — halting at session {SessionId} of {TenantId}",
|
||||
session.SessionId, tenantId);
|
||||
result.AbortedByBudget = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// Per-session kill-switch check so an emergency flip-ON halts immediately instead
|
||||
// of after the rest of this tenant's backlog. Uncached read — uniform behaviour
|
||||
// across scaled-out instances within seconds.
|
||||
@@ -224,12 +257,15 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
"SessionRetentionFanout: producer reported KillSwitchActive — aborting at session {SessionId} of {TenantId}",
|
||||
session.SessionId, tenantId);
|
||||
skipped++;
|
||||
result.SessionsSkipped++;
|
||||
result.AbortedByKillSwitch = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (outcome == SessionDeletionEnqueueOutcome.Enqueued) enqueued++;
|
||||
else skipped++;
|
||||
// Live counters on the shared result (in addition to the per-tenant locals used
|
||||
// for the audit entry) so the maintenance watchdog reports real progress.
|
||||
if (outcome == SessionDeletionEnqueueOutcome.Enqueued) { enqueued++; result.SessionsEnqueued++; }
|
||||
else { skipped++; result.SessionsSkipped++; }
|
||||
|
||||
processed++;
|
||||
|
||||
@@ -252,9 +288,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
tenantId, processed, skipped);
|
||||
}
|
||||
|
||||
result.SessionsEnqueued += enqueued;
|
||||
result.SessionsSkipped += skipped;
|
||||
|
||||
// result.SessionsEnqueued / SessionsSkipped are already bumped live inside the loop.
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantId,
|
||||
"SessionDeletionMaintenanceFanout",
|
||||
@@ -272,6 +306,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
{ "Enqueued", enqueued.ToString() },
|
||||
{ "Skipped", skipped.ToString() },
|
||||
{ "AbortedByKillSwitch", result.AbortedByKillSwitch.ToString() },
|
||||
{ "AbortedByBudget", result.AbortedByBudget.ToString() },
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
|
||||
@@ -112,9 +112,40 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
// ── Cascade-Delete Maintenance (Plan §5 PR6 / §16 R14) ─────────────────
|
||||
// Four event types dispatched by SessionDeletionMaintenanceFunction. Each is also
|
||||
// Event types dispatched by SessionDeletionMaintenanceFunction: Started, LongRunning,
|
||||
// LongRunningSevere, BudgetExceeded, SkippedLocked, Failed, Completed, FanoutSkipped
|
||||
// (+ StrandedQueued / Poisoned from the GCs and the cascade worker). Each is also
|
||||
// listed in OpsAlertRulesSection.tsx OPS_EVENT_TYPES (memory feedback_ops_event_types_dual_register).
|
||||
|
||||
/// <summary>
|
||||
/// Run started (timer tick or manual trigger). Emitted after the maintenance lease was
|
||||
/// acquired, so a lease-skip never masquerades as an active run — the Session Cleanup
|
||||
/// UI banner treats "latest Started newer than latest Completed/Failed" as run-active.
|
||||
/// </summary>
|
||||
public Task RecordSessionDeletionMaintenanceStartedAsync(string triggeredBy)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceStarted", OpsEventSeverity.Info,
|
||||
$"SessionDeletionMaintenance run started (triggered by {triggeredBy})",
|
||||
null, "System.Maintenance", new { triggeredBy });
|
||||
|
||||
/// <summary>
|
||||
/// The retention fanout stopped cleanly at the run-budget deadline. Not an error: the
|
||||
/// remaining backlog is picked up by the next run (12h cadence) or a manual trigger.
|
||||
/// Paired with a Completed event whose details carry <c>abortedByBudget=true</c>.
|
||||
/// </summary>
|
||||
public Task RecordSessionDeletionMaintenanceBudgetExceededAsync(int budgetMinutes, int tenantsProcessed, int sessionsEnqueued)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceBudgetExceeded", OpsEventSeverity.Warning,
|
||||
$"SessionDeletionMaintenance stopped at the {budgetMinutes}min run budget — tenants={tenantsProcessed} enqueued={sessionsEnqueued}; remaining backlog resumes on the next run",
|
||||
null, "System.Maintenance", new { budgetMinutes, tenantsProcessed, sessionsEnqueued });
|
||||
|
||||
/// <summary>
|
||||
/// A run (timer or manual) was skipped because another run holds the session-deletion
|
||||
/// maintenance lease. Mirrors <c>RecordCriticalTableBackupSkippedLockedAsync</c>.
|
||||
/// </summary>
|
||||
public Task RecordSessionDeletionMaintenanceSkippedLockedAsync(string triggeredBy)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceSkippedLocked", OpsEventSeverity.Info,
|
||||
$"SessionDeletionMaintenance skipped — another run holds the maintenance lease (triggeredBy={triggeredBy})",
|
||||
null, "System.Maintenance", new { reason = "lease held by another run", triggeredBy });
|
||||
|
||||
/// <summary>Watchdog: maintenance run still in flight 30 minutes after start. Warning-level early signal.</summary>
|
||||
public Task RecordSessionDeletionMaintenanceLongRunningAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceLongRunning", OpsEventSeverity.Warning,
|
||||
@@ -196,13 +227,13 @@ namespace AutopilotMonitor.Functions.Services
|
||||
public Task RecordSessionDeletionMaintenanceCompletedAsync(
|
||||
bool killSwitchActive, int tenantsProcessed, int sessionsEnqueued,
|
||||
int sessionsSkipped, int rateLimitedTenants, int blobsTtlGced, int preparingRowsCleared,
|
||||
int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch)
|
||||
int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch, bool abortedByBudget)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceCompleted", OpsEventSeverity.Info,
|
||||
$"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive}",
|
||||
$"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive} abortedByBudget={abortedByBudget}",
|
||||
null, "System.Maintenance", new {
|
||||
killSwitchActive, tenantsProcessed, sessionsEnqueued,
|
||||
sessionsSkipped, rateLimitedTenants, blobsTtlGced, preparingRowsCleared,
|
||||
strandedQueuedDetected, durationMs, abortedByKillSwitch,
|
||||
strandedQueuedDetected, durationMs, abortedByKillSwitch, abortedByBudget,
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -1154,6 +1154,20 @@ namespace AutopilotMonitor.Shared
|
||||
|
||||
/// <summary>Poison sibling of <see cref="CriticalTableBackup"/>.</summary>
|
||||
public const string CriticalTableBackupPoison = "critical-table-backup-jobs-poison";
|
||||
|
||||
/// <summary>
|
||||
/// Manual-trigger queue for the session-deletion maintenance run. Producer = HTTP
|
||||
/// trigger <c>/api/global/session-deletions/maintenance/trigger</c> (fail-hard);
|
||||
/// consumer = <c>SessionDeletionMaintenanceQueueWorker</c> (BackgroundService,
|
||||
/// BatchSize=1, VisibilityTimeout=60min — the run budget is 50min plus cushion).
|
||||
/// Carries a tiny envelope <c>{ triggeredBy }</c>; concurrency against the 12h timer
|
||||
/// is serialized by the session-deletion maintenance blob lease, not the queue.
|
||||
/// Poison-suffix <c>-poison</c>, max-dequeue 5.
|
||||
/// </summary>
|
||||
public const string SessionDeletionMaintenance = "session-deletion-maintenance";
|
||||
|
||||
/// <summary>Poison sibling of <see cref="SessionDeletionMaintenance"/>.</summary>
|
||||
public const string SessionDeletionMaintenancePoison = "session-deletion-maintenance-poison";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -27,6 +27,13 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
"SessionDeletionMaintenanceLongRunningSevere",
|
||||
"SessionDeletionMaintenanceFailed",
|
||||
"SessionDeletionMaintenanceFanoutSkipped",
|
||||
// Run lifecycle + 50min run budget + lease serialization (manual-trigger feature):
|
||||
// Started marks run begin (timer or manual, details.triggeredBy), BudgetExceeded is the
|
||||
// clean self-stop at the 50min budget (Warning), SkippedLocked means another run held the
|
||||
// maintenance lease. Backend helpers RecordSessionDeletionMaintenance{Started,BudgetExceeded,SkippedLocked}Async.
|
||||
"SessionDeletionMaintenanceStarted",
|
||||
"SessionDeletionMaintenanceBudgetExceeded",
|
||||
"SessionDeletionMaintenanceSkippedLocked",
|
||||
"SessionDeletionStrandedQueued",
|
||||
// PR-B audit consolidation: the per-session deletion_poisoned tenant-audit moved here so
|
||||
// tenant admins see only the lifecycle endpoints (deletion_started/completed/restored).
|
||||
|
||||
+223
@@ -0,0 +1,223 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { api } from "@/lib/api";
|
||||
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
|
||||
interface OpsEvent {
|
||||
id: string;
|
||||
eventType: string;
|
||||
severity: string;
|
||||
message: string;
|
||||
details: string | null;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
interface OpsEventsResponse {
|
||||
events: OpsEvent[];
|
||||
}
|
||||
|
||||
interface CompletedDetails {
|
||||
tenantsProcessed?: number;
|
||||
sessionsEnqueued?: number;
|
||||
durationMs?: number;
|
||||
abortedByBudget?: boolean;
|
||||
abortedByKillSwitch?: boolean;
|
||||
}
|
||||
|
||||
type BannerState =
|
||||
| { kind: "loading" }
|
||||
| { kind: "active"; since: string; triggeredBy: string }
|
||||
| { kind: "budget-exceeded"; at: string; details: CompletedDetails }
|
||||
| { kind: "failed"; at: string; message: string }
|
||||
| { kind: "idle"; at: string; details: CompletedDetails }
|
||||
| { kind: "none" }; // no lifecycle events in the window — render nothing
|
||||
|
||||
function parseDetails<T>(raw: string | null): T {
|
||||
if (!raw) return {} as T;
|
||||
try {
|
||||
return JSON.parse(raw) as T;
|
||||
} catch {
|
||||
return {} as T;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Status banner for the session-deletion maintenance run (12h retention fanout + GC sweeps).
|
||||
* Derives its state purely from the SessionDeletionMaintenance* ops events: the latest
|
||||
* Started newer than the latest terminal event (Completed/Failed) means a run is active;
|
||||
* otherwise the latest terminal event decides between budget-exceeded / failed / idle.
|
||||
* The "Run now" button POSTs the manual trigger (202 = queued, 409 = already active).
|
||||
*/
|
||||
export function MaintenanceStatusBanner({
|
||||
getAccessToken,
|
||||
setError,
|
||||
setSuccessMessage,
|
||||
}: {
|
||||
getAccessToken: (forceRefresh?: boolean) => Promise<string | null>;
|
||||
setError: (error: string | null) => void;
|
||||
setSuccessMessage: (message: string | null) => void;
|
||||
}) {
|
||||
const [state, setState] = useState<BannerState>({ kind: "loading" });
|
||||
const [triggering, setTriggering] = useState(false);
|
||||
const [refreshKey, setRefreshKey] = useState(0);
|
||||
|
||||
const fetchStatus = useCallback(async () => {
|
||||
try {
|
||||
const r = await authenticatedFetch(
|
||||
api.opsEvents.list("Maintenance", { pageSize: 100 }),
|
||||
getAccessToken,
|
||||
);
|
||||
if (!r.ok) throw new Error(`HTTP ${r.status} loading maintenance ops events`);
|
||||
const data = (await r.json()) as OpsEventsResponse;
|
||||
|
||||
const lifecycle = (data.events ?? []).filter((e) =>
|
||||
e.eventType.startsWith("SessionDeletionMaintenance"),
|
||||
);
|
||||
// Backend returns newest-first; keep it defensive anyway.
|
||||
lifecycle.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
|
||||
|
||||
const started = lifecycle.find((e) => e.eventType === "SessionDeletionMaintenanceStarted");
|
||||
const terminal = lifecycle.find(
|
||||
(e) =>
|
||||
e.eventType === "SessionDeletionMaintenanceCompleted" ||
|
||||
e.eventType === "SessionDeletionMaintenanceFailed",
|
||||
);
|
||||
|
||||
if (started && (!terminal || started.timestamp > terminal.timestamp)) {
|
||||
const details = parseDetails<{ triggeredBy?: string }>(started.details);
|
||||
setState({
|
||||
kind: "active",
|
||||
since: started.timestamp,
|
||||
triggeredBy: details.triggeredBy ?? "unknown",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!terminal) {
|
||||
setState({ kind: "none" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (terminal.eventType === "SessionDeletionMaintenanceFailed") {
|
||||
setState({ kind: "failed", at: terminal.timestamp, message: terminal.message });
|
||||
return;
|
||||
}
|
||||
|
||||
const details = parseDetails<CompletedDetails>(terminal.details);
|
||||
setState(
|
||||
details.abortedByBudget
|
||||
? { kind: "budget-exceeded", at: terminal.timestamp, details }
|
||||
: { kind: "idle", at: terminal.timestamp, details },
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError("Session expired; reload the page and try again.");
|
||||
}
|
||||
// Non-auth failures degrade silently — the banner is auxiliary to the tabs.
|
||||
setState({ kind: "none" });
|
||||
}
|
||||
}, [getAccessToken, setError]);
|
||||
|
||||
useEffect(() => {
|
||||
void fetchStatus();
|
||||
}, [fetchStatus, refreshKey]);
|
||||
|
||||
const triggerRun = useCallback(async () => {
|
||||
setTriggering(true);
|
||||
try {
|
||||
const r = await authenticatedFetch(api.sessionDeletions.triggerMaintenance(), getAccessToken, {
|
||||
method: "POST",
|
||||
});
|
||||
if (r.status === 202) {
|
||||
setSuccessMessage("Maintenance run queued — it will appear here as active shortly.");
|
||||
} else if (r.status === 409) {
|
||||
setSuccessMessage("A maintenance run is already active.");
|
||||
} else {
|
||||
const body = await r.text();
|
||||
throw new Error(`HTTP ${r.status}: ${body}`);
|
||||
}
|
||||
setRefreshKey((k) => k + 1);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) {
|
||||
setError("Session expired; reload the page and try again.");
|
||||
} else {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
} finally {
|
||||
setTriggering(false);
|
||||
}
|
||||
}, [getAccessToken, setError, setSuccessMessage]);
|
||||
|
||||
if (state.kind === "loading" || state.kind === "none") return null;
|
||||
|
||||
const runNowButton = (
|
||||
<button
|
||||
onClick={() => void triggerRun()}
|
||||
disabled={triggering}
|
||||
className="shrink-0 px-3 py-1.5 text-xs font-medium bg-purple-600 text-white rounded hover:bg-purple-700 disabled:opacity-50"
|
||||
>
|
||||
{triggering ? "Queuing…" : "Run now"}
|
||||
</button>
|
||||
);
|
||||
|
||||
if (state.kind === "active") {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-4 rounded-md border border-purple-200 dark:border-purple-800 bg-purple-50 dark:bg-purple-900/20 px-4 py-3">
|
||||
<p className="text-sm text-purple-800 dark:text-purple-200">
|
||||
<span className="font-medium">Maintenance run active</span> — started{" "}
|
||||
{new Date(state.since).toLocaleString()} (triggered by {state.triggeredBy}). Retention
|
||||
deletions enqueued by this run appear under In-Flight.
|
||||
</p>
|
||||
<button
|
||||
onClick={() => setRefreshKey((k) => k + 1)}
|
||||
className="shrink-0 px-3 py-1.5 text-xs font-medium border border-purple-300 dark:border-purple-700 text-purple-700 dark:text-purple-200 rounded hover:bg-purple-100 dark:hover:bg-purple-900/40"
|
||||
>
|
||||
Refresh
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (state.kind === "budget-exceeded") {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-4 rounded-md border border-amber-200 dark:border-amber-800 bg-amber-50 dark:bg-amber-900/20 px-4 py-3">
|
||||
<p className="text-sm text-amber-800 dark:text-amber-200">
|
||||
<span className="font-medium">Last run stopped at the 50-minute budget</span> (
|
||||
{new Date(state.at).toLocaleString()}) after processing {state.details.tenantsProcessed ?? 0}{" "}
|
||||
tenants ({state.details.sessionsEnqueued ?? 0} sessions enqueued). The remaining backlog
|
||||
resumes with the next scheduled run (every 12 h at 00:00 / 12:00 UTC) — or trigger a run
|
||||
now.
|
||||
</p>
|
||||
{runNowButton}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (state.kind === "failed") {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-4 rounded-md border border-red-200 dark:border-red-800 bg-red-50 dark:bg-red-900/20 px-4 py-3">
|
||||
<p className="text-sm text-red-800 dark:text-red-200">
|
||||
<span className="font-medium">Last maintenance run failed</span> (
|
||||
{new Date(state.at).toLocaleString()}): {state.message}
|
||||
</p>
|
||||
{runNowButton}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// idle — subtle last-run summary
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-4 rounded-md border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 px-4 py-3">
|
||||
<p className="text-sm text-gray-600 dark:text-gray-400">
|
||||
Last maintenance run completed {new Date(state.at).toLocaleString()} — tenants=
|
||||
{state.details.tenantsProcessed ?? 0}, enqueued={state.details.sessionsEnqueued ?? 0}
|
||||
{typeof state.details.durationMs === "number"
|
||||
? `, ${Math.round(state.details.durationMs / 60000)} min`
|
||||
: ""}
|
||||
. Next scheduled run: every 12 h at 00:00 / 12:00 UTC.
|
||||
</p>
|
||||
{runNowButton}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch"
|
||||
import { useAdminConfig } from "../../AdminConfigContext";
|
||||
import { AdminNotifications } from "../../AdminNotifications";
|
||||
import { DeletionPreviewModal } from "./components/DeletionPreviewModal";
|
||||
import { MaintenanceStatusBanner } from "./components/MaintenanceStatusBanner";
|
||||
import { RestoreBrowserTab } from "./components/RestoreBrowserTab";
|
||||
import { RestoreConfirmDialog } from "./components/RestoreConfirmDialog";
|
||||
|
||||
@@ -53,6 +54,12 @@ export default function SessionCleanupPage() {
|
||||
<main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
|
||||
<AdminNotifications />
|
||||
|
||||
<MaintenanceStatusBanner
|
||||
getAccessToken={getAccessToken}
|
||||
setError={setError}
|
||||
setSuccessMessage={setSuccessMessage}
|
||||
/>
|
||||
|
||||
{/* Tab bar */}
|
||||
<div className="border-b border-gray-200 dark:border-gray-700">
|
||||
<nav className="-mb-px flex gap-6 overflow-x-auto">
|
||||
|
||||
@@ -393,6 +393,12 @@ export const api = {
|
||||
// data" checkbox so the dropdown can hide tenants that have nothing to restore.
|
||||
tenantsWithManifests: () =>
|
||||
`${API_BASE_URL}/api/global/tenants-with-deletion-manifests`,
|
||||
|
||||
// Manual maintenance-run trigger (POST, GA-only): queues a one-off GC + retention-fanout
|
||||
// run. 202 = queued, 409 = a run is already active. Progress surfaces as
|
||||
// SessionDeletionMaintenance* ops events (MaintenanceStatusBanner reads those).
|
||||
triggerMaintenance: () =>
|
||||
`${API_BASE_URL}/api/global/session-deletions/maintenance/trigger`,
|
||||
},
|
||||
|
||||
// ── Hardware Rejection Insights (tenant-scoped, from distress data) ──────
|
||||
|
||||
Reference in New Issue
Block a user