feat(maintenance): session-deletion run budget, live watchdog counters, manual trigger

- Watchdog fix: FanoutResult is caller-allocated and mutated live per
  tenant/session, so 30/60min LongRunning warnings report real progress
  instead of the always-zero counters (today's warning showed 0/0 while
  5 tenants / 54 sessions were already done).
- 50min run budget: the retention fanout stops cleanly at the deadline
  (tenant boundary + per-session check) and reports AbortedByBudget —
  no more risk of the 60min Flex functionTimeout hard-aborting mid-run.
  New Warning OpsEvent SessionDeletionMaintenanceBudgetExceeded;
  Completed gains abortedByBudget.
- Blob-lease serialization: dedicated sentinel
  _lock/session-deletion-maintenance.lock (deletion-manifests container)
  via SessionDeletionMaintenanceLockStore + reused MaintenanceLeaseHolder;
  contended runs emit SessionDeletionMaintenanceSkippedLocked and return.
- Manual trigger: POST /api/global/session-deletions/maintenance/trigger
  (GlobalAdminOnly, policy catalog) -> fail-hard queue producer ->
  SessionDeletionMaintenanceQueueWorker runs the same RunCoreAsync body
  with triggeredBy = requesting admin; lease probe returns 409 while a
  run is active. New Started OpsEvent carries triggeredBy.
- Web: MaintenanceStatusBanner on the Session Cleanup page derives
  active / budget-exceeded / failed / idle from the lifecycle ops events
  and offers a Run-now button (202/409 handling). New event types
  dual-registered in OPS_EVENT_TYPES.
- Tests: 7 new (budget abort immediate/mid-tenant, live counter
  increments, watchdog live-progress regression, BudgetExceeded +
  Completed flag, SkippedLocked, Started triggeredBy); suites green
  (backend 2935, web 557, tsc clean).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-07-14 15:22:31 +02:00
parent 8a824d73fc
commit b0105b123a
17 changed files with 1014 additions and 99 deletions
@@ -46,10 +46,10 @@ public class SessionDeletionMaintenanceFunctionTests
harness.SeedSessionsByState(SessionDeletionState.Preparing); // no rows aged out in this scenario
harness.SeedSessionsByState(SessionDeletionState.Queued);
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
harness.BlobMock.Verify(b => b.DeleteDeletionManifestPairAsync("t", "s", "m", It.IsAny<CancellationToken>()), Times.Once);
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<CancellationToken>()), Times.Once);
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()), Times.Once);
// PR6 follow-up F3: lifecycle status now lives on OpsEvents, not AuditLogs (the latter
// would silently fail because PartitionKey requires a non-null tenantId).
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted");
@@ -62,12 +62,12 @@ public class SessionDeletionMaintenanceFunctionTests
harness.SetKillSwitch(true);
harness.SeedTtlBlobs(("t", "s", "m", DateTime.UtcNow.AddDays(-31)));
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
// GCs still ran — TTL sweep emitted the DeleteDeletionManifestPairAsync call.
harness.BlobMock.Verify(b => b.DeleteDeletionManifestPairAsync("t", "s", "m", It.IsAny<CancellationToken>()), Times.Once);
// Fanout was skipped.
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<CancellationToken>()), Times.Never);
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()), Times.Never);
// PR6 follow-up F3: fanout-skip + completion now both live on OpsEvents.
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceFanoutSkipped");
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted");
@@ -82,32 +82,55 @@ public class SessionDeletionMaintenanceFunctionTests
// but not severe.
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(500));
harness.SetKillSwitch(false);
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<CancellationToken>()))
.Returns(async (CancellationToken ct) =>
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
.Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) =>
{
r.TenantsProcessed = 1;
await Task.Delay(150, ct);
return new SessionRetentionFanoutService.FanoutResult { TenantsProcessed = 1, SessionsEnqueued = 0 };
});
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunningSevere");
}
[Fact]
public async Task RunCore_watchdog_reports_live_fanout_progress()
{
// Regression for the "tenants=0, enqueued=0" watchdog bug: the counters used to be
// assigned only AFTER the fanout returned, so every LongRunning warning showed zeros.
// The fanout now mutates the caller-supplied result incrementally; the watchdog snapshot
// must reflect that progress mid-run.
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromSeconds(5));
harness.SetKillSwitch(false);
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
.Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) =>
{
r.TenantsProcessed = 3;
r.SessionsEnqueued = 7;
await Task.Delay(200, ct); // long enough for the 50ms watchdog to fire mid-run
});
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
var warning = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
Assert.Contains("tenants=3", warning.Message);
Assert.Contains("enqueued=7", warning.Message);
}
[Fact]
public async Task RunCore_emits_LongRunningSevere_when_body_runs_past_severe_threshold()
{
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(150));
harness.SetKillSwitch(false);
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<CancellationToken>()))
.Returns(async (CancellationToken ct) =>
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
.Returns(async (SessionRetentionFanoutService.FanoutResult _, DateTime _, CancellationToken ct) =>
{
await Task.Delay(300, ct);
return new SessionRetentionFanoutService.FanoutResult();
});
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
// Both watchdogs fire: warning at 50ms, severe at 150ms.
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
@@ -126,7 +149,7 @@ public class SessionDeletionMaintenanceFunctionTests
("tenant-a", "session-y", "MANIFEST-2"),
("tenant-b", "session-z", "MANIFEST-3"));
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-x", It.IsAny<CancellationToken>()), Times.Once);
harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-y", It.IsAny<CancellationToken>()), Times.Once);
@@ -143,7 +166,7 @@ public class SessionDeletionMaintenanceFunctionTests
harness.SetKillSwitch(true);
harness.SeedExpiredTombstones(("tenant-a", "session-x", "M1"));
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-x", It.IsAny<CancellationToken>()),
Times.Once);
@@ -155,12 +178,63 @@ public class SessionDeletionMaintenanceFunctionTests
var harness = new Harness(watchdogWarn: TimeSpan.FromSeconds(5), watchdogSevere: TimeSpan.FromSeconds(10));
harness.SetKillSwitch(false);
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunningSevere");
}
// ────────────────────────────────────────────────────────────────────────── Run budget + lease + lifecycle ─
[Fact]
public async Task RunCore_emits_BudgetExceeded_and_flags_Completed_when_fanout_hits_the_budget()
{
var harness = new Harness();
harness.SetKillSwitch(false);
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
.Returns((SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken _) =>
{
r.TenantsProcessed = 12;
r.SessionsEnqueued = 40;
r.AbortedByBudget = true;
return Task.CompletedTask;
});
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
var budget = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceBudgetExceeded");
Assert.Contains("tenants=12", budget.Message);
var completed = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted");
Assert.Contains("abortedByBudget=True", completed.Message);
}
[Fact]
public async Task RunCore_emits_SkippedLocked_and_skips_everything_when_lease_is_held()
{
var harness = new Harness();
harness.LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new AutopilotMonitor.Functions.Services.Backup.LeaseHeldException("held", new Exception("409")));
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceSkippedLocked");
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceStarted");
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted");
harness.Fanout.Verify(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()), Times.Never);
}
[Fact]
public async Task RunCore_emits_Started_event_with_triggeredBy()
{
var harness = new Harness();
harness.SetKillSwitch(false);
await harness.Sut.RunCoreAsync("admin@contoso.com", CancellationToken.None);
var started = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceStarted");
Assert.Contains("admin@contoso.com", started.Message);
}
// ────────────────────────────────────────────────────────────────────────── Exception path ─
[Fact]
@@ -169,10 +243,10 @@ public class SessionDeletionMaintenanceFunctionTests
var harness = new Harness();
harness.SetKillSwitch(false);
// Make the fanout throw.
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<CancellationToken>()))
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new InvalidOperationException("fanout exploded"));
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.RunCoreAsync(CancellationToken.None));
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() => harness.Sut.RunCoreAsync("Test", CancellationToken.None));
Assert.Equal("fanout exploded", ex.Message);
// PR6 follow-up F3: the SessionDeletionMaintenanceFailed OpsEvent IS the audit. The prior
@@ -192,7 +266,7 @@ public class SessionDeletionMaintenanceFunctionTests
(TenantA, SessionA1, Manifest2, stuckSince),
(TenantB, SessionB1, Manifest2, stuckSince));
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
var stranded = harness.OpsEvents.Where(e => e.EventName == "SessionDeletionStrandedQueued").ToList();
Assert.Equal(2, stranded.Count);
@@ -206,7 +280,7 @@ public class SessionDeletionMaintenanceFunctionTests
var freshly = DateTime.UtcNow.AddMinutes(-5); // well under 30min
harness.SeedSessionsByState(SessionDeletionState.Queued, (TenantA, SessionA1, Manifest2, freshly));
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionStrandedQueued");
}
@@ -234,7 +308,7 @@ public class SessionDeletionMaintenanceFunctionTests
.Setup(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA1, Manifest1, It.IsAny<CancellationToken>()))
.ReturnsAsync(true);
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA1, Manifest1, It.IsAny<CancellationToken>()), Times.Once);
harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA2, Manifest1 + "X", It.IsAny<CancellationToken>()), Times.Never);
@@ -250,7 +324,7 @@ public class SessionDeletionMaintenanceFunctionTests
var recent = DateTime.UtcNow.AddMinutes(-5);
harness.SeedSessionsByState(SessionDeletionState.Preparing, (TenantA, SessionA1, Manifest1, recent));
await harness.Sut.RunCoreAsync(CancellationToken.None);
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Never);
}
@@ -263,6 +337,7 @@ public class SessionDeletionMaintenanceFunctionTests
public Mock<MaintenanceBlobStub> BlobMock { get; }
public Mock<AdminConfigurationService> AdminConfig { get; }
public Mock<SessionRetentionFanoutService> Fanout { get; }
public Mock<SessionDeletionMaintenanceLockStore> LockStore { get; }
public List<CapturedOpsEvent> OpsEvents { get; } = new();
public List<AuditEntry> AuditCalls { get; } = new();
public SessionDeletionMaintenanceFunction Sut { get; }
@@ -328,14 +403,24 @@ public class SessionDeletionMaintenanceFunctionTests
Mock.Of<ISessionDeletionEnqueuer>(),
AdminConfig.Object,
NullLogger<SessionRetentionFanoutService>.Instance);
Fanout.Setup(f => f.RunAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync(new SessionRetentionFanoutService.FanoutResult { TenantsProcessed = 0, SessionsEnqueued = 0 });
Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
.Returns(Task.CompletedTask);
// Lease store: default acquire succeeds with a mocked lease client. The renewal loop
// never ticks inside these millisecond-scale tests; DisposeAsync's release failure on
// the bare mock is swallowed by the holder's try/catch.
LockStore = new Mock<SessionDeletionMaintenanceLockStore>(
BlobMock.Object, NullLogger<SessionDeletionMaintenanceLockStore>.Instance);
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new Mock<Azure.Storage.Blobs.Specialized.BlobLeaseClient>().Object);
Sut = new SessionDeletionMaintenanceFunction(
StorageMock.Object, BlobMock.Object, AdminConfig.Object, opsService, maintRepo.Object, Fanout.Object,
LockStore.Object,
NullLogger<SessionDeletionMaintenanceFunction>.Instance,
watchdogWarning: watchdogWarn ?? TimeSpan.FromSeconds(30),
watchdogSevere: watchdogSevere ?? TimeSpan.FromMinutes(1));
watchdogSevere: watchdogSevere ?? TimeSpan.FromMinutes(1),
runBudget: TimeSpan.FromMinutes(50));
}
public void SetKillSwitch(bool active) =>
@@ -29,7 +29,7 @@ public class SessionRetentionFanoutServiceTests
harness.WithTenant(TenantA, retentionDays: 30,
sessions: new[] { Old("s1", 60), Old("s2", 45) });
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.Equal(2, result.SessionsEnqueued);
harness.Enqueuer.Verify(
@@ -56,7 +56,7 @@ public class SessionRetentionFanoutServiceTests
harness.MaintenanceRepo.Setup(m => m.GetSessionsOlderThanAsync(TenantB, It.Is<DateTime>(d => d <= DateTime.UtcNow.AddDays(-120)), It.IsAny<int>(), It.IsAny<bool>()))
.ReturnsAsync(new List<SessionSummary> { Summary(TenantB, "b2") });
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.Equal(2, result.TenantsProcessed);
Assert.Equal(2, result.SessionsEnqueued);
@@ -75,7 +75,7 @@ public class SessionRetentionFanoutServiceTests
many.Add(Summary(TenantA, $"s{i:000}"));
harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: many);
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.Equal(SessionRetentionFanoutService.MaxEnqueuesPerTenantPerRun, result.SessionsEnqueued);
Assert.Equal(1, result.RateLimitedTenants);
@@ -94,7 +94,7 @@ public class SessionRetentionFanoutServiceTests
var harness = new Harness();
harness.WithTenant(TenantA, retentionDays: 30, sessions: new[] { Old("s1", 60) });
await harness.Sut.RunAsync(CancellationToken.None);
await harness.RunAsync();
harness.MaintenanceRepo.Verify(
m => m.GetSessionsOlderThanAsync(
@@ -113,7 +113,7 @@ public class SessionRetentionFanoutServiceTests
var harness = new Harness();
harness.WithTenant(TenantA, retentionDays: 0, sessions: new[] { Old("s1", 365) });
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.Equal(0, result.SessionsEnqueued);
// GetSessionsOlderThanAsync must not have been called for that tenant.
@@ -129,7 +129,7 @@ public class SessionRetentionFanoutServiceTests
var harness = new Harness();
harness.WithTenant(TenantA, retentionDays: 180, sessions: new[] { Old("s1", 100) });
await harness.Sut.RunAsync(CancellationToken.None);
await harness.RunAsync();
harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(
TenantA,
@@ -143,7 +143,7 @@ public class SessionRetentionFanoutServiceTests
var harness = new Harness();
harness.WithTenant(TenantA, retentionDays: 180, sessions: new[] { Old("s1", 200) }, planTier: "enterprise");
await harness.Sut.RunAsync(CancellationToken.None);
await harness.RunAsync();
harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(
TenantA,
@@ -174,7 +174,7 @@ public class SessionRetentionFanoutServiceTests
return Task.FromResult(calls >= 4);
});
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.True(result.AbortedByKillSwitch);
Assert.Equal(2, result.SessionsEnqueued); // only sessions 0 and 1 made it through
@@ -203,7 +203,7 @@ public class SessionRetentionFanoutServiceTests
return Task.FromResult(calls >= 3);
});
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.True(result.AbortedByKillSwitch);
Assert.Equal(1, result.TenantsProcessed);
@@ -235,7 +235,7 @@ public class SessionRetentionFanoutServiceTests
: new SessionDeletionEnqueueResult { Outcome = SessionDeletionEnqueueOutcome.KillSwitchActive };
});
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.True(result.AbortedByKillSwitch);
Assert.Equal(1, result.SessionsEnqueued);
@@ -272,7 +272,7 @@ public class SessionRetentionFanoutServiceTests
: new SessionDeletionEnqueueResult { Outcome = SessionDeletionEnqueueOutcome.KillSwitchActive };
});
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.True(result.AbortedByKillSwitch);
Assert.Equal(1, result.TenantsProcessed); // tenant A completed its RunForTenantAsync; B never started
@@ -301,12 +301,79 @@ public class SessionRetentionFanoutServiceTests
ExistingState = SessionDeletionState.Running,
});
var result = await harness.Sut.RunAsync(CancellationToken.None);
var result = await harness.RunAsync();
Assert.Equal(0, result.SessionsEnqueued);
Assert.Equal(1, result.SessionsSkipped);
}
// ────────────────────────────────────────────────────────────────────────── Run budget ─
[Fact]
public async Task RunAsync_aborts_immediately_when_deadline_already_passed()
{
var harness = new Harness();
harness.WithTenant(TenantA, retentionDays: 30, sessions: new[] { Old("s1", 60) });
var result = await harness.RunAsync(deadlineUtc: DateTime.UtcNow.AddMinutes(-1));
Assert.True(result.AbortedByBudget);
Assert.Equal(0, result.TenantsProcessed);
Assert.Equal(0, result.SessionsEnqueued);
harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(TenantA, It.IsAny<DateTime>(), It.IsAny<int>(), It.IsAny<bool>()), Times.Never);
}
[Fact]
public async Task RunAsync_aborts_mid_tenant_when_budget_deadline_crosses()
{
// Scripted clock: the deadline crosses between session 1 and session 2 of tenant A —
// the remaining sessions of A and all of tenant B must be left untouched.
var baseTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var deadline = baseTime.AddMinutes(50);
var clockCalls = 0;
// Call sequence: tenant-A boundary (1), session s0 (2), session s1 (3), session s2 (4 → past deadline).
var harness = new Harness(utcNow: () => ++clockCalls >= 4 ? deadline.AddMinutes(1) : baseTime);
var sessions = new List<SessionSummary>();
for (int i = 0; i < 4; i++) sessions.Add(Summary(TenantA, $"s{i}"));
harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: sessions);
harness.WithTenant(TenantB, retentionDays: 30, sessions: new[] { Old("b1", 60) });
var result = await harness.RunAsync(deadlineUtc: deadline);
Assert.True(result.AbortedByBudget);
Assert.Equal(2, result.SessionsEnqueued); // only s0 and s1 made it through
Assert.Equal(1, result.TenantsProcessed); // tenant A returned normally, B never started
harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantA, "s2", It.IsAny<string>(), It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()), Times.Never);
harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantB, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<DeletionActor>(), It.IsAny<DeletionRetentionContext?>(), It.IsAny<CancellationToken>()), Times.Never);
harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(TenantB, It.IsAny<DateTime>(), It.IsAny<int>(), It.IsAny<bool>()), Times.Never);
}
[Fact]
public async Task RunAsync_mutates_the_shared_result_incrementally_during_the_run()
{
// Watchdog-progress contract: the caller-supplied FanoutResult must be bumped per
// session (not once at the end), so mid-run snapshots show real progress. The injected
// throttle runs between sessions — snapshot the counter there.
var snapshots = new List<int>();
var result = new SessionRetentionFanoutService.FanoutResult();
var harness = new Harness(throttle: (_, _) =>
{
snapshots.Add(result.SessionsEnqueued);
return Task.CompletedTask;
});
var sessions = new List<SessionSummary>();
for (int i = 0; i < 3; i++) sessions.Add(Summary(TenantA, $"s{i}"));
harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: sessions);
await harness.Sut.RunAsync(result, DateTime.UtcNow.AddMinutes(50), CancellationToken.None);
Assert.Equal(3, result.SessionsEnqueued);
// Throttle fires after s0 and s1 (not after the last session) — each snapshot must
// already reflect the enqueues so far.
Assert.Equal(new[] { 1, 2 }, snapshots);
}
// ============================================================ Helpers ====
private static SessionSummary Summary(string tenantId, string sessionId) =>
@@ -333,7 +400,7 @@ public class SessionRetentionFanoutServiceTests
private readonly List<string> _tenantIds = new();
public Harness()
public Harness(Func<TimeSpan, CancellationToken, Task>? throttle = null, Func<DateTime>? utcNow = null)
{
MaintenanceRepo = new Mock<IMaintenanceRepository>();
var memCache = new MemoryCache(new MemoryCacheOptions());
@@ -363,12 +430,24 @@ public class SessionRetentionFanoutServiceTests
.ReturnsAsync(true);
// Use internal ctor to inject a no-op throttle so the rate-limit test runs in real-time
// (50ms × 100 = 5s otherwise).
// (50ms × 100 = 5s otherwise) and an optional scripted clock for the budget tests.
Sut = new SessionRetentionFanoutService(
MaintenanceRepo.Object, TenantConfig.Object, Enqueuer.Object,
AdminConfig.Object,
NullLogger<SessionRetentionFanoutService>.Instance,
throttle: (_, _) => Task.CompletedTask);
throttle: throttle ?? ((_, _) => Task.CompletedTask),
utcNow: utcNow);
}
/// <summary>
/// Drives <see cref="SessionRetentionFanoutService.RunAsync"/> with a fresh result object
/// and a far-away default deadline (the budget is exercised by dedicated tests).
/// </summary>
public async Task<SessionRetentionFanoutService.FanoutResult> RunAsync(DateTime? deadlineUtc = null)
{
var result = new SessionRetentionFanoutService.FanoutResult();
await Sut.RunAsync(result, deadlineUtc ?? DateTime.UtcNow.AddMinutes(50), CancellationToken.None);
return result;
}
public void WithTenant(string tenantId, int retentionDays, SessionSummary[] sessions, string planTier = "free")
@@ -4,10 +4,12 @@ using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Deletion;
using Azure.Storage.Blobs.Specialized;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Extensions.Logging;
@@ -32,9 +34,18 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
/// <para>
/// 30-minute watchdog emits a Warning OpsEvent; 60-minute watchdog escalates to Error. The
/// 60-minute mark matches the Flex Consumption hard <c>functionTimeout</c> (host.json) so the
/// operator gets a Telegram alert before the Azure runtime abort. Cron <c>0 0 */12 * * *</c>
/// (every 12h at 00:00 / 12:00 UTC) — const per repo convention; change-and-redeploy if you
/// need a different cadence.
/// operator gets a Telegram alert before the Azure runtime abort. The retention fanout stops
/// itself at the <see cref="RunBudget"/> (50min) deadline so a growing backlog degrades into
/// a clean <c>BudgetExceeded</c> + partial sweep instead of a host abort. Cron
/// <c>0 0 */12 * * *</c> (every 12h at 00:00 / 12:00 UTC) — const per repo convention;
/// change-and-redeploy if you need a different cadence.
/// </para>
/// <para>
/// Runs are serialized by a blob lease (<see cref="SessionDeletionMaintenanceLockStore"/>):
/// the timer and the manual trigger (<c>POST /api/global/session-deletions/maintenance/trigger</c>
/// → <see cref="SessionDeletionMaintenanceQueueWorker"/>) both enter through
/// <see cref="RunCoreAsync"/>, which acquires the lease first and emits
/// <c>SkippedLocked</c> when another run is active.
/// </para>
/// </summary>
public sealed class SessionDeletionMaintenanceFunction
@@ -58,15 +69,21 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
internal static readonly TimeSpan WatchdogWarning = TimeSpan.FromMinutes(30);
internal static readonly TimeSpan WatchdogSevere = TimeSpan.FromMinutes(60);
// Run budget for the retention fanout (matches the critical-table backup convention):
// stop cleanly at 50min so the 60min host functionTimeout never aborts us mid-tenant.
internal static readonly TimeSpan RunBudget = TimeSpan.FromMinutes(50);
private readonly TableStorageService _storage;
private readonly BlobStorageService _blob;
private readonly AdminConfigurationService _adminConfig;
private readonly OpsEventService _opsEvents;
private readonly IMaintenanceRepository _maintenanceRepo;
private readonly SessionRetentionFanoutService _fanout;
private readonly SessionDeletionMaintenanceLockStore _lockStore;
private readonly ILogger<SessionDeletionMaintenanceFunction> _logger;
private readonly TimeSpan _watchdogWarning;
private readonly TimeSpan _watchdogSevere;
private readonly TimeSpan _runBudget;
public SessionDeletionMaintenanceFunction(
TableStorageService storage,
@@ -75,14 +92,16 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
OpsEventService opsEvents,
IMaintenanceRepository maintenanceRepo,
SessionRetentionFanoutService fanout,
SessionDeletionMaintenanceLockStore lockStore,
ILogger<SessionDeletionMaintenanceFunction> logger)
: this(storage, blob, adminConfig, opsEvents, maintenanceRepo, fanout, logger, WatchdogWarning, WatchdogSevere)
: this(storage, blob, adminConfig, opsEvents, maintenanceRepo, fanout, lockStore, logger, WatchdogWarning, WatchdogSevere, RunBudget)
{
}
/// <summary>
/// Test seam: lets unit tests inject sub-second watchdog thresholds so the function can be
/// driven end-to-end in milliseconds. Production code uses the static defaults (30min / 60min).
/// Test seam: lets unit tests inject sub-second watchdog thresholds and run budget so the
/// function can be driven end-to-end in milliseconds. Production code uses the static
/// defaults (30min / 60min / 50min).
/// </summary>
internal SessionDeletionMaintenanceFunction(
TableStorageService storage,
@@ -91,9 +110,11 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
OpsEventService opsEvents,
IMaintenanceRepository maintenanceRepo,
SessionRetentionFanoutService fanout,
SessionDeletionMaintenanceLockStore lockStore,
ILogger<SessionDeletionMaintenanceFunction> logger,
TimeSpan watchdogWarning,
TimeSpan watchdogSevere)
TimeSpan watchdogSevere,
TimeSpan runBudget)
{
_storage = storage;
_blob = blob;
@@ -101,63 +122,95 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
_opsEvents = opsEvents;
_maintenanceRepo = maintenanceRepo;
_fanout = fanout;
_lockStore = lockStore;
_logger = logger;
_watchdogWarning = watchdogWarning;
_watchdogSevere = watchdogSevere;
_runBudget = runBudget;
}
[Function("SessionDeletionMaintenance")]
public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken)
{
await RunCoreAsync(cancellationToken);
await RunCoreAsync("Timer", cancellationToken);
}
/// <summary>
/// Testable core. Separated from the TimerTrigger entry so unit tests can drive it
/// directly without a TimerInfo / FunctionContext.
/// directly without a TimerInfo / FunctionContext, and shared with the manual-trigger
/// queue worker. <paramref name="triggeredBy"/> is <c>"Timer"</c> for the cron path and
/// the requesting Global Admin's identifier for manual runs.
/// </summary>
internal async Task RunCoreAsync(CancellationToken cancellationToken)
internal async Task RunCoreAsync(string triggeredBy, CancellationToken cancellationToken)
{
_logger.LogInformation("SessionDeletionMaintenance: tick started at {Now:o}", DateTime.UtcNow);
var sw = Stopwatch.StartNew();
_logger.LogInformation("SessionDeletionMaintenance: tick started at {Now:o} (triggeredBy={TriggeredBy})", DateTime.UtcNow, triggeredBy);
using var watchdogCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
int sessionsEnqueuedSoFar = 0;
int tenantsProcessedSoFar = 0;
// Serialize timer vs manual trigger vs second host instance. Acquired BEFORE the
// Started OpsEvent so a lease-skip never masquerades as an active run to the UI.
BlobLeaseClient lease;
try
{
lease = await _lockStore.AcquireLeaseAsync(ct: cancellationToken);
}
catch (LeaseHeldException)
{
_logger.LogWarning(
"SessionDeletionMaintenance: lease held by another run — skipping (triggeredBy={TriggeredBy})",
triggeredBy);
await _opsEvents.RecordSessionDeletionMaintenanceSkippedLockedAsync(triggeredBy);
return;
}
var sw = Stopwatch.StartNew();
var deadlineUtc = DateTime.UtcNow + _runBudget;
// handlerCts: cancelled by the lease holder on renewal failure so the run never
// continues rogue under an expired lease. The body + watchdogs run off this token.
using var handlerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var leaseHolder = new MaintenanceLeaseHolder(lease, handlerCts, _logger);
using var watchdogCts = CancellationTokenSource.CreateLinkedTokenSource(handlerCts.Token);
// Live progress object: the fanout mutates it per tenant / per session, so the
// watchdog snapshots below report real progress mid-run (previously the counters
// were only assigned after the fanout returned — every warning showed 0/0).
var fanoutResult = new SessionRetentionFanoutService.FanoutResult();
// Watchdogs: each one auto-cancels when the body completes (we cancel watchdogCts in
// the finally). The await Task.WhenAny inside RunWatchdogAsync only emits if the body
// is still running at the threshold.
var watchdogWarn = RunWatchdogAsync(_watchdogWarning, severe: false, () => (tenantsProcessedSoFar, sessionsEnqueuedSoFar), watchdogCts.Token);
var watchdogSevere = RunWatchdogAsync(_watchdogSevere, severe: true, () => (tenantsProcessedSoFar, sessionsEnqueuedSoFar), watchdogCts.Token);
var watchdogWarn = RunWatchdogAsync(_watchdogWarning, severe: false, () => (fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued), watchdogCts.Token);
var watchdogSevere = RunWatchdogAsync(_watchdogSevere, severe: true, () => (fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued), watchdogCts.Token);
try
{
await _opsEvents.RecordSessionDeletionMaintenanceStartedAsync(triggeredBy);
var ct = handlerCts.Token;
// PR5 finding 1: uncached, fail-closed read so a flip-ON is honored within seconds
// across all Function-host instances.
var killSwitchActive = await _adminConfig.IsSessionDeletionKillSwitchActiveAsync();
// (1) Manifest-Blob-TTL sweep — runs even when kill-switch=true (Plan PR6 step 3).
var blobsTtlGced = await SweepManifestBlobsAsync(cancellationToken);
var blobsTtlGced = await SweepManifestBlobsAsync(ct);
// (2) Stale-Preparing GC — runs even when kill-switch=true.
var preparingRowsCleared = await GcStalePreparingAsync(cancellationToken);
var preparingRowsCleared = await GcStalePreparingAsync(ct);
// (3) Stranded-Queued detection (alert-only) — runs even when kill-switch=true.
var strandedQueuedDetected = await DetectStrandedQueuedAsync(cancellationToken);
var strandedQueuedDetected = await DetectStrandedQueuedAsync(ct);
// (3b) Tombstone-marker pruning — physically remove rows past their ExpiresAt
// (Codex F3). Runs even when kill-switch=true: these markers are short-lived
// race-shields, not policy-bound deletion artefacts. Failures are non-fatal
// because the in-flight Guard already treats expired markers as absent.
var tombstonesPruned = await PruneExpiredTombstonesAsync(cancellationToken);
var tombstonesPruned = await PruneExpiredTombstonesAsync(ct);
// (4) Retention fanout — skipped when kill-switch=true. Emit a Fanout-Skipped
// OpsEvent so the operator can fold the skip into the OpsEvents dashboard alongside
// the 30/60min watchdogs. PR6 follow-up F3: previously a LogAuditEntryAsync(null!)
// call that silently failed because the AuditLogs schema requires a non-null
// PartitionKey — OpsEvents tolerates null TenantId, so global-scope events live there.
SessionRetentionFanoutService.FanoutResult? fanoutResult = null;
if (killSwitchActive)
{
await _opsEvents.RecordSessionDeletionMaintenanceFanoutSkippedAsync(
@@ -165,32 +218,38 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
}
else
{
fanoutResult = await _fanout.RunAsync(cancellationToken);
tenantsProcessedSoFar = fanoutResult.TenantsProcessed;
sessionsEnqueuedSoFar = fanoutResult.SessionsEnqueued;
await _fanout.RunAsync(fanoutResult, deadlineUtc, ct);
if (fanoutResult.AbortedByBudget)
{
await _opsEvents.RecordSessionDeletionMaintenanceBudgetExceededAsync(
(int)_runBudget.TotalMinutes, fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued);
}
}
// (5) Completion OpsEvent — same shape regardless of kill-switch state, so dashboards
// can fold both paths together. PR6 follow-up F3.
await _opsEvents.RecordSessionDeletionMaintenanceCompletedAsync(
killSwitchActive: killSwitchActive,
tenantsProcessed: fanoutResult?.TenantsProcessed ?? 0,
sessionsEnqueued: fanoutResult?.SessionsEnqueued ?? 0,
sessionsSkipped: fanoutResult?.SessionsSkipped ?? 0,
rateLimitedTenants: fanoutResult?.RateLimitedTenants ?? 0,
tenantsProcessed: fanoutResult.TenantsProcessed,
sessionsEnqueued: fanoutResult.SessionsEnqueued,
sessionsSkipped: fanoutResult.SessionsSkipped,
rateLimitedTenants: fanoutResult.RateLimitedTenants,
blobsTtlGced: blobsTtlGced,
preparingRowsCleared: preparingRowsCleared,
strandedQueuedDetected: strandedQueuedDetected,
durationMs: (int)sw.ElapsedMilliseconds,
abortedByKillSwitch: fanoutResult?.AbortedByKillSwitch ?? false);
abortedByKillSwitch: fanoutResult.AbortedByKillSwitch,
abortedByBudget: fanoutResult.AbortedByBudget);
_logger.LogInformation(
"SessionDeletionMaintenance: completed in {Ms}ms — killSwitch={KillSwitch} tenants={Tenants} enqueued={Enqueued} skipped={Skipped} blobs={Blobs} preparing={Preparing} stranded={Stranded} tombstones={Tombstones}",
"SessionDeletionMaintenance: completed in {Ms}ms — killSwitch={KillSwitch} tenants={Tenants} enqueued={Enqueued} skipped={Skipped} blobs={Blobs} preparing={Preparing} stranded={Stranded} tombstones={Tombstones} abortedByBudget={Budget}",
sw.ElapsedMilliseconds, killSwitchActive,
fanoutResult?.TenantsProcessed ?? 0,
fanoutResult?.SessionsEnqueued ?? 0,
fanoutResult?.SessionsSkipped ?? 0,
blobsTtlGced, preparingRowsCleared, strandedQueuedDetected, tombstonesPruned);
fanoutResult.TenantsProcessed,
fanoutResult.SessionsEnqueued,
fanoutResult.SessionsSkipped,
blobsTtlGced, preparingRowsCleared, strandedQueuedDetected, tombstonesPruned,
fanoutResult.AbortedByBudget);
}
catch (Exception ex)
{
@@ -209,6 +268,10 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
// Observe the watchdog tasks so the OperationCanceledException doesn't get
// surfaced as an unhandled exception in the host log.
try { await Task.WhenAll(watchdogWarn, watchdogSevere); } catch (OperationCanceledException) { /* expected */ }
// Release the lease LAST — the Failed/Completed OpsEvents above are written while
// the run is still exclusive, matching the backup worker's ordering.
await leaseHolder.DisposeAsync();
}
}
@@ -0,0 +1,105 @@
using System;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Functions.Services.Deletion;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Maintenance
{
/// <summary>
/// <c>POST /api/global/session-deletions/maintenance/trigger</c> — manual GA trigger for a
/// one-off session-deletion maintenance run (GC sweeps + retention fanout), the on-demand
/// counterpart to the 12h <see cref="SessionDeletionMaintenanceFunction"/> timer. GA-only via
/// <c>EndpointAccessPolicyCatalog</c>; authentication + authorization are enforced by
/// <c>PolicyEnforcementMiddleware</c>, the body does not re-check.
/// <para>
/// Semantics (mirrors <c>TriggerCriticalTableBackupFunction</c>'s fail-hard producer): a
/// lease probe first rejects with <c>409 RunAlreadyActive</c> while a run is in flight; on
/// successful enqueue we return <c>202 Accepted</c>; on enqueue failure <c>500</c> so the
/// operator never sees a hollow success. Run progress surfaces via the
/// <c>SessionDeletionMaintenance*</c> OpsEvents (Session Cleanup banner + Ops Events page).
/// </para>
/// </summary>
public class TriggerSessionDeletionMaintenanceFunction
{
private readonly ISessionDeletionMaintenanceTriggerProducer _producer;
private readonly SessionDeletionMaintenanceLockStore _lockStore;
private readonly ILogger<TriggerSessionDeletionMaintenanceFunction> _logger;
public TriggerSessionDeletionMaintenanceFunction(
ISessionDeletionMaintenanceTriggerProducer producer,
SessionDeletionMaintenanceLockStore lockStore,
ILogger<TriggerSessionDeletionMaintenanceFunction> logger)
{
_producer = producer;
_lockStore = lockStore;
_logger = logger;
}
[Function("TriggerSessionDeletionMaintenance")]
public async Task<HttpResponseData> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/session-deletions/maintenance/trigger")] HttpRequestData req)
{
var actor = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin";
var ct = req.FunctionContext.CancellationToken;
// Acquire+release probe: a held lease means a run is active RIGHT NOW — tell the
// operator instead of queuing a message that would only produce a SkippedLocked
// event. Racing the probe (run starts between probe and worker pickup) is benign:
// the worker's own lease acquire lands on the SkippedLocked path.
try
{
var lease = await _lockStore.AcquireLeaseAsync(ct: ct);
try { await lease.ReleaseAsync(cancellationToken: ct); }
catch (Exception ex)
{
// Release failure is harmless — the 60s lease auto-expires; the worker's
// acquire retries land within the queue's poll cadence.
_logger.LogWarning(ex, "TriggerSessionDeletionMaintenance: probe-lease release failed (auto-expires in 60s)");
}
}
catch (LeaseHeldException)
{
return await WriteJsonAsync(req, HttpStatusCode.Conflict, new
{
error = "RunAlreadyActive",
message = "A session-deletion maintenance run is already active — check the Session Cleanup page for progress.",
});
}
try
{
await _producer.EnqueueAsync(new SessionDeletionMaintenanceTriggerEnvelope { TriggeredBy = actor }, ct);
}
catch (Exception ex)
{
_logger.LogError(ex, "TriggerSessionDeletionMaintenance: enqueue failed (triggeredBy={TriggeredBy})", actor);
return await WriteJsonAsync(req, HttpStatusCode.InternalServerError, new
{
error = "EnqueueFailed",
message = "Failed to enqueue the maintenance run — please retry.",
});
}
_logger.LogInformation("TriggerSessionDeletionMaintenance: run queued (triggeredBy={TriggeredBy})", actor);
return await WriteJsonAsync(req, HttpStatusCode.Accepted, new
{
message = "Maintenance run queued — progress surfaces as SessionDeletionMaintenance* ops events.",
triggeredBy = actor,
});
}
private static async Task<HttpResponseData> WriteJsonAsync(HttpRequestData req, HttpStatusCode status, object body)
{
var response = req.CreateResponse(status);
response.Headers.Add("Content-Type", "application/json; charset=utf-8");
await response.WriteStringAsync(JsonSerializer.Serialize(body));
return response;
}
}
}
@@ -189,6 +189,17 @@ builder.Services.AddHostedService<AutopilotMonitor.Functions.Services.Deletion.S
// retention fanout (V2 + legacy), manifest-blob TTL sweep, stale-Preparing GC, and stranded-
// Queued detection. Independent cadence + kill-switch from the generic 2h Maintenance timer.
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionRetentionFanoutService>();
// Manual-trigger path: blob-lease lock store (serializes timer vs manual runs), fail-hard
// queue producer for POST /api/global/session-deletions/maintenance/trigger, and the
// BackgroundService worker that consumes trigger envelopes. The maintenance Function class is
// registered explicitly so the worker can invoke the same RunCoreAsync body the timer uses.
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Deletion.SessionDeletionMaintenanceLockStore>();
builder.Services.AddSingleton<
AutopilotMonitor.Functions.Services.Deletion.ISessionDeletionMaintenanceTriggerProducer,
AutopilotMonitor.Functions.Services.Deletion.AzureQueueSessionDeletionMaintenanceTriggerProducer>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Functions.Maintenance.SessionDeletionMaintenanceFunction>();
builder.Services.AddHostedService<
AutopilotMonitor.Functions.Services.Deletion.SessionDeletionMaintenanceQueueWorker>();
// Tenant-offboarding cascade worker (Plan Rev 9). Consumes per-tenant envelopes off
// `tenant-offboarding`, drives the §3 Phase 2 sequence (cascade enqueue → drain → SafeWipe →
@@ -328,6 +328,8 @@ public static class EndpointAccessPolicyCatalog
// Platform-operational (cascade-delete restore prep) — GA + read-only Reader only, NOT delegated.
new("GET", "global/tenants/{tenantId}/deletion-manifests", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true),
new("GET", "global/session-deletions", EndpointPolicy.GlobalReadOrAdmin),
// Manual maintenance-run trigger (GC sweeps + retention fanout) — mutating, GA only.
new("POST", "global/session-deletions/maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/raw/sessions", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
new("GET", "global/raw/events", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam),
// Raw table/log access stays GlobalAdminOnly: query_table can dump arbitrary tables
@@ -0,0 +1,61 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Queueing;
using AutopilotMonitor.Shared;
using Azure.Storage.Queues;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace AutopilotMonitor.Functions.Services.Deletion
{
/// <summary>
/// Azure-Storage-Queue producer for the manual session-deletion-maintenance trigger.
/// Mirrors <c>AzureQueueCriticalTableBackupProducer</c>: Managed Identity preferred via
/// <see cref="QueueClientFactory"/>, fail-hard semantics (SendMessageAsync errors propagate
/// so the HTTP trigger returns 5xx instead of a hollow 202).
/// </summary>
public sealed class AzureQueueSessionDeletionMaintenanceTriggerProducer : ISessionDeletionMaintenanceTriggerProducer
{
private readonly QueueClient _queueClient;
private readonly ILogger<AzureQueueSessionDeletionMaintenanceTriggerProducer> _logger;
private int _queueEnsured;
public AzureQueueSessionDeletionMaintenanceTriggerProducer(
QueueClientFactory queueFactory,
ILogger<AzureQueueSessionDeletionMaintenanceTriggerProducer> logger)
{
_logger = logger;
_queueClient = queueFactory.Create(Constants.QueueNames.SessionDeletionMaintenance);
}
public async Task EnqueueAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct = default)
{
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
if (string.IsNullOrEmpty(envelope.TriggeredBy)) throw new ArgumentException("TriggeredBy required", nameof(envelope));
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
// Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and returns 500.
var body = JsonConvert.SerializeObject(envelope);
await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false);
_logger.LogInformation("SessionDeletionMaintenance trigger enqueued (triggeredBy={TriggeredBy})", envelope.TriggeredBy);
}
private async Task EnsureQueueExistsAsync(CancellationToken ct)
{
if (_queueEnsured == 1) return;
try
{
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
Interlocked.Exchange(ref _queueEnsured, 1);
}
catch (Exception ex)
{
// Ensure failure during enqueue is itself a fail-hard signal — surface to the caller.
_logger.LogError(ex, "SessionDeletionMaintenance queue CreateIfNotExists failed");
throw;
}
}
}
}
@@ -0,0 +1,26 @@
using System.Threading;
using System.Threading.Tasks;
namespace AutopilotMonitor.Functions.Services.Deletion
{
/// <summary>
/// Envelope for a single manual session-deletion-maintenance trigger message. Tiny on
/// purpose — the run has no per-job state row; the OpsEvents lifecycle
/// (Started / BudgetExceeded / SkippedLocked / Completed / Failed) is the status surface.
/// </summary>
public sealed class SessionDeletionMaintenanceTriggerEnvelope
{
/// <summary>Identifier of the requesting Global Admin (UPN) — flows into the Started OpsEvent.</summary>
public string TriggeredBy { get; set; } = string.Empty;
}
/// <summary>
/// Producer for the <c>session-deletion-maintenance</c> queue. <b>Fail-hard</b>:
/// SendMessageAsync exceptions propagate so the HTTP trigger can return 5xx — silent
/// enqueue loss would leave the operator with a 202 and no run ever starting.
/// </summary>
public interface ISessionDeletionMaintenanceTriggerProducer
{
Task EnqueueAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct = default);
}
}
@@ -0,0 +1,107 @@
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Shared;
using Azure;
using Azure.Storage.Blobs.Models;
using Azure.Storage.Blobs.Specialized;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Deletion
{
/// <summary>
/// Blob-lease lock for the session-deletion maintenance run. Serializes the 12h timer
/// against the manual HTTP trigger (and against a second host instance) — without it a
/// manual run racing the timer would double-run the GC sweeps and the retention fanout.
/// <para>
/// Deliberately a DEDICATED sentinel (<see cref="LockBlobName"/> in the
/// <c>deletion-manifests</c> container) rather than the backup sentinel in
/// <see cref="BlobBackupStore"/> — sharing that lease would serialize backups against the
/// retention fanout, which have no reason to exclude each other. Lease mechanics
/// (60s lease + <see cref="MaintenanceLeaseHolder"/> 45s renewal, <see cref="LeaseHeldException"/>
/// on contention) are identical to the backup path.
/// </para>
/// <para>
/// The <c>deletion-manifests</c> container has a 30-day lifecycle policy that may delete an
/// idle sentinel; <see cref="AcquireLeaseAsync"/> re-ensures it on every acquire, so the
/// lock self-heals. The manifest-TTL sweep skips the sentinel because <c>_lock/…</c> does
/// not parse as a manifest blob name.
/// </para>
/// </summary>
public class SessionDeletionMaintenanceLockStore
{
/// <summary>Sentinel blob path — leased by the timer / manual-trigger worker to serialize runs.</summary>
public const string LockBlobName = "_lock/session-deletion-maintenance.lock";
/// <summary>Lease duration (Azure spec allows 15..60s for renewable leases).</summary>
public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60);
private readonly BlobStorageService _blobs;
private readonly ILogger<SessionDeletionMaintenanceLockStore> _logger;
private int _containerEnsured;
public SessionDeletionMaintenanceLockStore(
BlobStorageService blobs,
ILogger<SessionDeletionMaintenanceLockStore> logger)
{
_blobs = blobs;
_logger = logger;
}
/// <summary>
/// Attempts to acquire the session-deletion maintenance lease. Throws
/// <see cref="LeaseHeldException"/> on 409 (another holder); other errors propagate.
/// Caller is responsible for renewing (via <see cref="MaintenanceLeaseHolder"/>) and
/// releasing. <c>virtual</c> so tests can stub it.
/// </summary>
public virtual async Task<BlobLeaseClient> AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default)
{
await EnsureSentinelAsync(ct).ConfigureAwait(false);
var blob = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests).GetBlobClient(LockBlobName);
var leaseClient = blob.GetBlobLeaseClient();
try
{
await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false);
return leaseClient;
}
catch (RequestFailedException ex) when (ex.Status == 409)
{
throw new LeaseHeldException("session-deletion maintenance lease is held by another run", ex);
}
}
/// <summary>
/// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob
/// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409
/// BlobAlreadyExists and 412 ConditionNotMet are the steady state.
/// </summary>
private async Task EnsureSentinelAsync(CancellationToken ct)
{
var container = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests);
if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0)
{
await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
Interlocked.Exchange(ref _containerEnsured, 1);
}
var blob = container.GetBlobClient(LockBlobName);
var options = new BlobUploadOptions
{
HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" },
Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All },
};
try
{
using var empty = new MemoryStream(Array.Empty<byte>());
await blob.UploadAsync(empty, options, ct).ConfigureAwait(false);
}
catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412)
{
// Sentinel exists already — steady state.
}
}
}
}
@@ -0,0 +1,53 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Functions.Maintenance;
using AutopilotMonitor.Functions.Services.Queueing;
using AutopilotMonitor.Shared;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Deletion
{
/// <summary>
/// Background worker for the <c>session-deletion-maintenance</c> queue — the manual-trigger
/// counterpart to the 12h timer. Each message runs the full
/// <see cref="SessionDeletionMaintenanceFunction.RunCoreAsync"/> body with the requesting
/// admin as <c>triggeredBy</c>.
/// <para>
/// Concurrency against the timer (and a second host instance) is handled INSIDE
/// <c>RunCoreAsync</c> via the session-deletion maintenance blob lease: a lease-held run
/// returns normally (SkippedLocked OpsEvent), so the message is deleted rather than
/// retried — the operator asked for "a run", and one is already active.
/// </para>
/// <para>
/// BatchSize=1 + VisibilityTimeout=60min (run budget is 50min plus cushion — same rationale
/// as <c>CriticalTableBackupQueueWorker</c>). Real exceptions rethrow → visibility-timeout
/// retry → poison after <see cref="QueuePollingWorkerBase.MaxDequeueCount"/>.
/// </para>
/// </summary>
public sealed class SessionDeletionMaintenanceQueueWorker : QueuePollingWorker<SessionDeletionMaintenanceTriggerEnvelope>
{
private readonly SessionDeletionMaintenanceFunction _maintenance;
public SessionDeletionMaintenanceQueueWorker(
QueueClientFactory queueFactory,
SessionDeletionMaintenanceFunction maintenance,
ILogger<SessionDeletionMaintenanceQueueWorker> logger)
: base(queueFactory, Constants.QueueNames.SessionDeletionMaintenance, logger, Constants.QueueNames.SessionDeletionMaintenancePoison)
{
_maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance));
}
protected override int BatchSize => 1;
protected override TimeSpan VisibilityTimeout => TimeSpan.FromMinutes(60);
protected override bool TryValidate(SessionDeletionMaintenanceTriggerEnvelope envelope)
=> !string.IsNullOrEmpty(envelope.TriggeredBy);
protected override string DescribeForLog(SessionDeletionMaintenanceTriggerEnvelope envelope)
=> $"triggeredBy={envelope.TriggeredBy}";
protected override Task HandleAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct)
=> _maintenance.RunCoreAsync(envelope.TriggeredBy, ct);
}
}
@@ -34,6 +34,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
private readonly AdminConfigurationService _adminConfig;
private readonly ILogger<SessionRetentionFanoutService> _logger;
private readonly Func<TimeSpan, CancellationToken, Task> _throttle;
private readonly Func<DateTime> _utcNow;
public SessionRetentionFanoutService(
IMaintenanceRepository maintenanceRepo,
@@ -41,13 +42,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion
ISessionDeletionEnqueuer enqueuer,
AdminConfigurationService adminConfig,
ILogger<SessionRetentionFanoutService> logger)
: this(maintenanceRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay)
: this(maintenanceRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay, utcNow: () => DateTime.UtcNow)
{
}
/// <summary>
/// Test seam — tests inject a no-op throttle to make the rate-limit loop exercise its
/// counter without waiting 50ms × N in real time.
/// counter without waiting 50ms × N in real time, and a scripted clock so the run-budget
/// deadline can be crossed deterministically.
/// </summary>
internal SessionRetentionFanoutService(
IMaintenanceRepository maintenanceRepo,
@@ -55,7 +57,8 @@ namespace AutopilotMonitor.Functions.Services.Deletion
ISessionDeletionEnqueuer enqueuer,
AdminConfigurationService adminConfig,
ILogger<SessionRetentionFanoutService> logger,
Func<TimeSpan, CancellationToken, Task> throttle)
Func<TimeSpan, CancellationToken, Task> throttle,
Func<DateTime>? utcNow = null)
{
_maintenanceRepo = maintenanceRepo;
_tenantConfig = tenantConfig;
@@ -63,11 +66,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion
_adminConfig = adminConfig;
_logger = logger;
_throttle = throttle;
_utcNow = utcNow ?? (() => DateTime.UtcNow);
}
/// <summary>
/// Aggregate result of a single fanout invocation; used for the per-tenant audit + the
/// completion summary OpsEvent.
/// completion summary OpsEvent. The caller allocates it and <see cref="RunAsync"/> mutates
/// it incrementally (per tenant / per session), so watchdog snapshots taken while the
/// fanout is still running read real progress instead of zeros.
/// </summary>
public sealed class FanoutResult
{
@@ -76,6 +82,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
public int SessionsSkipped { get; set; } // already locked / poisoned / kill-switch / etc.
public int RateLimitedTenants { get; set; } // tenants that hit MaxEnqueuesPerTenantPerRun
public bool AbortedByKillSwitch { get; set; } // kill-switch flipped mid-run
public bool AbortedByBudget { get; set; } // run-budget deadline crossed mid-run
}
/// <summary>
@@ -83,16 +90,32 @@ namespace AutopilotMonitor.Functions.Services.Deletion
/// Each tenant is processed independently; an exception on tenant A is logged and the loop
/// continues with tenant B (matches the existing maintenance behaviour: per-tenant
/// failures must not cascade).
/// <para>
/// <paramref name="deadlineUtc"/> is the run-budget cutoff: once crossed, the fanout stops
/// cleanly at the next tenant/session boundary and sets <see cref="FanoutResult.AbortedByBudget"/>
/// — the remaining backlog is picked up by the next run. This keeps the maintenance
/// function comfortably inside the Flex Consumption 60min <c>functionTimeout</c> instead of
/// being hard-aborted by the host mid-tenant.
/// </para>
/// </summary>
public virtual async Task<FanoutResult> RunAsync(CancellationToken cancellationToken)
public virtual async Task RunAsync(FanoutResult result, DateTime deadlineUtc, CancellationToken cancellationToken)
{
var result = new FanoutResult();
var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync().ConfigureAwait(false);
foreach (var tenantId in tenantIds)
{
cancellationToken.ThrowIfCancellationRequested();
// Run-budget check at the tenant boundary — cheapest clean stopping point.
if (_utcNow() >= deadlineUtc)
{
_logger.LogWarning(
"SessionRetentionFanout: run budget exhausted — halting before tenant {TenantId} (deadline {Deadline:o})",
tenantId, deadlineUtc);
result.AbortedByBudget = true;
break;
}
// Per-tenant kill-switch check so a flip-ON mid-run halts the remaining tenants.
// The maintenance function gates entry; this gates iteration.
if (await _adminConfig.IsSessionDeletionKillSwitchActiveAsync().ConfigureAwait(false))
@@ -106,7 +129,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
try
{
await RunForTenantAsync(tenantId, result, cancellationToken).ConfigureAwait(false);
await RunForTenantAsync(tenantId, deadlineUtc, result, cancellationToken).ConfigureAwait(false);
result.TenantsProcessed++;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -118,22 +141,20 @@ namespace AutopilotMonitor.Functions.Services.Deletion
_logger.LogError(ex, "SessionRetentionFanout failed for tenant {TenantId} — continuing with next tenant", tenantId);
}
// The inner loop sets this when the producer's Step 0 reports KillSwitchActive
// mid-tenant. That's the authoritative fail-closed outcome — stop touching the
// remaining tenants instead of relying on the next pre-check to catch the flip.
if (result.AbortedByKillSwitch)
// The inner loop sets these when the producer's Step 0 reports KillSwitchActive
// mid-tenant, or when the budget deadline is crossed mid-tenant. Both are
// authoritative stop signals — don't touch the remaining tenants.
if (result.AbortedByKillSwitch || result.AbortedByBudget)
{
_logger.LogWarning(
"SessionRetentionFanout: aborting outer loop after tenant {TenantId} reported KillSwitchActive from producer",
tenantId);
"SessionRetentionFanout: aborting outer loop after tenant {TenantId} (killSwitch={KillSwitch}, budget={Budget})",
tenantId, result.AbortedByKillSwitch, result.AbortedByBudget);
break;
}
}
return result;
}
private async Task RunForTenantAsync(string tenantId, FanoutResult result, CancellationToken cancellationToken)
private async Task RunForTenantAsync(string tenantId, DateTime deadlineUtc, FanoutResult result, CancellationToken cancellationToken)
{
var config = await _tenantConfig.GetConfigurationAsync(tenantId).ConfigureAwait(false);
// Edition retention cap (fail-closed backstop): the stored value is clamped to the
@@ -200,6 +221,18 @@ namespace AutopilotMonitor.Functions.Services.Deletion
break;
}
// Run-budget check per session — a single tenant's batch can take tens of
// minutes (manifest build ~7-25s per session), so the tenant-boundary check
// alone would not stop a run drifting into the host's hard functionTimeout.
if (_utcNow() >= deadlineUtc)
{
_logger.LogWarning(
"SessionRetentionFanout: run budget exhausted mid-tenant — halting at session {SessionId} of {TenantId}",
session.SessionId, tenantId);
result.AbortedByBudget = true;
break;
}
// Per-session kill-switch check so an emergency flip-ON halts immediately instead
// of after the rest of this tenant's backlog. Uncached read — uniform behaviour
// across scaled-out instances within seconds.
@@ -224,12 +257,15 @@ namespace AutopilotMonitor.Functions.Services.Deletion
"SessionRetentionFanout: producer reported KillSwitchActive — aborting at session {SessionId} of {TenantId}",
session.SessionId, tenantId);
skipped++;
result.SessionsSkipped++;
result.AbortedByKillSwitch = true;
break;
}
if (outcome == SessionDeletionEnqueueOutcome.Enqueued) enqueued++;
else skipped++;
// Live counters on the shared result (in addition to the per-tenant locals used
// for the audit entry) so the maintenance watchdog reports real progress.
if (outcome == SessionDeletionEnqueueOutcome.Enqueued) { enqueued++; result.SessionsEnqueued++; }
else { skipped++; result.SessionsSkipped++; }
processed++;
@@ -252,9 +288,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
tenantId, processed, skipped);
}
result.SessionsEnqueued += enqueued;
result.SessionsSkipped += skipped;
// result.SessionsEnqueued / SessionsSkipped are already bumped live inside the loop.
await _maintenanceRepo.LogAuditEntryAsync(
tenantId,
"SessionDeletionMaintenanceFanout",
@@ -272,6 +306,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
{ "Enqueued", enqueued.ToString() },
{ "Skipped", skipped.ToString() },
{ "AbortedByKillSwitch", result.AbortedByKillSwitch.ToString() },
{ "AbortedByBudget", result.AbortedByBudget.ToString() },
}).ConfigureAwait(false);
_logger.LogInformation(
@@ -112,9 +112,40 @@ namespace AutopilotMonitor.Functions.Services
}
// ── Cascade-Delete Maintenance (Plan §5 PR6 / §16 R14) ─────────────────
// Four event types dispatched by SessionDeletionMaintenanceFunction. Each is also
// Event types dispatched by SessionDeletionMaintenanceFunction: Started, LongRunning,
// LongRunningSevere, BudgetExceeded, SkippedLocked, Failed, Completed, FanoutSkipped
// (+ StrandedQueued / Poisoned from the GCs and the cascade worker). Each is also
// listed in OpsAlertRulesSection.tsx OPS_EVENT_TYPES (memory feedback_ops_event_types_dual_register).
/// <summary>
/// Run started (timer tick or manual trigger). Emitted after the maintenance lease was
/// acquired, so a lease-skip never masquerades as an active run — the Session Cleanup
/// UI banner treats "latest Started newer than latest Completed/Failed" as run-active.
/// </summary>
public Task RecordSessionDeletionMaintenanceStartedAsync(string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceStarted", OpsEventSeverity.Info,
$"SessionDeletionMaintenance run started (triggered by {triggeredBy})",
null, "System.Maintenance", new { triggeredBy });
/// <summary>
/// The retention fanout stopped cleanly at the run-budget deadline. Not an error: the
/// remaining backlog is picked up by the next run (12h cadence) or a manual trigger.
/// Paired with a Completed event whose details carry <c>abortedByBudget=true</c>.
/// </summary>
public Task RecordSessionDeletionMaintenanceBudgetExceededAsync(int budgetMinutes, int tenantsProcessed, int sessionsEnqueued)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceBudgetExceeded", OpsEventSeverity.Warning,
$"SessionDeletionMaintenance stopped at the {budgetMinutes}min run budget — tenants={tenantsProcessed} enqueued={sessionsEnqueued}; remaining backlog resumes on the next run",
null, "System.Maintenance", new { budgetMinutes, tenantsProcessed, sessionsEnqueued });
/// <summary>
/// A run (timer or manual) was skipped because another run holds the session-deletion
/// maintenance lease. Mirrors <c>RecordCriticalTableBackupSkippedLockedAsync</c>.
/// </summary>
public Task RecordSessionDeletionMaintenanceSkippedLockedAsync(string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceSkippedLocked", OpsEventSeverity.Info,
$"SessionDeletionMaintenance skipped — another run holds the maintenance lease (triggeredBy={triggeredBy})",
null, "System.Maintenance", new { reason = "lease held by another run", triggeredBy });
/// <summary>Watchdog: maintenance run still in flight 30 minutes after start. Warning-level early signal.</summary>
public Task RecordSessionDeletionMaintenanceLongRunningAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceLongRunning", OpsEventSeverity.Warning,
@@ -196,13 +227,13 @@ namespace AutopilotMonitor.Functions.Services
public Task RecordSessionDeletionMaintenanceCompletedAsync(
bool killSwitchActive, int tenantsProcessed, int sessionsEnqueued,
int sessionsSkipped, int rateLimitedTenants, int blobsTtlGced, int preparingRowsCleared,
int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch)
int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch, bool abortedByBudget)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceCompleted", OpsEventSeverity.Info,
$"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive}",
$"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive} abortedByBudget={abortedByBudget}",
null, "System.Maintenance", new {
killSwitchActive, tenantsProcessed, sessionsEnqueued,
sessionsSkipped, rateLimitedTenants, blobsTtlGced, preparingRowsCleared,
strandedQueuedDetected, durationMs, abortedByKillSwitch,
strandedQueuedDetected, durationMs, abortedByKillSwitch, abortedByBudget,
});
/// <summary>
@@ -1154,6 +1154,20 @@ namespace AutopilotMonitor.Shared
/// <summary>Poison sibling of <see cref="CriticalTableBackup"/>.</summary>
public const string CriticalTableBackupPoison = "critical-table-backup-jobs-poison";
/// <summary>
/// Manual-trigger queue for the session-deletion maintenance run. Producer = HTTP
/// trigger <c>/api/global/session-deletions/maintenance/trigger</c> (fail-hard);
/// consumer = <c>SessionDeletionMaintenanceQueueWorker</c> (BackgroundService,
/// BatchSize=1, VisibilityTimeout=60min — the run budget is 50min plus cushion).
/// Carries a tiny envelope <c>{ triggeredBy }</c>; concurrency against the 12h timer
/// is serialized by the session-deletion maintenance blob lease, not the queue.
/// Poison-suffix <c>-poison</c>, max-dequeue 5.
/// </summary>
public const string SessionDeletionMaintenance = "session-deletion-maintenance";
/// <summary>Poison sibling of <see cref="SessionDeletionMaintenance"/>.</summary>
public const string SessionDeletionMaintenancePoison = "session-deletion-maintenance-poison";
}
/// <summary>
@@ -27,6 +27,13 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
"SessionDeletionMaintenanceLongRunningSevere",
"SessionDeletionMaintenanceFailed",
"SessionDeletionMaintenanceFanoutSkipped",
// Run lifecycle + 50min run budget + lease serialization (manual-trigger feature):
// Started marks run begin (timer or manual, details.triggeredBy), BudgetExceeded is the
// clean self-stop at the 50min budget (Warning), SkippedLocked means another run held the
// maintenance lease. Backend helpers RecordSessionDeletionMaintenance{Started,BudgetExceeded,SkippedLocked}Async.
"SessionDeletionMaintenanceStarted",
"SessionDeletionMaintenanceBudgetExceeded",
"SessionDeletionMaintenanceSkippedLocked",
"SessionDeletionStrandedQueued",
// PR-B audit consolidation: the per-session deletion_poisoned tenant-audit moved here so
// tenant admins see only the lifecycle endpoints (deletion_started/completed/restored).
@@ -0,0 +1,223 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import { api } from "@/lib/api";
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
interface OpsEvent {
id: string;
eventType: string;
severity: string;
message: string;
details: string | null;
timestamp: string;
}
interface OpsEventsResponse {
events: OpsEvent[];
}
interface CompletedDetails {
tenantsProcessed?: number;
sessionsEnqueued?: number;
durationMs?: number;
abortedByBudget?: boolean;
abortedByKillSwitch?: boolean;
}
type BannerState =
| { kind: "loading" }
| { kind: "active"; since: string; triggeredBy: string }
| { kind: "budget-exceeded"; at: string; details: CompletedDetails }
| { kind: "failed"; at: string; message: string }
| { kind: "idle"; at: string; details: CompletedDetails }
| { kind: "none" }; // no lifecycle events in the window — render nothing
function parseDetails<T>(raw: string | null): T {
if (!raw) return {} as T;
try {
return JSON.parse(raw) as T;
} catch {
return {} as T;
}
}
/**
* Status banner for the session-deletion maintenance run (12h retention fanout + GC sweeps).
* Derives its state purely from the SessionDeletionMaintenance* ops events: the latest
* Started newer than the latest terminal event (Completed/Failed) means a run is active;
* otherwise the latest terminal event decides between budget-exceeded / failed / idle.
* The "Run now" button POSTs the manual trigger (202 = queued, 409 = already active).
*/
export function MaintenanceStatusBanner({
getAccessToken,
setError,
setSuccessMessage,
}: {
getAccessToken: (forceRefresh?: boolean) => Promise<string | null>;
setError: (error: string | null) => void;
setSuccessMessage: (message: string | null) => void;
}) {
const [state, setState] = useState<BannerState>({ kind: "loading" });
const [triggering, setTriggering] = useState(false);
const [refreshKey, setRefreshKey] = useState(0);
const fetchStatus = useCallback(async () => {
try {
const r = await authenticatedFetch(
api.opsEvents.list("Maintenance", { pageSize: 100 }),
getAccessToken,
);
if (!r.ok) throw new Error(`HTTP ${r.status} loading maintenance ops events`);
const data = (await r.json()) as OpsEventsResponse;
const lifecycle = (data.events ?? []).filter((e) =>
e.eventType.startsWith("SessionDeletionMaintenance"),
);
// Backend returns newest-first; keep it defensive anyway.
lifecycle.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1));
const started = lifecycle.find((e) => e.eventType === "SessionDeletionMaintenanceStarted");
const terminal = lifecycle.find(
(e) =>
e.eventType === "SessionDeletionMaintenanceCompleted" ||
e.eventType === "SessionDeletionMaintenanceFailed",
);
if (started && (!terminal || started.timestamp > terminal.timestamp)) {
const details = parseDetails<{ triggeredBy?: string }>(started.details);
setState({
kind: "active",
since: started.timestamp,
triggeredBy: details.triggeredBy ?? "unknown",
});
return;
}
if (!terminal) {
setState({ kind: "none" });
return;
}
if (terminal.eventType === "SessionDeletionMaintenanceFailed") {
setState({ kind: "failed", at: terminal.timestamp, message: terminal.message });
return;
}
const details = parseDetails<CompletedDetails>(terminal.details);
setState(
details.abortedByBudget
? { kind: "budget-exceeded", at: terminal.timestamp, details }
: { kind: "idle", at: terminal.timestamp, details },
);
} catch (err) {
if (err instanceof TokenExpiredError) {
setError("Session expired; reload the page and try again.");
}
// Non-auth failures degrade silently — the banner is auxiliary to the tabs.
setState({ kind: "none" });
}
}, [getAccessToken, setError]);
useEffect(() => {
void fetchStatus();
}, [fetchStatus, refreshKey]);
const triggerRun = useCallback(async () => {
setTriggering(true);
try {
const r = await authenticatedFetch(api.sessionDeletions.triggerMaintenance(), getAccessToken, {
method: "POST",
});
if (r.status === 202) {
setSuccessMessage("Maintenance run queued — it will appear here as active shortly.");
} else if (r.status === 409) {
setSuccessMessage("A maintenance run is already active.");
} else {
const body = await r.text();
throw new Error(`HTTP ${r.status}: ${body}`);
}
setRefreshKey((k) => k + 1);
} catch (err) {
if (err instanceof TokenExpiredError) {
setError("Session expired; reload the page and try again.");
} else {
setError(err instanceof Error ? err.message : String(err));
}
} finally {
setTriggering(false);
}
}, [getAccessToken, setError, setSuccessMessage]);
if (state.kind === "loading" || state.kind === "none") return null;
const runNowButton = (
<button
onClick={() => void triggerRun()}
disabled={triggering}
className="shrink-0 px-3 py-1.5 text-xs font-medium bg-purple-600 text-white rounded hover:bg-purple-700 disabled:opacity-50"
>
{triggering ? "Queuing…" : "Run now"}
</button>
);
if (state.kind === "active") {
return (
<div className="flex items-center justify-between gap-4 rounded-md border border-purple-200 dark:border-purple-800 bg-purple-50 dark:bg-purple-900/20 px-4 py-3">
<p className="text-sm text-purple-800 dark:text-purple-200">
<span className="font-medium">Maintenance run active</span> — started{" "}
{new Date(state.since).toLocaleString()} (triggered by {state.triggeredBy}). Retention
deletions enqueued by this run appear under In-Flight.
</p>
<button
onClick={() => setRefreshKey((k) => k + 1)}
className="shrink-0 px-3 py-1.5 text-xs font-medium border border-purple-300 dark:border-purple-700 text-purple-700 dark:text-purple-200 rounded hover:bg-purple-100 dark:hover:bg-purple-900/40"
>
Refresh
</button>
</div>
);
}
if (state.kind === "budget-exceeded") {
return (
<div className="flex items-center justify-between gap-4 rounded-md border border-amber-200 dark:border-amber-800 bg-amber-50 dark:bg-amber-900/20 px-4 py-3">
<p className="text-sm text-amber-800 dark:text-amber-200">
<span className="font-medium">Last run stopped at the 50-minute budget</span> (
{new Date(state.at).toLocaleString()}) after processing {state.details.tenantsProcessed ?? 0}{" "}
tenants ({state.details.sessionsEnqueued ?? 0} sessions enqueued). The remaining backlog
resumes with the next scheduled run (every 12 h at 00:00 / 12:00 UTC) — or trigger a run
now.
</p>
{runNowButton}
</div>
);
}
if (state.kind === "failed") {
return (
<div className="flex items-center justify-between gap-4 rounded-md border border-red-200 dark:border-red-800 bg-red-50 dark:bg-red-900/20 px-4 py-3">
<p className="text-sm text-red-800 dark:text-red-200">
<span className="font-medium">Last maintenance run failed</span> (
{new Date(state.at).toLocaleString()}): {state.message}
</p>
{runNowButton}
</div>
);
}
// idle — subtle last-run summary
return (
<div className="flex items-center justify-between gap-4 rounded-md border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 px-4 py-3">
<p className="text-sm text-gray-600 dark:text-gray-400">
Last maintenance run completed {new Date(state.at).toLocaleString()} — tenants=
{state.details.tenantsProcessed ?? 0}, enqueued={state.details.sessionsEnqueued ?? 0}
{typeof state.details.durationMs === "number"
? `, ${Math.round(state.details.durationMs / 60000)} min`
: ""}
. Next scheduled run: every 12 h at 00:00 / 12:00 UTC.
</p>
{runNowButton}
</div>
);
}
@@ -7,6 +7,7 @@ import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch"
import { useAdminConfig } from "../../AdminConfigContext";
import { AdminNotifications } from "../../AdminNotifications";
import { DeletionPreviewModal } from "./components/DeletionPreviewModal";
import { MaintenanceStatusBanner } from "./components/MaintenanceStatusBanner";
import { RestoreBrowserTab } from "./components/RestoreBrowserTab";
import { RestoreConfirmDialog } from "./components/RestoreConfirmDialog";
@@ -53,6 +54,12 @@ export default function SessionCleanupPage() {
<main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
<AdminNotifications />
<MaintenanceStatusBanner
getAccessToken={getAccessToken}
setError={setError}
setSuccessMessage={setSuccessMessage}
/>
{/* Tab bar */}
<div className="border-b border-gray-200 dark:border-gray-700">
<nav className="-mb-px flex gap-6 overflow-x-auto">
+6
View File
@@ -393,6 +393,12 @@ export const api = {
// data" checkbox so the dropdown can hide tenants that have nothing to restore.
tenantsWithManifests: () =>
`${API_BASE_URL}/api/global/tenants-with-deletion-manifests`,
// Manual maintenance-run trigger (POST, GA-only): queues a one-off GC + retention-fanout
// run. 202 = queued, 409 = a run is already active. Progress surfaces as
// SessionDeletionMaintenance* ops events (MaintenanceStatusBanner reads those).
triggerMaintenance: () =>
`${API_BASE_URL}/api/global/session-deletions/maintenance/trigger`,
},
// ── Hardware Rejection Insights (tenant-scoped, from distress data) ──────