From b0105b123adfa09365cb678d6b3d37e2f530f5ee Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Tue, 14 Jul 2026 15:22:31 +0200 Subject: [PATCH] feat(maintenance): session-deletion run budget, live watchdog counters, manual trigger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Watchdog fix: FanoutResult is caller-allocated and mutated live per tenant/session, so 30/60min LongRunning warnings report real progress instead of the always-zero counters (today's warning showed 0/0 while 5 tenants / 54 sessions were already done). - 50min run budget: the retention fanout stops cleanly at the deadline (tenant boundary + per-session check) and reports AbortedByBudget — no more risk of the 60min Flex functionTimeout hard-aborting mid-run. New Warning OpsEvent SessionDeletionMaintenanceBudgetExceeded; Completed gains abortedByBudget. - Blob-lease serialization: dedicated sentinel _lock/session-deletion-maintenance.lock (deletion-manifests container) via SessionDeletionMaintenanceLockStore + reused MaintenanceLeaseHolder; contended runs emit SessionDeletionMaintenanceSkippedLocked and return. - Manual trigger: POST /api/global/session-deletions/maintenance/trigger (GlobalAdminOnly, policy catalog) -> fail-hard queue producer -> SessionDeletionMaintenanceQueueWorker runs the same RunCoreAsync body with triggeredBy = requesting admin; lease probe returns 409 while a run is active. New Started OpsEvent carries triggeredBy. - Web: MaintenanceStatusBanner on the Session Cleanup page derives active / budget-exceeded / failed / idle from the lifecycle ops events and offers a Run-now button (202/409 handling). New event types dual-registered in OPS_EVENT_TYPES. - Tests: 7 new (budget abort immediate/mid-tenant, live counter increments, watchdog live-progress regression, BudgetExceeded + Completed flag, SkippedLocked, Started triggeredBy); suites green (backend 2935, web 557, tsc clean). Co-Authored-By: Claude Fable 5 --- ...SessionDeletionMaintenanceFunctionTests.cs | 133 +++++++++-- .../SessionRetentionFanoutServiceTests.cs | 109 +++++++-- .../SessionDeletionMaintenanceFunction.cs | 133 ++++++++--- ...iggerSessionDeletionMaintenanceFunction.cs | 105 +++++++++ .../AutopilotMonitor.Functions/Program.cs | 11 + .../Security/EndpointAccessPolicyCatalog.cs | 2 + ...ssionDeletionMaintenanceTriggerProducer.cs | 61 +++++ ...ssionDeletionMaintenanceTriggerProducer.cs | 26 ++ .../SessionDeletionMaintenanceLockStore.cs | 107 +++++++++ .../SessionDeletionMaintenanceQueueWorker.cs | 53 +++++ .../Deletion/SessionRetentionFanoutService.cs | 77 ++++-- .../Services/OpsEventService.cs | 39 ++- .../AutopilotMonitor.Shared/Constants.cs | 14 ++ .../admin/components/OpsAlertRulesSection.tsx | 7 + .../components/MaintenanceStatusBanner.tsx | 223 ++++++++++++++++++ .../app/admin/ops/session-cleanup/page.tsx | 7 + src/Web/autopilot-monitor-web/lib/api.ts | 6 + 17 files changed, 1014 insertions(+), 99 deletions(-) create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/TriggerSessionDeletionMaintenanceFunction.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Deletion/AzureQueueSessionDeletionMaintenanceTriggerProducer.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Deletion/ISessionDeletionMaintenanceTriggerProducer.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceLockStore.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceQueueWorker.cs create mode 100644 src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/MaintenanceStatusBanner.tsx diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs index b5c2a941..6794658c 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs @@ -46,10 +46,10 @@ public class SessionDeletionMaintenanceFunctionTests harness.SeedSessionsByState(SessionDeletionState.Preparing); // no rows aged out in this scenario harness.SeedSessionsByState(SessionDeletionState.Queued); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); harness.BlobMock.Verify(b => b.DeleteDeletionManifestPairAsync("t", "s", "m", It.IsAny()), Times.Once); - harness.Fanout.Verify(f => f.RunAsync(It.IsAny()), Times.Once); + harness.Fanout.Verify(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); // PR6 follow-up F3: lifecycle status now lives on OpsEvents, not AuditLogs (the latter // would silently fail because PartitionKey requires a non-null tenantId). Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted"); @@ -62,12 +62,12 @@ public class SessionDeletionMaintenanceFunctionTests harness.SetKillSwitch(true); harness.SeedTtlBlobs(("t", "s", "m", DateTime.UtcNow.AddDays(-31))); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); // GCs still ran — TTL sweep emitted the DeleteDeletionManifestPairAsync call. harness.BlobMock.Verify(b => b.DeleteDeletionManifestPairAsync("t", "s", "m", It.IsAny()), Times.Once); // Fanout was skipped. - harness.Fanout.Verify(f => f.RunAsync(It.IsAny()), Times.Never); + harness.Fanout.Verify(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); // PR6 follow-up F3: fanout-skip + completion now both live on OpsEvents. Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceFanoutSkipped"); Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted"); @@ -82,32 +82,55 @@ public class SessionDeletionMaintenanceFunctionTests // but not severe. var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(500)); harness.SetKillSwitch(false); - harness.Fanout.Setup(f => f.RunAsync(It.IsAny())) - .Returns(async (CancellationToken ct) => + harness.Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) => { + r.TenantsProcessed = 1; await Task.Delay(150, ct); - return new SessionRetentionFanoutService.FanoutResult { TenantsProcessed = 1, SessionsEnqueued = 0 }; }); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning"); Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunningSevere"); } + [Fact] + public async Task RunCore_watchdog_reports_live_fanout_progress() + { + // Regression for the "tenants=0, enqueued=0" watchdog bug: the counters used to be + // assigned only AFTER the fanout returned, so every LongRunning warning showed zeros. + // The fanout now mutates the caller-supplied result incrementally; the watchdog snapshot + // must reflect that progress mid-run. + var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromSeconds(5)); + harness.SetKillSwitch(false); + harness.Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) => + { + r.TenantsProcessed = 3; + r.SessionsEnqueued = 7; + await Task.Delay(200, ct); // long enough for the 50ms watchdog to fire mid-run + }); + + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); + + var warning = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning"); + Assert.Contains("tenants=3", warning.Message); + Assert.Contains("enqueued=7", warning.Message); + } + [Fact] public async Task RunCore_emits_LongRunningSevere_when_body_runs_past_severe_threshold() { var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(150)); harness.SetKillSwitch(false); - harness.Fanout.Setup(f => f.RunAsync(It.IsAny())) - .Returns(async (CancellationToken ct) => + harness.Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (SessionRetentionFanoutService.FanoutResult _, DateTime _, CancellationToken ct) => { await Task.Delay(300, ct); - return new SessionRetentionFanoutService.FanoutResult(); }); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); // Both watchdogs fire: warning at 50ms, severe at 150ms. Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning"); @@ -126,7 +149,7 @@ public class SessionDeletionMaintenanceFunctionTests ("tenant-a", "session-y", "MANIFEST-2"), ("tenant-b", "session-z", "MANIFEST-3")); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-x", It.IsAny()), Times.Once); harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-y", It.IsAny()), Times.Once); @@ -143,7 +166,7 @@ public class SessionDeletionMaintenanceFunctionTests harness.SetKillSwitch(true); harness.SeedExpiredTombstones(("tenant-a", "session-x", "M1")); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); harness.StorageMock.Verify(s => s.DeleteSessionTombstoneAsync("tenant-a", "session-x", It.IsAny()), Times.Once); @@ -155,12 +178,63 @@ public class SessionDeletionMaintenanceFunctionTests var harness = new Harness(watchdogWarn: TimeSpan.FromSeconds(5), watchdogSevere: TimeSpan.FromSeconds(10)); harness.SetKillSwitch(false); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning"); Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunningSevere"); } + // ────────────────────────────────────────────────────────────────────────── Run budget + lease + lifecycle ─ + + [Fact] + public async Task RunCore_emits_BudgetExceeded_and_flags_Completed_when_fanout_hits_the_budget() + { + var harness = new Harness(); + harness.SetKillSwitch(false); + harness.Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken _) => + { + r.TenantsProcessed = 12; + r.SessionsEnqueued = 40; + r.AbortedByBudget = true; + return Task.CompletedTask; + }); + + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); + + var budget = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceBudgetExceeded"); + Assert.Contains("tenants=12", budget.Message); + var completed = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted"); + Assert.Contains("abortedByBudget=True", completed.Message); + } + + [Fact] + public async Task RunCore_emits_SkippedLocked_and_skips_everything_when_lease_is_held() + { + var harness = new Harness(); + harness.LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new AutopilotMonitor.Functions.Services.Backup.LeaseHeldException("held", new Exception("409"))); + + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); + + Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceSkippedLocked"); + Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceStarted"); + Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted"); + harness.Fanout.Verify(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunCore_emits_Started_event_with_triggeredBy() + { + var harness = new Harness(); + harness.SetKillSwitch(false); + + await harness.Sut.RunCoreAsync("admin@contoso.com", CancellationToken.None); + + var started = Assert.Single(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceStarted"); + Assert.Contains("admin@contoso.com", started.Message); + } + // ────────────────────────────────────────────────────────────────────────── Exception path ─ [Fact] @@ -169,10 +243,10 @@ public class SessionDeletionMaintenanceFunctionTests var harness = new Harness(); harness.SetKillSwitch(false); // Make the fanout throw. - harness.Fanout.Setup(f => f.RunAsync(It.IsAny())) + harness.Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ThrowsAsync(new InvalidOperationException("fanout exploded")); - var ex = await Assert.ThrowsAsync(() => harness.Sut.RunCoreAsync(CancellationToken.None)); + var ex = await Assert.ThrowsAsync(() => harness.Sut.RunCoreAsync("Test", CancellationToken.None)); Assert.Equal("fanout exploded", ex.Message); // PR6 follow-up F3: the SessionDeletionMaintenanceFailed OpsEvent IS the audit. The prior @@ -192,7 +266,7 @@ public class SessionDeletionMaintenanceFunctionTests (TenantA, SessionA1, Manifest2, stuckSince), (TenantB, SessionB1, Manifest2, stuckSince)); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); var stranded = harness.OpsEvents.Where(e => e.EventName == "SessionDeletionStrandedQueued").ToList(); Assert.Equal(2, stranded.Count); @@ -206,7 +280,7 @@ public class SessionDeletionMaintenanceFunctionTests var freshly = DateTime.UtcNow.AddMinutes(-5); // well under 30min harness.SeedSessionsByState(SessionDeletionState.Queued, (TenantA, SessionA1, Manifest2, freshly)); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); Assert.DoesNotContain(harness.OpsEvents, e => e.EventName == "SessionDeletionStrandedQueued"); } @@ -234,7 +308,7 @@ public class SessionDeletionMaintenanceFunctionTests .Setup(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA1, Manifest1, It.IsAny())) .ReturnsAsync(true); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA1, Manifest1, It.IsAny()), Times.Once); harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(TenantA, SessionA2, Manifest1 + "X", It.IsAny()), Times.Never); @@ -250,7 +324,7 @@ public class SessionDeletionMaintenanceFunctionTests var recent = DateTime.UtcNow.AddMinutes(-5); harness.SeedSessionsByState(SessionDeletionState.Preparing, (TenantA, SessionA1, Manifest1, recent)); - await harness.Sut.RunCoreAsync(CancellationToken.None); + await harness.Sut.RunCoreAsync("Test", CancellationToken.None); harness.StorageMock.Verify(s => s.RevertStalePreparingToNoneAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); } @@ -263,6 +337,7 @@ public class SessionDeletionMaintenanceFunctionTests public Mock BlobMock { get; } public Mock AdminConfig { get; } public Mock Fanout { get; } + public Mock LockStore { get; } public List OpsEvents { get; } = new(); public List AuditCalls { get; } = new(); public SessionDeletionMaintenanceFunction Sut { get; } @@ -328,14 +403,24 @@ public class SessionDeletionMaintenanceFunctionTests Mock.Of(), AdminConfig.Object, NullLogger.Instance); - Fanout.Setup(f => f.RunAsync(It.IsAny())) - .ReturnsAsync(new SessionRetentionFanoutService.FanoutResult { TenantsProcessed = 0, SessionsEnqueued = 0 }); + Fanout.Setup(f => f.RunAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // Lease store: default acquire succeeds with a mocked lease client. The renewal loop + // never ticks inside these millisecond-scale tests; DisposeAsync's release failure on + // the bare mock is swallowed by the holder's try/catch. + LockStore = new Mock( + BlobMock.Object, NullLogger.Instance); + LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new Mock().Object); Sut = new SessionDeletionMaintenanceFunction( StorageMock.Object, BlobMock.Object, AdminConfig.Object, opsService, maintRepo.Object, Fanout.Object, + LockStore.Object, NullLogger.Instance, watchdogWarning: watchdogWarn ?? TimeSpan.FromSeconds(30), - watchdogSevere: watchdogSevere ?? TimeSpan.FromMinutes(1)); + watchdogSevere: watchdogSevere ?? TimeSpan.FromMinutes(1), + runBudget: TimeSpan.FromMinutes(50)); } public void SetKillSwitch(bool active) => diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionRetentionFanoutServiceTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionRetentionFanoutServiceTests.cs index 7015832b..cd3d0cf8 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionRetentionFanoutServiceTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionRetentionFanoutServiceTests.cs @@ -29,7 +29,7 @@ public class SessionRetentionFanoutServiceTests harness.WithTenant(TenantA, retentionDays: 30, sessions: new[] { Old("s1", 60), Old("s2", 45) }); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.Equal(2, result.SessionsEnqueued); harness.Enqueuer.Verify( @@ -56,7 +56,7 @@ public class SessionRetentionFanoutServiceTests harness.MaintenanceRepo.Setup(m => m.GetSessionsOlderThanAsync(TenantB, It.Is(d => d <= DateTime.UtcNow.AddDays(-120)), It.IsAny(), It.IsAny())) .ReturnsAsync(new List { Summary(TenantB, "b2") }); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.Equal(2, result.TenantsProcessed); Assert.Equal(2, result.SessionsEnqueued); @@ -75,7 +75,7 @@ public class SessionRetentionFanoutServiceTests many.Add(Summary(TenantA, $"s{i:000}")); harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: many); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.Equal(SessionRetentionFanoutService.MaxEnqueuesPerTenantPerRun, result.SessionsEnqueued); Assert.Equal(1, result.RateLimitedTenants); @@ -94,7 +94,7 @@ public class SessionRetentionFanoutServiceTests var harness = new Harness(); harness.WithTenant(TenantA, retentionDays: 30, sessions: new[] { Old("s1", 60) }); - await harness.Sut.RunAsync(CancellationToken.None); + await harness.RunAsync(); harness.MaintenanceRepo.Verify( m => m.GetSessionsOlderThanAsync( @@ -113,7 +113,7 @@ public class SessionRetentionFanoutServiceTests var harness = new Harness(); harness.WithTenant(TenantA, retentionDays: 0, sessions: new[] { Old("s1", 365) }); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.Equal(0, result.SessionsEnqueued); // GetSessionsOlderThanAsync must not have been called for that tenant. @@ -129,7 +129,7 @@ public class SessionRetentionFanoutServiceTests var harness = new Harness(); harness.WithTenant(TenantA, retentionDays: 180, sessions: new[] { Old("s1", 100) }); - await harness.Sut.RunAsync(CancellationToken.None); + await harness.RunAsync(); harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync( TenantA, @@ -143,7 +143,7 @@ public class SessionRetentionFanoutServiceTests var harness = new Harness(); harness.WithTenant(TenantA, retentionDays: 180, sessions: new[] { Old("s1", 200) }, planTier: "enterprise"); - await harness.Sut.RunAsync(CancellationToken.None); + await harness.RunAsync(); harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync( TenantA, @@ -174,7 +174,7 @@ public class SessionRetentionFanoutServiceTests return Task.FromResult(calls >= 4); }); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.True(result.AbortedByKillSwitch); Assert.Equal(2, result.SessionsEnqueued); // only sessions 0 and 1 made it through @@ -203,7 +203,7 @@ public class SessionRetentionFanoutServiceTests return Task.FromResult(calls >= 3); }); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.True(result.AbortedByKillSwitch); Assert.Equal(1, result.TenantsProcessed); @@ -235,7 +235,7 @@ public class SessionRetentionFanoutServiceTests : new SessionDeletionEnqueueResult { Outcome = SessionDeletionEnqueueOutcome.KillSwitchActive }; }); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.True(result.AbortedByKillSwitch); Assert.Equal(1, result.SessionsEnqueued); @@ -272,7 +272,7 @@ public class SessionRetentionFanoutServiceTests : new SessionDeletionEnqueueResult { Outcome = SessionDeletionEnqueueOutcome.KillSwitchActive }; }); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.True(result.AbortedByKillSwitch); Assert.Equal(1, result.TenantsProcessed); // tenant A completed its RunForTenantAsync; B never started @@ -301,12 +301,79 @@ public class SessionRetentionFanoutServiceTests ExistingState = SessionDeletionState.Running, }); - var result = await harness.Sut.RunAsync(CancellationToken.None); + var result = await harness.RunAsync(); Assert.Equal(0, result.SessionsEnqueued); Assert.Equal(1, result.SessionsSkipped); } + // ────────────────────────────────────────────────────────────────────────── Run budget ─ + + [Fact] + public async Task RunAsync_aborts_immediately_when_deadline_already_passed() + { + var harness = new Harness(); + harness.WithTenant(TenantA, retentionDays: 30, sessions: new[] { Old("s1", 60) }); + + var result = await harness.RunAsync(deadlineUtc: DateTime.UtcNow.AddMinutes(-1)); + + Assert.True(result.AbortedByBudget); + Assert.Equal(0, result.TenantsProcessed); + Assert.Equal(0, result.SessionsEnqueued); + harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(TenantA, It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsync_aborts_mid_tenant_when_budget_deadline_crosses() + { + // Scripted clock: the deadline crosses between session 1 and session 2 of tenant A — + // the remaining sessions of A and all of tenant B must be left untouched. + var baseTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var deadline = baseTime.AddMinutes(50); + var clockCalls = 0; + // Call sequence: tenant-A boundary (1), session s0 (2), session s1 (3), session s2 (4 → past deadline). + var harness = new Harness(utcNow: () => ++clockCalls >= 4 ? deadline.AddMinutes(1) : baseTime); + + var sessions = new List(); + for (int i = 0; i < 4; i++) sessions.Add(Summary(TenantA, $"s{i}")); + harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: sessions); + harness.WithTenant(TenantB, retentionDays: 30, sessions: new[] { Old("b1", 60) }); + + var result = await harness.RunAsync(deadlineUtc: deadline); + + Assert.True(result.AbortedByBudget); + Assert.Equal(2, result.SessionsEnqueued); // only s0 and s1 made it through + Assert.Equal(1, result.TenantsProcessed); // tenant A returned normally, B never started + harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantA, "s2", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantB, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(TenantB, It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsync_mutates_the_shared_result_incrementally_during_the_run() + { + // Watchdog-progress contract: the caller-supplied FanoutResult must be bumped per + // session (not once at the end), so mid-run snapshots show real progress. The injected + // throttle runs between sessions — snapshot the counter there. + var snapshots = new List(); + var result = new SessionRetentionFanoutService.FanoutResult(); + var harness = new Harness(throttle: (_, _) => + { + snapshots.Add(result.SessionsEnqueued); + return Task.CompletedTask; + }); + var sessions = new List(); + for (int i = 0; i < 3; i++) sessions.Add(Summary(TenantA, $"s{i}")); + harness.WithTenantOverride(TenantA, retentionDays: 30, sessions: sessions); + + await harness.Sut.RunAsync(result, DateTime.UtcNow.AddMinutes(50), CancellationToken.None); + + Assert.Equal(3, result.SessionsEnqueued); + // Throttle fires after s0 and s1 (not after the last session) — each snapshot must + // already reflect the enqueues so far. + Assert.Equal(new[] { 1, 2 }, snapshots); + } + // ============================================================ Helpers ==== private static SessionSummary Summary(string tenantId, string sessionId) => @@ -333,7 +400,7 @@ public class SessionRetentionFanoutServiceTests private readonly List _tenantIds = new(); - public Harness() + public Harness(Func? throttle = null, Func? utcNow = null) { MaintenanceRepo = new Mock(); var memCache = new MemoryCache(new MemoryCacheOptions()); @@ -363,12 +430,24 @@ public class SessionRetentionFanoutServiceTests .ReturnsAsync(true); // Use internal ctor to inject a no-op throttle so the rate-limit test runs in real-time - // (50ms × 100 = 5s otherwise). + // (50ms × 100 = 5s otherwise) and an optional scripted clock for the budget tests. Sut = new SessionRetentionFanoutService( MaintenanceRepo.Object, TenantConfig.Object, Enqueuer.Object, AdminConfig.Object, NullLogger.Instance, - throttle: (_, _) => Task.CompletedTask); + throttle: throttle ?? ((_, _) => Task.CompletedTask), + utcNow: utcNow); + } + + /// + /// Drives with a fresh result object + /// and a far-away default deadline (the budget is exercised by dedicated tests). + /// + public async Task RunAsync(DateTime? deadlineUtc = null) + { + var result = new SessionRetentionFanoutService.FanoutResult(); + await Sut.RunAsync(result, deadlineUtc ?? DateTime.UtcNow.AddMinutes(50), CancellationToken.None); + return result; } public void WithTenant(string tenantId, int retentionDays, SessionSummary[] sessions, string planTier = "free") diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs index e9dedece..3f48a504 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs @@ -4,10 +4,12 @@ using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Functions.Services.Backup; using AutopilotMonitor.Functions.Services.Deletion; using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models.Deletion; +using Azure.Storage.Blobs.Specialized; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; @@ -32,9 +34,18 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance /// /// 30-minute watchdog emits a Warning OpsEvent; 60-minute watchdog escalates to Error. The /// 60-minute mark matches the Flex Consumption hard functionTimeout (host.json) so the - /// operator gets a Telegram alert before the Azure runtime abort. Cron 0 0 */12 * * * - /// (every 12h at 00:00 / 12:00 UTC) — const per repo convention; change-and-redeploy if you - /// need a different cadence. + /// operator gets a Telegram alert before the Azure runtime abort. The retention fanout stops + /// itself at the (50min) deadline so a growing backlog degrades into + /// a clean BudgetExceeded + partial sweep instead of a host abort. Cron + /// 0 0 */12 * * * (every 12h at 00:00 / 12:00 UTC) — const per repo convention; + /// change-and-redeploy if you need a different cadence. + /// + /// + /// Runs are serialized by a blob lease (): + /// the timer and the manual trigger (POST /api/global/session-deletions/maintenance/trigger + /// → ) both enter through + /// , which acquires the lease first and emits + /// SkippedLocked when another run is active. /// /// public sealed class SessionDeletionMaintenanceFunction @@ -58,15 +69,21 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance internal static readonly TimeSpan WatchdogWarning = TimeSpan.FromMinutes(30); internal static readonly TimeSpan WatchdogSevere = TimeSpan.FromMinutes(60); + // Run budget for the retention fanout (matches the critical-table backup convention): + // stop cleanly at 50min so the 60min host functionTimeout never aborts us mid-tenant. + internal static readonly TimeSpan RunBudget = TimeSpan.FromMinutes(50); + private readonly TableStorageService _storage; private readonly BlobStorageService _blob; private readonly AdminConfigurationService _adminConfig; private readonly OpsEventService _opsEvents; private readonly IMaintenanceRepository _maintenanceRepo; private readonly SessionRetentionFanoutService _fanout; + private readonly SessionDeletionMaintenanceLockStore _lockStore; private readonly ILogger _logger; private readonly TimeSpan _watchdogWarning; private readonly TimeSpan _watchdogSevere; + private readonly TimeSpan _runBudget; public SessionDeletionMaintenanceFunction( TableStorageService storage, @@ -75,14 +92,16 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance OpsEventService opsEvents, IMaintenanceRepository maintenanceRepo, SessionRetentionFanoutService fanout, + SessionDeletionMaintenanceLockStore lockStore, ILogger logger) - : this(storage, blob, adminConfig, opsEvents, maintenanceRepo, fanout, logger, WatchdogWarning, WatchdogSevere) + : this(storage, blob, adminConfig, opsEvents, maintenanceRepo, fanout, lockStore, logger, WatchdogWarning, WatchdogSevere, RunBudget) { } /// - /// Test seam: lets unit tests inject sub-second watchdog thresholds so the function can be - /// driven end-to-end in milliseconds. Production code uses the static defaults (30min / 60min). + /// Test seam: lets unit tests inject sub-second watchdog thresholds and run budget so the + /// function can be driven end-to-end in milliseconds. Production code uses the static + /// defaults (30min / 60min / 50min). /// internal SessionDeletionMaintenanceFunction( TableStorageService storage, @@ -91,9 +110,11 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance OpsEventService opsEvents, IMaintenanceRepository maintenanceRepo, SessionRetentionFanoutService fanout, + SessionDeletionMaintenanceLockStore lockStore, ILogger logger, TimeSpan watchdogWarning, - TimeSpan watchdogSevere) + TimeSpan watchdogSevere, + TimeSpan runBudget) { _storage = storage; _blob = blob; @@ -101,63 +122,95 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance _opsEvents = opsEvents; _maintenanceRepo = maintenanceRepo; _fanout = fanout; + _lockStore = lockStore; _logger = logger; _watchdogWarning = watchdogWarning; _watchdogSevere = watchdogSevere; + _runBudget = runBudget; } [Function("SessionDeletionMaintenance")] public async Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken) { - await RunCoreAsync(cancellationToken); + await RunCoreAsync("Timer", cancellationToken); } /// /// Testable core. Separated from the TimerTrigger entry so unit tests can drive it - /// directly without a TimerInfo / FunctionContext. + /// directly without a TimerInfo / FunctionContext, and shared with the manual-trigger + /// queue worker. is "Timer" for the cron path and + /// the requesting Global Admin's identifier for manual runs. /// - internal async Task RunCoreAsync(CancellationToken cancellationToken) + internal async Task RunCoreAsync(string triggeredBy, CancellationToken cancellationToken) { - _logger.LogInformation("SessionDeletionMaintenance: tick started at {Now:o}", DateTime.UtcNow); - var sw = Stopwatch.StartNew(); + _logger.LogInformation("SessionDeletionMaintenance: tick started at {Now:o} (triggeredBy={TriggeredBy})", DateTime.UtcNow, triggeredBy); - using var watchdogCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - int sessionsEnqueuedSoFar = 0; - int tenantsProcessedSoFar = 0; + // Serialize timer vs manual trigger vs second host instance. Acquired BEFORE the + // Started OpsEvent so a lease-skip never masquerades as an active run to the UI. + BlobLeaseClient lease; + try + { + lease = await _lockStore.AcquireLeaseAsync(ct: cancellationToken); + } + catch (LeaseHeldException) + { + _logger.LogWarning( + "SessionDeletionMaintenance: lease held by another run — skipping (triggeredBy={TriggeredBy})", + triggeredBy); + await _opsEvents.RecordSessionDeletionMaintenanceSkippedLockedAsync(triggeredBy); + return; + } + + var sw = Stopwatch.StartNew(); + var deadlineUtc = DateTime.UtcNow + _runBudget; + + // handlerCts: cancelled by the lease holder on renewal failure so the run never + // continues rogue under an expired lease. The body + watchdogs run off this token. + using var handlerCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var leaseHolder = new MaintenanceLeaseHolder(lease, handlerCts, _logger); + using var watchdogCts = CancellationTokenSource.CreateLinkedTokenSource(handlerCts.Token); + + // Live progress object: the fanout mutates it per tenant / per session, so the + // watchdog snapshots below report real progress mid-run (previously the counters + // were only assigned after the fanout returned — every warning showed 0/0). + var fanoutResult = new SessionRetentionFanoutService.FanoutResult(); // Watchdogs: each one auto-cancels when the body completes (we cancel watchdogCts in // the finally). The await Task.WhenAny inside RunWatchdogAsync only emits if the body // is still running at the threshold. - var watchdogWarn = RunWatchdogAsync(_watchdogWarning, severe: false, () => (tenantsProcessedSoFar, sessionsEnqueuedSoFar), watchdogCts.Token); - var watchdogSevere = RunWatchdogAsync(_watchdogSevere, severe: true, () => (tenantsProcessedSoFar, sessionsEnqueuedSoFar), watchdogCts.Token); + var watchdogWarn = RunWatchdogAsync(_watchdogWarning, severe: false, () => (fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued), watchdogCts.Token); + var watchdogSevere = RunWatchdogAsync(_watchdogSevere, severe: true, () => (fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued), watchdogCts.Token); try { + await _opsEvents.RecordSessionDeletionMaintenanceStartedAsync(triggeredBy); + + var ct = handlerCts.Token; + // PR5 finding 1: uncached, fail-closed read so a flip-ON is honored within seconds // across all Function-host instances. var killSwitchActive = await _adminConfig.IsSessionDeletionKillSwitchActiveAsync(); // (1) Manifest-Blob-TTL sweep — runs even when kill-switch=true (Plan PR6 step 3). - var blobsTtlGced = await SweepManifestBlobsAsync(cancellationToken); + var blobsTtlGced = await SweepManifestBlobsAsync(ct); // (2) Stale-Preparing GC — runs even when kill-switch=true. - var preparingRowsCleared = await GcStalePreparingAsync(cancellationToken); + var preparingRowsCleared = await GcStalePreparingAsync(ct); // (3) Stranded-Queued detection (alert-only) — runs even when kill-switch=true. - var strandedQueuedDetected = await DetectStrandedQueuedAsync(cancellationToken); + var strandedQueuedDetected = await DetectStrandedQueuedAsync(ct); // (3b) Tombstone-marker pruning — physically remove rows past their ExpiresAt // (Codex F3). Runs even when kill-switch=true: these markers are short-lived // race-shields, not policy-bound deletion artefacts. Failures are non-fatal // because the in-flight Guard already treats expired markers as absent. - var tombstonesPruned = await PruneExpiredTombstonesAsync(cancellationToken); + var tombstonesPruned = await PruneExpiredTombstonesAsync(ct); // (4) Retention fanout — skipped when kill-switch=true. Emit a Fanout-Skipped // OpsEvent so the operator can fold the skip into the OpsEvents dashboard alongside // the 30/60min watchdogs. PR6 follow-up F3: previously a LogAuditEntryAsync(null!) // call that silently failed because the AuditLogs schema requires a non-null // PartitionKey — OpsEvents tolerates null TenantId, so global-scope events live there. - SessionRetentionFanoutService.FanoutResult? fanoutResult = null; if (killSwitchActive) { await _opsEvents.RecordSessionDeletionMaintenanceFanoutSkippedAsync( @@ -165,32 +218,38 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance } else { - fanoutResult = await _fanout.RunAsync(cancellationToken); - tenantsProcessedSoFar = fanoutResult.TenantsProcessed; - sessionsEnqueuedSoFar = fanoutResult.SessionsEnqueued; + await _fanout.RunAsync(fanoutResult, deadlineUtc, ct); + + if (fanoutResult.AbortedByBudget) + { + await _opsEvents.RecordSessionDeletionMaintenanceBudgetExceededAsync( + (int)_runBudget.TotalMinutes, fanoutResult.TenantsProcessed, fanoutResult.SessionsEnqueued); + } } // (5) Completion OpsEvent — same shape regardless of kill-switch state, so dashboards // can fold both paths together. PR6 follow-up F3. await _opsEvents.RecordSessionDeletionMaintenanceCompletedAsync( killSwitchActive: killSwitchActive, - tenantsProcessed: fanoutResult?.TenantsProcessed ?? 0, - sessionsEnqueued: fanoutResult?.SessionsEnqueued ?? 0, - sessionsSkipped: fanoutResult?.SessionsSkipped ?? 0, - rateLimitedTenants: fanoutResult?.RateLimitedTenants ?? 0, + tenantsProcessed: fanoutResult.TenantsProcessed, + sessionsEnqueued: fanoutResult.SessionsEnqueued, + sessionsSkipped: fanoutResult.SessionsSkipped, + rateLimitedTenants: fanoutResult.RateLimitedTenants, blobsTtlGced: blobsTtlGced, preparingRowsCleared: preparingRowsCleared, strandedQueuedDetected: strandedQueuedDetected, durationMs: (int)sw.ElapsedMilliseconds, - abortedByKillSwitch: fanoutResult?.AbortedByKillSwitch ?? false); + abortedByKillSwitch: fanoutResult.AbortedByKillSwitch, + abortedByBudget: fanoutResult.AbortedByBudget); _logger.LogInformation( - "SessionDeletionMaintenance: completed in {Ms}ms — killSwitch={KillSwitch} tenants={Tenants} enqueued={Enqueued} skipped={Skipped} blobs={Blobs} preparing={Preparing} stranded={Stranded} tombstones={Tombstones}", + "SessionDeletionMaintenance: completed in {Ms}ms — killSwitch={KillSwitch} tenants={Tenants} enqueued={Enqueued} skipped={Skipped} blobs={Blobs} preparing={Preparing} stranded={Stranded} tombstones={Tombstones} abortedByBudget={Budget}", sw.ElapsedMilliseconds, killSwitchActive, - fanoutResult?.TenantsProcessed ?? 0, - fanoutResult?.SessionsEnqueued ?? 0, - fanoutResult?.SessionsSkipped ?? 0, - blobsTtlGced, preparingRowsCleared, strandedQueuedDetected, tombstonesPruned); + fanoutResult.TenantsProcessed, + fanoutResult.SessionsEnqueued, + fanoutResult.SessionsSkipped, + blobsTtlGced, preparingRowsCleared, strandedQueuedDetected, tombstonesPruned, + fanoutResult.AbortedByBudget); } catch (Exception ex) { @@ -209,6 +268,10 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance // Observe the watchdog tasks so the OperationCanceledException doesn't get // surfaced as an unhandled exception in the host log. try { await Task.WhenAll(watchdogWarn, watchdogSevere); } catch (OperationCanceledException) { /* expected */ } + + // Release the lease LAST — the Failed/Completed OpsEvents above are written while + // the run is still exclusive, matching the backup worker's ordering. + await leaseHolder.DisposeAsync(); } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/TriggerSessionDeletionMaintenanceFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/TriggerSessionDeletionMaintenanceFunction.cs new file mode 100644 index 00000000..5252d60b --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/TriggerSessionDeletionMaintenanceFunction.cs @@ -0,0 +1,105 @@ +using System; +using System.Net; +using System.Text.Json; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Services.Backup; +using AutopilotMonitor.Functions.Services.Deletion; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Maintenance +{ + /// + /// POST /api/global/session-deletions/maintenance/trigger — manual GA trigger for a + /// one-off session-deletion maintenance run (GC sweeps + retention fanout), the on-demand + /// counterpart to the 12h timer. GA-only via + /// EndpointAccessPolicyCatalog; authentication + authorization are enforced by + /// PolicyEnforcementMiddleware, the body does not re-check. + /// + /// Semantics (mirrors TriggerCriticalTableBackupFunction's fail-hard producer): a + /// lease probe first rejects with 409 RunAlreadyActive while a run is in flight; on + /// successful enqueue we return 202 Accepted; on enqueue failure 500 so the + /// operator never sees a hollow success. Run progress surfaces via the + /// SessionDeletionMaintenance* OpsEvents (Session Cleanup banner + Ops Events page). + /// + /// + public class TriggerSessionDeletionMaintenanceFunction + { + private readonly ISessionDeletionMaintenanceTriggerProducer _producer; + private readonly SessionDeletionMaintenanceLockStore _lockStore; + private readonly ILogger _logger; + + public TriggerSessionDeletionMaintenanceFunction( + ISessionDeletionMaintenanceTriggerProducer producer, + SessionDeletionMaintenanceLockStore lockStore, + ILogger logger) + { + _producer = producer; + _lockStore = lockStore; + _logger = logger; + } + + [Function("TriggerSessionDeletionMaintenance")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/session-deletions/maintenance/trigger")] HttpRequestData req) + { + var actor = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin"; + var ct = req.FunctionContext.CancellationToken; + + // Acquire+release probe: a held lease means a run is active RIGHT NOW — tell the + // operator instead of queuing a message that would only produce a SkippedLocked + // event. Racing the probe (run starts between probe and worker pickup) is benign: + // the worker's own lease acquire lands on the SkippedLocked path. + try + { + var lease = await _lockStore.AcquireLeaseAsync(ct: ct); + try { await lease.ReleaseAsync(cancellationToken: ct); } + catch (Exception ex) + { + // Release failure is harmless — the 60s lease auto-expires; the worker's + // acquire retries land within the queue's poll cadence. + _logger.LogWarning(ex, "TriggerSessionDeletionMaintenance: probe-lease release failed (auto-expires in 60s)"); + } + } + catch (LeaseHeldException) + { + return await WriteJsonAsync(req, HttpStatusCode.Conflict, new + { + error = "RunAlreadyActive", + message = "A session-deletion maintenance run is already active — check the Session Cleanup page for progress.", + }); + } + + try + { + await _producer.EnqueueAsync(new SessionDeletionMaintenanceTriggerEnvelope { TriggeredBy = actor }, ct); + } + catch (Exception ex) + { + _logger.LogError(ex, "TriggerSessionDeletionMaintenance: enqueue failed (triggeredBy={TriggeredBy})", actor); + return await WriteJsonAsync(req, HttpStatusCode.InternalServerError, new + { + error = "EnqueueFailed", + message = "Failed to enqueue the maintenance run — please retry.", + }); + } + + _logger.LogInformation("TriggerSessionDeletionMaintenance: run queued (triggeredBy={TriggeredBy})", actor); + return await WriteJsonAsync(req, HttpStatusCode.Accepted, new + { + message = "Maintenance run queued — progress surfaces as SessionDeletionMaintenance* ops events.", + triggeredBy = actor, + }); + } + + private static async Task WriteJsonAsync(HttpRequestData req, HttpStatusCode status, object body) + { + var response = req.CreateResponse(status); + response.Headers.Add("Content-Type", "application/json; charset=utf-8"); + await response.WriteStringAsync(JsonSerializer.Serialize(body)); + return response; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index fd11485a..4f61c7fd 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -189,6 +189,17 @@ builder.Services.AddHostedService(); +// Manual-trigger path: blob-lease lock store (serializes timer vs manual runs), fail-hard +// queue producer for POST /api/global/session-deletions/maintenance/trigger, and the +// BackgroundService worker that consumes trigger envelopes. The maintenance Function class is +// registered explicitly so the worker can invoke the same RunCoreAsync body the timer uses. +builder.Services.AddSingleton(); +builder.Services.AddSingleton< + AutopilotMonitor.Functions.Services.Deletion.ISessionDeletionMaintenanceTriggerProducer, + AutopilotMonitor.Functions.Services.Deletion.AzureQueueSessionDeletionMaintenanceTriggerProducer>(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService< + AutopilotMonitor.Functions.Services.Deletion.SessionDeletionMaintenanceQueueWorker>(); // Tenant-offboarding cascade worker (Plan Rev 9). Consumes per-tenant envelopes off // `tenant-offboarding`, drives the §3 Phase 2 sequence (cascade enqueue → drain → SafeWipe → diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index 50b60db2..e312aca8 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -328,6 +328,8 @@ public static class EndpointAccessPolicyCatalog // Platform-operational (cascade-delete restore prep) — GA + read-only Reader only, NOT delegated. new("GET", "global/tenants/{tenantId}/deletion-manifests", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.RouteParam, excludeDelegated: true), new("GET", "global/session-deletions", EndpointPolicy.GlobalReadOrAdmin), + // Manual maintenance-run trigger (GC sweeps + retention fanout) — mutating, GA only. + new("POST", "global/session-deletions/maintenance/trigger", EndpointPolicy.GlobalAdminOnly), new("GET", "global/raw/sessions", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), new("GET", "global/raw/events", EndpointPolicy.GlobalReadOrAdmin, TenantScoping.QueryParam), // Raw table/log access stays GlobalAdminOnly: query_table can dump arbitrary tables diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/AzureQueueSessionDeletionMaintenanceTriggerProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/AzureQueueSessionDeletionMaintenanceTriggerProducer.cs new file mode 100644 index 00000000..226b323b --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/AzureQueueSessionDeletionMaintenanceTriggerProducer.cs @@ -0,0 +1,61 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Queueing; +using AutopilotMonitor.Shared; +using Azure.Storage.Queues; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Functions.Services.Deletion +{ + /// + /// Azure-Storage-Queue producer for the manual session-deletion-maintenance trigger. + /// Mirrors AzureQueueCriticalTableBackupProducer: Managed Identity preferred via + /// , fail-hard semantics (SendMessageAsync errors propagate + /// so the HTTP trigger returns 5xx instead of a hollow 202). + /// + public sealed class AzureQueueSessionDeletionMaintenanceTriggerProducer : ISessionDeletionMaintenanceTriggerProducer + { + private readonly QueueClient _queueClient; + private readonly ILogger _logger; + private int _queueEnsured; + + public AzureQueueSessionDeletionMaintenanceTriggerProducer( + QueueClientFactory queueFactory, + ILogger logger) + { + _logger = logger; + _queueClient = queueFactory.Create(Constants.QueueNames.SessionDeletionMaintenance); + } + + public async Task EnqueueAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct = default) + { + if (envelope == null) throw new ArgumentNullException(nameof(envelope)); + if (string.IsNullOrEmpty(envelope.TriggeredBy)) throw new ArgumentException("TriggeredBy required", nameof(envelope)); + + await EnsureQueueExistsAsync(ct).ConfigureAwait(false); + + // Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and returns 500. + var body = JsonConvert.SerializeObject(envelope); + await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false); + _logger.LogInformation("SessionDeletionMaintenance trigger enqueued (triggeredBy={TriggeredBy})", envelope.TriggeredBy); + } + + private async Task EnsureQueueExistsAsync(CancellationToken ct) + { + if (_queueEnsured == 1) return; + try + { + await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false); + Interlocked.Exchange(ref _queueEnsured, 1); + } + catch (Exception ex) + { + // Ensure failure during enqueue is itself a fail-hard signal — surface to the caller. + _logger.LogError(ex, "SessionDeletionMaintenance queue CreateIfNotExists failed"); + throw; + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/ISessionDeletionMaintenanceTriggerProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/ISessionDeletionMaintenanceTriggerProducer.cs new file mode 100644 index 00000000..b5c36fd1 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/ISessionDeletionMaintenanceTriggerProducer.cs @@ -0,0 +1,26 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace AutopilotMonitor.Functions.Services.Deletion +{ + /// + /// Envelope for a single manual session-deletion-maintenance trigger message. Tiny on + /// purpose — the run has no per-job state row; the OpsEvents lifecycle + /// (Started / BudgetExceeded / SkippedLocked / Completed / Failed) is the status surface. + /// + public sealed class SessionDeletionMaintenanceTriggerEnvelope + { + /// Identifier of the requesting Global Admin (UPN) — flows into the Started OpsEvent. + public string TriggeredBy { get; set; } = string.Empty; + } + + /// + /// Producer for the session-deletion-maintenance queue. Fail-hard: + /// SendMessageAsync exceptions propagate so the HTTP trigger can return 5xx — silent + /// enqueue loss would leave the operator with a 202 and no run ever starting. + /// + public interface ISessionDeletionMaintenanceTriggerProducer + { + Task EnqueueAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct = default); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceLockStore.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceLockStore.cs new file mode 100644 index 00000000..e370894f --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceLockStore.cs @@ -0,0 +1,107 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Backup; +using AutopilotMonitor.Shared; +using Azure; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Deletion +{ + /// + /// Blob-lease lock for the session-deletion maintenance run. Serializes the 12h timer + /// against the manual HTTP trigger (and against a second host instance) — without it a + /// manual run racing the timer would double-run the GC sweeps and the retention fanout. + /// + /// Deliberately a DEDICATED sentinel ( in the + /// deletion-manifests container) rather than the backup sentinel in + /// — sharing that lease would serialize backups against the + /// retention fanout, which have no reason to exclude each other. Lease mechanics + /// (60s lease + 45s renewal, + /// on contention) are identical to the backup path. + /// + /// + /// The deletion-manifests container has a 30-day lifecycle policy that may delete an + /// idle sentinel; re-ensures it on every acquire, so the + /// lock self-heals. The manifest-TTL sweep skips the sentinel because _lock/… does + /// not parse as a manifest blob name. + /// + /// + public class SessionDeletionMaintenanceLockStore + { + /// Sentinel blob path — leased by the timer / manual-trigger worker to serialize runs. + public const string LockBlobName = "_lock/session-deletion-maintenance.lock"; + + /// Lease duration (Azure spec allows 15..60s for renewable leases). + public static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(60); + + private readonly BlobStorageService _blobs; + private readonly ILogger _logger; + private int _containerEnsured; + + public SessionDeletionMaintenanceLockStore( + BlobStorageService blobs, + ILogger logger) + { + _blobs = blobs; + _logger = logger; + } + + /// + /// Attempts to acquire the session-deletion maintenance lease. Throws + /// on 409 (another holder); other errors propagate. + /// Caller is responsible for renewing (via ) and + /// releasing. virtual so tests can stub it. + /// + public virtual async Task AcquireLeaseAsync(TimeSpan? leaseDuration = null, CancellationToken ct = default) + { + await EnsureSentinelAsync(ct).ConfigureAwait(false); + + var blob = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests).GetBlobClient(LockBlobName); + var leaseClient = blob.GetBlobLeaseClient(); + try + { + await leaseClient.AcquireAsync(leaseDuration ?? LeaseDuration, conditions: null, ct).ConfigureAwait(false); + return leaseClient; + } + catch (RequestFailedException ex) when (ex.Status == 409) + { + throw new LeaseHeldException("session-deletion maintenance lease is held by another run", ex); + } + } + + /// + /// Idempotently ensures the sentinel blob exists — an AcquireLease on a missing blob + /// throws 404. If-None-Match=* lets concurrent callers cooperate; both 409 + /// BlobAlreadyExists and 412 ConditionNotMet are the steady state. + /// + private async Task EnsureSentinelAsync(CancellationToken ct) + { + var container = _blobs.GetContainerClient(Constants.BlobContainers.DeletionManifests); + if (Interlocked.CompareExchange(ref _containerEnsured, 0, 0) == 0) + { + await container.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false); + Interlocked.Exchange(ref _containerEnsured, 1); + } + + var blob = container.GetBlobClient(LockBlobName); + var options = new BlobUploadOptions + { + HttpHeaders = new BlobHttpHeaders { ContentType = "application/octet-stream" }, + Conditions = new BlobRequestConditions { IfNoneMatch = ETag.All }, + }; + try + { + using var empty = new MemoryStream(Array.Empty()); + await blob.UploadAsync(empty, options, ct).ConfigureAwait(false); + } + catch (RequestFailedException ex) when (ex.Status == 409 || ex.Status == 412) + { + // Sentinel exists already — steady state. + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceQueueWorker.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceQueueWorker.cs new file mode 100644 index 00000000..a390f6a4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionMaintenanceQueueWorker.cs @@ -0,0 +1,53 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Functions.Maintenance; +using AutopilotMonitor.Functions.Services.Queueing; +using AutopilotMonitor.Shared; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Deletion +{ + /// + /// Background worker for the session-deletion-maintenance queue — the manual-trigger + /// counterpart to the 12h timer. Each message runs the full + /// body with the requesting + /// admin as triggeredBy. + /// + /// Concurrency against the timer (and a second host instance) is handled INSIDE + /// RunCoreAsync via the session-deletion maintenance blob lease: a lease-held run + /// returns normally (SkippedLocked OpsEvent), so the message is deleted rather than + /// retried — the operator asked for "a run", and one is already active. + /// + /// + /// BatchSize=1 + VisibilityTimeout=60min (run budget is 50min plus cushion — same rationale + /// as CriticalTableBackupQueueWorker). Real exceptions rethrow → visibility-timeout + /// retry → poison after . + /// + /// + public sealed class SessionDeletionMaintenanceQueueWorker : QueuePollingWorker + { + private readonly SessionDeletionMaintenanceFunction _maintenance; + + public SessionDeletionMaintenanceQueueWorker( + QueueClientFactory queueFactory, + SessionDeletionMaintenanceFunction maintenance, + ILogger logger) + : base(queueFactory, Constants.QueueNames.SessionDeletionMaintenance, logger, Constants.QueueNames.SessionDeletionMaintenancePoison) + { + _maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance)); + } + + protected override int BatchSize => 1; + protected override TimeSpan VisibilityTimeout => TimeSpan.FromMinutes(60); + + protected override bool TryValidate(SessionDeletionMaintenanceTriggerEnvelope envelope) + => !string.IsNullOrEmpty(envelope.TriggeredBy); + + protected override string DescribeForLog(SessionDeletionMaintenanceTriggerEnvelope envelope) + => $"triggeredBy={envelope.TriggeredBy}"; + + protected override Task HandleAsync(SessionDeletionMaintenanceTriggerEnvelope envelope, CancellationToken ct) + => _maintenance.RunCoreAsync(envelope.TriggeredBy, ct); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs index f14923d9..5b66337b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs @@ -34,6 +34,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion private readonly AdminConfigurationService _adminConfig; private readonly ILogger _logger; private readonly Func _throttle; + private readonly Func _utcNow; public SessionRetentionFanoutService( IMaintenanceRepository maintenanceRepo, @@ -41,13 +42,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion ISessionDeletionEnqueuer enqueuer, AdminConfigurationService adminConfig, ILogger logger) - : this(maintenanceRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay) + : this(maintenanceRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay, utcNow: () => DateTime.UtcNow) { } /// /// Test seam — tests inject a no-op throttle to make the rate-limit loop exercise its - /// counter without waiting 50ms × N in real time. + /// counter without waiting 50ms × N in real time, and a scripted clock so the run-budget + /// deadline can be crossed deterministically. /// internal SessionRetentionFanoutService( IMaintenanceRepository maintenanceRepo, @@ -55,7 +57,8 @@ namespace AutopilotMonitor.Functions.Services.Deletion ISessionDeletionEnqueuer enqueuer, AdminConfigurationService adminConfig, ILogger logger, - Func throttle) + Func throttle, + Func? utcNow = null) { _maintenanceRepo = maintenanceRepo; _tenantConfig = tenantConfig; @@ -63,11 +66,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion _adminConfig = adminConfig; _logger = logger; _throttle = throttle; + _utcNow = utcNow ?? (() => DateTime.UtcNow); } /// /// Aggregate result of a single fanout invocation; used for the per-tenant audit + the - /// completion summary OpsEvent. + /// completion summary OpsEvent. The caller allocates it and mutates + /// it incrementally (per tenant / per session), so watchdog snapshots taken while the + /// fanout is still running read real progress instead of zeros. /// public sealed class FanoutResult { @@ -76,6 +82,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion public int SessionsSkipped { get; set; } // already locked / poisoned / kill-switch / etc. public int RateLimitedTenants { get; set; } // tenants that hit MaxEnqueuesPerTenantPerRun public bool AbortedByKillSwitch { get; set; } // kill-switch flipped mid-run + public bool AbortedByBudget { get; set; } // run-budget deadline crossed mid-run } /// @@ -83,16 +90,32 @@ namespace AutopilotMonitor.Functions.Services.Deletion /// Each tenant is processed independently; an exception on tenant A is logged and the loop /// continues with tenant B (matches the existing maintenance behaviour: per-tenant /// failures must not cascade). + /// + /// is the run-budget cutoff: once crossed, the fanout stops + /// cleanly at the next tenant/session boundary and sets + /// — the remaining backlog is picked up by the next run. This keeps the maintenance + /// function comfortably inside the Flex Consumption 60min functionTimeout instead of + /// being hard-aborted by the host mid-tenant. + /// /// - public virtual async Task RunAsync(CancellationToken cancellationToken) + public virtual async Task RunAsync(FanoutResult result, DateTime deadlineUtc, CancellationToken cancellationToken) { - var result = new FanoutResult(); var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync().ConfigureAwait(false); foreach (var tenantId in tenantIds) { cancellationToken.ThrowIfCancellationRequested(); + // Run-budget check at the tenant boundary — cheapest clean stopping point. + if (_utcNow() >= deadlineUtc) + { + _logger.LogWarning( + "SessionRetentionFanout: run budget exhausted — halting before tenant {TenantId} (deadline {Deadline:o})", + tenantId, deadlineUtc); + result.AbortedByBudget = true; + break; + } + // Per-tenant kill-switch check so a flip-ON mid-run halts the remaining tenants. // The maintenance function gates entry; this gates iteration. if (await _adminConfig.IsSessionDeletionKillSwitchActiveAsync().ConfigureAwait(false)) @@ -106,7 +129,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion try { - await RunForTenantAsync(tenantId, result, cancellationToken).ConfigureAwait(false); + await RunForTenantAsync(tenantId, deadlineUtc, result, cancellationToken).ConfigureAwait(false); result.TenantsProcessed++; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -118,22 +141,20 @@ namespace AutopilotMonitor.Functions.Services.Deletion _logger.LogError(ex, "SessionRetentionFanout failed for tenant {TenantId} — continuing with next tenant", tenantId); } - // The inner loop sets this when the producer's Step 0 reports KillSwitchActive - // mid-tenant. That's the authoritative fail-closed outcome — stop touching the - // remaining tenants instead of relying on the next pre-check to catch the flip. - if (result.AbortedByKillSwitch) + // The inner loop sets these when the producer's Step 0 reports KillSwitchActive + // mid-tenant, or when the budget deadline is crossed mid-tenant. Both are + // authoritative stop signals — don't touch the remaining tenants. + if (result.AbortedByKillSwitch || result.AbortedByBudget) { _logger.LogWarning( - "SessionRetentionFanout: aborting outer loop after tenant {TenantId} reported KillSwitchActive from producer", - tenantId); + "SessionRetentionFanout: aborting outer loop after tenant {TenantId} (killSwitch={KillSwitch}, budget={Budget})", + tenantId, result.AbortedByKillSwitch, result.AbortedByBudget); break; } } - - return result; } - private async Task RunForTenantAsync(string tenantId, FanoutResult result, CancellationToken cancellationToken) + private async Task RunForTenantAsync(string tenantId, DateTime deadlineUtc, FanoutResult result, CancellationToken cancellationToken) { var config = await _tenantConfig.GetConfigurationAsync(tenantId).ConfigureAwait(false); // Edition retention cap (fail-closed backstop): the stored value is clamped to the @@ -200,6 +221,18 @@ namespace AutopilotMonitor.Functions.Services.Deletion break; } + // Run-budget check per session — a single tenant's batch can take tens of + // minutes (manifest build ~7-25s per session), so the tenant-boundary check + // alone would not stop a run drifting into the host's hard functionTimeout. + if (_utcNow() >= deadlineUtc) + { + _logger.LogWarning( + "SessionRetentionFanout: run budget exhausted mid-tenant — halting at session {SessionId} of {TenantId}", + session.SessionId, tenantId); + result.AbortedByBudget = true; + break; + } + // Per-session kill-switch check so an emergency flip-ON halts immediately instead // of after the rest of this tenant's backlog. Uncached read — uniform behaviour // across scaled-out instances within seconds. @@ -224,12 +257,15 @@ namespace AutopilotMonitor.Functions.Services.Deletion "SessionRetentionFanout: producer reported KillSwitchActive — aborting at session {SessionId} of {TenantId}", session.SessionId, tenantId); skipped++; + result.SessionsSkipped++; result.AbortedByKillSwitch = true; break; } - if (outcome == SessionDeletionEnqueueOutcome.Enqueued) enqueued++; - else skipped++; + // Live counters on the shared result (in addition to the per-tenant locals used + // for the audit entry) so the maintenance watchdog reports real progress. + if (outcome == SessionDeletionEnqueueOutcome.Enqueued) { enqueued++; result.SessionsEnqueued++; } + else { skipped++; result.SessionsSkipped++; } processed++; @@ -252,9 +288,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion tenantId, processed, skipped); } - result.SessionsEnqueued += enqueued; - result.SessionsSkipped += skipped; - + // result.SessionsEnqueued / SessionsSkipped are already bumped live inside the loop. await _maintenanceRepo.LogAuditEntryAsync( tenantId, "SessionDeletionMaintenanceFanout", @@ -272,6 +306,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion { "Enqueued", enqueued.ToString() }, { "Skipped", skipped.ToString() }, { "AbortedByKillSwitch", result.AbortedByKillSwitch.ToString() }, + { "AbortedByBudget", result.AbortedByBudget.ToString() }, }).ConfigureAwait(false); _logger.LogInformation( diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 54fae30f..e16e7aa4 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -112,9 +112,40 @@ namespace AutopilotMonitor.Functions.Services } // ── Cascade-Delete Maintenance (Plan §5 PR6 / §16 R14) ───────────────── - // Four event types dispatched by SessionDeletionMaintenanceFunction. Each is also + // Event types dispatched by SessionDeletionMaintenanceFunction: Started, LongRunning, + // LongRunningSevere, BudgetExceeded, SkippedLocked, Failed, Completed, FanoutSkipped + // (+ StrandedQueued / Poisoned from the GCs and the cascade worker). Each is also // listed in OpsAlertRulesSection.tsx OPS_EVENT_TYPES (memory feedback_ops_event_types_dual_register). + /// + /// Run started (timer tick or manual trigger). Emitted after the maintenance lease was + /// acquired, so a lease-skip never masquerades as an active run — the Session Cleanup + /// UI banner treats "latest Started newer than latest Completed/Failed" as run-active. + /// + public Task RecordSessionDeletionMaintenanceStartedAsync(string triggeredBy) + => WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceStarted", OpsEventSeverity.Info, + $"SessionDeletionMaintenance run started (triggered by {triggeredBy})", + null, "System.Maintenance", new { triggeredBy }); + + /// + /// The retention fanout stopped cleanly at the run-budget deadline. Not an error: the + /// remaining backlog is picked up by the next run (12h cadence) or a manual trigger. + /// Paired with a Completed event whose details carry abortedByBudget=true. + /// + public Task RecordSessionDeletionMaintenanceBudgetExceededAsync(int budgetMinutes, int tenantsProcessed, int sessionsEnqueued) + => WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceBudgetExceeded", OpsEventSeverity.Warning, + $"SessionDeletionMaintenance stopped at the {budgetMinutes}min run budget — tenants={tenantsProcessed} enqueued={sessionsEnqueued}; remaining backlog resumes on the next run", + null, "System.Maintenance", new { budgetMinutes, tenantsProcessed, sessionsEnqueued }); + + /// + /// A run (timer or manual) was skipped because another run holds the session-deletion + /// maintenance lease. Mirrors RecordCriticalTableBackupSkippedLockedAsync. + /// + public Task RecordSessionDeletionMaintenanceSkippedLockedAsync(string triggeredBy) + => WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceSkippedLocked", OpsEventSeverity.Info, + $"SessionDeletionMaintenance skipped — another run holds the maintenance lease (triggeredBy={triggeredBy})", + null, "System.Maintenance", new { reason = "lease held by another run", triggeredBy }); + /// Watchdog: maintenance run still in flight 30 minutes after start. Warning-level early signal. public Task RecordSessionDeletionMaintenanceLongRunningAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued) => WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceLongRunning", OpsEventSeverity.Warning, @@ -196,13 +227,13 @@ namespace AutopilotMonitor.Functions.Services public Task RecordSessionDeletionMaintenanceCompletedAsync( bool killSwitchActive, int tenantsProcessed, int sessionsEnqueued, int sessionsSkipped, int rateLimitedTenants, int blobsTtlGced, int preparingRowsCleared, - int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch) + int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch, bool abortedByBudget) => WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceCompleted", OpsEventSeverity.Info, - $"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive}", + $"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive} abortedByBudget={abortedByBudget}", null, "System.Maintenance", new { killSwitchActive, tenantsProcessed, sessionsEnqueued, sessionsSkipped, rateLimitedTenants, blobsTtlGced, preparingRowsCleared, - strandedQueuedDetected, durationMs, abortedByKillSwitch, + strandedQueuedDetected, durationMs, abortedByKillSwitch, abortedByBudget, }); /// diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 06fe95a1..5f5b1dbe 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -1154,6 +1154,20 @@ namespace AutopilotMonitor.Shared /// Poison sibling of . public const string CriticalTableBackupPoison = "critical-table-backup-jobs-poison"; + + /// + /// Manual-trigger queue for the session-deletion maintenance run. Producer = HTTP + /// trigger /api/global/session-deletions/maintenance/trigger (fail-hard); + /// consumer = SessionDeletionMaintenanceQueueWorker (BackgroundService, + /// BatchSize=1, VisibilityTimeout=60min — the run budget is 50min plus cushion). + /// Carries a tiny envelope { triggeredBy }; concurrency against the 12h timer + /// is serialized by the session-deletion maintenance blob lease, not the queue. + /// Poison-suffix -poison, max-dequeue 5. + /// + public const string SessionDeletionMaintenance = "session-deletion-maintenance"; + + /// Poison sibling of . + public const string SessionDeletionMaintenancePoison = "session-deletion-maintenance-poison"; } /// diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index 921415f7..0a0f4863 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -27,6 +27,13 @@ const OPS_EVENT_TYPES: Record = { "SessionDeletionMaintenanceLongRunningSevere", "SessionDeletionMaintenanceFailed", "SessionDeletionMaintenanceFanoutSkipped", + // Run lifecycle + 50min run budget + lease serialization (manual-trigger feature): + // Started marks run begin (timer or manual, details.triggeredBy), BudgetExceeded is the + // clean self-stop at the 50min budget (Warning), SkippedLocked means another run held the + // maintenance lease. Backend helpers RecordSessionDeletionMaintenance{Started,BudgetExceeded,SkippedLocked}Async. + "SessionDeletionMaintenanceStarted", + "SessionDeletionMaintenanceBudgetExceeded", + "SessionDeletionMaintenanceSkippedLocked", "SessionDeletionStrandedQueued", // PR-B audit consolidation: the per-session deletion_poisoned tenant-audit moved here so // tenant admins see only the lifecycle endpoints (deletion_started/completed/restored). diff --git a/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/MaintenanceStatusBanner.tsx b/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/MaintenanceStatusBanner.tsx new file mode 100644 index 00000000..aec46d9b --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/MaintenanceStatusBanner.tsx @@ -0,0 +1,223 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { api } from "@/lib/api"; +import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch"; + +interface OpsEvent { + id: string; + eventType: string; + severity: string; + message: string; + details: string | null; + timestamp: string; +} + +interface OpsEventsResponse { + events: OpsEvent[]; +} + +interface CompletedDetails { + tenantsProcessed?: number; + sessionsEnqueued?: number; + durationMs?: number; + abortedByBudget?: boolean; + abortedByKillSwitch?: boolean; +} + +type BannerState = + | { kind: "loading" } + | { kind: "active"; since: string; triggeredBy: string } + | { kind: "budget-exceeded"; at: string; details: CompletedDetails } + | { kind: "failed"; at: string; message: string } + | { kind: "idle"; at: string; details: CompletedDetails } + | { kind: "none" }; // no lifecycle events in the window — render nothing + +function parseDetails(raw: string | null): T { + if (!raw) return {} as T; + try { + return JSON.parse(raw) as T; + } catch { + return {} as T; + } +} + +/** + * Status banner for the session-deletion maintenance run (12h retention fanout + GC sweeps). + * Derives its state purely from the SessionDeletionMaintenance* ops events: the latest + * Started newer than the latest terminal event (Completed/Failed) means a run is active; + * otherwise the latest terminal event decides between budget-exceeded / failed / idle. + * The "Run now" button POSTs the manual trigger (202 = queued, 409 = already active). + */ +export function MaintenanceStatusBanner({ + getAccessToken, + setError, + setSuccessMessage, +}: { + getAccessToken: (forceRefresh?: boolean) => Promise; + setError: (error: string | null) => void; + setSuccessMessage: (message: string | null) => void; +}) { + const [state, setState] = useState({ kind: "loading" }); + const [triggering, setTriggering] = useState(false); + const [refreshKey, setRefreshKey] = useState(0); + + const fetchStatus = useCallback(async () => { + try { + const r = await authenticatedFetch( + api.opsEvents.list("Maintenance", { pageSize: 100 }), + getAccessToken, + ); + if (!r.ok) throw new Error(`HTTP ${r.status} loading maintenance ops events`); + const data = (await r.json()) as OpsEventsResponse; + + const lifecycle = (data.events ?? []).filter((e) => + e.eventType.startsWith("SessionDeletionMaintenance"), + ); + // Backend returns newest-first; keep it defensive anyway. + lifecycle.sort((a, b) => (a.timestamp < b.timestamp ? 1 : -1)); + + const started = lifecycle.find((e) => e.eventType === "SessionDeletionMaintenanceStarted"); + const terminal = lifecycle.find( + (e) => + e.eventType === "SessionDeletionMaintenanceCompleted" || + e.eventType === "SessionDeletionMaintenanceFailed", + ); + + if (started && (!terminal || started.timestamp > terminal.timestamp)) { + const details = parseDetails<{ triggeredBy?: string }>(started.details); + setState({ + kind: "active", + since: started.timestamp, + triggeredBy: details.triggeredBy ?? "unknown", + }); + return; + } + + if (!terminal) { + setState({ kind: "none" }); + return; + } + + if (terminal.eventType === "SessionDeletionMaintenanceFailed") { + setState({ kind: "failed", at: terminal.timestamp, message: terminal.message }); + return; + } + + const details = parseDetails(terminal.details); + setState( + details.abortedByBudget + ? { kind: "budget-exceeded", at: terminal.timestamp, details } + : { kind: "idle", at: terminal.timestamp, details }, + ); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError("Session expired; reload the page and try again."); + } + // Non-auth failures degrade silently — the banner is auxiliary to the tabs. + setState({ kind: "none" }); + } + }, [getAccessToken, setError]); + + useEffect(() => { + void fetchStatus(); + }, [fetchStatus, refreshKey]); + + const triggerRun = useCallback(async () => { + setTriggering(true); + try { + const r = await authenticatedFetch(api.sessionDeletions.triggerMaintenance(), getAccessToken, { + method: "POST", + }); + if (r.status === 202) { + setSuccessMessage("Maintenance run queued — it will appear here as active shortly."); + } else if (r.status === 409) { + setSuccessMessage("A maintenance run is already active."); + } else { + const body = await r.text(); + throw new Error(`HTTP ${r.status}: ${body}`); + } + setRefreshKey((k) => k + 1); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError("Session expired; reload the page and try again."); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setTriggering(false); + } + }, [getAccessToken, setError, setSuccessMessage]); + + if (state.kind === "loading" || state.kind === "none") return null; + + const runNowButton = ( + + ); + + if (state.kind === "active") { + return ( +
+

+ Maintenance run active — started{" "} + {new Date(state.since).toLocaleString()} (triggered by {state.triggeredBy}). Retention + deletions enqueued by this run appear under In-Flight. +

+ +
+ ); + } + + if (state.kind === "budget-exceeded") { + return ( +
+

+ Last run stopped at the 50-minute budget ( + {new Date(state.at).toLocaleString()}) after processing {state.details.tenantsProcessed ?? 0}{" "} + tenants ({state.details.sessionsEnqueued ?? 0} sessions enqueued). The remaining backlog + resumes with the next scheduled run (every 12 h at 00:00 / 12:00 UTC) — or trigger a run + now. +

+ {runNowButton} +
+ ); + } + + if (state.kind === "failed") { + return ( +
+

+ Last maintenance run failed ( + {new Date(state.at).toLocaleString()}): {state.message} +

+ {runNowButton} +
+ ); + } + + // idle — subtle last-run summary + return ( +
+

+ Last maintenance run completed {new Date(state.at).toLocaleString()} — tenants= + {state.details.tenantsProcessed ?? 0}, enqueued={state.details.sessionsEnqueued ?? 0} + {typeof state.details.durationMs === "number" + ? `, ${Math.round(state.details.durationMs / 60000)} min` + : ""} + . Next scheduled run: every 12 h at 00:00 / 12:00 UTC. +

+ {runNowButton} +
+ ); +} diff --git a/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/page.tsx b/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/page.tsx index bb54348d..e99c6b72 100644 --- a/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/page.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/page.tsx @@ -7,6 +7,7 @@ import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch" import { useAdminConfig } from "../../AdminConfigContext"; import { AdminNotifications } from "../../AdminNotifications"; import { DeletionPreviewModal } from "./components/DeletionPreviewModal"; +import { MaintenanceStatusBanner } from "./components/MaintenanceStatusBanner"; import { RestoreBrowserTab } from "./components/RestoreBrowserTab"; import { RestoreConfirmDialog } from "./components/RestoreConfirmDialog"; @@ -53,6 +54,12 @@ export default function SessionCleanupPage() {
+ + {/* Tab bar */}