diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs index 07fd56eb..d19e8d12 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionMaintenanceFunctionTests.cs @@ -50,7 +50,9 @@ public class SessionDeletionMaintenanceFunctionTests harness.BlobMock.Verify(b => b.DeleteDeletionManifestPairAsync("t", "s", "m", It.IsAny()), Times.Once); harness.Fanout.Verify(f => f.RunAsync(It.IsAny()), Times.Once); - Assert.Contains(harness.AuditCalls, a => a.Action == "session_deletion_maintenance_completed"); + // PR6 follow-up F3: lifecycle status now lives on OpsEvents, not AuditLogs (the latter + // would silently fail because PartitionKey requires a non-null tenantId). + Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted"); } [Fact] @@ -66,10 +68,9 @@ public class SessionDeletionMaintenanceFunctionTests harness.BlobMock.Verify(b => b.DeleteDeletionManifestPairAsync("t", "s", "m", It.IsAny()), Times.Once); // Fanout was skipped. harness.Fanout.Verify(f => f.RunAsync(It.IsAny()), Times.Never); - // Skip audit emitted with kill-switch reason. - Assert.Contains(harness.AuditCalls, a => a.Action == "session_deletion_maintenance_fanout_skipped_killswitch"); - // Completion audit still emitted so dashboards fold the run into the timeline. - Assert.Contains(harness.AuditCalls, a => a.Action == "session_deletion_maintenance_completed"); + // PR6 follow-up F3: fanout-skip + completion now both live on OpsEvents. + Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceFanoutSkipped"); + Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceCompleted"); } // ────────────────────────────────────────────────────────────────────────── Watchdog OpsEvents ─ @@ -139,8 +140,9 @@ public class SessionDeletionMaintenanceFunctionTests var ex = await Assert.ThrowsAsync(() => harness.Sut.RunCoreAsync(CancellationToken.None)); Assert.Equal("fanout exploded", ex.Message); + // PR6 follow-up F3: the SessionDeletionMaintenanceFailed OpsEvent IS the audit. The prior + // parallel LogAuditEntryAsync(null!) call was a silent no-op and has been removed. Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceFailed"); - Assert.Contains(harness.AuditCalls, a => a.Action == "session_deletion_maintenance_failed"); } // ────────────────────────────────────────────────────────────────────────── Stranded-Queued detection ─ @@ -284,6 +286,7 @@ public class SessionDeletionMaintenanceFunctionTests Mock.Of(), new Mock(Mock.Of(), NullLogger.Instance, memCache).Object, Mock.Of(), + AdminConfig.Object, NullLogger.Instance); Fanout.Setup(f => f.RunAsync(It.IsAny())) .ReturnsAsync(new SessionRetentionFanoutService.FanoutResult { TenantsProcessed = 0, SessionsEnqueued = 0 }); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionRetentionFanoutServiceTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionRetentionFanoutServiceTests.cs index 08fd7965..38e65f90 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionRetentionFanoutServiceTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionRetentionFanoutServiceTests.cs @@ -138,6 +138,115 @@ public class SessionRetentionFanoutServiceTests harness.MaintenanceRepo.Verify(m => m.GetSessionsOlderThanAsync(TenantA, It.IsAny()), Times.Never); } + // ────────────────────────────────────────────────────────────────────────── PR6 follow-up F1 ─ + + [Fact] + public async Task RunAsync_legacy_path_skips_locked_session_WITHOUT_touching_side_tables() + { + // PR6 follow-up F1: pre-read DeletionState before any side-table mutation. Without this + // gate the legacy path would delete Events / RuleResults / AppSummaries and only discover + // the V2 cascade lock at the tombstone CAS — leaving the V2 cascade's manifest claiming + // rows that were silently removed in the meantime. + var harness = new Harness(); + harness.WithTenant(TenantA, retentionDays: 30, v2: false, sessions: new[] { Old("s1", 60) }); + // Override the default fresh-read: this session is locked by a V2 cascade. + harness.SessionRepo.Setup(r => r.GetSessionAsync(TenantA, "s1")) + .ReturnsAsync(new SessionSummary + { + TenantId = TenantA, + SessionId = "s1", + DeletionState = SessionDeletionState.Queued, + PendingDeletionManifestId = "EXISTING-MANIFEST", + }); + + var result = await harness.Sut.RunAsync(CancellationToken.None); + + Assert.Equal(1, result.SessionsSkipped); + Assert.Equal(0, result.SessionsLegacyDeleted); + // Critical assertion: NONE of the side-table delete helpers were invoked. + harness.MaintenanceRepo.Verify(m => m.DeleteSessionEventsAsync(It.IsAny(), It.IsAny()), Times.Never); + harness.MaintenanceRepo.Verify(m => m.DeleteSessionRuleResultsAsync(It.IsAny(), It.IsAny()), Times.Never); + harness.MaintenanceRepo.Verify(m => m.DeleteSessionAppInstallSummariesAsync(It.IsAny(), It.IsAny()), Times.Never); + harness.SessionRepo.Verify(r => r.DeleteSessionAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsync_legacy_path_skips_when_session_already_removed_during_preread() + { + // Edge case: row disappeared between GetSessionsOlderThan and the pre-read. Treat as skip + // (no side-table work needed); no error. + var harness = new Harness(); + harness.WithTenant(TenantA, retentionDays: 30, v2: false, sessions: new[] { Old("s1", 60) }); + harness.SessionRepo.Setup(r => r.GetSessionAsync(TenantA, "s1")).ReturnsAsync((SessionSummary?)null); + + var result = await harness.Sut.RunAsync(CancellationToken.None); + + Assert.Equal(1, result.SessionsSkipped); + harness.MaintenanceRepo.Verify(m => m.DeleteSessionEventsAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + // ────────────────────────────────────────────────────────────────────────── PR6 follow-up F2 ─ + + [Fact] + public async Task RunAsync_aborts_mid_loop_when_kill_switch_flips_on() + { + // PR6 follow-up F2: per-session kill-switch check halts the fanout immediately when the + // emergency switch flips, instead of finishing the rest of this tenant's backlog. + var harness = new Harness(); + var manySessions = new List(); + for (int i = 0; i < 5; i++) manySessions.Add(Summary(TenantA, $"s{i}")); + harness.WithTenantOverride(TenantA, retentionDays: 30, v2: true, sessions: manySessions); + + // First 2 calls return false (kill-switch off), then true → fanout aborts at the 3rd iteration. + var calls = 0; + harness.AdminConfig.Setup(a => a.IsSessionDeletionKillSwitchActiveAsync()) + .Returns(() => + { + calls++; + // Sequence: tenant-entry probe (call 1, false), then per-session probes: + // session 0 (call 2, false), session 1 (call 3, false), session 2 (call 4, TRUE — abort). + return Task.FromResult(calls >= 4); + }); + + var result = await harness.Sut.RunAsync(CancellationToken.None); + + Assert.True(result.AbortedByKillSwitch); + Assert.Equal(2, result.SessionsEnqueued); // only sessions 0 and 1 made it through + // Confirm only the first two enqueues fired. + harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantA, "s0", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantA, "s1", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantA, "s2", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantA, "s4", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsync_aborts_at_tenant_boundary_when_kill_switch_flips_between_tenants() + { + // Kill-switch flips on between tenant A and tenant B → tenant B is never started. + var harness = new Harness(); + harness.WithTenant(TenantA, retentionDays: 30, v2: true, sessions: new[] { Old("a1", 60) }); + harness.WithTenant(TenantB, retentionDays: 30, v2: true, sessions: new[] { Old("b1", 60) }); + + var calls = 0; + harness.AdminConfig.Setup(a => a.IsSessionDeletionKillSwitchActiveAsync()) + .Returns(() => + { + calls++; + // Tenant A: entry probe (call 1, false), session probe (call 2, false), enqueue runs. + // Tenant B: entry probe (call 3, TRUE — abort). + return Task.FromResult(calls >= 3); + }); + + var result = await harness.Sut.RunAsync(CancellationToken.None); + + Assert.True(result.AbortedByKillSwitch); + Assert.Equal(1, result.TenantsProcessed); + harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantA, "a1", It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); + harness.Enqueuer.Verify(e => e.EnqueueAsync(TenantB, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + // ─────────────────────────────────────────────────────────────────────────────────────────────── + [Fact] public async Task RunAsync_counts_AlreadyInFlight_outcome_as_skip() { @@ -180,6 +289,7 @@ public class SessionRetentionFanoutServiceTests public Mock SessionRepo { get; } public Mock TenantConfig { get; } public Mock Enqueuer { get; } + public Mock AdminConfig { get; } public SessionRetentionFanoutService Sut { get; } private readonly List _tenantIds = new(); @@ -193,6 +303,17 @@ public class SessionRetentionFanoutServiceTests Mock.Of(), NullLogger.Instance, memCache); Enqueuer = new Mock(); + // PR6 follow-up F1: legacy path now pre-reads the Sessions row via ISessionRepository + // to check DeletionState. Default: row exists with DeletionState=None so legacy proceeds. + SessionRepo.Setup(r => r.GetSessionAsync(It.IsAny(), It.IsAny())) + .Returns((tenant, sid) => Task.FromResult( + new SessionSummary { TenantId = tenant, SessionId = sid, DeletionState = SessionDeletionState.None })); + + // PR6 follow-up F2: per-session kill-switch check inside the fanout loop. + AdminConfig = new Mock( + Mock.Of(), NullLogger.Instance, memCache); + AdminConfig.Setup(a => a.IsSessionDeletionKillSwitchActiveAsync()).ReturnsAsync(false); + // Default: every enqueue succeeds. Enqueuer.Setup(e => e.EnqueueAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) @@ -213,6 +334,7 @@ public class SessionRetentionFanoutServiceTests // (50ms × 100 = 5s otherwise). Sut = new SessionRetentionFanoutService( MaintenanceRepo.Object, SessionRepo.Object, TenantConfig.Object, Enqueuer.Object, + AdminConfig.Object, NullLogger.Instance, throttle: (_, _) => Task.CompletedTask); } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs index 425b7981..2963dd3a 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs @@ -146,24 +146,16 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance // (3) Stranded-Queued detection (alert-only) — runs even when kill-switch=true. var strandedQueuedDetected = await DetectStrandedQueuedAsync(cancellationToken); - // (4) Retention fanout — skipped when kill-switch=true. Audit the skip so the - // operator sees in MCP/admin logs that the fanout was deliberately bypassed. + // (4) Retention fanout — skipped when kill-switch=true. Emit a Fanout-Skipped + // OpsEvent so the operator can fold the skip into the OpsEvents dashboard alongside + // the 30/60min watchdogs. PR6 follow-up F3: previously a LogAuditEntryAsync(null!) + // call that silently failed because the AuditLogs schema requires a non-null + // PartitionKey — OpsEvents tolerates null TenantId, so global-scope events live there. SessionRetentionFanoutService.FanoutResult? fanoutResult = null; if (killSwitchActive) { - await _maintenanceRepo.LogAuditEntryAsync( - tenantId: null!, - action: "session_deletion_maintenance_fanout_skipped_killswitch", - entityType: "Maintenance", - entityId: "fanout", - performedBy: "System.Maintenance", - details: new Dictionary - { - { "Reason", "SessionDeletionKillSwitch is active" }, - { "BlobsTtlGced", blobsTtlGced.ToString() }, - { "PreparingRowsCleared", preparingRowsCleared.ToString() }, - { "StrandedQueuedDetected", strandedQueuedDetected.ToString() }, - }); + await _opsEvents.RecordSessionDeletionMaintenanceFanoutSkippedAsync( + blobsTtlGced, preparingRowsCleared, strandedQueuedDetected); } else { @@ -172,26 +164,20 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance sessionsEnqueuedSoFar = fanoutResult.SessionsEnqueued + fanoutResult.SessionsLegacyDeleted; } - // (5) Completion audit — same shape regardless of kill-switch state, so dashboards can fold both paths together. - await _maintenanceRepo.LogAuditEntryAsync( - tenantId: null!, - action: "session_deletion_maintenance_completed", - entityType: "Maintenance", - entityId: $"run-{DateTime.UtcNow:yyyyMMddHHmmss}", - performedBy: "System.Maintenance", - details: new Dictionary - { - { "KillSwitchActive", killSwitchActive.ToString() }, - { "TenantsProcessed", (fanoutResult?.TenantsProcessed ?? 0).ToString() }, - { "SessionsEnqueued", (fanoutResult?.SessionsEnqueued ?? 0).ToString() }, - { "SessionsLegacyDeleted", (fanoutResult?.SessionsLegacyDeleted ?? 0).ToString() }, - { "SessionsSkipped", (fanoutResult?.SessionsSkipped ?? 0).ToString() }, - { "RateLimitedTenants", (fanoutResult?.RateLimitedTenants ?? 0).ToString() }, - { "BlobsTtlGced", blobsTtlGced.ToString() }, - { "PreparingRowsCleared", preparingRowsCleared.ToString() }, - { "StrandedQueuedDetected", strandedQueuedDetected.ToString() }, - { "DurationMs", sw.ElapsedMilliseconds.ToString() }, - }); + // (5) Completion OpsEvent — same shape regardless of kill-switch state, so dashboards + // can fold both paths together. PR6 follow-up F3. + await _opsEvents.RecordSessionDeletionMaintenanceCompletedAsync( + killSwitchActive: killSwitchActive, + tenantsProcessed: fanoutResult?.TenantsProcessed ?? 0, + sessionsEnqueued: fanoutResult?.SessionsEnqueued ?? 0, + sessionsLegacyDeleted: fanoutResult?.SessionsLegacyDeleted ?? 0, + sessionsSkipped: fanoutResult?.SessionsSkipped ?? 0, + rateLimitedTenants: fanoutResult?.RateLimitedTenants ?? 0, + blobsTtlGced: blobsTtlGced, + preparingRowsCleared: preparingRowsCleared, + strandedQueuedDetected: strandedQueuedDetected, + durationMs: (int)sw.ElapsedMilliseconds, + abortedByKillSwitch: fanoutResult?.AbortedByKillSwitch ?? false); _logger.LogInformation( "SessionDeletionMaintenance: completed in {Ms}ms — killSwitch={KillSwitch} tenants={Tenants} enqueued={Enqueued} legacy={Legacy} skipped={Skipped} blobs={Blobs} preparing={Preparing} stranded={Stranded}", @@ -204,21 +190,13 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance } catch (Exception ex) { - // Failure path (Plan §5 PR6): OpsEvent → audit → re-throw so the runtime records it. + // Failure path (Plan §5 PR6) — emit the SessionDeletionMaintenanceFailed OpsEvent + // (captures exceptionType + message + stack preview as event details) and re-throw + // so the Azure Functions runtime records the failure. PR6 follow-up F3: the prior + // parallel LogAuditEntryAsync(null!) call was a silent no-op because AuditLogs + // requires a non-null PartitionKey; the OpsEvent IS the authoritative audit record. var stackPreview = ex.StackTrace is { Length: > 2048 } s ? s.Substring(0, 2048) : ex.StackTrace ?? string.Empty; await _opsEvents.RecordSessionDeletionMaintenanceFailedAsync(ex.GetType().FullName ?? ex.GetType().Name, ex.Message, stackPreview); - await _maintenanceRepo.LogAuditEntryAsync( - tenantId: null!, - action: "session_deletion_maintenance_failed", - entityType: "Maintenance", - entityId: $"run-{DateTime.UtcNow:yyyyMMddHHmmss}", - performedBy: "System.Maintenance", - details: new Dictionary - { - { "ExceptionType", ex.GetType().FullName ?? ex.GetType().Name }, - { "Message", ex.Message }, - { "DurationMs", sw.ElapsedMilliseconds.ToString() }, - }); throw; } finally diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs index 75759ae2..16ad2356 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs @@ -40,6 +40,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion private readonly ISessionRepository _sessionRepo; private readonly TenantConfigurationService _tenantConfig; private readonly ISessionDeletionEnqueuer _enqueuer; + private readonly AdminConfigurationService _adminConfig; private readonly ILogger _logger; private readonly Func _throttle; @@ -48,8 +49,9 @@ namespace AutopilotMonitor.Functions.Services.Deletion ISessionRepository sessionRepo, TenantConfigurationService tenantConfig, ISessionDeletionEnqueuer enqueuer, + AdminConfigurationService adminConfig, ILogger logger) - : this(maintenanceRepo, sessionRepo, tenantConfig, enqueuer, logger, throttle: Task.Delay) + : this(maintenanceRepo, sessionRepo, tenantConfig, enqueuer, adminConfig, logger, throttle: Task.Delay) { } @@ -62,6 +64,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion ISessionRepository sessionRepo, TenantConfigurationService tenantConfig, ISessionDeletionEnqueuer enqueuer, + AdminConfigurationService adminConfig, ILogger logger, Func throttle) { @@ -69,6 +72,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion _sessionRepo = sessionRepo; _tenantConfig = tenantConfig; _enqueuer = enqueuer; + _adminConfig = adminConfig; _logger = logger; _throttle = throttle; } @@ -84,6 +88,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion public int SessionsLegacyDeleted { get; set; } // flag-OFF path: direct-delete succeeded public int SessionsSkipped { get; set; } // already locked / poisoned / kill-switch / etc. public int RateLimitedTenants { get; set; } // tenants that hit MaxEnqueuesPerTenantPerRun + public bool AbortedByKillSwitch { get; set; } // PR6 follow-up F2: kill-switch flipped mid-run } /// @@ -100,6 +105,18 @@ namespace AutopilotMonitor.Functions.Services.Deletion foreach (var tenantId in tenantIds) { cancellationToken.ThrowIfCancellationRequested(); + + // PR6 follow-up F2: per-tenant kill-switch check so a flip-ON mid-run halts the + // remaining tenants. The maintenance function gates entry; this gates iteration. + if (await _adminConfig.IsSessionDeletionKillSwitchActiveAsync().ConfigureAwait(false)) + { + _logger.LogWarning( + "SessionRetentionFanout: kill-switch flipped on mid-run — halting before tenant {TenantId}", + tenantId); + result.AbortedByKillSwitch = true; + break; + } + try { await RunForTenantAsync(tenantId, result, cancellationToken).ConfigureAwait(false); @@ -158,6 +175,18 @@ namespace AutopilotMonitor.Functions.Services.Deletion break; } + // PR6 follow-up F2: per-session kill-switch check so an emergency flip-ON halts + // immediately instead of after the rest of this tenant's backlog. Uncached read + // (PR5 F1) — uniform behaviour across scaled-out instances within seconds. + if (await _adminConfig.IsSessionDeletionKillSwitchActiveAsync().ConfigureAwait(false)) + { + _logger.LogWarning( + "SessionRetentionFanout: kill-switch flipped on mid-tenant — halting at session {SessionId} of {TenantId}", + session.SessionId, tenantId); + result.AbortedByKillSwitch = true; + break; + } + if (useV2Cascade) { var outcome = await EnqueueV2Async(session, cancellationToken).ConfigureAwait(false); @@ -198,6 +227,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion { "Enqueued", enqueued.ToString() }, { "LegacyDeleted", legacyDeleted.ToString() }, { "Skipped", skipped.ToString() }, + { "AbortedByKillSwitch", result.AbortedByKillSwitch.ToString() }, }).ConfigureAwait(false); _logger.LogInformation( @@ -257,11 +287,19 @@ namespace AutopilotMonitor.Functions.Services.Deletion /// /// Legacy direct-delete path — invoked when the tenant has not yet been migrated to the V2 - /// cascade (per-tenant flag OFF). Reuses the exact sequence - /// MaintenanceService.CleanupOldDataAsync ran before PR6 extracted it: delete - /// events / rule-results / app-summaries, then delete the Sessions row itself. PR5 - /// finding 2 added a CAS guard into DeleteSessionAsync, so a V2 cascade locked by - /// a parallel admin click during this run will refuse to be tombstoned here. + /// cascade (per-tenant flag OFF). Sequence: + /// + /// Pre-read the Sessions row (PR6 follow-up F1). If DeletionState != None + /// the V2 cascade owns the session — return false without touching ANY side table. + /// Without this gate, we would delete Events / RuleResults / AppInstallSummaries + /// and only discover the lock at the tombstone CAS, leaving the V2 cascade with a + /// half-eaten session. + /// Delete events / rule-results / app-summaries. + /// Delete the Sessions row via the CAS-guarded + /// (PR5 finding 2) — second-line + /// protection against the T1-T3 race where another instance CAS-claimed the row + /// after our pre-read but before our side-table mutations. + /// /// private async Task LegacyDeleteAsync(SessionSummary session, CancellationToken cancellationToken) { @@ -269,18 +307,38 @@ namespace AutopilotMonitor.Functions.Services.Deletion var tenantId = session.TenantId; var sessionId = session.SessionId; + // F1: gate side-table mutations on DeletionState. The CAS guard inside DeleteSessionAsync + // is the storage-atomic backstop (PR5 finding 2), but it fires AFTER the side-table + // deletes. By the time it returns false we've already orphaned events from the V2 + // cascade's planned manifest. Reading the Sessions row up-front is cheap (one Get) and + // skips ~3 side-table deletes when the lock is held. + var refreshed = await _sessionRepo.GetSessionAsync(tenantId, sessionId).ConfigureAwait(false); + if (refreshed == null) + { + // Row already gone — nothing to clean up. Treat as skip (no events were ours to delete either). + return false; + } + if (SessionDeletionState.IsLocked(refreshed.DeletionState)) + { + _logger.LogInformation( + "Retention fanout (legacy): session {SessionId} is locked by a V2 cascade (state={State}, manifestId={ManifestId}) — skipping ALL deletes, V2 worker owns the cascade", + sessionId, refreshed.DeletionState, refreshed.PendingDeletionManifestId); + return false; + } + _ = await _maintenanceRepo.DeleteSessionEventsAsync(tenantId, sessionId).ConfigureAwait(false); _ = await _maintenanceRepo.DeleteSessionRuleResultsAsync(tenantId, sessionId).ConfigureAwait(false); _ = await _maintenanceRepo.DeleteSessionAppInstallSummariesAsync(tenantId, sessionId).ConfigureAwait(false); - // CAS guard inside DeleteSessionAsync (PR5 finding 2) refuses if DeletionState is locked. - // A `false` here means either 404 (already gone) is treated as success internally, OR - // a concurrent V2 cascade claimed the row. The latter is a skip, not a failure. + // CAS guard inside DeleteSessionAsync (PR5 finding 2) refuses if DeletionState is locked + // — backstop for the T1-T3 race after our pre-read above. A `false` here means either + // 404 (already gone) is treated as success internally, OR a concurrent V2 cascade + // claimed the row in the brief window between our pre-read and the tombstone. var deleted = await _sessionRepo.DeleteSessionAsync(tenantId, sessionId).ConfigureAwait(false); if (!deleted) { _logger.LogWarning( - "Retention fanout (legacy): session {SessionId} could not be tombstoned — likely claimed by a concurrent V2 cascade. Will retry next run", + "Retention fanout (legacy): session {SessionId} could not be tombstoned — likely claimed by a concurrent V2 cascade. Side tables already deleted; cascade is idempotent so V2 will absorb the difference. Will retry next run", sessionId); } return deleted; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 30647eb0..eb545887 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -98,6 +98,36 @@ namespace AutopilotMonitor.Functions.Services $"Session {sessionId} stuck in DeletionState=Queued since {queuedSince:o} (manifestId={manifestId})", tenantId, "System.Maintenance", new { tenantId, sessionId, queuedSince = queuedSince.ToString("o"), manifestId }); + /// + /// Successful end of a + /// run — records the per-block totals so dashboards can fold the cadence into the timeline. + /// PR6 follow-up F3: replaces the prior LogAuditEntryAsync(null!, ...) call, which + /// silently failed because the AuditLogs schema requires a non-null PartitionKey (tenantId). + /// + public Task RecordSessionDeletionMaintenanceCompletedAsync( + bool killSwitchActive, int tenantsProcessed, int sessionsEnqueued, int sessionsLegacyDeleted, + int sessionsSkipped, int rateLimitedTenants, int blobsTtlGced, int preparingRowsCleared, + int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch) + => WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceCompleted", OpsEventSeverity.Info, + $"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} legacy={sessionsLegacyDeleted} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive}", + null, "System.Maintenance", new { + killSwitchActive, tenantsProcessed, sessionsEnqueued, sessionsLegacyDeleted, + sessionsSkipped, rateLimitedTenants, blobsTtlGced, preparingRowsCleared, + strandedQueuedDetected, durationMs, abortedByKillSwitch, + }); + + /// + /// Records that the retention fanout half of a + /// run was skipped because the global kill-switch was active at entry. The three GCs + /// (manifest TTL sweep, stale-Preparing, stranded-Queued) still ran — see the totals on + /// the paired event. + /// + public Task RecordSessionDeletionMaintenanceFanoutSkippedAsync( + int blobsTtlGced, int preparingRowsCleared, int strandedQueuedDetected) + => WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceFanoutSkipped", OpsEventSeverity.Info, + $"SessionDeletionMaintenance fanout skipped (kill-switch active) — GCs ran: blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected}", + null, "System.Maintenance", new { reason = "SessionDeletionKillSwitch", blobsTtlGced, preparingRowsCleared, strandedQueuedDetected }); + // ── Security ─────────────────────────────────────────────────────────── public Task RecordDeviceBlockedAsync(string tenantId, string serialNumber, string reason, string blockedBy) diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index bb1e0e5a..440668d1 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -13,10 +13,16 @@ const OPS_EVENT_TYPES: Record = { "SessionTimeouts", // Cascade-delete maintenance (PR6) — dual-register per memory feedback_ops_event_types_dual_register. // Dispatched by SessionDeletionMaintenanceFunction; backend helpers in OpsEventService.cs are - // RecordSessionDeletionMaintenance{LongRunning,LongRunningSevere,Failed}Async + RecordSessionDeletionStrandedQueuedAsync. + // RecordSessionDeletionMaintenance{LongRunning,LongRunningSevere,Failed,Completed,FanoutSkipped}Async + + // RecordSessionDeletionStrandedQueuedAsync. The Completed + FanoutSkipped events landed with the + // PR6 follow-up (F3) — they replaced the AuditLogs-based lifecycle audits that silently failed + // because the schema requires a non-null PartitionKey (tenantId), but the maintenance lifecycle + // events are global-scope. + "SessionDeletionMaintenanceCompleted", "SessionDeletionMaintenanceLongRunning", "SessionDeletionMaintenanceLongRunningSevere", "SessionDeletionMaintenanceFailed", + "SessionDeletionMaintenanceFanoutSkipped", "SessionDeletionStrandedQueued", ], Security: [