From 09bfe7a1bc4bff9e236a7482a6d9ccd72a0da39c Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Wed, 13 May 2026 20:09:50 +0200 Subject: [PATCH] feat(backend,web): admin session-cleanup UI for V2 cascade-delete + codex follow-ups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operationalises the 7-PR backend cascade-delete pipeline that landed via commits c8658b9f..921994e8: per-tenant V2 opt-in switch, audit-trail consolidation, and a Global-Admin-only Session Cleanup page to track in-flight / poisoned / stranded cascades and trigger restores. == PR-A: per-tenant V2 enrollment == - UpdateTenantConfigurationFunction extends the GA-only gate list with EnableCascadeDeleteV2 so tenant admins cannot self-opt-in / out of the snapshot+restore pathway. - TenantManagementSection.tsx adds a purple-bordered "Enable V2 Cascade Delete" checkbox in the GA edit modal and a "Cascade V2" badge on the tenant card so rollout state is visible at a glance. == PR-B: audit consolidation == - SessionDeletionHandler drops the 15-per-cascade deletion_step_completed / deletion_step_failed / deletion_verification_failed audit emissions. Step telemetry lives in DeletionProgress + structured logs; tenant admins now see just deletion_started + deletion_completed + deletion_restored. - SessionDeletionWorker no longer writes a deletion_poisoned tenant audit; the signal moved to a new SessionDeletionPoisoned OpsEvent (operator-bound, Telegram-routable, dual-registered in OpsAlertRulesSection.tsx catalog). IMaintenanceRepository dependency removed from the worker constructor. - SessionRestoreService stops emitting the deletion_manifest_downloaded breadcrumb; deletion_restored is the operator-visible outcome. == PR-C: Session Cleanup admin page == - New /admin/ops/session-cleanup route (gated by app/admin/layout.tsx's ProtectedRoute requireGlobalAdmin). Sidebar entry under ga-ops alongside Maintenance. Four URL-state tabs: In-Flight / Poisoned / Stranded Queued / Activity (links to existing /audit page). - Backend reads: GetSessionDeletionsListFunction (GET /api/global/session-deletions?state=&strandedSinceMinutes=) + GetSessionDeletionManifestFunction (GET /api/admin/sessions/{id}/deletion-manifest?tenantId=&manifestId=&mode=). Both registered as GlobalAdminOnly in EndpointAccessPolicyCatalog. - DeletionPreviewModal reads the persisted snapshot via the stored-manifest endpoint (NOT a fresh dry-run via the existing /delete/preview, which would enumerate live-but-partially-deleted data for in-flight cascades). Renders manifest counts, snapshot SHA, worker progress (CompletedSteps, VerificationDone, TombstoneStarted, aggregate decrements), and the last verification failure with residual sample. Download button uses authenticatedFetch + Blob + Object URL so the Bearer token is sent. - RestoreConfirmDialog drops the Full/Partial radio (mode is auto-selected by SessionRestoreService based on cascade state — forcing the wrong mode would corrupt state). Reason field threads through to deletion_restored audit details. Dry-run default = true and previews the auto-selected mode. == Codex review follow-ups (across four rounds) == - High: tenantId is explicitly accepted on /delete/preview (query) and /restore (body); GA cross-tenant resolution prefers the explicit value over JWT TargetTenantId, falling back to SessionsIndex lookup. Without this, a GA acting on a session in another tenant got their JWT home tenant. - High: new stored-manifest endpoint replaces the dry-run preview for in-flight / poisoned rows; previous fix built a fresh manifest against already-partially-deleted data and misrepresented the cascade. - Medium: SessionDeletionPoisoned OpsEvent now carries failureType + failureMessage + observedResidualCount + a small residual preview, persisted by the handler into DeletionProgress (LastFailureType, LastFailureMessage, LastObservedResidualCount, LastResidualSampleJson) before the throw so the worker can include them. - Medium: dry-run restore reports the AUTO-SELECTED mode (full/partial) so the dialog's preview reflects what the real run will do, instead of the literal string "dryRun". - Medium: residual count field renamed to observedResidualCount with an honest XML doc — CascadeVerificationService caps at 50 per table AND short-circuits after the first failing table, so the number is a lower bound. UI surfaces "N of 50+ observed rows" when N hits the cap. - Medium: three-layer cap defense on the OpsEvent residual sample to stay under the 4096-char TableOpsEventRepository truncation: max 5 entries (OpsEventResidualSamplePreviewSize), per-field max 96 chars with `…` marker (OpsEventResidualKeyMaxChars), total preview budget 1200 chars (OpsEventResidualPreviewBudgetChars) with trailing-entry drop. Full sample stays in the progress blob for the admin UI. - Low: estimatedSnapshotBytes field name aligned end-to-end; activity tab text clarifies audit-action vs OpsEvent distinction. Tests: - Backend xUnit 1538/1538 pass (up from 1510 baseline; +28 across the new Session Cleanup endpoints, OpsEvent enrichment, restore-mode reporting, cap-defense unit + integration tests). - Web vitest 347/347 pass. - TypeScript strict + dotnet build clean. Deliberately deferred: - xUnit tests for GetSessionDeletionsListFunction + GetSessionDeletionManifestFunction (no HttpRequestData mock harness in repo; would be a separate refactor). - vitest for DeletionPreviewModal + RestoreConfirmDialog (defer until first field issue). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../SessionDeletionHandlerTests.cs | 115 +++- .../SessionDeletionPr4cCodexFixesTests.cs | 22 +- .../SessionDeletionWorkerTests.cs | 341 +++++++++++- .../SessionRestoreServiceTests.cs | 72 ++- .../UpdateTenantConfigurationFunction.cs | 6 +- .../GetSessionDeletePreviewFunction.cs | 26 +- .../GetSessionDeletionManifestFunction.cs | 242 +++++++++ .../GetSessionDeletionsListFunction.cs | 143 +++++ .../Sessions/RestoreSessionFunction.cs | 78 ++- .../Security/EndpointAccessPolicyCatalog.cs | 2 + .../Deletion/SessionDeletionHandler.cs | 188 ++++--- .../Deletion/SessionDeletionWorker.cs | 231 +++++++- .../Deletion/SessionRestoreService.cs | 57 +- .../Services/OpsEventService.cs | 48 ++ .../Models/Deletion/DeletionProgress.cs | 86 +++ .../admin/components/OpsAlertRulesSection.tsx | 4 + .../components/TenantManagementSection.tsx | 27 + .../components/DeletionPreviewModal.tsx | 302 +++++++++++ .../components/RestoreConfirmDialog.tsx | 180 +++++++ .../app/admin/ops/session-cleanup/page.tsx | 506 ++++++++++++++++++ src/Web/autopilot-monitor-web/lib/api.ts | 28 + .../lib/globalNavConfig.tsx | 1 + 22 files changed, 2519 insertions(+), 186 deletions(-) create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletionManifestFunction.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletionsListFunction.cs create mode 100644 src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/DeletionPreviewModal.tsx create mode 100644 src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/RestoreConfirmDialog.tsx create mode 100644 src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/page.tsx diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs index cf251857..7656f580 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs @@ -61,9 +61,13 @@ public class SessionDeletionHandlerTests Assert.Single(harness.SignalR.SessionDeletedCalls); Assert.Equal((TenantId, SessionId), harness.SignalR.SessionDeletedCalls[0]); - // Final audit asserts both step-completed entries AND deletion_completed. + // PR-B audit consolidation: only the lifecycle endpoint deletion_completed lands in the + // tenant audit. Step-level progress lives in DeletionProgress (CompletedSteps); step + // success goes to structured logs only. Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_completed"); - Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_step_completed"); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_step_completed"); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_step_failed"); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_verification_failed"); } [Fact] @@ -223,7 +227,95 @@ public class SessionDeletionHandlerTests Constants.TableNames.Sessions, It.IsAny>(), It.IsAny()), Times.Never); Assert.Empty(harness.SignalR.SessionDeletedCalls); - Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_verification_failed"); + // PR-B audit consolidation: verification failure surfaces via the throw → worker + // → SessionDeletionPoisoned OpsEvent path. Tenant audit stays silent — the absence of + // deletion_completed is the tenant-visible signal. + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_verification_failed"); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_completed"); + } + + [Fact] + public async Task Verification_failure_persists_LastFailure_into_DeletionProgress_for_OpsEvent_enrichment() + { + // Codex F4: the SessionDeletionPoisoned OpsEvent used to carry only queue-side metadata + // (dequeueCount + manifestId). The handler now persists failureType + a residual sample + // into DeletionProgress before throwing, so the worker can include the root cause when + // it transitions the cascade to Poisoned. This test pins the persistence-before-throw + // ordering and the field projection. + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + harness.Verifier.Setup(v => v.VerifyAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CascadeVerificationResult( + isClean: false, + residuals: new List + { + new CascadeResidualKey("Events", "ghost-pk", "ghost-rk-1"), + new CascadeResidualKey("Signals", "ghost-pk", "ghost-rk-2"), + })); + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + + Assert.Equal("verification_residuals", harness.ProgressFake.LastFailureType); + Assert.Contains("observed residual row(s)", harness.ProgressFake.LastFailureMessage ?? string.Empty); + Assert.False(string.IsNullOrEmpty(harness.ProgressFake.LastResidualSampleJson)); + // The handler persists whatever the verifier returned — which is the OBSERVED count, not + // the true total. CascadeVerificationService caps at 50 per table; here we feed it 2. + Assert.Equal(2, harness.ProgressFake.LastObservedResidualCount); + // Sample is JSON-encoded array; deserialize and assert the cap-respecting shape. + var parsed = System.Text.Json.JsonDocument.Parse(harness.ProgressFake.LastResidualSampleJson!); + Assert.Equal(System.Text.Json.JsonValueKind.Array, parsed.RootElement.ValueKind); + Assert.Equal(2, parsed.RootElement.GetArrayLength()); + } + + [Fact] + public async Task Verification_failure_persists_OBSERVED_residual_count_matching_verifier_cap() + { + // Codex F2 round-3: the verifier (CascadeVerificationService) caps each table at + // MaxResidualSampleSize=50 AND short-circuits after the first failing table, so + // verification.Residuals.Count is the OBSERVED count, not the true total. We honestly + // model that — the handler persists what it actually receives. The UI / OpsEvent layer + // surfaces this as a "≥ N" lower bound when N equals the cap. + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + // Simulate the production behaviour: the verifier already capped at 50 internally and + // returned exactly that many entries; the real residual count in the database may be + // larger but the handler cannot know that. + const int observedAtCap = 50; + var residualsAtCap = Enumerable.Range(0, observedAtCap) + .Select(i => new CascadeResidualKey("Events", "pk", $"ghost-{i}")) + .ToList(); + harness.Verifier.Setup(v => v.VerifyAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CascadeVerificationResult(isClean: false, residuals: residualsAtCap)); + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + + Assert.Equal(observedAtCap, harness.ProgressFake.LastObservedResidualCount); + // Sample length matches the observed count (both share the same cap today). The fields + // are kept separate because a future verifier upgrade may decouple them. + var sample = System.Text.Json.JsonDocument.Parse(harness.ProgressFake.LastResidualSampleJson!); + Assert.Equal(observedAtCap, sample.RootElement.GetArrayLength()); + } + + [Fact] + public async Task Step_exception_persists_LastFailure_into_DeletionProgress_for_OpsEvent_enrichment() + { + // Codex F4: same contract for the step-exception path. Step failures bubble up via the + // worker's retry/poison loop, so the durable OpsEvent record must carry failureType + + // failureMessage so operators can triage without grovelling through App Insights. + var harness = new Harness(); + harness.SetHappyPath(); + harness.SetFullSessionManifest(); + harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( + Constants.TableNames.Events, It.IsAny>(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(503, "ServiceUnavailable")); + + await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); + + Assert.Equal("step_exception", harness.ProgressFake.LastFailureType); + Assert.Contains("RequestFailedException", harness.ProgressFake.LastFailureMessage ?? string.Empty); + Assert.Null(harness.ProgressFake.LastResidualSampleJson); // step path has no residuals } [Fact] @@ -416,19 +508,24 @@ public class SessionDeletionHandlerTests } [Fact] - public async Task Handler_audits_deletion_step_failed_when_table_delete_throws() + public async Task Handler_rethrows_step_failure_without_emitting_tenant_audit() { + // PR-B audit consolidation: step failures used to write deletion_step_failed + + // deletion_step_completed into the tenant audit. Both now route to structured logs; + // the failure surfaces to the worker which routes the poisoned cascade to the + // SessionDeletionPoisoned OpsEvent. Tenant-visible signal is the absence of + // deletion_completed. var harness = new Harness(); harness.SetHappyPath(); harness.SetFullSessionManifest(); - // First delete throws — Handler must audit and rethrow. harness.Storage.Setup(s => s.DeleteByExactKeysInBatchesAsync( Constants.TableNames.Events, It.IsAny>(), It.IsAny())) .ThrowsAsync(new RequestFailedException(503, "ServiceUnavailable")); await Assert.ThrowsAsync(() => harness.Sut.HandleAsync(harness.Envelope)); - Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_step_failed"); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_step_failed"); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_step_completed"); Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_completed"); Assert.Empty(harness.SignalR.SessionDeletedCalls); } @@ -555,6 +652,12 @@ public class SessionDeletionHandlerTests ProgressFake.RestoreReIncrementsApplied = p.RestoreReIncrementsApplied == null ? null : new HashSet(p.RestoreReIncrementsApplied, StringComparer.Ordinal); ProgressFake.TombstoneStarted = p.TombstoneStarted; + // PR-B Codex F4: propagate the new last-failure fields so tests can verify + // the handler persists them before throwing on step / verification failures. + ProgressFake.LastFailureType = p.LastFailureType; + ProgressFake.LastFailureMessage = p.LastFailureMessage; + ProgressFake.LastObservedResidualCount = p.LastObservedResidualCount; + ProgressFake.LastResidualSampleJson = p.LastResidualSampleJson; return "\"0xFAKE_ETAG_2\""; }); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs index 48241bcf..ceca3bea 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionPr4cCodexFixesTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Azure; @@ -650,10 +651,29 @@ public class SessionDeletionPr4cCodexFixesTests It.IsAny(), It.IsAny(), It.IsAny?>())) .ReturnsAsync(true); + // PR-B audit consolidation: SessionDeletionWorker no longer takes IMaintenanceRepository; + // poisoned cascades flow through OpsEventService. Build a fire-and-forget instance + // backed by a noop repository — F5 tests assert CAS ordering, not the OpsEvent itself. + var opsRepoStub = new Mock(); + opsRepoStub.Setup(r => r.SaveOpsEventAsync(It.IsAny())).Returns(Task.CompletedTask); + var alertDispatchStub = new OpsAlertDispatchService( + AdminConfig.Object, + new TelegramNotificationService(new HttpClient(), Mock.Of(), NullLogger.Instance), + new AutopilotMonitor.Functions.Services.Notifications.WebhookNotificationService(new HttpClient(), NullLogger.Instance), + NullLogger.Instance); + var opsServiceStub = new OpsEventService(opsRepoStub.Object, NullLogger.Instance, alertDispatchStub); + + // PR-B Codex F4 follow-up: worker now reads DeletionProgress on the poison path. + // F5 tests assert CAS ordering, not the OpsEvent payload, so a noop blob suffices. + blobMock.Setup(b => b.DownloadDeletionProgressAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(((AutopilotMonitor.Shared.Models.Deletion.DeletionProgress, string))( + new AutopilotMonitor.Shared.Models.Deletion.DeletionProgress(), "etag-stub")); + Sut = new SessionDeletionWorker( MainQueue.Object, PoisonQueue.Object, handlerMock.Object, StorageMock.Object, - AdminConfig.Object, Maintenance.Object, + AdminConfig.Object, blobMock.Object, opsServiceStub, NullLogger.Instance, heartbeatInterval: TimeSpan.FromMilliseconds(200), pollInterval: TimeSpan.FromMilliseconds(50)); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionWorkerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionWorkerTests.cs index 746c10a2..ae20c160 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionWorkerTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionWorkerTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Azure; @@ -9,6 +10,7 @@ using Azure.Storage.Queues; using Azure.Storage.Queues.Models; using AutopilotMonitor.Functions.Services; using AutopilotMonitor.Functions.Services.Deletion; +using AutopilotMonitor.Functions.Services.Notifications; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; using AutopilotMonitor.Shared.Models.Deletion; @@ -49,7 +51,7 @@ public class SessionDeletionWorkerTests } [Fact] - public async Task Worker_moves_message_to_poison_after_max_dequeue_and_audits() + public async Task Worker_moves_message_to_poison_after_max_dequeue_and_emits_ops_event() { var harness = new Harness(); var envelope = new SessionDeletionEnvelope @@ -68,9 +70,256 @@ public class SessionDeletionWorkerTests harness.MainQueue.Verify(q => q.DeleteMessageAsync( It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeastOnce); - // Audit deletion_poisoned written with the parsed envelope's manifestId. - Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_poisoned" - && a.Details != null && a.Details["manifestId"] == ManifestId); + // PR-B audit consolidation: deletion_poisoned audit moved to SessionDeletionPoisoned + // OpsEvent (operator-bound, Telegram-routable). Tenant-audit row is no longer written. + var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned"); + Assert.Equal(TenantId, poisoned.TenantId); + Assert.Equal(SessionId, poisoned.SessionId); + Assert.Equal(ManifestId, poisoned.ManifestId); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_poisoned"); + // Codex F4: poison OpsEvent carries the handler's last-failure breadcrumb from the + // DeletionProgress blob (harness seeds a verification-failure scenario by default). + Assert.Contains("verification_residuals", poisoned.RawDetails); + Assert.Contains("residual row(s)", poisoned.RawDetails); + // Codex F2 round-3: OpsEvent carries the verifier's OBSERVED residual count, not a + // hypothetical "true" total — CascadeVerificationService short-circuits at the first + // failing table and caps each table's sample at MaxResidualSampleSize. The degenerate + // residualSampleSize key was dropped because it always equalled observedResidualCount. + Assert.Contains("\"observedResidualCount\":2", poisoned.RawDetails); + Assert.DoesNotContain("\"residualSampleSize\":", poisoned.RawDetails); + Assert.DoesNotContain("\"residualCount\":", poisoned.RawDetails); // old, misleading key + } + + [Fact] + public async Task Worker_OpsEvent_carries_observed_count_at_verifier_cap_for_large_failures() + { + // Codex F2 round-3: when the real residual mountain is bigger than the verifier cap, + // verification.Residuals.Count = MaxResidualSampleSize (50) and that's the only number + // we have. The worker forwards exactly that. UI surface treats "==cap" as a "≥" lower + // bound; the OpsEvent itself stays honest and just reports what was observed. + var harness = new Harness(); + var envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1); + + const int capObservation = 50; + var sample = string.Join(",", Enumerable.Range(0, capObservation) + .Select(i => $"{{\"table\":\"Events\",\"pk\":\"t\",\"rk\":\"r{i}\"}}")); + harness.BlobMock.Setup(b => b.DownloadDeletionProgressAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(((AutopilotMonitor.Shared.Models.Deletion.DeletionProgress, string))( + new AutopilotMonitor.Shared.Models.Deletion.DeletionProgress + { + LastFailureType = "verification_residuals", + LastFailureMessage = $"{capObservation} observed residual row(s); refusing to tombstone", + LastObservedResidualCount = capObservation, + LastResidualSampleJson = $"[{sample}]", + }, + "etag-mock")); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned"); + Assert.Contains($"\"observedResidualCount\":{capObservation}", poisoned.RawDetails); + // OpsEvents table truncates Details at 4096 chars (TableOpsEventRepository.cs); the + // preview cap (OpsEventResidualSamplePreviewSize=5) keeps us safely under that even + // with a 50-entry progress-blob sample. + Assert.True(poisoned.RawDetails.Length < 4096, + $"OpsEvent Details exceeded 4096 chars ({poisoned.RawDetails.Length}); the table repository will truncate mid-JSON."); + // Preview JSON has at most OpsEventResidualSamplePreviewSize entries. + var match = System.Text.RegularExpressions.Regex.Match( + poisoned.RawDetails, + "\"residualSamplePreviewJson\":\"(?(?:[^\"\\\\]|\\\\.)*)\""); + Assert.True(match.Success, "Expected residualSamplePreviewJson key in OpsEvent details."); + var previewJson = System.Text.RegularExpressions.Regex.Unescape(match.Groups["json"].Value); + using var previewDoc = System.Text.Json.JsonDocument.Parse(previewJson); + Assert.InRange(previewDoc.RootElement.GetArrayLength(), 1, + AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualSamplePreviewSize); + } + + [Fact] + public async Task Worker_OpsEvent_falls_back_to_sample_length_when_observed_count_missing_pre_followup_blob() + { + // Back-compat: a progress blob written before LastObservedResidualCount existed (this + // field was added in a Codex follow-up to PR-B) still produces a useful OpsEvent — the + // worker derives the number from the sample-array length so observers don't see a null + // observedResidualCount when reading historical poisoned cascades. + var harness = new Harness(); + var envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1); + + harness.BlobMock.Setup(b => b.DownloadDeletionProgressAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(((AutopilotMonitor.Shared.Models.Deletion.DeletionProgress, string))( + new AutopilotMonitor.Shared.Models.Deletion.DeletionProgress + { + LastFailureType = "verification_residuals", + LastFailureMessage = "observed residual row(s); refusing to tombstone", + LastObservedResidualCount = null, // pre-followup blob — field did not exist + LastResidualSampleJson = "[{\"table\":\"Events\",\"pk\":\"t\",\"rk\":\"r1\"},{\"table\":\"Events\",\"pk\":\"t\",\"rk\":\"r2\"},{\"table\":\"Events\",\"pk\":\"t\",\"rk\":\"r3\"}]", + }, + "etag-mock")); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned"); + Assert.Contains("\"observedResidualCount\":3", poisoned.RawDetails); + } + + [Fact] + public void ShrinkResidualSampleForOpsEvent_caps_array_at_preview_size() + { + // Unit-level pin on the helper so the cap can't silently drift if someone changes the + // constant or the serialization plumbing. + var full = string.Join(",", Enumerable.Range(0, 50) + .Select(i => $"{{\"table\":\"Events\",\"pk\":\"tenant\",\"rk\":\"row-{i}\"}}")); + var shrunk = SessionDeletionWorker.ShrinkResidualSampleForOpsEvent($"[{full}]"); + Assert.NotNull(shrunk); + using var doc = System.Text.Json.JsonDocument.Parse(shrunk!); + Assert.Equal(System.Text.Json.JsonValueKind.Array, doc.RootElement.ValueKind); + Assert.Equal( + AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualSamplePreviewSize, + doc.RootElement.GetArrayLength()); + } + + [Fact] + public void ShrinkResidualSampleForOpsEvent_returns_null_for_malformed_input() + { + Assert.Null(SessionDeletionWorker.ShrinkResidualSampleForOpsEvent(null)); + Assert.Null(SessionDeletionWorker.ShrinkResidualSampleForOpsEvent("")); + Assert.Null(SessionDeletionWorker.ShrinkResidualSampleForOpsEvent("not-json")); + Assert.Null(SessionDeletionWorker.ShrinkResidualSampleForOpsEvent("{\"not\":\"an array\"}")); + } + + [Fact] + public void ShrinkResidualSampleForOpsEvent_trims_per_field_with_ellipsis_marker() + { + // Codex follow-up: a single entry with a 200-char composite PK could on its own consume + // most of the OpsEvent Details budget. The helper trims each field to + // OpsEventResidualKeyMaxChars and appends `…` so the truncation is operator-visible. + var maxKeyChars = AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualKeyMaxChars; + var longPk = new string('p', maxKeyChars * 3); + var longRk = new string('r', maxKeyChars * 3); + var input = $"[{{\"table\":\"Events\",\"pk\":\"{longPk}\",\"rk\":\"{longRk}\"}}]"; + + var shrunk = SessionDeletionWorker.ShrinkResidualSampleForOpsEvent(input); + Assert.NotNull(shrunk); + + using var doc = System.Text.Json.JsonDocument.Parse(shrunk!); + var first = doc.RootElement[0]; + var pk = first.GetProperty("pk").GetString()!; + var rk = first.GetProperty("rk").GetString()!; + Assert.Equal(maxKeyChars, pk.Length); + Assert.Equal(maxKeyChars, rk.Length); + Assert.EndsWith("…", pk); + Assert.EndsWith("…", rk); + Assert.StartsWith("ppp", pk); + Assert.StartsWith("rrr", rk); + } + + [Fact] + public void ShrinkResidualSampleForOpsEvent_short_fields_pass_through_unchanged() + { + // Don't paint truncation markers on values that fit naturally. + var input = "[{\"table\":\"Events\",\"pk\":\"short-pk\",\"rk\":\"short-rk\"}]"; + var shrunk = SessionDeletionWorker.ShrinkResidualSampleForOpsEvent(input); + Assert.NotNull(shrunk); + using var doc = System.Text.Json.JsonDocument.Parse(shrunk!); + var first = doc.RootElement[0]; + Assert.Equal("short-pk", first.GetProperty("pk").GetString()); + Assert.Equal("short-rk", first.GetProperty("rk").GetString()); + } + + [Fact] + public void ShrinkResidualSampleForOpsEvent_drops_trailing_entries_when_over_total_budget() + { + // Even with per-field trimming, max-length entries can collectively exceed the preview + // budget (~1200 chars). The helper drops trailing entries one at a time until the JSON + // fits — operators always see SOMETHING parseable, the full sample stays in the blob. + var maxKeyChars = AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualKeyMaxChars; + var budget = AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualPreviewBudgetChars; + var maxEntries = AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualSamplePreviewSize; + // Each entry takes ~(20 table + 96 pk + 96 rk + 30 JSON shape) ≈ 240 chars; 5 entries + // would land at ~1200, right at the budget. Make table/pk/rk all max-length so we land + // slightly OVER and force a drop. + var longTable = new string('T', maxKeyChars); + var longPk = new string('p', maxKeyChars); + var longRk = new string('r', maxKeyChars); + var entry = $"{{\"table\":\"{longTable}\",\"pk\":\"{longPk}\",\"rk\":\"{longRk}\"}}"; + var input = "[" + string.Join(",", Enumerable.Range(0, maxEntries).Select(_ => entry)) + "]"; + + var shrunk = SessionDeletionWorker.ShrinkResidualSampleForOpsEvent(input); + Assert.NotNull(shrunk); + Assert.True(shrunk!.Length <= budget, + $"Preview exceeded total budget: {shrunk.Length} > {budget}."); + + using var doc = System.Text.Json.JsonDocument.Parse(shrunk!); + var kept = doc.RootElement.GetArrayLength(); + // We must have kept at least one entry (a single entry is small enough to fit), and + // dropped at least one (otherwise the budget wasn't actually enforced). + Assert.InRange(kept, 1, maxEntries - 1); + } + + [Fact] + public async Task Worker_OpsEvent_stays_under_4096_chars_with_worst_case_long_keys() + { + // Codex end-to-end pin: the full pipeline (handler → progress blob → worker shrink → + // OpsEvent JSON envelope → table-row truncate) must NEVER produce a Details string > 4096 + // chars, because TableOpsEventRepository hard-truncates at that boundary and would corrupt + // the JSON mid-string. We simulate the worst case: max-length PK + RK on every sample + // entry, plus a max-length failureMessage. + var harness = new Harness(); + var envelope = new SessionDeletionEnvelope + { + TenantId = TenantId, SessionId = SessionId, ManifestId = ManifestId, + Reason = "admin_delete", EnqueuedAt = DateTime.UtcNow, + }; + harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: SessionDeletionWorker.MaxDequeueCount + 1); + + // 50 sample entries × ~200-char composite keys = ~10 KB blob; the worker must defend. + const int progressSampleSize = 50; + const int hugeKeyChars = 240; + var hugeTable = new string('T', hugeKeyChars); + var hugePk = new string('p', hugeKeyChars); + var hugeRk = new string('r', hugeKeyChars); + var sample = string.Join(",", Enumerable.Range(0, progressSampleSize) + .Select(_ => $"{{\"table\":\"{hugeTable}\",\"pk\":\"{hugePk}\",\"rk\":\"{hugeRk}\"}}")); + // Max-length failure message too. + var longMessage = new string('m', 1024); + + harness.BlobMock.Setup(b => b.DownloadDeletionProgressAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(((AutopilotMonitor.Shared.Models.Deletion.DeletionProgress, string))( + new AutopilotMonitor.Shared.Models.Deletion.DeletionProgress + { + LastFailureType = "verification_residuals", + LastFailureMessage = longMessage, + LastObservedResidualCount = progressSampleSize, + LastResidualSampleJson = $"[{sample}]", + }, + "etag-mock")); + + await harness.RunForAsync(TimeSpan.FromMilliseconds(500)); + + var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned"); + Assert.True(poisoned.RawDetails.Length < 4096, + $"OpsEvent Details exceeded 4096 chars ({poisoned.RawDetails.Length}); TableOpsEventRepository would truncate mid-JSON."); + + // Sanity check that the preview is still parseable as JSON after the trim+budget pass. + var match = System.Text.RegularExpressions.Regex.Match( + poisoned.RawDetails, + "\"residualSamplePreviewJson\":\"(?(?:[^\"\\\\]|\\\\.)*)\""); + Assert.True(match.Success, "Expected residualSamplePreviewJson key in OpsEvent details."); + var previewJson = System.Text.RegularExpressions.Regex.Unescape(match.Groups["json"].Value); + using var previewDoc = System.Text.Json.JsonDocument.Parse(previewJson); + Assert.Equal(System.Text.Json.JsonValueKind.Array, previewDoc.RootElement.ValueKind); } [Fact] @@ -198,7 +447,7 @@ public class SessionDeletionWorkerTests harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny(), It.IsAny()), Times.AtLeastOnce); - Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_poisoned"); + Assert.Contains(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned"); } [Fact] @@ -280,8 +529,11 @@ public class SessionDeletionWorkerTests public Mock PoisonQueue { get; } public Mock HandlerMock { get; } public Mock AdminConfig { get; } - public Mock Maintenance { get; } + // PR-B audit consolidation: deletion_poisoned tenant audit moved to SessionDeletionPoisoned + // OpsEvent. We keep AuditCalls as a passthrough capture (always empty for the worker + // post-PR-B) so existing assertions like DoesNotContain still compile. public List AuditCalls { get; } = new List(); + public List OpsEventCalls { get; } = new List(); public SessionDeletionWorker Sut { get; } private readonly Queue _pendingMessages = new Queue(); @@ -362,6 +614,24 @@ public class SessionDeletionWorkerTests var blobMock = new Mock( new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"), NullLogger.Instance, false); + // PR-B Codex F4 follow-up: worker reads DeletionProgress on the poison path so the + // SessionDeletionPoisoned OpsEvent can include the handler's last-failure breadcrumb. + // Default mock returns a populated progress to exercise the enrichment path. + blobMock.Setup(b => b.DownloadDeletionProgressAsync( + It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(((AutopilotMonitor.Shared.Models.Deletion.DeletionProgress, string))( + new AutopilotMonitor.Shared.Models.Deletion.DeletionProgress + { + LastFailureType = "verification_residuals", + LastFailureMessage = "2 observed residual row(s)", + // Codex F2 round-3: the persisted count is the verifier's OBSERVED count + // (capped at MaxResidualSampleSize=50 per table + short-circuit at first + // failing table). Default mock uses a low number so the field matches the + // sample length; the cap-hit test below exercises the lower-bound case. + LastObservedResidualCount = 2, + LastResidualSampleJson = "[{\"table\":\"Events\",\"pk\":\"t\",\"rk\":\"r\"},{\"table\":\"Signals\",\"pk\":\"t\",\"rk\":\"r2\"}]", + }, + "etag-mock")); var verifierMock = new Mock( Mock.Of(), NullLogger.Instance); @@ -384,28 +654,37 @@ public class SessionDeletionWorkerTests AdminConfig.Setup(a => a.IsSessionDeletionKillSwitchActiveAsync()) .ReturnsAsync(false); - Maintenance = new Mock(); - Maintenance.Setup(m => m.LogAuditEntryAsync( - It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), It.IsAny(), It.IsAny?>())) - .Returns?>( - (tenantId, action, entityType, entityId, performedBy, details) => - { - AuditCalls.Add(new AuditEntry(tenantId, action, entityType, entityId, performedBy, details)); - return Task.FromResult(true); - }); + // PR-B audit consolidation: build a real OpsEventService backed by a captured-list + // mock repository so the worker's RecordSessionDeletionPoisonedAsync call is + // observable as an OpsEventEntry on the OpsEventCalls list. + var opsRepo = new Mock(); + opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny())) + .Returns(e => + { + OpsEventCalls.Add(CapturedOpsEvent.From(e)); + return Task.CompletedTask; + }); + var alertDispatch = new OpsAlertDispatchService( + AdminConfig.Object, + new TelegramNotificationService(new HttpClient(), Mock.Of(), NullLogger.Instance), + new AutopilotMonitor.Functions.Services.Notifications.WebhookNotificationService(new HttpClient(), NullLogger.Instance), + NullLogger.Instance); + var opsService = new OpsEventService(opsRepo.Object, NullLogger.Instance, alertDispatch); Sut = new SessionDeletionWorker( MainQueue.Object, PoisonQueue.Object, HandlerMock.Object, storageMock.Object, - AdminConfig.Object, Maintenance.Object, + AdminConfig.Object, blobMock.Object, opsService, NullLogger.Instance, heartbeatInterval: heartbeatInterval ?? TimeSpan.FromMilliseconds(200), pollInterval: TimeSpan.FromMilliseconds(50)); StorageMock = storageMock; + BlobMock = blobMock; } + public Mock BlobMock { get; private set; } = null!; + public Mock StorageMock { get; private set; } = null!; public void SetKillSwitch(bool active) @@ -445,4 +724,32 @@ public class SessionDeletionWorkerTests private sealed record AuditEntry( string TenantId, string Action, string EntityType, string EntityId, string PerformedBy, Dictionary? Details); + + /// + /// Flattened OpsEventEntry projection for assertions. + the other + /// strongly-typed accessors are pulled from Details JSON so tests don't have to + /// re-deserialise. + /// + private sealed record CapturedOpsEvent( + string EventType, string Severity, string Message, + string? TenantId, string? SessionId, string? ManifestId, string RawDetails) + { + public static CapturedOpsEvent From(OpsEventEntry e) + { + string? tenantId = e.TenantId; + string? sessionId = null; + string? manifestId = null; + if (!string.IsNullOrEmpty(e.Details)) + { + try + { + using var doc = System.Text.Json.JsonDocument.Parse(e.Details!); + if (doc.RootElement.TryGetProperty("sessionId", out var s)) sessionId = s.GetString(); + if (doc.RootElement.TryGetProperty("manifestId", out var m)) manifestId = m.GetString(); + } + catch (System.Text.Json.JsonException) { /* tolerate malformed test payloads */ } + } + return new CapturedOpsEvent(e.EventType, e.Severity, e.Message, tenantId, sessionId, manifestId, e.Details ?? string.Empty); + } + } } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionRestoreServiceTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionRestoreServiceTests.cs index e93ed192..2fa7abb8 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionRestoreServiceTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionRestoreServiceTests.cs @@ -31,6 +31,45 @@ public class SessionRestoreServiceTests // ============================================================ Full restore (8 facts) ==== + [Fact] + public async Task Restore_persists_operator_reason_into_audit_details() + { + // Codex follow-up: the Session Cleanup admin dialog sends a free-text `reason` so the + // operator's intent is recoverable from the tenant audit trail. Pre-fix the backend + // accepted the field but the service dropped it on the floor. + var harness = new Harness(); + harness.SetCompletedCascade(); + + await harness.Sut.RestoreAsync( + TenantId, SessionId, ManifestId, + dryRun: false, actor: "ga@example.com", + operatorReason: "customer ticket ABC-123 — accidental delete"); + + var restored = Assert.Single(harness.AuditCalls, a => a.Action == "deletion_restored"); + Assert.NotNull(restored.Details); + Assert.True(restored.Details!.TryGetValue("reason", out var reasonValue)); + Assert.Equal("customer ticket ABC-123 — accidental delete", reasonValue); + } + + [Fact] + public async Task Restore_omits_reason_detail_when_caller_supplied_none() + { + // Sister to the above — a null/empty reason must NOT add a stray `reason=""` key, since + // downstream consumers (audit-log UI filters, kusto queries) treat the key's presence as + // signal that the operator deliberately recorded intent. + var harness = new Harness(); + harness.SetCompletedCascade(); + + await harness.Sut.RestoreAsync( + TenantId, SessionId, ManifestId, + dryRun: false, actor: "ga@example.com", + operatorReason: null); + + var restored = Assert.Single(harness.AuditCalls, a => a.Action == "deletion_restored"); + Assert.NotNull(restored.Details); + Assert.False(restored.Details!.ContainsKey("reason")); + } + [Fact] public async Task Restore_full_round_trips_session_after_completed_cascade() { @@ -118,18 +157,20 @@ public class SessionRestoreServiceTests } [Fact] - public async Task Restore_full_audits_manifest_download_before_any_read() + public async Task Restore_full_does_not_emit_internal_breadcrumb_audit() { + // PR-B audit consolidation: deletion_manifest_downloaded was an internal step-breadcrumb + // emitted before any read. It doubled the tenant audit row count without adding signal + // — the deletion_restored audit on success (or the absence of it on the reject path) + // is the operator-relevant outcome. var harness = new Harness(); harness.SetCompletedCascade(); - // Force a post-download reject path so we can verify the audit fired BEFORE the reject. harness.Blob.Setup(b => b.DownloadDeletionManifestWithShaAsync(TenantId, SessionId, ManifestId, It.IsAny())) .ThrowsAsync(new InvalidDataException("corruption")); await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: false, actor: "ga@example.com"); - Assert.Contains(harness.AuditCalls, a => a.Action == "deletion_manifest_downloaded" - && a.Details != null && a.Details["reason"] == "restore_attempt"); + Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_manifest_downloaded"); } [Fact] @@ -186,7 +227,10 @@ public class SessionRestoreServiceTests var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: true, actor: "ga@example.com"); Assert.Equal(SessionRestoreOutcome.DryRunOk, result.Outcome); - Assert.Equal("dryRun", result.Mode); + // Codex follow-up: dry-run must report the auto-selected mode (full / partial) so the + // admin dialog can preview it. The DryRunOk Outcome is the dry-run signal — Mode is the + // operator-visible preview of what the real run would do. + Assert.Equal("full", result.Mode); Assert.NotEmpty(result.WouldRestoreByTable); // No live writes. harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync( @@ -199,6 +243,24 @@ public class SessionRestoreServiceTests Assert.DoesNotContain(harness.AuditCalls, a => a.Action == "deletion_restored"); } + [Fact] + public async Task Restore_partial_dryRun_reports_partial_mode_for_poisoned_cascade() + { + // Codex follow-up symmetry test: dry-run on a Poisoned session must preview "partial", + // mirroring the completed-cascade test above that previews "full". + var harness = new Harness(); + harness.SetPoisonedCascade(); + harness.Progress.CompletedSteps.UnionWith(new[] { 1, 2, 3, 4 }); + + var result = await harness.Sut.RestoreAsync(TenantId, SessionId, ManifestId, dryRun: true, actor: "ga@example.com"); + + Assert.Equal(SessionRestoreOutcome.DryRunOk, result.Outcome); + Assert.Equal("partial", result.Mode); + harness.Storage.Verify(s => s.RestoreRowsByExactKeysInBatchesAsync( + It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Never); + } + // ============================================================ Partial restore (6 facts) ==== [Fact] diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Config/UpdateTenantConfigurationFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Config/UpdateTenantConfigurationFunction.cs index 695ed665..75ddf0cf 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Config/UpdateTenantConfigurationFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Config/UpdateTenantConfigurationFunction.cs @@ -95,7 +95,8 @@ namespace AutopilotMonitor.Functions.Functions.Config config.CustomRateLimitRequestsPerMinute != existingConfig.CustomRateLimitRequestsPerMinute || config.RateLimitRequestsPerMinute != existingConfig.RateLimitRequestsPerMinute || config.Disabled != existingConfig.Disabled || - config.ValidateDeviceAssociation != existingConfig.ValidateDeviceAssociation) + config.ValidateDeviceAssociation != existingConfig.ValidateDeviceAssociation || + config.EnableCascadeDeleteV2 != existingConfig.EnableCascadeDeleteV2) { _logger.LogWarning( "Tenant Admin {User} attempted to modify GA-only fields for tenant {TenantId}", @@ -116,6 +117,9 @@ namespace AutopilotMonitor.Functions.Functions.Config // RequestContext + TenantConfigurationService). Tracked in // memory/project_devprep_followups.md. config.ValidateDeviceAssociation = existingConfig.ValidateDeviceAssociation; + // V2 cascade-delete pipeline activation is a rollout decision; tenant + // admins cannot self-opt-in (or out) of the snapshot+restore pathway. + config.EnableCascadeDeleteV2 = existingConfig.EnableCascadeDeleteV2; } // Safety: if GA gate is off, force UnrestrictedMode to false diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletePreviewFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletePreviewFunction.cs index 6b155472..78c74a89 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletePreviewFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletePreviewFunction.cs @@ -72,12 +72,28 @@ namespace AutopilotMonitor.Functions.Functions.Sessions return bad; } - // Resolve tenantId: prefer the JWT-derived target, fall back to SessionsIndex lookup - // (Global Admin can access any session by sessionId without knowing the tenantId). - var tenantId = requestCtx.TargetTenantId; - if (string.IsNullOrEmpty(tenantId) || string.Equals(tenantId, "global", StringComparison.OrdinalIgnoreCase)) + // Resolve tenantId. Three-tier fallback ordered by Codex F1 review: + // 1. Explicit ?tenantId= on GA calls — only path that lets a Global Admin + // preview a session in a tenant other than their JWT home tenant. + // 2. JWT TargetTenantId, where the policy middleware mirrors the JWT tid for + // non-RouteParam routes. Sufficient for tenant admins acting on their own tenant. + // 3. SessionsIndex lookup keyed by sessionId — the historical fallback for the + // "global" sentinel; preserved so existing callers without ?tenantId still work. + var explicitTenantId = query["tenantId"]; + string tenantId; + if (requestCtx.IsGlobalAdmin + && !string.IsNullOrEmpty(explicitTenantId) + && Guid.TryParse(explicitTenantId, out _)) { - tenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId); + tenantId = explicitTenantId!; + } + else + { + tenantId = requestCtx.TargetTenantId; + if (string.IsNullOrEmpty(tenantId) || string.Equals(tenantId, "global", StringComparison.OrdinalIgnoreCase)) + { + tenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId) ?? string.Empty; + } } if (string.IsNullOrEmpty(tenantId)) { diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletionManifestFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletionManifestFunction.cs new file mode 100644 index 00000000..cd215667 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletionManifestFunction.cs @@ -0,0 +1,242 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Text.Json; +using System.Threading.Tasks; +using System.Web; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Sessions +{ + /// + /// GET /api/admin/sessions/{sessionId}/deletion-manifest?tenantId=&manifestId=&mode=... — + /// reads the persisted cascade-delete snapshot + progress for an in-flight / poisoned cascade. + /// Distinct from , which builds a fresh dry-run + /// manifest from current data: for a running cascade that view is misleading because the live + /// data is already partially deleted. The Session Cleanup admin page uses THIS endpoint to + /// show the operator the actual snapshot the worker captured at cascade start. + /// + /// GA-only enforcement comes from EndpointAccessPolicyCatalog. Cross-tenant access is + /// the design intent — the caller passes the row's tenantId from the listing endpoint; + /// no SessionsIndex fallback because the index row is often the first thing the cascade + /// deletes, so we cannot rely on it for a Running/Poisoned manifest. + /// + /// + public class GetSessionDeletionManifestFunction + { + private readonly ILogger _logger; + private readonly BlobStorageService _blob; + + public GetSessionDeletionManifestFunction( + ILogger logger, + BlobStorageService blob) + { + _logger = logger; + _blob = blob; + } + + [Function("GetSessionDeletionManifest")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "admin/sessions/{sessionId}/deletion-manifest")] + HttpRequestData req, + string sessionId) + { + if (string.IsNullOrWhiteSpace(sessionId)) + { + return await BadRequest(req, "sessionId is required"); + } + + var query = HttpUtility.ParseQueryString(req.Url.Query ?? string.Empty); + var tenantId = query["tenantId"]; + var manifestId = query["manifestId"]; + var mode = (query["mode"] ?? "summary").ToLowerInvariant(); + if (string.IsNullOrWhiteSpace(tenantId) || !Guid.TryParse(tenantId, out _)) + { + return await BadRequest(req, "Query parameter 'tenantId' must be a GUID."); + } + if (string.IsNullOrWhiteSpace(manifestId)) + { + return await BadRequest(req, "Query parameter 'manifestId' is required."); + } + if (mode != "summary" && mode != "full" && mode != "download") + { + return await BadRequest(req, "mode must be one of: summary, full, download"); + } + + DeletionManifest manifest; + string snapshotSha; + try + { + (manifest, snapshotSha) = await _blob.DownloadDeletionManifestWithShaAsync( + tenantId!, sessionId, manifestId!, req.FunctionContext.CancellationToken); + } + catch (Azure.RequestFailedException rfe) when (rfe.Status == 404) + { + var notFound = req.CreateResponse(HttpStatusCode.NotFound); + await notFound.WriteAsJsonAsync(new + { + success = false, + message = $"Manifest blob not found for tenant={tenantId} session={sessionId} manifestId={manifestId}.", + }); + return notFound; + } + catch (InvalidDataException ex) + { + _logger.LogError(ex, + "GetSessionDeletionManifest: snapshot integrity failure for tenant={TenantId} session={SessionId} manifestId={ManifestId}", + tenantId, sessionId, manifestId); + var corrupt = req.CreateResponse(HttpStatusCode.Conflict); + await corrupt.WriteAsJsonAsync(new + { + success = false, + message = "Snapshot SHA-256 mismatch — the manifest blob has been tampered with or the producer crashed.", + error = ex.Message, + }); + return corrupt; + } + catch (Exception ex) + { + _logger.LogError(ex, + "GetSessionDeletionManifest: unexpected error for tenant={TenantId} session={SessionId} manifestId={ManifestId}", + tenantId, sessionId, manifestId); + var error = req.CreateResponse(HttpStatusCode.InternalServerError); + await error.WriteAsJsonAsync(new { success = false, message = "Internal server error" }); + return error; + } + + // Progress blob — best-effort. A Preparing-phase manifest may exist without a progress + // blob (the producer hasn't reached the upload step yet); surface as null and let the + // UI render "no progress yet" instead of failing the whole request. + DeletionProgress? progress = null; + try + { + var (p, _) = await _blob.DownloadDeletionProgressAsync( + tenantId!, sessionId, manifestId!, req.FunctionContext.CancellationToken); + progress = p; + } + catch (Azure.RequestFailedException rfe) when (rfe.Status == 404) + { + _logger.LogInformation( + "GetSessionDeletionManifest: no progress blob yet (likely Preparing phase) for tenant={TenantId} session={SessionId} manifestId={ManifestId}", + tenantId, sessionId, manifestId); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "GetSessionDeletionManifest: failed to read progress blob — returning manifest only. tenant={TenantId} session={SessionId} manifestId={ManifestId}", + tenantId, sessionId, manifestId); + } + + return mode switch + { + "full" => await BuildFullResponse(req, manifest, progress, snapshotSha), + "download" => await BuildDownloadResponse(req, manifest, tenantId!, sessionId, manifestId!), + _ => await BuildSummaryResponse(req, manifest, progress, snapshotSha), + }; + } + + private static async Task BadRequest(HttpRequestData req, string message) + { + var bad = req.CreateResponse(HttpStatusCode.BadRequest); + await bad.WriteAsJsonAsync(new { success = false, message }); + return bad; + } + + private static async Task BuildSummaryResponse( + HttpRequestData req, DeletionManifest manifest, DeletionProgress? progress, string snapshotSha) + { + long totalRowCount = 0; + var sampleKeys = new Dictionary>(StringComparer.Ordinal); + foreach (var step in manifest.Steps) + { + totalRowCount += step.RowCount; + if (step.Rows.Count == 0) continue; + var key = step.Table ?? step.Step ?? $"order_{step.Order}"; + sampleKeys[key] = step.Rows.Take(5).Select(r => (object)new { pk = r.Pk, rk = r.Rk }).ToList(); + } + + // Serialize once to report bytes — same projection the producer would upload (modulo gzip). + var serializedBytes = JsonSerializer.SerializeToUtf8Bytes(manifest, DeletionManifestJson.SerializerOptions); + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new + { + success = true, + mode = "summary", + source = "stored", + manifestId = manifest.ManifestId, + schemaHash = manifest.SchemaHash, + snapshotSha256 = snapshotSha, + estimatedRowCount = totalRowCount, + estimatedSnapshotBytes = serializedBytes.LongLength, + preflightCounts = manifest.PreflightCounts, + sampleKeys, + progress = progress == null ? null : new + { + progress.SnapshotSha256, + completedStepOrders = progress.CompletedSteps, + progress.VerificationDone, + progress.TombstoneStarted, + progress.CompletedAt, + aggregateDecrementsApplied = progress.AggregateDecrementsApplied?.Count ?? 0, + restoreReIncrementsApplied = progress.RestoreReIncrementsApplied?.Count ?? 0, + progress.LastFailureType, + progress.LastFailureMessage, + // Codex F2 round-3: this is the verifier's OBSERVED count, capped at the + // sample size — UI surfaces it as a lower bound when it equals the cap. + progress.LastObservedResidualCount, + progress.LastResidualSampleJson, + }, + }); + return response; + } + + private static async Task BuildFullResponse( + HttpRequestData req, DeletionManifest manifest, DeletionProgress? progress, string snapshotSha) + { + var response = req.CreateResponse(HttpStatusCode.OK); + response.Headers.Add("Content-Type", "application/json"); + response.Headers.Add("X-Deletion-Manifest-Source", "stored"); + response.Headers.Add("X-Deletion-Manifest-Snapshot-Sha256", snapshotSha); + response.Headers.Add("X-Deletion-Manifest-Has-Progress", (progress != null).ToString()); + var json = JsonSerializer.SerializeToUtf8Bytes(new + { + manifest, + progress, + snapshotSha256 = snapshotSha, + }, DeletionManifestJson.SerializerOptions); + await response.Body.WriteAsync(json, 0, json.Length); + return response; + } + + private static async Task BuildDownloadResponse( + HttpRequestData req, DeletionManifest manifest, string tenantId, string sessionId, string manifestId) + { + var json = JsonSerializer.SerializeToUtf8Bytes(manifest, DeletionManifestJson.SerializerOptions); + byte[] gzipped; + using (var output = new MemoryStream()) + { + using (var gzip = new GZipStream(output, CompressionLevel.Optimal, leaveOpen: true)) + { + await gzip.WriteAsync(json, 0, json.Length); + } + gzipped = output.ToArray(); + } + + var fileName = $"{tenantId}_{sessionId}_{manifestId}.snapshot.json.gz"; + var response = req.CreateResponse(HttpStatusCode.OK); + response.Headers.Add("Content-Type", "application/gzip"); + response.Headers.Add("Content-Disposition", $"attachment; filename=\"{fileName}\""); + await response.Body.WriteAsync(gzipped, 0, gzipped.Length); + return response; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletionsListFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletionsListFunction.cs new file mode 100644 index 00000000..b0351408 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/GetSessionDeletionsListFunction.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading.Tasks; +using System.Web; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.Models.Deletion; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Sessions +{ + /// + /// Cross-tenant read-only listing of sessions currently in a cascade-deletion state. + /// Feeds the Global-Admin "Session Cleanup" page (PR-C) — one call per tab: + /// + /// ?state=Preparing|Queued|Running — In-Flight tab. + /// ?state=Poisoned — Poisoned tab; rows here need an operator restore. + /// ?state=Queued&strandedSinceMinutes=30 — Stranded tab; matches the + /// SessionDeletionStrandedQueued OpsEvent watchdog (plan §10). + /// + /// + /// Cross-partition scan via . + /// Acceptable here for the same reason the 12h maintenance function uses it: the row count + /// for any non-None state is typically < 10 in steady state (poisoned/stranded peak higher + /// during incidents). No pagination — page-size pagination would complicate the cross-state + /// fan-out the UI needs and the absolute upper bound is bounded by per-tenant retention. + /// + /// + /// GA-only enforcement comes from EndpointAccessPolicyCatalog (route registered as + /// GlobalAdminOnly); the function body does not re-check. + /// + /// + public class GetSessionDeletionsListFunction + { + private readonly ILogger _logger; + private readonly TableStorageService _storage; + + public GetSessionDeletionsListFunction( + ILogger logger, + TableStorageService storage) + { + _logger = logger; + _storage = storage; + } + + [Function("GetSessionDeletionsList")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/session-deletions")] + HttpRequestData req) + { + try + { + var query = HttpUtility.ParseQueryString(req.Url.Query ?? string.Empty); + var state = query["state"] ?? string.Empty; + if (!IsAllowedState(state)) + { + var bad = req.CreateResponse(HttpStatusCode.BadRequest); + await bad.WriteAsJsonAsync(new + { + success = false, + message = "Query parameter 'state' must be one of: Preparing, Queued, Running, Poisoned.", + }); + return bad; + } + + int? strandedSinceMinutes = null; + var strandedRaw = query["strandedSinceMinutes"]; + if (!string.IsNullOrEmpty(strandedRaw)) + { + if (state != SessionDeletionState.Queued) + { + var bad = req.CreateResponse(HttpStatusCode.BadRequest); + await bad.WriteAsJsonAsync(new + { + success = false, + message = "strandedSinceMinutes is only valid with state=Queued.", + }); + return bad; + } + if (!int.TryParse(strandedRaw, out var minutes) || minutes <= 0 || minutes > 10080) + { + var bad = req.CreateResponse(HttpStatusCode.BadRequest); + await bad.WriteAsJsonAsync(new + { + success = false, + message = "strandedSinceMinutes must be a positive integer up to 10080 (7 days).", + }); + return bad; + } + strandedSinceMinutes = minutes; + } + + var cutoffUtc = strandedSinceMinutes.HasValue + ? (DateTimeOffset?)DateTimeOffset.UtcNow.AddMinutes(-strandedSinceMinutes.Value) + : null; + + var now = DateTimeOffset.UtcNow; + var sessions = new List(); + await foreach (var entity in _storage.GetSessionsByDeletionStateAsync(state, req.FunctionContext.CancellationToken)) + { + var ts = entity.Timestamp ?? now; + if (cutoffUtc.HasValue && ts > cutoffUtc.Value) continue; + + var ageMinutes = (int)Math.Floor((now - ts).TotalMinutes); + sessions.Add(new + { + tenantId = entity.PartitionKey, + sessionId = entity.RowKey, + deletionState = entity.GetString("DeletionState") ?? state, + manifestId = entity.GetString("PendingDeletionManifestId") ?? string.Empty, + timestamp = ts.UtcDateTime.ToString("o"), + ageMinutes, + }); + } + + return await req.OkAsync(new + { + success = true, + state, + strandedSinceMinutes, + count = sessions.Count, + sessions, + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "GetSessionDeletionsList: unexpected error (state={State})", req.Url.Query); + var err = req.CreateResponse(HttpStatusCode.InternalServerError); + await err.WriteAsJsonAsync(new { success = false, message = "Internal server error" }); + return err; + } + } + + private static bool IsAllowedState(string state) => + state == SessionDeletionState.Preparing + || state == SessionDeletionState.Queued + || state == SessionDeletionState.Running + || state == SessionDeletionState.Poisoned; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RestoreSessionFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RestoreSessionFunction.cs index a2829440..948cc46f 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RestoreSessionFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RestoreSessionFunction.cs @@ -72,44 +72,74 @@ namespace AutopilotMonitor.Functions.Functions.Sessions new { success = false, message = "manifestId is required in the request body." }); } - // Tenant resolution: prefer JWT TargetTenantId, fall back to SessionsIndex lookup - // (Global Admin can target any session by id alone). Mirrors the preview endpoint. + // Tenant resolution. Codex F1 review: an explicit body.TenantId from a GA caller is + // the only path that lets a Global Admin restore a session in a tenant other than + // their JWT home tenant. Order: + // 1. Explicit body.TenantId on GA calls (cross-tenant restore — the common case + // when restoring from the Session Cleanup admin page). + // 2. JWT TargetTenantId (single-tenant operators). + // 3. SessionsIndex lookup keyed by sessionId (rare but preserved). + // 4. body.TenantId again — last-resort fallback for the "Sessions row gone" path + // where neither the index nor the JWT tenant can resolve the target. var requestCtx = req.GetRequestContext(); var actorEmail = TenantHelper.GetUserIdentifier(req); - var tenantId = requestCtx.TargetTenantId; - if (string.IsNullOrEmpty(tenantId) || string.Equals(tenantId, "global", StringComparison.OrdinalIgnoreCase)) + string tenantId; + if (requestCtx.IsGlobalAdmin + && !string.IsNullOrWhiteSpace(body.TenantId) + && Guid.TryParse(body.TenantId, out _)) { - tenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId); + tenantId = body.TenantId!; } - if (string.IsNullOrEmpty(tenantId)) + else { - // Sessions row is gone — but a full restore IS legal in that case. We need a way - // to determine the tenantId without the SessionsIndex (which is also gone). - // The manifest blob path encodes tenantId, but the operator only gives us - // sessionId + manifestId. Require the operator to scope the call by providing - // the tenantId in the request body (or via the X-Target-Tenant header). - if (string.IsNullOrWhiteSpace(body.TenantId)) + tenantId = requestCtx.TargetTenantId; + if (string.IsNullOrEmpty(tenantId) || string.Equals(tenantId, "global", StringComparison.OrdinalIgnoreCase)) { - return await WriteJsonAsync(req, HttpStatusCode.NotFound, new - { - success = false, - message = $"Session {sessionId} not found in any active SessionsIndex entry. " + - "If the cascade completed (Sessions row gone), pass the tenantId explicitly in the request body." - }); + tenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId) ?? string.Empty; + } + if (string.IsNullOrEmpty(tenantId)) + { + // Sessions row is gone — but a full restore IS legal in that case. The + // manifest blob path encodes tenantId, but the operator only gives us + // sessionId + manifestId. Require the operator to scope the call by passing + // the tenantId in the request body (or via the X-Target-Tenant header). + if (string.IsNullOrWhiteSpace(body.TenantId)) + { + return await WriteJsonAsync(req, HttpStatusCode.NotFound, new + { + success = false, + message = $"Session {sessionId} not found in any active SessionsIndex entry. " + + "If the cascade completed (Sessions row gone), pass the tenantId explicitly in the request body." + }); + } + tenantId = body.TenantId!; } - tenantId = body.TenantId; } _logger.LogInformation( "RestoreSession: tenant={TenantId} session={SessionId} manifestId={ManifestId} dryRun={DryRun} actor={Actor}", tenantId, sessionId, body.ManifestId, body.DryRun, actorEmail); + // Trim + cap operator-supplied reason so it can't bloat the audit-row PartitionKey + // payload or be used to smuggle large blobs through. 1024 chars matches the cap the + // handler uses for LastFailureMessage in DeletionProgress (same compliance bucket). + var reason = body.Reason; + if (!string.IsNullOrWhiteSpace(reason)) + { + reason = reason!.Trim(); + if (reason.Length > 1024) reason = reason.Substring(0, 1024); + } + else + { + reason = null; + } + SessionRestoreResult result; try { result = await _restoreService.RestoreAsync( tenantId, sessionId, body.ManifestId, - body.DryRun, actorEmail); + body.DryRun, actorEmail, reason); } catch (Exception ex) { @@ -175,6 +205,14 @@ namespace AutopilotMonitor.Functions.Functions.Sessions /// should provide tenantId explicitly in this scenario. /// public string? TenantId { get; set; } + /// + /// Optional free-text justification. Persisted into the deletion_restored audit + /// row's reason detail so the operator's intent (customer ticket, compliance + /// request, …) is recoverable from the tenant audit trail. Codex follow-up to the + /// PR-B audit consolidation: previously the UI sent this but the backend dropped it. + /// Trimmed + capped at 1024 chars by the function before being passed to the service. + /// + public string? Reason { get; set; } } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index b4b42d2f..a8cabea3 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -215,6 +215,8 @@ public static class EndpointAccessPolicyCatalog new("GET", "sessions/{sessionId}/reducer-verification", EndpointPolicy.GlobalAdminOnly), new("GET", "admin/sessions/{sessionId}/delete/preview", EndpointPolicy.GlobalAdminOnly), new("POST", "admin/sessions/{sessionId}/restore", EndpointPolicy.GlobalAdminOnly), + new("GET", "admin/sessions/{sessionId}/deletion-manifest", EndpointPolicy.GlobalAdminOnly), + new("GET", "global/session-deletions", EndpointPolicy.GlobalAdminOnly), new("GET", "global/raw/sessions", EndpointPolicy.GlobalAdminOnly), new("GET", "global/raw/events", EndpointPolicy.GlobalAdminOnly), new("GET", "global/raw/events/search", EndpointPolicy.GlobalAdminOnly), diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs index 62e5cbb6..d300ac3b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionHandler.cs @@ -42,10 +42,9 @@ namespace AutopilotMonitor.Functions.Services.Deletion internal const int ProgressCasMaxAttempts = 10; internal static readonly TimeSpan ProgressCasMaxWallClock = TimeSpan.FromSeconds(60); - // Cap on the per-table audit-detail `residualKeys` sample — paired with - // CascadeVerificationService.MaxResidualSampleSize. The Dictionary entry - // holds a small JSON array of "{table}/{pk}/{rk}" strings. - internal const int AuditResidualSampleSize = 50; + // Verification-failure sample cap moved to DeletionProgressConstants so the + // log line, the persisted progress-blob field, and the downstream OpsEvent all + // use the same projection. private readonly TableStorageService _storage; private readonly BlobStorageService _blob; @@ -163,7 +162,18 @@ namespace AutopilotMonitor.Functions.Services.Deletion } catch (Exception ex) when (!(ex is OperationCanceledException)) { - await AuditStepFailedAsync(tenantId, sessionId, manifestId, step, ex, cancellationToken); + LogStepFailed(tenantId, sessionId, manifestId, step, ex); + // PR-B Codex F4: persist the failure context into the progress blob BEFORE + // rethrowing so the worker's poison-emit path can include the root cause in + // the SessionDeletionPoisoned OpsEvent. Best-effort — a persist failure is + // logged and swallowed; the original step exception is the one that matters. + (progress, etag) = await TryPersistFailureContextAsync( + tenantId, sessionId, manifestId, progress, etag, + failureType: "step_exception", + failureMessage: $"step {step.Order} ({step.Class}{(string.IsNullOrEmpty(step.Table) ? "" : $", {step.Table}")}): {ex.GetType().Name}: {ex.Message}", + observedResidualCount: null, + residualSampleJson: null, + cancellationToken); throw; } @@ -175,14 +185,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion isAlreadyApplied: p => p.CompletedSteps.Contains(step.Order), cancellationToken); - await _maintenanceRepo.LogAuditEntryAsync( - tenantId, - action: "deletion_step_completed", - entityType: "Session", - entityId: sessionId, - performedBy: "system", - details: BuildStepCompletedDetails(manifestId, step, result, aggregateProcessed, sw.ElapsedMilliseconds - stepStart)) - .ConfigureAwait(false); + // PR-B audit consolidation: step-level progress lives in DeletionProgress + // (CompletedSteps + AggregateDecrementsApplied + TombstoneStarted) — the admin + // Session Cleanup page reads from there. Structured log for App Insights only. + _logger.LogInformation( + "SessionDeletionHandler step complete: tenant={TenantId} session={SessionId} manifestId={ManifestId} stepOrder={StepOrder} class={Class} table={Table} attempted={Attempted} deletedNow={DeletedNow} alreadyMissing={AlreadyMissing} aggregateProcessed={AggregateProcessed} stepDurationMs={StepDurationMs}", + tenantId, sessionId, manifestId, step.Order, step.Class, step.Table ?? string.Empty, + result.Attempted, result.DeletedNow, result.AlreadyMissing, aggregateProcessed, + sw.ElapsedMilliseconds - stepStart); } // (5) Live verification (§1 P4). Outcome dictates whether tombstone runs at all. @@ -191,14 +201,28 @@ namespace AutopilotMonitor.Functions.Services.Deletion var verification = await _verifier.VerifyAsync(manifest, cancellationToken); if (!verification.IsClean) { - await _maintenanceRepo.LogAuditEntryAsync( - tenantId, - action: "deletion_verification_failed", - entityType: "Session", - entityId: sessionId, - performedBy: "system", - details: BuildVerificationFailedDetails(manifestId, verification.Residuals)) - .ConfigureAwait(false); + // PR-B audit consolidation: verification residuals are diagnostic detail for + // operators on the Session Cleanup page (read from the throw → poison path's + // SessionDeletionPoisoned OpsEvent). Tenant admins see deletion_completed + // absence as the signal; no tenant-scope audit row needed. + LogVerificationFailed(tenantId, sessionId, manifestId, verification.Residuals); + + // PR-B Codex F4: persist failure type + residual sample into the progress blob + // so the worker's poison-emit path can include it in the OpsEvent (durable + // operator record), not just structured logs which roll off. + // Codex F2 round-3: the persisted count is the verifier's OBSERVED count — + // CascadeVerificationService stops at MaxResidualSampleSize per table AND at + // the first failing table, so the real residual mountain may be larger. We + // do not pay the cost of an exhaustive count on the hot path because the + // recovery action (poison → operator restore) doesn't need exact magnitudes. + var residualSampleJson = BuildResidualSampleJson(verification.Residuals); + (progress, etag) = await TryPersistFailureContextAsync( + tenantId, sessionId, manifestId, progress, etag, + failureType: "verification_residuals", + failureMessage: $"{verification.Residuals.Count} observed residual row(s); refusing to tombstone (verifier short-circuits at first failing table)", + observedResidualCount: verification.Residuals.Count, + residualSampleJson: residualSampleJson, + cancellationToken); throw new InvalidOperationException( $"Cascade verification found {verification.Residuals.Count} residual row(s) for " + @@ -543,49 +567,70 @@ namespace AutopilotMonitor.Functions.Services.Deletion // ============================================================ Audit details builders ==== - private static Dictionary BuildStepCompletedDetails( - string manifestId, DeletionStep step, DeletionBatchResult tableResult, int aggregateProcessed, long stepDurationMs) + private void LogVerificationFailed( + string tenantId, string sessionId, string manifestId, IReadOnlyList residuals) { - var details = new Dictionary(StringComparer.Ordinal) - { - ["manifestId"] = manifestId, - ["stepOrder"] = step.Order.ToString(System.Globalization.CultureInfo.InvariantCulture), - ["class"] = step.Class, - ["stepDurationMs"] = stepDurationMs.ToString(System.Globalization.CultureInfo.InvariantCulture), - }; - if (!string.IsNullOrEmpty(step.Table)) - { - details["table"] = step.Table!; - } - if (!string.IsNullOrEmpty(step.Step)) - { - details["syntheticStep"] = step.Step!; - } - if (step.Class == DeletionStepClass.Aggregate) - { - details["decrementsProcessed"] = aggregateProcessed.ToString(System.Globalization.CultureInfo.InvariantCulture); - } - else - { - details["attempted"] = tableResult.Attempted.ToString(System.Globalization.CultureInfo.InvariantCulture); - details["deletedNow"] = tableResult.DeletedNow.ToString(System.Globalization.CultureInfo.InvariantCulture); - details["alreadyMissing"] = tableResult.AlreadyMissing.ToString(System.Globalization.CultureInfo.InvariantCulture); - } - return details; + _logger.LogError( + "SessionDeletionHandler verification failed: tenant={TenantId} session={SessionId} manifestId={ManifestId} residualCount={ResidualCount} sampleKeys={SampleKeys}", + tenantId, sessionId, manifestId, residuals.Count, BuildResidualSampleJson(residuals)); } - private static Dictionary BuildVerificationFailedDetails( - string manifestId, IReadOnlyList residuals) + /// + /// JSON-encoded sample of residual table/pk/rk triples, capped at + /// . Shared by the + /// log-only sink (App Insights) and the durable progress-blob sink that feeds the + /// SessionDeletionPoisoned OpsEvent. + /// + internal static string BuildResidualSampleJson(IReadOnlyList residuals) { var sample = residuals - .Take(AuditResidualSampleSize) + .Take(DeletionProgressConstants.VerificationResidualSampleSize) .Select(r => new { table = r.Table, pk = r.Pk, rk = r.Rk }); - return new Dictionary(StringComparer.Ordinal) + return JsonSerializer.Serialize(sample); + } + + /// + /// Persists / + /// / to the progress blob via the + /// same ETag-CAS path the other progress mutations use. Best-effort — a CAS failure here + /// is logged and swallowed because the caller's next step is to rethrow the underlying + /// failure, and burying that exception under "progress write failed" would obscure the + /// real cause. Returns the latest (progress, etag) tuple so the caller can keep the + /// captured ETag in sync for any further progress writes downstream (the verification + /// path does no further writes; the step-exception path does none either before throw). + /// Failure messages are clipped to 1024 chars before persistence. + /// + private async Task<(DeletionProgress, string)> TryPersistFailureContextAsync( + string tenantId, string sessionId, string manifestId, + DeletionProgress progress, string etag, + string failureType, string failureMessage, + int? observedResidualCount, string? residualSampleJson, + CancellationToken ct) + { + var clippedMessage = failureMessage.Length > 1024 ? failureMessage.Substring(0, 1024) : failureMessage; + try { - ["manifestId"] = manifestId, - ["residualCount"] = residuals.Count.ToString(System.Globalization.CultureInfo.InvariantCulture), - ["residualKeys"] = JsonSerializer.Serialize(sample), - }; + return await UpdateProgressWithRetryAsync( + tenantId, sessionId, manifestId, progress, etag, + mutate: p => + { + p.LastFailureType = failureType; + p.LastFailureMessage = clippedMessage; + p.LastObservedResidualCount = observedResidualCount; + p.LastResidualSampleJson = residualSampleJson; + }, + isAlreadyApplied: p => + string.Equals(p.LastFailureType, failureType, StringComparison.Ordinal) + && string.Equals(p.LastFailureMessage, clippedMessage, StringComparison.Ordinal), + ct); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "SessionDeletionHandler: failed to persist failure context (tenant={TenantId} session={SessionId} manifestId={ManifestId} failureType={FailureType}) — proceeding with rethrow", + tenantId, sessionId, manifestId, failureType); + return (progress, etag); + } } private static Dictionary BuildCompletedDetails( @@ -602,27 +647,16 @@ namespace AutopilotMonitor.Functions.Services.Deletion }; } - private Task AuditStepFailedAsync( - string tenantId, string sessionId, string manifestId, DeletionStep step, Exception ex, CancellationToken _) + private void LogStepFailed( + string tenantId, string sessionId, string manifestId, DeletionStep step, Exception ex) { - var details = new Dictionary(StringComparer.Ordinal) - { - ["manifestId"] = manifestId, - ["stepOrder"] = step.Order.ToString(System.Globalization.CultureInfo.InvariantCulture), - ["class"] = step.Class, - ["exceptionType"] = ex.GetType().FullName ?? "Unknown", - // Trim error message to a reasonable size for the audit row. - ["error"] = ex.Message.Length > 1024 ? ex.Message.Substring(0, 1024) : ex.Message, - }; - if (!string.IsNullOrEmpty(step.Table)) details["table"] = step.Table!; - - return _maintenanceRepo.LogAuditEntryAsync( - tenantId, - action: "deletion_step_failed", - entityType: "Session", - entityId: sessionId, - performedBy: "system", - details: details); + // PR-B audit consolidation: step failures bubble up to the worker → poison path → + // SessionDeletionPoisoned OpsEvent. Diagnostic detail goes to App Insights so the + // Session Cleanup admin page (or a kusto query) can drill in by manifestId. + _logger.LogError(ex, + "SessionDeletionHandler step failed: tenant={TenantId} session={SessionId} manifestId={ManifestId} stepOrder={StepOrder} class={Class} table={Table} exceptionType={ExceptionType}", + tenantId, sessionId, manifestId, step.Order, step.Class, step.Table ?? string.Empty, + ex.GetType().FullName ?? "Unknown"); } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionWorker.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionWorker.cs index 9d9c8db8..0af95d97 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionWorker.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionDeletionWorker.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Azure.Identity; @@ -7,7 +6,6 @@ using Azure.Storage.Queues; using Azure.Storage.Queues.Models; using AutopilotMonitor.Functions.Services; using AutopilotMonitor.Shared; -using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models.Deletion; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; @@ -30,10 +28,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion /// calls UpdateMessageAsync every to extend the /// message visibility by . Prevents queue re-delivery /// from spawning a parallel worker on the same manifest while a step is in flight. - /// Poison audit: max-dequeue exhaustion writes a deletion_poisoned - /// audit entry alongside the poison-queue move so the operator can correlate the - /// poisoned cascade with its manifest. The Sessions row's DeletionState is NOT - /// auto-cleared — operator action via restore-from-poisoned (§13, PR4b) is required. + /// Poison OpsEvent: max-dequeue exhaustion records a + /// SessionDeletionPoisoned OpsEvent alongside the poison-queue move so the + /// operator can correlate the poisoned cascade with its manifest (and wire Telegram + /// routing). PR-B audit consolidation: this used to be a per-tenant + /// deletion_poisoned audit but tenant admins should only see the lifecycle + /// endpoints (started / completed / restored), so the signal moved to OpsEvents. + /// The Sessions row's DeletionState is NOT auto-cleared — operator action via + /// restore-from-poisoned (§13, PR4b) is required. /// /// /// The worker is NOT registered in Program.cs in PR4 — that wires up in PR5 @@ -81,7 +83,8 @@ namespace AutopilotMonitor.Functions.Services.Deletion private readonly SessionDeletionHandler _handler; private readonly TableStorageService _storage; private readonly AdminConfigurationService _adminConfig; - private readonly IMaintenanceRepository _maintenanceRepo; + private readonly BlobStorageService _blob; + private readonly OpsEventService _opsEvents; private readonly ILogger _logger; public SessionDeletionWorker( @@ -89,13 +92,15 @@ namespace AutopilotMonitor.Functions.Services.Deletion SessionDeletionHandler handler, TableStorageService storage, AdminConfigurationService adminConfig, - IMaintenanceRepository maintenanceRepo, + BlobStorageService blob, + OpsEventService opsEvents, ILogger logger) { _handler = handler ?? throw new ArgumentNullException(nameof(handler)); _storage = storage ?? throw new ArgumentNullException(nameof(storage)); _adminConfig = adminConfig ?? throw new ArgumentNullException(nameof(adminConfig)); - _maintenanceRepo = maintenanceRepo ?? throw new ArgumentNullException(nameof(maintenanceRepo)); + _blob = blob ?? throw new ArgumentNullException(nameof(blob)); + _opsEvents = opsEvents ?? throw new ArgumentNullException(nameof(opsEvents)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _heartbeatInterval = DefaultHeartbeatInterval; _pollInterval = DefaultPollInterval; @@ -147,7 +152,8 @@ namespace AutopilotMonitor.Functions.Services.Deletion SessionDeletionHandler handler, TableStorageService storage, AdminConfigurationService adminConfig, - IMaintenanceRepository maintenanceRepo, + BlobStorageService blob, + OpsEventService opsEvents, ILogger logger, TimeSpan? heartbeatInterval = null, TimeSpan? pollInterval = null) @@ -157,7 +163,8 @@ namespace AutopilotMonitor.Functions.Services.Deletion _handler = handler ?? throw new ArgumentNullException(nameof(handler)); _storage = storage ?? throw new ArgumentNullException(nameof(storage)); _adminConfig = adminConfig ?? throw new ArgumentNullException(nameof(adminConfig)); - _maintenanceRepo = maintenanceRepo ?? throw new ArgumentNullException(nameof(maintenanceRepo)); + _blob = blob ?? throw new ArgumentNullException(nameof(blob)); + _opsEvents = opsEvents ?? throw new ArgumentNullException(nameof(opsEvents)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _heartbeatInterval = heartbeatInterval ?? DefaultHeartbeatInterval; _pollInterval = pollInterval ?? DefaultPollInterval; @@ -358,31 +365,203 @@ namespace AutopilotMonitor.Functions.Services.Deletion if (envelope != null && !string.IsNullOrEmpty(envelope.TenantId) && !string.IsNullOrEmpty(envelope.SessionId)) { + // PR-B audit consolidation: poisoned cascades surface as a global-scope OpsEvent + // (operator-bound, Telegram-routable) rather than a per-tenant audit. Tenant admins + // observe deletion via the lifecycle audits (started / completed / restored). + // + // Codex F4 follow-up: pull the handler's last-failure breadcrumb out of the + // DeletionProgress blob so the OpsEvent carries the root cause (failureType + + // residual sample), not just queue-side metadata. Blob read is best-effort — + // a missing blob, transient 5xx, or pre-followup progress without LastFailure* + // fields still produces a useful OpsEvent, just without root-cause data. + string? failureType = null; + string? failureMessage = null; + int? observedResidualCount = null; + string? residualSamplePreviewJson = null; + if (!string.IsNullOrEmpty(envelope.ManifestId)) + { + try + { + var (progress, _) = await _blob.DownloadDeletionProgressAsync( + envelope.TenantId, envelope.SessionId, envelope.ManifestId!, ct) + .ConfigureAwait(false); + failureType = string.IsNullOrEmpty(progress.LastFailureType) ? null : progress.LastFailureType; + failureMessage = string.IsNullOrEmpty(progress.LastFailureMessage) ? null : progress.LastFailureMessage; + // Codex F2 round-3: this is the verifier's OBSERVED count + // (CascadeVerificationService caps at MaxResidualSampleSize per table AND + // short-circuits after the first failing table — the real residual count + // may be larger). For pre-followup progress blobs lacking the field, fall + // back to the sample-array length so the OpsEvent still carries a number. + observedResidualCount = progress.LastObservedResidualCount + ?? ExtractResidualSampleArrayLength(progress.LastResidualSampleJson); + // Codex F2 round-2: shrink the residual sample further before embedding + // it in the OpsEvent. The OpsEvents table truncates Details at 4096 chars + // (TableOpsEventRepository.cs); the full 50-entry sample plus the other + // payload fields can blow past that and leave the JSON mid-string-corrupt + // — operators end up reading garbled raw text instead of structured data. + // The Session Cleanup admin page already shows the full sample by reading + // the progress blob directly, so the OpsEvent only needs a preview. + residualSamplePreviewJson = ShrinkResidualSampleForOpsEvent(progress.LastResidualSampleJson); + } + catch (Exception blobEx) + { + _logger.LogWarning(blobEx, + "SessionDeletionWorker: failed to read DeletionProgress for poison enrichment (tenant={Tenant} session={Session} manifestId={Manifest}) — OpsEvent will lack root-cause data", + envelope.TenantId, envelope.SessionId, envelope.ManifestId); + } + } + try { - await _maintenanceRepo.LogAuditEntryAsync( + await _opsEvents.RecordSessionDeletionPoisonedAsync( envelope.TenantId, - action: "deletion_poisoned", - entityType: "Session", - entityId: envelope.SessionId, - performedBy: "system", - details: new Dictionary(StringComparer.Ordinal) - { - ["manifestId"] = envelope.ManifestId ?? string.Empty, - ["reason"] = envelope.Reason ?? string.Empty, - ["messageId"] = msg.MessageId, - ["dequeueCount"] = (msg.DequeueCount - 1).ToString(System.Globalization.CultureInfo.InvariantCulture), - }).ConfigureAwait(false); + envelope.SessionId, + envelope.ManifestId ?? string.Empty, + envelope.Reason ?? string.Empty, + msg.MessageId, + (int)(msg.DequeueCount - 1), + failureType, + failureMessage, + observedResidualCount, + residualSamplePreviewJson).ConfigureAwait(false); } - catch (Exception auditEx) + catch (Exception opsEx) { - _logger.LogError(auditEx, - "SessionDeletionWorker: deletion_poisoned audit log write failed for tenant={Tenant} session={Session}", + _logger.LogError(opsEx, + "SessionDeletionWorker: SessionDeletionPoisoned OpsEvent write failed for tenant={Tenant} session={Session}", envelope.TenantId, envelope.SessionId); } } } + /// + /// Back-compat fallback used only when + /// is missing (progress blobs written before the field was introduced). Parses the JSON + /// sample array length and uses it as a best-effort substitute for the observed count. + /// Returns null on parse errors or missing data. + /// + internal static int? ExtractResidualSampleArrayLength(string? residualSampleJson) + { + if (string.IsNullOrEmpty(residualSampleJson)) return null; + try + { + using var doc = System.Text.Json.JsonDocument.Parse(residualSampleJson!); + if (doc.RootElement.ValueKind == System.Text.Json.JsonValueKind.Array) + { + return doc.RootElement.GetArrayLength(); + } + } + catch (System.Text.Json.JsonException) + { + // Tolerate malformed payloads — the breadcrumb itself still goes through. + } + return null; + } + + /// + /// Trims the progress-blob's residual sample for safe embedding in the + /// SessionDeletionPoisoned OpsEvent's Details column. Three-layer defense + /// against the 4096-char Azure Table truncation discovered by Codex review: + /// + /// Take at most + /// entries (drop tail, the rest stays in the progress blob for the admin UI). + /// Trim each table / pk / rk to + /// + /// with a trailing … marker — a single composite key can be 200+ chars and + /// would otherwise consume most of the budget on its own. + /// If the serialized JSON still exceeds + /// , + /// drop trailing entries until it fits. Worst case (all entries too long) returns + /// "[]"; operators still get failureType + observedResidualCount and can + /// fetch the full sample from the progress blob. + /// + /// Returns null when there's nothing to embed or the input cannot be parsed. + /// + internal static string? ShrinkResidualSampleForOpsEvent(string? fullSampleJson) + { + if (string.IsNullOrEmpty(fullSampleJson)) return null; + + List entries; + try + { + using var doc = System.Text.Json.JsonDocument.Parse(fullSampleJson!); + if (doc.RootElement.ValueKind != System.Text.Json.JsonValueKind.Array) + { + return null; + } + entries = new List(); + var max = AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualSamplePreviewSize; + foreach (var element in doc.RootElement.EnumerateArray()) + { + if (entries.Count >= max) break; + if (element.ValueKind != System.Text.Json.JsonValueKind.Object) continue; + entries.Add(new ResidualPreviewEntry + { + Table = TrimResidualKeyField(ReadStringProperty(element, "table")), + Pk = TrimResidualKeyField(ReadStringProperty(element, "pk")), + Rk = TrimResidualKeyField(ReadStringProperty(element, "rk")), + }); + } + } + catch (System.Text.Json.JsonException) + { + return null; + } + + // Total-budget pass: serialize, then drop trailing entries until under the cap. + // Worst case (one over-budget entry) ends at "[]" — still valid JSON, and the full + // sample remains in the progress blob. + var budget = AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualPreviewBudgetChars; + while (true) + { + var serialized = SerializeResidualPreview(entries); + if (serialized.Length <= budget || entries.Count == 0) return serialized; + entries.RemoveAt(entries.Count - 1); + } + } + + private static string SerializeResidualPreview(List entries) + { + using var stream = new System.IO.MemoryStream(); + using (var writer = new System.Text.Json.Utf8JsonWriter(stream)) + { + writer.WriteStartArray(); + foreach (var e in entries) + { + writer.WriteStartObject(); + writer.WriteString("table", e.Table); + writer.WriteString("pk", e.Pk); + writer.WriteString("rk", e.Rk); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + } + return System.Text.Encoding.UTF8.GetString(stream.ToArray()); + } + + private static string TrimResidualKeyField(string? raw) + { + if (string.IsNullOrEmpty(raw)) return string.Empty; + var max = AutopilotMonitor.Shared.Models.Deletion.DeletionProgressConstants.OpsEventResidualKeyMaxChars; + if (raw!.Length <= max) return raw; + // -1 makes room for the trailing ellipsis so the marked-truncated string fits in + // the same budget as a max-length untrimmed one. + return raw.Substring(0, max - 1) + "…"; + } + + private static string? ReadStringProperty(System.Text.Json.JsonElement obj, string propertyName) + { + if (!obj.TryGetProperty(propertyName, out var value)) return null; + return value.ValueKind == System.Text.Json.JsonValueKind.String ? value.GetString() : null; + } + + private struct ResidualPreviewEntry + { + public string Table; + public string Pk; + public string Rk; + } + private async Task SafeDeleteAsync(QueueMessage msg, CancellationToken ct) => await SafeDeleteAsync(msg.MessageId, msg.PopReceipt, ct).ConfigureAwait(false); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRestoreService.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRestoreService.cs index 0ccd031b..02ca6c98 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRestoreService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRestoreService.cs @@ -90,6 +90,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion public virtual async Task RestoreAsync( string tenantId, string sessionId, string manifestId, bool dryRun, string actor, + string? operatorReason = null, CancellationToken ct = default) { if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("tenantId is required", nameof(tenantId)); @@ -100,8 +101,13 @@ namespace AutopilotMonitor.Functions.Services.Deletion var sw = Stopwatch.StartNew(); var result = new SessionRestoreResult(); - // (1) §13.1 — audit BEFORE any read. - await AuditManifestDownloadedAsync(tenantId, sessionId, manifestId, actor, ct).ConfigureAwait(false); + // (1) PR-B audit consolidation: deletion_manifest_downloaded was an internal-step + // breadcrumb that doubled the tenant audit row count without adding signal. The + // operator-relevant outcome — deletion_restored — already records the actor + + // manifestId once the restore completes (or surfaces failure via the throw path). + _logger.LogInformation( + "SessionRestoreService: starting restore for tenant={TenantId} session={SessionId} manifestId={ManifestId} actor={Actor}", + tenantId, sessionId, manifestId, actor); // (2) Download manifest (SHA-verified) + progress. PR4c F6: use the with-Sha variant // and verify the snapshot↔progress binding once both blobs are loaded (plan §3). @@ -207,10 +213,14 @@ namespace AutopilotMonitor.Functions.Services.Deletion } // (4) Dry-run path: simulate by counting what each step would restore, no writes. + // Codex follow-up: report the AUTO-SELECTED mode (full / partial) here so the admin + // UI can show "Mode (auto-selected): partial" before the real run. The Outcome of + // DryRunOk is the dry-run signal — using mode="dryRun" hid the operator-visible + // mode preview the dialog was built to surface. if (dryRun) { result.Outcome = SessionRestoreOutcome.DryRunOk; - result.Mode = "dryRun"; + result.Mode = mode; foreach (var step in manifest.Steps) { if (step.Class == DeletionStepClass.Aggregate || step.Class == DeletionStepClass.Final) continue; @@ -265,7 +275,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion result.DurationMs = sw.ElapsedMilliseconds; // (6) Audit completion. - await AuditRestoredAsync(tenantId, sessionId, manifestId, mode, actor, result, ct).ConfigureAwait(false); + await AuditRestoredAsync(tenantId, sessionId, manifestId, mode, actor, operatorReason, result, ct).ConfigureAwait(false); _logger.LogInformation( "SessionRestoreService completed: tenant={Tenant} session={Session} manifestId={ManifestId} mode={Mode} durationMs={Duration}", @@ -578,40 +588,31 @@ namespace AutopilotMonitor.Functions.Services.Deletion map[key] = map.TryGetValue(key, out var c) ? c + delta : delta; } - private async Task AuditManifestDownloadedAsync(string tenantId, string sessionId, string manifestId, string actor, CancellationToken ct) - { - await _maintenanceRepo.LogAuditEntryAsync( - tenantId, - action: "deletion_manifest_downloaded", - entityType: "Session", - entityId: sessionId, - performedBy: actor, - details: new Dictionary(StringComparer.Ordinal) - { - ["manifestId"] = manifestId, - ["reason"] = "restore_attempt", - }).ConfigureAwait(false); - } - private async Task AuditRestoredAsync( string tenantId, string sessionId, string manifestId, string mode, string actor, + string? operatorReason, SessionRestoreResult result, CancellationToken ct) { + var details = new Dictionary(StringComparer.Ordinal) + { + ["manifestId"] = manifestId, + ["mode"] = mode, + ["rowsRestoredByTable"] = JsonSerializer.Serialize(result.RowsRestoredByTable), + ["rowsSkippedByTable"] = JsonSerializer.Serialize(result.RowsSkippedByTable), + ["inventoryReIncrements"] = result.InventoryReIncrements.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["durationMs"] = result.DurationMs.ToString(System.Globalization.CultureInfo.InvariantCulture), + }; + if (!string.IsNullOrEmpty(operatorReason)) + { + details["reason"] = operatorReason!; + } await _maintenanceRepo.LogAuditEntryAsync( tenantId, action: "deletion_restored", entityType: "Session", entityId: sessionId, performedBy: actor, - details: new Dictionary(StringComparer.Ordinal) - { - ["manifestId"] = manifestId, - ["mode"] = mode, - ["rowsRestoredByTable"] = JsonSerializer.Serialize(result.RowsRestoredByTable), - ["rowsSkippedByTable"] = JsonSerializer.Serialize(result.RowsSkippedByTable), - ["inventoryReIncrements"] = result.InventoryReIncrements.ToString(System.Globalization.CultureInfo.InvariantCulture), - ["durationMs"] = result.DurationMs.ToString(System.Globalization.CultureInfo.InvariantCulture), - }).ConfigureAwait(false); + details: details).ConfigureAwait(false); } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index eb545887..4a7371e0 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -98,6 +98,54 @@ namespace AutopilotMonitor.Functions.Services $"Session {sessionId} stuck in DeletionState=Queued since {queuedSince:o} (manifestId={manifestId})", tenantId, "System.Maintenance", new { tenantId, sessionId, queuedSince = queuedSince.ToString("o"), manifestId }); + /// + /// Cascade max-dequeue exhaustion (PR-B follow-up): the worker has moved the envelope to + /// the poison queue and CAS-transitioned the Sessions row to . + /// Replaces the prior deletion_poisoned tenant audit — tenant admins see only the + /// lifecycle endpoints (deletion_started, deletion_completed, deletion_restored), + /// while operators get this OpsEvent for the Session Cleanup admin page + Telegram routing. + /// + /// Codex follow-ups F4 + F2: / + /// / / + /// are populated from fields the handler writes before + /// throwing. Worker reads the progress blob in its poison path and passes whatever is + /// present (all are nullable so a worker that pre-dates the progress-schema bump still + /// emits a useful event, just without root-cause data). + /// + /// + /// is the verifier's observed count, not + /// the true total: CascadeVerificationService caps at + /// rows per table + /// and short-circuits after the first failing table. Operators reading this number should + /// treat it as a lower bound, especially when it equals the cap. + /// + /// + /// is a small (≤ + /// ) preview that + /// fits under the OpsEvents table's 4096-char Details truncation. The full progress-blob + /// sample (up to + /// entries) is available via the Session Cleanup admin page's stored-manifest modal. + /// + /// + public Task RecordSessionDeletionPoisonedAsync( + string tenantId, string sessionId, string manifestId, string reason, string messageId, int dequeueCount, + string? failureType = null, string? failureMessage = null, + int? observedResidualCount = null, string? residualSamplePreviewJson = null) + { + var cause = !string.IsNullOrEmpty(failureType) + ? $" — cause: {failureType}{(string.IsNullOrEmpty(failureMessage) ? "" : $" ({failureMessage})")}" + : string.Empty; + return WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionPoisoned", OpsEventSeverity.Error, + $"Session {sessionId} cascade poisoned after {dequeueCount} attempts (manifestId={manifestId}){cause}", + tenantId, "System.Maintenance", + new + { + tenantId, sessionId, manifestId, reason, messageId, dequeueCount, + failureType, failureMessage, observedResidualCount, + residualSamplePreviewJson, + }); + } + /// /// Successful end of a /// run — records the per-block totals so dashboards can fold the cadence into the timeline. diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionProgress.cs b/src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionProgress.cs index 87dc4428..8d2087f1 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionProgress.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Deletion/DeletionProgress.cs @@ -59,5 +59,91 @@ namespace AutopilotMonitor.Shared.Models.Deletion /// behaviour for genuine bugs (Sessions row removed outside the cascade). /// public bool TombstoneStarted { get; set; } + + // ============================================================ PR-B follow-up ==== + // Codex F4 (Medium): PR-B routed step / verification failures out of the per-tenant + // audit (intentional — tenants don't need them), but the SessionDeletionPoisoned + // OpsEvent was left with only queue-side data (dequeueCount + manifestId). The actual + // root cause was only visible in App Insights structured logs, which is a problem for + // long-tail operator forensics (logs roll off, the OpsEvent is the durable record). + // These three nullable fields are written by the handler immediately before it throws + // and read by the worker when it transitions the cascade to Poisoned, so the + // SessionDeletionPoisoned OpsEvent carries the failure context that explains WHY. + // + // All three are nullable / default-empty so blobs written by pre-PR-B-followup workers + // deserialize cleanly. The worker's poison-emit path defends against missing values + // with explicit string.IsNullOrEmpty checks before attaching them to the OpsEvent. + + /// Classification of the most recent failure that triggered a re-queue. Examples: + /// "verification_residuals", "step_exception". Null until the first failure. + public string? LastFailureType { get; set; } + + /// Short, human-readable description of the most recent failure (truncated to + /// 1024 chars by the writer). Suitable for OpsEvent embedding and operator triage. + public string? LastFailureMessage { get; set; } + + /// + /// Verification-failure path only: how many residual rows the verifier observed + /// before short-circuiting. This is NOT guaranteed to be the true residual count + /// — CascadeVerificationService stops both at + /// per table AND + /// after the first failing table, so the real count may be higher. Operators treating + /// this number as a blast-radius estimate should add "≥" mentally when it equals the cap. + /// Null on step-exception failures and on pre-followup progress blobs. + /// + public int? LastObservedResidualCount { get; set; } + + /// + /// Verification-failure path only: JSON-encoded sample of residual {table, pk, rk} + /// triples (capped at ). + /// Empty/null on step-exception failures. The array length always matches + /// for blobs written by the current handler + /// because the verifier and the sample share the same cap; the two fields existed + /// historically because the design anticipated decoupling them when the verifier becomes + /// exhaustive. + /// + public string? LastResidualSampleJson { get; set; } + } + + /// Constants relating to serialization caps. + public static class DeletionProgressConstants + { + /// + /// Cap on the verification residual sample written into the progress blob. Matches the + /// in-memory log cap so the two payloads stay aligned. The progress blob has no size + /// budget worth worrying about; this is the size operators see when they fetch the + /// stored manifest from the admin Session Cleanup page. + /// + public const int VerificationResidualSampleSize = 50; + + /// + /// Maximum entries kept in the residual preview the worker embeds in the + /// SessionDeletionPoisoned OpsEvent. The OpsEvents table truncates the entire + /// Details column at 4096 chars (TableOpsEventRepository.cs); the full + /// 50-entry sample plus the other JSON-payload fields can blow past that and leave the + /// OpsEvent details mid-string-corrupt, which makes the admin UI fall back to raw text + /// rendering. The OpsEvent is a Telegram-routable summary, not the forensic record — + /// operators get the full sample from DeletionProgress.LastResidualSampleJson + /// via the Session Cleanup page's stored-manifest modal. + /// + public const int OpsEventResidualSamplePreviewSize = 5; + + /// + /// Per-field length cap (table / pk / rk) inside the OpsEvent residual preview. Without + /// this, an entry with a 200-char composite PK could on its own consume most of the + /// 4096-char Details budget. Anything trimmed gets a trailing … marker so the + /// truncation is visible to operators reading the OpsEvent. + /// + public const int OpsEventResidualKeyMaxChars = 96; + + /// + /// Total character budget for the OpsEvent residual preview JSON (before it gets + /// JSON-string-escaped into the outer Details column). Sized so that, after + /// escape inflation (~1.5×) + the other Details fields (failureMessage + tenantId + + /// manifestId + …, ~1500 chars), the resulting Details string stays comfortably + /// under the 4096-char Azure Table column cap. If the per-entry trims still leave the + /// JSON over budget, trailing entries are dropped one at a time until it fits. + /// + public const int OpsEventResidualPreviewBudgetChars = 1200; } } diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index 440668d1..5a26ea9d 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -24,6 +24,10 @@ const OPS_EVENT_TYPES: Record = { "SessionDeletionMaintenanceFailed", "SessionDeletionMaintenanceFanoutSkipped", "SessionDeletionStrandedQueued", + // PR-B audit consolidation: the per-session deletion_poisoned tenant-audit moved here so + // tenant admins see only the lifecycle endpoints (deletion_started/completed/restored). + // Operators wire Telegram on this event + read it in the Session Cleanup admin page. + "SessionDeletionPoisoned", ], Security: [ "DeviceBlocked", diff --git a/src/Web/autopilot-monitor-web/app/admin/components/TenantManagementSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/TenantManagementSection.tsx index 67d4d4c6..dee78e75 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/TenantManagementSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/TenantManagementSection.tsx @@ -21,6 +21,7 @@ export interface TenantConfiguration { bootstrapTokenEnabled?: boolean; unrestrictedModeEnabled?: boolean; dataRetentionDays: number; + enableCascadeDeleteV2?: boolean; sessionTimeoutHours: number; planTier?: string; } @@ -351,6 +352,14 @@ export function TenantManagementSection({ Ready )} + {tenant.enableCascadeDeleteV2 && ( + + Cascade V2 + + )}
+
+ +
+ {loading &&

Loading stored manifest…

} + {error && ( +
+ {error} +
+ )} + {!loading && !error && summary && ( + <> +
+ + + + +
+ + {progress && ( +
+

Worker progress

+
+ Completed steps: {completedCount} + Verification done: {progress.verificationDone ? "yes" : "no"} + Tombstone started: {progress.tombstoneStarted ? "yes" : "no"} + Aggregate decrements: {progress.aggregateDecrementsApplied ?? 0} +
+ {progress.lastFailureType && ( +
+
Last failure: {progress.lastFailureType}
+ {progress.lastFailureMessage && ( +
{progress.lastFailureMessage}
+ )} + {failureSample && failureSample.length > 0 && (() => { + const sampleSize = failureSample.length; + const observed = progress.lastObservedResidualCount ?? sampleSize; + const capHit = observed >= VERIFIER_OBSERVATION_CAP; + return ( +
+ + Residual sample ({sampleSize} of {observed}{capHit ? "+" : ""} observed row{observed === 1 ? "" : "s"}) + +
    + {failureSample.slice(0, 20).map((r, i) => ( +
  • {r.table}: {r.pk} / {r.rk}
    • + ))} +
+ {capHit && ( +

+ The verifier short-circuits at {VERIFIER_OBSERVATION_CAP} rows per + table and after the first failing table — the real residual count + may be higher. Use the cascade-restore path to recover; do not rely + on this number as an exact blast-radius estimate. +

+ )} +
+ ); + })()} +
+ )} +
+ )} + + {summary.preflightCounts && ( +
+

Rows in snapshot (by table)

+
+ + + + + + + + + {Object.entries(summary.preflightCounts).map(([table, count]) => ( + + + + + ))} + +
TableCount
{table}{count}
+
+
+ )} + + )} +
+ +
+ + +
+ + + ); +} + +function Field({ label, value, mono }: { label: string; value: string; mono?: boolean }) { + return ( +
+

{label}

+

{value}

+
+ ); +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / 1024 / 1024).toFixed(2)} MB`; +} diff --git a/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/RestoreConfirmDialog.tsx b/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/RestoreConfirmDialog.tsx new file mode 100644 index 00000000..4f578113 --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/components/RestoreConfirmDialog.tsx @@ -0,0 +1,180 @@ +"use client"; + +import { useState } from "react"; +import { api } from "@/lib/api"; +import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch"; + +interface RestoreConfirmDialogProps { + tenantId: string; + sessionId: string; + manifestId: string; + onClose: () => void; + onRestored: () => void; + getAccessToken: (forceRefresh?: boolean) => Promise; +} + +export function RestoreConfirmDialog({ + tenantId, + sessionId, + manifestId, + onClose, + onRestored, + getAccessToken, +}: RestoreConfirmDialogProps) { + const [reason, setReason] = useState(""); + const [dryRun, setDryRun] = useState(true); + const [submitting, setSubmitting] = useState(false); + const [error, setError] = useState(null); + const [outcome, setOutcome] = useState<{ outcome: string; mode: string | null; dryRun: boolean } | null>(null); + + const handleSubmit = async () => { + setSubmitting(true); + setError(null); + setOutcome(null); + try { + const resp = await authenticatedFetch( + api.sessionDeletions.restore(sessionId), + getAccessToken, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + // Codex F1: explicit tenantId is mandatory for cross-tenant GA restores. + // Codex follow-up: backend selects the restore mode automatically based on the + // cascade state (Full / Partial); operators cannot — and must not — force a Full + // restore against a Poisoned session or vice versa. The reason is persisted into + // the deletion_restored audit row as `details.reason`. + body: JSON.stringify({ + tenantId, + manifestId, + reason: reason.trim() || null, + dryRun, + }), + }, + ); + const body = await resp.json().catch(() => ({})); + if (!resp.ok) { + throw new Error(body?.message ?? `Restore failed: HTTP ${resp.status}`); + } + setOutcome({ + outcome: body?.outcome ?? "Completed", + mode: typeof body?.mode === "string" ? body.mode : null, + dryRun, + }); + if (!dryRun) { + // Real restore done — let the parent refresh its list. + onRestored(); + } + } catch (err) { + if (err instanceof TokenExpiredError) { + setError("Session expired; reload the page and try again."); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setSubmitting(false); + } + }; + + return ( +
+
e.stopPropagation()} + > +
+
+ + + +
+
+

Restore session from snapshot

+

{tenantId} / {sessionId}

+
+
+ +
+
+

+ Mode is selected automatically based on the cascade state at execution time: +

+
    +
  • Partial — Poisoned cascade: replays only the rows the worker had already removed and re-increments inventory counters.
    • +
  • Full — Completed cascade (Sessions row gone): inserts every row from the snapshot and resets DeletionState=None.
    • +
+

+ Forcing the wrong mode would corrupt state, so the choice is not exposed. Run with dry-run first to confirm the predicted mode. +

+
+ +
+ + setReason(e.target.value)} + placeholder="e.g. customer support — accidental delete" + maxLength={200} + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded text-sm text-gray-900 dark:text-white bg-white dark:bg-gray-900 focus:outline-none focus:ring-2 focus:ring-blue-500" + /> +

+ Stored on the deletion_restored audit row. +

+
+ + + + {outcome && ( +
+ Outcome: {outcome.outcome} + {outcome.mode && ( + <> · Mode (auto-selected): {outcome.mode} + )} + {outcome.dryRun ? " · dry-run — no rows written" : ""} +
+ )} + {error && ( +
+ {error} +
+ )} +
+ +
+ + +
+
+
+ ); +} diff --git a/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/page.tsx b/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/page.tsx new file mode 100644 index 00000000..03d84f2f --- /dev/null +++ b/src/Web/autopilot-monitor-web/app/admin/ops/session-cleanup/page.tsx @@ -0,0 +1,506 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import Link from "next/link"; +import { api } from "@/lib/api"; +import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch"; +import { useAdminConfig } from "../../AdminConfigContext"; +import { AdminNotifications } from "../../AdminNotifications"; +import { DeletionPreviewModal } from "./components/DeletionPreviewModal"; +import { RestoreConfirmDialog } from "./components/RestoreConfirmDialog"; + +type CascadeState = "Preparing" | "Queued" | "Running" | "Poisoned"; + +interface SessionDeletionRow { + tenantId: string; + sessionId: string; + deletionState: CascadeState | string; + manifestId: string; + timestamp: string; + ageMinutes: number; +} + +interface SessionDeletionListResponse { + success: boolean; + state: string; + strandedSinceMinutes: number | null; + count: number; + sessions: SessionDeletionRow[]; +} + +type Tab = "in-flight" | "poisoned" | "stranded" | "activity"; + +const STRANDED_THRESHOLD_MINUTES = 30; + +const IN_FLIGHT_STATES: CascadeState[] = ["Preparing", "Queued", "Running"]; + +export default function SessionCleanupPage() { + const { getAccessToken, setError, setSuccessMessage } = useAdminConfig(); + const [activeTab, setActiveTab] = useState("in-flight"); + + return ( +
+
+
+

Session Cleanup

+

+ V2 cascade-delete observability — In-Flight, Poisoned, Stranded queue, and lifecycle + activity. Operator actions: preview the planned manifest, or restore from snapshot. +

+
+
+
+ + + {/* Tab bar */} +
+ +
+ + {activeTab === "in-flight" && ( + + )} + {activeTab === "poisoned" && ( + + )} + {activeTab === "stranded" && ( + + )} + {activeTab === "activity" && } +
+
+ ); +} + +function TabButton({ + active, + onClick, + children, +}: { + active: boolean; + onClick: () => void; + children: React.ReactNode; +}) { + return ( + + ); +} + +// ── In-Flight tab ──────────────────────────────────────────────────────────── + +function InFlightTab({ + getAccessToken, + setError, + setSuccessMessage, +}: { + getAccessToken: (forceRefresh?: boolean) => Promise; + setError: (error: string | null) => void; + setSuccessMessage: (message: string | null) => void; +}) { + const [rows, setRows] = useState([]); + const [loading, setLoading] = useState(true); + const [refreshKey, setRefreshKey] = useState(0); + const [previewing, setPreviewing] = useState(null); + + const fetchAll = useCallback(async () => { + setLoading(true); + try { + const results = await Promise.all( + IN_FLIGHT_STATES.map((state) => + authenticatedFetch(api.sessionDeletions.list(state), getAccessToken).then(async (r) => { + if (!r.ok) throw new Error(`HTTP ${r.status} for state=${state}`); + return (await r.json()) as SessionDeletionListResponse; + }), + ), + ); + const all = results.flatMap((r) => r.sessions); + // Sort by age desc (oldest first → likely the one needing attention). + all.sort((a, b) => b.ageMinutes - a.ageMinutes); + setRows(all); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError("Session expired; reload the page and try again."); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setLoading(false); + } + }, [getAccessToken, setError]); + + useEffect(() => { + void fetchAll(); + }, [fetchAll, refreshKey]); + + return ( + <> + setRefreshKey((k) => k + 1)} + renderRowActions={(row) => ( + + )} + stateColors={{ + Preparing: "bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-200", + Queued: "bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-200", + Running: "bg-purple-100 text-purple-800 dark:bg-purple-900/30 dark:text-purple-200", + }} + emptyMessage="No active cascades." + successMessage={null} + /> + {previewing && ( + setPreviewing(null)} + getAccessToken={getAccessToken} + /> + )} + + ); +} + +// ── Poisoned tab ───────────────────────────────────────────────────────────── + +function PoisonedTab({ + getAccessToken, + setError, + setSuccessMessage, +}: { + getAccessToken: (forceRefresh?: boolean) => Promise; + setError: (error: string | null) => void; + setSuccessMessage: (message: string | null) => void; +}) { + const [rows, setRows] = useState([]); + const [loading, setLoading] = useState(true); + const [refreshKey, setRefreshKey] = useState(0); + const [previewing, setPreviewing] = useState(null); + const [restoring, setRestoring] = useState(null); + + const fetchPoisoned = useCallback(async () => { + setLoading(true); + try { + const resp = await authenticatedFetch(api.sessionDeletions.list("Poisoned"), getAccessToken); + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + const json = (await resp.json()) as SessionDeletionListResponse; + const sorted = [...json.sessions].sort((a, b) => b.ageMinutes - a.ageMinutes); + setRows(sorted); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError("Session expired; reload the page and try again."); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setLoading(false); + } + }, [getAccessToken, setError]); + + useEffect(() => { + void fetchPoisoned(); + }, [fetchPoisoned, refreshKey]); + + return ( + <> + setRefreshKey((k) => k + 1)} + renderRowActions={(row) => ( +
+ + +
+ )} + stateColors={{ + Poisoned: "bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200", + }} + emptyMessage="No poisoned cascades — V2 is running clean." + successMessage={null} + /> + {previewing && ( + setPreviewing(null)} + getAccessToken={getAccessToken} + /> + )} + {restoring && ( + setRestoring(null)} + onRestored={() => { + setRestoring(null); + setSuccessMessage(`Session ${restoring.sessionId} restore submitted.`); + setRefreshKey((k) => k + 1); + }} + getAccessToken={getAccessToken} + /> + )} + + ); +} + +// ── Stranded tab ───────────────────────────────────────────────────────────── + +function StrandedTab({ + getAccessToken, + setError, +}: { + getAccessToken: (forceRefresh?: boolean) => Promise; + setError: (error: string | null) => void; +}) { + const [rows, setRows] = useState([]); + const [loading, setLoading] = useState(true); + const [refreshKey, setRefreshKey] = useState(0); + const [previewing, setPreviewing] = useState(null); + + const fetchStranded = useCallback(async () => { + setLoading(true); + try { + const resp = await authenticatedFetch( + api.sessionDeletions.list("Queued", STRANDED_THRESHOLD_MINUTES), + getAccessToken, + ); + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + const json = (await resp.json()) as SessionDeletionListResponse; + const sorted = [...json.sessions].sort((a, b) => b.ageMinutes - a.ageMinutes); + setRows(sorted); + } catch (err) { + if (err instanceof TokenExpiredError) { + setError("Session expired; reload the page and try again."); + } else { + setError(err instanceof Error ? err.message : String(err)); + } + } finally { + setLoading(false); + } + }, [getAccessToken, setError]); + + useEffect(() => { + void fetchStranded(); + }, [fetchStranded, refreshKey]); + + return ( + <> + setRefreshKey((k) => k + 1)} + renderRowActions={(row) => ( + + )} + stateColors={{ + Queued: "bg-yellow-100 text-yellow-800 dark:bg-yellow-900/30 dark:text-yellow-200", + }} + emptyMessage="No stranded queued cascades." + successMessage={null} + /> + {previewing && ( + setPreviewing(null)} + getAccessToken={getAccessToken} + /> + )} + + ); +} + +// ── Activity tab ───────────────────────────────────────────────────────────── + +function ActivityTab() { + return ( +
+

Cascade-Delete Activity

+

+ Tenant audit-log action strings for V2 cascade lifecycle: + deletion_started, + deletion_completed, + deletion_restored. + Retention-driven cascades land as + SessionDeletionMaintenanceFanout + per tenant (aggregate counts) plus the individual deletion_started/deletion_completed rows. +

+

+ Step-level progress (per-table deletes, verification residuals) lives in DeletionProgress + blobs and structured logs — not in the tenant audit trail. Operator-scoped signals are + emitted as OpsEvents instead: + SessionDeletionPoisoned + (max-dequeue or verification failure — see Poisoned tab), + SessionDeletionStrandedQueued + (Stranded tab), + SessionDeletionMaintenanceFanoutSkipped + (kill-switch active during a retention sweep). All available on the Ops Events page and + wirable for Telegram alerts. +

+ + Open audit log → + +
+ ); +} + +// ── Shared list component ──────────────────────────────────────────────────── + +function SessionList({ + title, + description, + rows, + loading, + onRefresh, + renderRowActions, + stateColors, + emptyMessage, +}: { + title: string; + description: string; + rows: SessionDeletionRow[]; + loading: boolean; + onRefresh: () => void; + renderRowActions: (row: SessionDeletionRow) => React.ReactNode; + stateColors: Record; + emptyMessage: string; + successMessage: string | null; +}) { + return ( +
+
+
+

{title}

+

{description}

+
+ +
+ + {loading && ( +
Loading…
+ )} + {!loading && rows.length === 0 && ( +
{emptyMessage}
+ )} + {!loading && rows.length > 0 && ( +
+ + + + + + + + + + + + + {rows.map((row) => ( + + + + + + + + + ))} + +
TenantSessionStateManifestAgeActions
{row.tenantId}{row.sessionId} + + {row.deletionState} + + + {row.manifestId || "—"} + + {formatAge(row.ageMinutes)} + {renderRowActions(row)}
+
+ )} +
+ ); +} + +function formatAge(minutes: number): string { + if (minutes < 60) return `${minutes}m`; + if (minutes < 60 * 24) return `${Math.floor(minutes / 60)}h ${minutes % 60}m`; + const days = Math.floor(minutes / (60 * 24)); + const hours = Math.floor((minutes - days * 60 * 24) / 60); + return `${days}d ${hours}h`; +} diff --git a/src/Web/autopilot-monitor-web/lib/api.ts b/src/Web/autopilot-monitor-web/lib/api.ts index 0c71a622..4960e42a 100644 --- a/src/Web/autopilot-monitor-web/lib/api.ts +++ b/src/Web/autopilot-monitor-web/lib/api.ts @@ -311,6 +311,34 @@ export const api = { list: () => `${API_BASE_URL}/api/global/distress-reports`, }, + // ── Session Cascade Deletion (Global-Admin Session Cleanup page) ────────── + sessionDeletions: { + list: (state: "Preparing" | "Queued" | "Running" | "Poisoned", strandedSinceMinutes?: number) => + `${API_BASE_URL}/api/global/session-deletions${qs({ + state, + strandedSinceMinutes: strandedSinceMinutes?.toString(), + })}`, + // Reads the persisted manifest + progress blob for an in-flight / poisoned cascade. + // This is what the Session Cleanup page uses — NOT the dry-run preview below. + storedManifest: ( + sessionId: string, + tenantId: string, + manifestId: string, + mode: "summary" | "full" | "download" = "summary", + ) => + `${API_BASE_URL}/api/admin/sessions/${encodeURIComponent(sessionId)}/deletion-manifest${qs({ + tenantId, + manifestId, + mode, + })}`, + // Dry-run builder that enumerates current data — only meaningful BEFORE a cascade starts. + // The Session Cleanup page should use storedManifest instead. + preview: (sessionId: string, tenantId: string, mode: "summary" | "full" | "download" = "summary") => + `${API_BASE_URL}/api/admin/sessions/${encodeURIComponent(sessionId)}/delete/preview${qs({ tenantId, mode })}`, + restore: (sessionId: string) => + `${API_BASE_URL}/api/admin/sessions/${encodeURIComponent(sessionId)}/restore`, + }, + // ── Hardware Rejection Insights (tenant-scoped, from distress data) ────── distress: { hardwareRejected: () => `${API_BASE_URL}/api/audit/hardware-rejected`, diff --git a/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx b/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx index 367d8b93..6890886e 100644 --- a/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx +++ b/src/Web/autopilot-monitor-web/lib/globalNavConfig.tsx @@ -262,6 +262,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [ id: "ga-ops", label: "Ops", icon: , items: [ { id: "ga-maintenance", label: "Maintenance", href: "/admin/ops" }, + { id: "ga-session-cleanup", label: "Session Cleanup", href: "/admin/ops/session-cleanup" }, ], }, {