From c8ce7cdb37b97c80e3bc8d69e1c5f5d9f705dd9c Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Sat, 5 Sep 2026 04:27:48 +0200 Subject: [PATCH] backend: remove the V2 index tables and their dual-write pipeline (D-196) The five cross-session index tables (SessionsByTerminal, SessionsByStage, DeadEndsByReason, ClassifierVerdictsByIdLevel, SignalsByKind) were write-only since M5.d: no endpoint, MCP tool or web page ever read them, while the dual-write cost a serial queue send per signal/transition on the ingest path, a 5-13 min full scan of Signals + DecisionTransitions every two hours and five property-filter scans per session deletion (audit 2026-09-05 F05). Removed: the tables from TableNames.All (schema hash changes, next cold start runs a full pass without them), the telemetry-index-reconcile queue name, producer, envelope, handler, queue function and 2h timer, the cross-partition Timestamp scans on the signal/transition repositories, the EnableIndexDualWrite flag (model, config mapping, admin UI, generated wire types), cascade steps 11-15 (later steps renumbered 11-18), the deletion wrappers and the offboarding wipe entries. DiscriminatorPkProp stays as a step class so manifests written before this change still resume. Order of operations: deploy first, then delete the queue pair and the five tables in the portal (B-88h). Deleting the tables before the deploy 500s every delete preview/enqueue and poisons offboarding. Submodules: .claude/commands (import_session.py), internal (D-196, docs, plan). Co-Authored-By: Claude Fable 5.1 --- .claude/commands | 2 +- internal | 2 +- ...nConfigurationEnableIndexDualWriteTests.cs | 34 -- .../DeletionManifestBuilderTests.cs | 60 +--- .../HealthCheckServicePoisonQueueTests.cs | 10 +- .../IndexDualWriteConsistencyTests.cs | 285 ---------------- .../IndexReconcileEnvelopeFactoryTests.cs | 167 --------- ...ndexReconcileEnvelopeSerializationTests.cs | 102 ------ .../IndexReconcileHandlerTests.cs | 321 ------------------ .../IndexReconcileTimerTests.cs | 156 --------- .../IndexRowKeysTests.cs | 153 --------- .../SessionDeletionGuardTests.cs | 4 +- .../SessionDeletionHandlerTests.cs | 10 +- .../TableIndexRepositoryMappingTests.cs | 167 --------- .../DataAccess/DataAccessServiceExtensions.cs | 1 - .../DataAccess/TableStorage/IndexRowKeys.cs | 87 ----- .../TableStorage/TableConfigRepository.cs | 2 - .../TableDecisionTransitionRepository.cs | 27 -- .../TableStorage/TableIndexRepository.cs | 175 ---------- .../TableStorage/TableSignalRepository.cs | 29 -- .../Functions/Admin/IndexReconcileTimer.cs | 100 ------ .../Ingest/IngestTelemetryFunction.cs | 14 - .../SessionDeletionMaintenanceFunction.cs | 2 +- .../Maintenance/SessionSweepFunction.cs | 2 +- .../Queue/IndexReconcileQueueFunction.cs | 44 --- .../AutopilotMonitor.Functions/Program.cs | 19 +- .../Services/AdminConfigurationService.cs | 2 +- ...zureQueueAnalyzeOnEnrollmentEndProducer.cs | 4 +- .../Deletion/DeletionManifestBuilder.cs | 53 +-- .../Deletion/ISessionDeletionEnqueuer.cs | 2 +- .../AzureQueueIndexReconcileProducer.cs | 113 ------ .../Indexing/IndexReconcileEnvelopeFactory.cs | 75 ---- .../Indexing/IndexReconcileHandler.cs | 203 ----------- .../Offboarding/TenantOffboardingHandler.cs | 5 - .../Services/TableStorageService.Deletion.cs | 22 -- .../src/generated/wire-types.generated.ts | 2 - .../AutopilotMonitor.Shared/Constants.cs | 23 -- .../IDecisionTransitionRepository.cs | 14 - .../DataAccess/IIndexReconcileProducer.cs | 29 -- .../DataAccess/IIndexTableRepository.cs | 43 --- .../DataAccess/ISignalRepository.cs | 14 - .../Models/Config/AdminConfiguration.cs | 10 - .../Models/DecisionTransitionRecord.cs | 8 +- .../Models/IndexReconcileEnvelope.cs | 56 --- .../Models/IndexRecords.cs | 125 ------- .../app/admin/AdminConfigContext.tsx | 9 +- .../components/AdminConfigSettingsSection.tsx | 32 -- .../sections/SectionGlobalSettings.tsx | 3 - .../utils/shared-manifests.generated.ts | 1 - .../utils/shared-manifests.json | 9 - .../utils/wire-types.generated.ts | 2 - 51 files changed, 50 insertions(+), 2784 deletions(-) delete mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/AdminConfigurationEnableIndexDualWriteTests.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/IndexDualWriteConsistencyTests.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileEnvelopeFactoryTests.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileEnvelopeSerializationTests.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileHandlerTests.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileTimerTests.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/IndexRowKeysTests.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/TableIndexRepositoryMappingTests.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/IndexRowKeys.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableIndexRepository.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Admin/IndexReconcileTimer.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Queue/IndexReconcileQueueFunction.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Indexing/AzureQueueIndexReconcileProducer.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Indexing/IndexReconcileEnvelopeFactory.cs delete mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Indexing/IndexReconcileHandler.cs delete mode 100644 src/Shared/AutopilotMonitor.Shared/DataAccess/IIndexReconcileProducer.cs delete mode 100644 src/Shared/AutopilotMonitor.Shared/DataAccess/IIndexTableRepository.cs delete mode 100644 src/Shared/AutopilotMonitor.Shared/Models/IndexReconcileEnvelope.cs delete mode 100644 src/Shared/AutopilotMonitor.Shared/Models/IndexRecords.cs diff --git a/.claude/commands b/.claude/commands index 6b39d14e..11d9864e 160000 --- a/.claude/commands +++ b/.claude/commands @@ -1 +1 @@ -Subproject commit 6b39d14ed0c3aa174f85949451fae3c39bc56d3c +Subproject commit 11d9864e88b4d69996a573acababa8d0ff6f54f2 diff --git a/internal b/internal index fe88d8bd..d3deed71 160000 --- a/internal +++ b/internal @@ -1 +1 @@ -Subproject commit fe88d8bd4ae2834e9ac31a95e1c36bcab309830e +Subproject commit d3deed71fba0a24b4c7eddc392d99e41fb3dfbf1 diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AdminConfigurationEnableIndexDualWriteTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AdminConfigurationEnableIndexDualWriteTests.cs deleted file mode 100644 index 081e198d..00000000 --- a/src/Backend/AutopilotMonitor.Functions.Tests/AdminConfigurationEnableIndexDualWriteTests.cs +++ /dev/null @@ -1,34 +0,0 @@ -using AutopilotMonitor.Shared.Models; - -namespace AutopilotMonitor.Functions.Tests; - -/// -/// Default-value + round-trip tests for -/// (Plan §M5.d). The flag gates the index-table dual-write rollout — defaulting to false -/// keeps pre-M5.d behaviour bit-exact and lets operators flip it on deliberately. -/// -public class AdminConfigurationEnableIndexDualWriteTests -{ - [Fact] - public void EnableIndexDualWrite_defaults_to_false_on_new_config() - { - var cfg = new AdminConfiguration(); - Assert.False(cfg.EnableIndexDualWrite); - } - - [Fact] - public void EnableIndexDualWrite_defaults_to_false_on_CreateDefault() - { - // CreateDefault() is called when no admin-config row exists yet — if it somehow - // enabled dual-write it would break the "default-off" contract immediately. - var cfg = AdminConfiguration.CreateDefault(); - Assert.False(cfg.EnableIndexDualWrite); - } - - [Fact] - public void EnableIndexDualWrite_true_persists_on_the_config() - { - var cfg = new AdminConfiguration { EnableIndexDualWrite = true }; - Assert.True(cfg.EnableIndexDualWrite); - } -} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/DeletionManifestBuilderTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/DeletionManifestBuilderTests.cs index 15d68f37..71a1e3be 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/DeletionManifestBuilderTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/DeletionManifestBuilderTests.cs @@ -46,12 +46,7 @@ public class DeletionManifestBuilderTests .WithSignals(15) .WithDecisionTransitions(4) .WithEventTypeIndex(5) - .WithCveIndex(2) - .WithSessionsByTerminal(1) - .WithSessionsByStage(3) - .WithDeadEndsByReason(0) - .WithClassifierVerdictsByIdLevel(2) - .WithSignalsByKind(11)); + .WithCveIndex(2)); var manifest = await NewBuilder(reader).BuildAsync( TenantId, SessionId, "admin_delete", @@ -69,15 +64,10 @@ public class DeletionManifestBuilderTests Assert.Equal(4, c["decisionTransitions"]); Assert.Equal(5, c["eventTypeIndex"]); Assert.Equal(2, c["cveIndex"]); - Assert.Equal(1, c["sessionsByTerminal"]); - Assert.Equal(3, c["sessionsByStage"]); - Assert.Equal(0, c["deadEndsByReason"]); - Assert.Equal(2, c["classifierVerdictsByIdLevel"]); - Assert.Equal(11, c["signalsByKind"]); - // 20 cascade tables (incl. SessionTimeBreakdowns F1 PR2, 3 SessionAnnotations lanes + - // SessionTenantLookup P6.1) + tombstone = 21 steps; no inventory steps when no contributions row. - Assert.Equal(21, manifest.Steps.Count); + // 15 cascade tables (incl. SessionTimeBreakdowns F1 PR2, 3 SessionAnnotations lanes + + // SessionTenantLookup P6.1) + tombstone = 16 steps; no inventory steps when no contributions row. + Assert.Equal(16, manifest.Steps.Count); Assert.Equal(DeletionStepClass.Final, manifest.Steps.Last().Class); } @@ -99,11 +89,6 @@ public class DeletionManifestBuilderTests .WithDecisionTransitions(1, propValue: "transitions_marker") .WithEventTypeIndex(1, propValue: "eti_marker") .WithCveIndex(1, propValue: "cve_marker") - .WithSessionsByTerminal(1, propValue: "terminal_marker") - .WithSessionsByStage(1, propValue: "stage_marker") - .WithDeadEndsByReason(1, propValue: "deadend_marker") - .WithClassifierVerdictsByIdLevel(1, propValue: "classifier_marker") - .WithSignalsByKind(1, propValue: "signalkind_marker") .WithSessionTimeBreakdown(propValue: "breakdown_marker") .WithSessionAnnotations(propValue: "annotation_marker") .WithSessionTenantLookup(propValue: "lookup_marker")); @@ -149,9 +134,9 @@ public class DeletionManifestBuilderTests new DeletionRetentionContext { TenantRetentionDays = 90 }); var nonTombstoneSteps = manifest.Steps.Where(s => s.Class != DeletionStepClass.Final).ToList(); - // Without a contributions row, neither inventory step is emitted → exactly 20 cascade-table steps - // (16 + the 3 SessionAnnotations lane steps + SessionTenantLookup). - Assert.Equal(20, nonTombstoneSteps.Count); + // Without a contributions row, neither inventory step is emitted → exactly 15 cascade-table steps + // (11 + the 3 SessionAnnotations lane steps + SessionTenantLookup). + Assert.Equal(15, nonTombstoneSteps.Count); foreach (var step in nonTombstoneSteps) { Assert.Equal(0, step.RowCount); @@ -180,8 +165,8 @@ public class DeletionManifestBuilderTests new DeletionActor { Type = "admin", Actor = "alice@example.com" }, new DeletionRetentionContext { TenantRetentionDays = 90 }); - // Expect: 20 cascade + 2 inventory + 1 tombstone = 23 steps. - Assert.Equal(23, manifest.Steps.Count); + // Expect: 15 cascade + 2 inventory + 1 tombstone = 18 steps. + Assert.Equal(18, manifest.Steps.Count); var aggregate = manifest.Steps.Single(s => s.Class == DeletionStepClass.Aggregate); Assert.Equal(DeletionStepNames.SoftwareInventoryDecrement, aggregate.Step); @@ -310,8 +295,7 @@ public class DeletionManifestBuilderTests .WithSessionsRow() .WithSessionsIndexRow() .WithEvents(3) - .WithSignals(5) - .WithSessionsByTerminal(1)); + .WithSignals(5)); var first = await NewBuilder(reader).BuildAsync(TenantId, SessionId, "admin_delete", new DeletionActor { Type = "admin", Actor = "alice@example.com" }, @@ -345,7 +329,6 @@ public class DeletionManifestBuilderTests .WithSessionsIndexRow() .WithEvents(2, propValue: "events_check") .WithRuleResults(1, propValue: "rules_check") - .WithSessionsByTerminal(1, propValue: "terminal_check") .WithSessionInventoryContributions(MakeContributionsRow(new List { new DeletionDecrementKey { Vendor = "Contoso", Name = "App", Version = "1.0" }, @@ -778,12 +761,6 @@ public class DeletionManifestBuilderTests return this; } - public ReaderSeed WithSessionsByTerminal(int n, string propValue = "term") => SeedDiscriminatorPkProp(Constants.TableNames.SessionsByTerminal, n, propValue, "Failed"); - public ReaderSeed WithSessionsByStage(int n, string propValue = "stg") => SeedDiscriminatorPkProp(Constants.TableNames.SessionsByStage, n, propValue, "AccountSetup"); - public ReaderSeed WithDeadEndsByReason(int n, string propValue = "de") => SeedDiscriminatorPkProp(Constants.TableNames.DeadEndsByReason, n, propValue, "Timeout"); - public ReaderSeed WithClassifierVerdictsByIdLevel(int n, string propValue = "cls") => SeedDiscriminatorPkProp(Constants.TableNames.ClassifierVerdictsByIdLevel, n, propValue, "EspApps_High"); - public ReaderSeed WithSignalsByKind(int n, string propValue = "sk") => SeedDiscriminatorPkProp(Constants.TableNames.SignalsByKind, n, propValue, "EspPhaseChanged"); - public ReaderSeed WithSessionAnnotations(string propValue = "anno") { // One row per lane — the builder emits one exact-RK step per lane. @@ -887,22 +864,5 @@ public class DeletionManifestBuilderTests return this; } - private ReaderSeed SeedDiscriminatorPkProp(string tableName, int n, string propValue, string discriminator) - { - // PK = {tenantId}_{discriminator}, server-side filter on PK prefix + SessionId == sessionId. - var rows = new List(n); - for (int i = 0; i < n; i++) - { - var e = new TableEntity($"{TenantId}_{discriminator}", $"{tableName}_{i:D5}") - { - ["Marker"] = propValue, - ["SessionId"] = SessionId, - }; - e.ETag = new ETag("0x" + tableName + i); - rows.Add(e); - } - _query[tableName] = rows; - return this; - } } } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServicePoisonQueueTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServicePoisonQueueTests.cs index dc8d9667..3fb3d20f 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServicePoisonQueueTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/HealthCheckServicePoisonQueueTests.cs @@ -75,7 +75,7 @@ public class HealthCheckServicePoisonQueueTests var probe = new FakePoisonQueueProbe(); probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 3; probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0; - probe.Counts[Constants.QueueNames.TelemetryIndexReconcile + "-poison"] = 0; + probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0; var svc = BuildService(probe); var check = await svc.CheckPoisonQueuesAsync(); @@ -94,7 +94,7 @@ public class HealthCheckServicePoisonQueueTests var probe = new FakePoisonQueueProbe(); probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 15; probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0; - probe.Counts[Constants.QueueNames.TelemetryIndexReconcile + "-poison"] = 2; + probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 2; var svc = BuildService(probe); var check = await svc.CheckPoisonQueuesAsync(); @@ -117,7 +117,7 @@ public class HealthCheckServicePoisonQueueTests } }; probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 0; - probe.Counts[Constants.QueueNames.TelemetryIndexReconcile + "-poison"] = 0; + probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0; var svc = BuildService(probe); var check = await svc.CheckPoisonQueuesAsync(); @@ -134,7 +134,7 @@ public class HealthCheckServicePoisonQueueTests var probe = new FakePoisonQueueProbe(); probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 4; probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0; - probe.Counts[Constants.QueueNames.TelemetryIndexReconcile + "-poison"] = 0; + probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0; // warn=5 → 4 messages now classifies as healthy (would be warning under defaults) var svc = BuildService(probe, new Dictionary @@ -153,7 +153,7 @@ public class HealthCheckServicePoisonQueueTests var probe = new FakePoisonQueueProbe(); probe.Counts[Constants.QueueNames.AnalyzeOnEnrollmentEnd + "-poison"] = 1; probe.Counts[Constants.QueueNames.VulnerabilityCorrelate + "-poison"] = 0; - probe.Counts[Constants.QueueNames.TelemetryIndexReconcile + "-poison"] = 0; + probe.Counts[Constants.QueueNames.TenantAutoApprove + "-poison"] = 0; var svc = BuildService(probe); var check = await svc.CheckPoisonQueuesAsync(); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IndexDualWriteConsistencyTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IndexDualWriteConsistencyTests.cs deleted file mode 100644 index 36be3e64..00000000 --- a/src/Backend/AutopilotMonitor.Functions.Tests/IndexDualWriteConsistencyTests.cs +++ /dev/null @@ -1,285 +0,0 @@ -using AutopilotMonitor.Functions.DataAccess.TableStorage; -using AutopilotMonitor.Functions.Services.Indexing; -using AutopilotMonitor.Shared.DataAccess; -using AutopilotMonitor.Shared.Models; -using Microsoft.Extensions.Logging.Abstractions; - -namespace AutopilotMonitor.Functions.Tests; - -/// -/// M5 release-gate (Plan §M5-Pflicht line 1395): per-index-table PK/RK consistency -/// between primary rows and their derived index rows. For each index table, walk a -/// canonical primary row through the Factory + Handler pipeline and verify that the -/// captured index record carries enough back-ref information to reconstruct the -/// primary's (PartitionKey, RowKey) via / -/// key builders. -/// -/// Also asserts pipeline idempotency: running the same primary row through twice -/// produces identical upsert keys — the consumer's UpsertReplace batches stay -/// deduplicating even under timer-triggered resubmits. -/// -/// -public class IndexDualWriteConsistencyTests -{ - private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; - private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; - - private static readonly DateTime Occurred = - new(2026, 4, 21, 14, 7, 33, DateTimeKind.Utc); - - private static (IndexReconcileHandler handler, FakeIndexTableRepository repo) Sut() - { - var repo = new FakeIndexTableRepository(); - var handler = new IndexReconcileHandler(repo, NullLogger.Instance); - return (handler, repo); - } - - // ============================================================ SignalsByKind ↔ Signals - - [Fact] - public async Task SignalsByKind_row_carries_back_ref_to_primary_Signals_row() - { - var primary = new SignalRecord - { - TenantId = TenantId, - SessionId = SessionId, - SessionSignalOrdinal = 42, - Kind = "EspTerminalFailure", - OccurredAtUtc = Occurred, - SourceOrigin = "EspAndHelloTrackerAdapter", - }; - - var (handler, repo) = Sut(); - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromSignal(primary)); - - var index = Assert.Single(repo.SignalsByKind); - - // Back-ref fields on the index row must reconstruct the primary PK/RK. - var reconstructedPk = TableSignalRepository.BuildPartitionKey(index.TenantId, index.SessionId); - var reconstructedRk = TableSignalRepository.BuildRowKey(index.SessionSignalOrdinal); - - Assert.Equal( - TableSignalRepository.BuildPartitionKey(primary.TenantId, primary.SessionId), - reconstructedPk); - Assert.Equal( - TableSignalRepository.BuildRowKey(primary.SessionSignalOrdinal), - reconstructedRk); - } - - // ============================================================ SessionsByTerminal ↔ DecisionTransitions - - [Fact] - public async Task SessionsByTerminal_row_carries_back_ref_to_primary_DecisionTransitions_row() - { - var primary = TerminalTransition(); - - var (handler, repo) = Sut(); - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromDecisionTransition(primary)); - - var index = Assert.Single(repo.SessionsByTerminal); - - var reconstructedPk = TableDecisionTransitionRepository.BuildPartitionKey(index.TenantId, index.SessionId); - var reconstructedRk = TableDecisionTransitionRepository.BuildRowKey(index.StepIndex); - - Assert.Equal( - TableDecisionTransitionRepository.BuildPartitionKey(primary.TenantId, primary.SessionId), - reconstructedPk); - Assert.Equal( - TableDecisionTransitionRepository.BuildRowKey(primary.StepIndex), - reconstructedRk); - } - - [Fact] - public async Task SessionsByStage_row_carries_back_ref_to_primary_DecisionTransitions_row() - { - var primary = new DecisionTransitionRecord - { - TenantId = TenantId, - SessionId = SessionId, - StepIndex = 3, - FromStage = "AwaitingEsp", - ToStage = "EspInProgress", - Taken = true, - IsTerminal = false, - OccurredAtUtc = Occurred, - }; - - var (handler, repo) = Sut(); - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromDecisionTransition(primary)); - - var index = Assert.Single(repo.SessionsByStage); - - var reconstructedPk = TableDecisionTransitionRepository.BuildPartitionKey(index.TenantId, index.SessionId); - var reconstructedRk = TableDecisionTransitionRepository.BuildRowKey(index.StepIndex); - - Assert.Equal( - TableDecisionTransitionRepository.BuildPartitionKey(primary.TenantId, primary.SessionId), - reconstructedPk); - Assert.Equal( - TableDecisionTransitionRepository.BuildRowKey(primary.StepIndex), - reconstructedRk); - } - - [Fact] - public async Task DeadEndsByReason_row_carries_back_ref_to_primary_DecisionTransitions_row() - { - var primary = new DecisionTransitionRecord - { - TenantId = TenantId, - SessionId = SessionId, - StepIndex = 5, - FromStage = "EspInProgress", - ToStage = "EspInProgress", - Taken = false, - IsTerminal = false, - DeadEndReason = "hybrid_reboot_gate_blocking", - OccurredAtUtc = Occurred, - }; - - var (handler, repo) = Sut(); - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromDecisionTransition(primary)); - - var index = Assert.Single(repo.DeadEndsByReason); - - Assert.Equal( - TableDecisionTransitionRepository.BuildPartitionKey(primary.TenantId, primary.SessionId), - TableDecisionTransitionRepository.BuildPartitionKey(index.TenantId, index.SessionId)); - Assert.Equal( - TableDecisionTransitionRepository.BuildRowKey(primary.StepIndex), - TableDecisionTransitionRepository.BuildRowKey(index.StepIndex)); - } - - [Fact] - public async Task ClassifierVerdictsByIdLevel_row_carries_back_ref_to_primary_DecisionTransitions_row() - { - var primary = new DecisionTransitionRecord - { - TenantId = TenantId, - SessionId = SessionId, - StepIndex = 7, - FromStage = "EspInProgress", - ToStage = "WhiteGloveSealed", - Taken = true, - IsTerminal = true, - ClassifierVerdictId = "whiteglove-sealing", - ClassifierHypothesisLevel = "Strong", - OccurredAtUtc = Occurred, - }; - - var (handler, repo) = Sut(); - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromDecisionTransition(primary)); - - var index = Assert.Single(repo.ClassifierVerdictsByIdLevel); - - Assert.Equal( - TableDecisionTransitionRepository.BuildPartitionKey(primary.TenantId, primary.SessionId), - TableDecisionTransitionRepository.BuildPartitionKey(index.TenantId, index.SessionId)); - Assert.Equal( - TableDecisionTransitionRepository.BuildRowKey(primary.StepIndex), - TableDecisionTransitionRepository.BuildRowKey(index.StepIndex)); - } - - // ============================================================ pipeline idempotency - - [Fact] - public async Task Resubmitting_same_primary_rows_produces_same_index_keys() - { - // Simulates the M5.d.4 timer re-enqueueing primary rows it already handled — - // the handler's index-row keys must be key-stable so the real TableIndexRepository's - // UpsertReplace collapses duplicates rather than appending rows. - var (handler, repo) = Sut(); - - var signal = new SignalRecord - { - TenantId = TenantId, - SessionId = SessionId, - SessionSignalOrdinal = 1, - Kind = "DesktopArrived", - OccurredAtUtc = Occurred, - }; - var transition = TerminalTransition(); - - // First pass: initial ingest fan-out. - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromSignal(signal)); - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromDecisionTransition(transition)); - - // Second pass: 2h-timer reconcile resubmits the same primaries. - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromSignal(signal)); - await handler.HandleAsync(IndexReconcileEnvelopeFactory.FromDecisionTransition(transition)); - - // The fake repo captures one row per call, but all duplicated rows must map to the - // same (PK, RK) — the real Azure Tables Upsert would collapse them. - Assert.Equal(2, repo.SignalsByKind.Count); - var signalKeys = repo.SignalsByKind - .Select(r => ( - Pk: IndexRowKeys.BuildSignalsByKindPk(r.TenantId, r.SignalKind), - Rk: IndexRowKeys.BuildSignalsByKindRk(r.OccurredAtUtc, r.SessionId, r.SessionSignalOrdinal))) - .Distinct() - .ToList(); - Assert.Single(signalKeys); - - Assert.Equal(2, repo.SessionsByTerminal.Count); - var terminalKeys = repo.SessionsByTerminal - .Select(r => ( - Pk: IndexRowKeys.BuildSessionsByTerminalPk(r.TenantId, r.TerminalStage), - Rk: IndexRowKeys.BuildSessionsByTerminalRk(r.OccurredAtUtc, r.SessionId))) - .Distinct() - .ToList(); - Assert.Single(terminalKeys); - - Assert.Equal(2, repo.SessionsByStage.Count); - var stageKeys = repo.SessionsByStage - .Select(r => ( - Pk: IndexRowKeys.BuildSessionsByStagePk(r.TenantId, r.Stage), - Rk: IndexRowKeys.BuildSessionsByStageRk(r.LastUpdatedUtc, r.SessionId))) - .Distinct() - .ToList(); - Assert.Single(stageKeys); - - Assert.Equal(2, repo.ClassifierVerdictsByIdLevel.Count); - var verdictKeys = repo.ClassifierVerdictsByIdLevel - .Select(r => ( - Pk: IndexRowKeys.BuildClassifierVerdictsByIdLevelPk(r.TenantId, r.ClassifierId, r.HypothesisLevel), - Rk: IndexRowKeys.BuildClassifierVerdictsByIdLevelRk(r.OccurredAtUtc, r.SessionId, r.StepIndex))) - .Distinct() - .ToList(); - Assert.Single(verdictKeys); - } - - // ============================================================ helpers - - private static DecisionTransitionRecord TerminalTransition() - => new() - { - TenantId = TenantId, - SessionId = SessionId, - StepIndex = 7, - FromStage = "EspInProgress", - ToStage = "WhiteGloveSealed", - Taken = true, - IsTerminal = true, - ClassifierVerdictId = "whiteglove-sealing", - ClassifierHypothesisLevel = "Strong", - OccurredAtUtc = Occurred, - }; - - private sealed class FakeIndexTableRepository : IIndexTableRepository - { - public List SessionsByTerminal { get; } = new(); - public List SessionsByStage { get; } = new(); - public List DeadEndsByReason { get; } = new(); - public List ClassifierVerdictsByIdLevel { get; } = new(); - public List SignalsByKind { get; } = new(); - - public Task StoreSessionsByTerminalAsync(IReadOnlyList records, CancellationToken ct = default) - { SessionsByTerminal.AddRange(records); return Task.FromResult(records.Count); } - public Task StoreSessionsByStageAsync(IReadOnlyList records, CancellationToken ct = default) - { SessionsByStage.AddRange(records); return Task.FromResult(records.Count); } - public Task StoreDeadEndsByReasonAsync(IReadOnlyList records, CancellationToken ct = default) - { DeadEndsByReason.AddRange(records); return Task.FromResult(records.Count); } - public Task StoreClassifierVerdictsByIdLevelAsync(IReadOnlyList records, CancellationToken ct = default) - { ClassifierVerdictsByIdLevel.AddRange(records); return Task.FromResult(records.Count); } - public Task StoreSignalsByKindAsync(IReadOnlyList records, CancellationToken ct = default) - { SignalsByKind.AddRange(records); return Task.FromResult(records.Count); } - } -} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileEnvelopeFactoryTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileEnvelopeFactoryTests.cs deleted file mode 100644 index 36b88258..00000000 --- a/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileEnvelopeFactoryTests.cs +++ /dev/null @@ -1,167 +0,0 @@ -using AutopilotMonitor.Functions.Services.Indexing; -using AutopilotMonitor.Shared.Models; - -namespace AutopilotMonitor.Functions.Tests; - -/// -/// Pure tests for (Plan §2.8, §M5.d). -/// Verifies the shape-only projection from primary record → queue envelope — the -/// consumer (M5.d.3) relies on these fields being copied verbatim. -/// -public class IndexReconcileEnvelopeFactoryTests -{ - private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; - private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; - - private static readonly DateTime Occurred = - new(2026, 4, 21, 14, 7, 33, DateTimeKind.Utc); - - // ============================================================ FromSignal - - [Fact] - public void FromSignal_copies_discriminators_and_back_ref() - { - var record = new SignalRecord - { - TenantId = TenantId, - SessionId = SessionId, - SessionSignalOrdinal = 42, - SessionTraceOrdinal = 117, - Kind = "EspTerminalFailure", - KindSchemaVersion = 2, - OccurredAtUtc = Occurred, - SourceOrigin = "EspAndHelloTrackerAdapter", - PayloadJson = "{}", - }; - - var envelope = IndexReconcileEnvelopeFactory.FromSignal(record); - - Assert.Equal("1", envelope.EnvelopeVersion); - Assert.Equal("Signal", envelope.SourceKind); - Assert.Equal(TenantId, envelope.TenantId); - Assert.Equal(SessionId, envelope.SessionId); - Assert.Equal(Occurred, envelope.OccurredAtUtc); - Assert.Equal(42L, envelope.SessionSignalOrdinal); - Assert.Equal("EspTerminalFailure", envelope.SignalKind); - Assert.Equal("EspAndHelloTrackerAdapter", envelope.SourceOrigin); - - // DecisionTransition-only fields stay null on Signal envelopes. - Assert.Null(envelope.StepIndex); - Assert.Null(envelope.FromStage); - Assert.Null(envelope.ToStage); - Assert.Null(envelope.Taken); - Assert.Null(envelope.IsTerminal); - Assert.Null(envelope.DeadEndReason); - Assert.Null(envelope.ClassifierVerdictId); - Assert.Null(envelope.ClassifierHypothesisLevel); - } - - // ============================================================ FromDecisionTransition - - [Fact] - public void FromDecisionTransition_taken_terminal_with_classifier_copies_all_discriminators() - { - var record = new DecisionTransitionRecord - { - TenantId = TenantId, - SessionId = SessionId, - StepIndex = 7, - SessionTraceOrdinal = 118, - SignalOrdinalRef = 42, - OccurredAtUtc = Occurred, - Trigger = "EspExiting", - FromStage = "EspInProgress", - ToStage = "WhiteGloveSealed", - Taken = true, - DeadEndReason = null, - ReducerVersion = "1.0.0", - IsTerminal = true, - ClassifierVerdictId = "whiteglove-sealing", - ClassifierHypothesisLevel = "Strong", - PayloadJson = "{}", - }; - - var envelope = IndexReconcileEnvelopeFactory.FromDecisionTransition(record); - - Assert.Equal("1", envelope.EnvelopeVersion); - Assert.Equal("DecisionTransition", envelope.SourceKind); - Assert.Equal(TenantId, envelope.TenantId); - Assert.Equal(SessionId, envelope.SessionId); - Assert.Equal(Occurred, envelope.OccurredAtUtc); - Assert.Equal(7, envelope.StepIndex); - Assert.Equal("EspInProgress", envelope.FromStage); - Assert.Equal("WhiteGloveSealed", envelope.ToStage); - Assert.Equal(true, envelope.Taken); - Assert.Equal(true, envelope.IsTerminal); - Assert.Null(envelope.DeadEndReason); - Assert.Equal("whiteglove-sealing", envelope.ClassifierVerdictId); - Assert.Equal("Strong", envelope.ClassifierHypothesisLevel); - - // Signal-only fields stay null on DecisionTransition envelopes. - Assert.Null(envelope.SessionSignalOrdinal); - Assert.Null(envelope.SignalKind); - Assert.Null(envelope.SourceOrigin); - } - - [Fact] - public void FromDecisionTransition_dead_end_carries_reason_without_classifier_fields() - { - var record = new DecisionTransitionRecord - { - TenantId = TenantId, - SessionId = SessionId, - StepIndex = 3, - OccurredAtUtc = Occurred, - Trigger = "EspExiting", - FromStage = "EspInProgress", - ToStage = "EspInProgress", - Taken = false, - DeadEndReason = "hybrid_reboot_gate_blocking", - ReducerVersion = "1.0.0", - IsTerminal = false, - PayloadJson = "{}", - }; - - var envelope = IndexReconcileEnvelopeFactory.FromDecisionTransition(record); - - Assert.Equal(false, envelope.Taken); - Assert.Equal(false, envelope.IsTerminal); - Assert.Equal("hybrid_reboot_gate_blocking", envelope.DeadEndReason); - Assert.Null(envelope.ClassifierVerdictId); - Assert.Null(envelope.ClassifierHypothesisLevel); - } - - // ============================================================ BuildBatch - - [Fact] - public void BuildBatch_preserves_input_order_and_kind_tags() - { - var signals = new List - { - new() { TenantId = TenantId, SessionId = SessionId, SessionSignalOrdinal = 1, Kind = "EspPhaseChanged", OccurredAtUtc = Occurred }, - new() { TenantId = TenantId, SessionId = SessionId, SessionSignalOrdinal = 2, Kind = "DesktopArrived", OccurredAtUtc = Occurred }, - }; - var transitions = new List - { - new() { TenantId = TenantId, SessionId = SessionId, StepIndex = 1, FromStage = "A", ToStage = "B", Taken = true, OccurredAtUtc = Occurred }, - }; - - var envelopes = IndexReconcileEnvelopeFactory.BuildBatch(signals, transitions); - - Assert.Equal(3, envelopes.Count); - Assert.Equal("Signal", envelopes[0].SourceKind); - Assert.Equal(1L, envelopes[0].SessionSignalOrdinal); - Assert.Equal("Signal", envelopes[1].SourceKind); - Assert.Equal(2L, envelopes[1].SessionSignalOrdinal); - Assert.Equal("DecisionTransition", envelopes[2].SourceKind); - Assert.Equal(1, envelopes[2].StepIndex); - } - - [Fact] - public void BuildBatch_returns_empty_for_null_or_empty_inputs() - { - Assert.Empty(IndexReconcileEnvelopeFactory.BuildBatch( - new List(), new List())); - Assert.Empty(IndexReconcileEnvelopeFactory.BuildBatch(null!, null!)); - } -} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileEnvelopeSerializationTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileEnvelopeSerializationTests.cs deleted file mode 100644 index 9386bf06..00000000 --- a/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileEnvelopeSerializationTests.cs +++ /dev/null @@ -1,102 +0,0 @@ -using AutopilotMonitor.Shared.Models; -using Newtonsoft.Json; - -namespace AutopilotMonitor.Functions.Tests; - -/// -/// JSON round-trip tests for (Plan §2.8, §M5.d). -/// The producer (M5.d.2) serializes with Newtonsoft; the queue-triggered consumer -/// (M5.d.3) must deserialize the same shape back — verified here so the contract -/// doesn't drift silently as fields are added. -/// -public class IndexReconcileEnvelopeSerializationTests -{ - private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; - private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; - - private static readonly DateTime Occurred = - new(2026, 4, 21, 14, 7, 33, DateTimeKind.Utc); - - [Fact] - public void Signal_envelope_round_trips_through_newtonsoft() - { - var original = new IndexReconcileEnvelope - { - EnvelopeVersion = "1", - SourceKind = "Signal", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - SessionSignalOrdinal = 42, - SignalKind = "EspTerminalFailure", - SourceOrigin = "EspAndHelloTrackerAdapter", - }; - - var json = JsonConvert.SerializeObject(original); - var restored = JsonConvert.DeserializeObject(json); - - Assert.NotNull(restored); - Assert.Equal("1", restored!.EnvelopeVersion); - Assert.Equal("Signal", restored.SourceKind); - Assert.Equal(TenantId, restored.TenantId); - Assert.Equal(SessionId, restored.SessionId); - Assert.Equal(Occurred, restored.OccurredAtUtc.ToUniversalTime()); - Assert.Equal(42L, restored.SessionSignalOrdinal); - Assert.Equal("EspTerminalFailure", restored.SignalKind); - Assert.Equal("EspAndHelloTrackerAdapter", restored.SourceOrigin); - Assert.Null(restored.StepIndex); - Assert.Null(restored.ClassifierVerdictId); - } - - [Fact] - public void DecisionTransition_envelope_round_trips_all_discriminators() - { - var original = new IndexReconcileEnvelope - { - EnvelopeVersion = "1", - SourceKind = "DecisionTransition", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - StepIndex = 7, - FromStage = "EspInProgress", - ToStage = "WhiteGloveSealed", - Taken = true, - IsTerminal = true, - DeadEndReason = null, - ClassifierVerdictId = "whiteglove-sealing", - ClassifierHypothesisLevel = "Strong", - }; - - var json = JsonConvert.SerializeObject(original); - var restored = JsonConvert.DeserializeObject(json); - - Assert.NotNull(restored); - Assert.Equal("DecisionTransition", restored!.SourceKind); - Assert.Equal(7, restored.StepIndex); - Assert.Equal("EspInProgress", restored.FromStage); - Assert.Equal("WhiteGloveSealed", restored.ToStage); - Assert.Equal(true, restored.Taken); - Assert.Equal(true, restored.IsTerminal); - Assert.Null(restored.DeadEndReason); - Assert.Equal("whiteglove-sealing", restored.ClassifierVerdictId); - Assert.Equal("Strong", restored.ClassifierHypothesisLevel); - Assert.Null(restored.SessionSignalOrdinal); - } - - [Fact] - public void Deserialization_tolerates_missing_optional_fields() - { - // A minimal envelope from a consumer-compatible-but-older writer should still parse. - const string minimal = - "{\"envelopeVersion\":\"1\",\"sourceKind\":\"Signal\",\"tenantId\":\"t\",\"sessionId\":\"s\",\"occurredAtUtc\":\"2026-04-21T14:07:33Z\"}"; - - var restored = JsonConvert.DeserializeObject(minimal); - - Assert.NotNull(restored); - Assert.Equal("Signal", restored!.SourceKind); - Assert.Null(restored.SessionSignalOrdinal); - Assert.Null(restored.SignalKind); - Assert.Null(restored.StepIndex); - } -} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileHandlerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileHandlerTests.cs deleted file mode 100644 index 0f5f08a6..00000000 --- a/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileHandlerTests.cs +++ /dev/null @@ -1,321 +0,0 @@ -using AutopilotMonitor.Functions.Services.Indexing; -using AutopilotMonitor.Shared.DataAccess; -using AutopilotMonitor.Shared.Models; -using Microsoft.Extensions.Logging.Abstractions; - -namespace AutopilotMonitor.Functions.Tests; - -/// -/// Routing tests for (Plan §2.8, §M5.d.3). Verifies -/// the 0–3 fan-out decisions per envelope shape using a capture-only fake repo — -/// the real TableIndexRepository mapping is covered by . -/// -public class IndexReconcileHandlerTests -{ - private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; - private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; - - private static readonly DateTime Occurred = - new(2026, 4, 21, 14, 7, 33, DateTimeKind.Utc); - - private static (IndexReconcileHandler handler, FakeIndexTableRepository repo) Sut() - { - var repo = new FakeIndexTableRepository(); - var handler = new IndexReconcileHandler(repo, NullLogger.Instance); - return (handler, repo); - } - - // ============================================================ Signal routing - - [Fact] - public async Task Signal_envelope_writes_exactly_one_SignalsByKind_row() - { - var (handler, repo) = Sut(); - - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "Signal", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - SessionSignalOrdinal = 42, - SignalKind = "EspTerminalFailure", - SourceOrigin = "EspAndHelloTrackerAdapter", - }); - - Assert.Single(repo.SignalsByKind); - var row = repo.SignalsByKind[0]; - Assert.Equal(TenantId, row.TenantId); - Assert.Equal(SessionId, row.SessionId); - Assert.Equal("EspTerminalFailure", row.SignalKind); - Assert.Equal(42L, row.SessionSignalOrdinal); - Assert.Equal("EspAndHelloTrackerAdapter", row.SourceOrigin); - Assert.Equal(Occurred, row.OccurredAtUtc); - - // Other tables untouched. - Assert.Empty(repo.SessionsByTerminal); - Assert.Empty(repo.SessionsByStage); - Assert.Empty(repo.DeadEndsByReason); - Assert.Empty(repo.ClassifierVerdictsByIdLevel); - } - - [Fact] - public async Task Signal_envelope_missing_ordinal_is_dropped() - { - var (handler, repo) = Sut(); - - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "Signal", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - SessionSignalOrdinal = null, - SignalKind = "EspPhaseChanged", - }); - - Assert.Empty(repo.SignalsByKind); - } - - [Fact] - public async Task Signal_envelope_missing_kind_is_dropped() - { - var (handler, repo) = Sut(); - - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "Signal", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - SessionSignalOrdinal = 1, - SignalKind = string.Empty, - }); - - Assert.Empty(repo.SignalsByKind); - } - - // ============================================================ DecisionTransition routing — all 4 paths - - [Fact] - public async Task DecisionTransition_terminal_taken_with_classifier_writes_three_index_rows() - { - var (handler, repo) = Sut(); - - // WhiteGlove-sealing: Taken=true, IsTerminal=true, classifier verdict attached. - // Expected: SessionsByTerminal + SessionsByStage + ClassifierVerdictsByIdLevel (3 rows). - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "DecisionTransition", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - StepIndex = 7, - FromStage = "EspInProgress", - ToStage = "WhiteGloveSealed", - Taken = true, - IsTerminal = true, - ClassifierVerdictId = "whiteglove-sealing", - ClassifierHypothesisLevel = "Strong", - }); - - Assert.Single(repo.SessionsByTerminal); - Assert.Equal("WhiteGloveSealed", repo.SessionsByTerminal[0].TerminalStage); - Assert.Equal(7, repo.SessionsByTerminal[0].StepIndex); - - Assert.Single(repo.SessionsByStage); - Assert.Equal("WhiteGloveSealed", repo.SessionsByStage[0].Stage); - Assert.Equal(Occurred, repo.SessionsByStage[0].LastUpdatedUtc); - - Assert.Single(repo.ClassifierVerdictsByIdLevel); - Assert.Equal("whiteglove-sealing", repo.ClassifierVerdictsByIdLevel[0].ClassifierId); - Assert.Equal("Strong", repo.ClassifierVerdictsByIdLevel[0].HypothesisLevel); - - // Dead-end path not triggered for Taken=true. - Assert.Empty(repo.DeadEndsByReason); - // Signal path never touched on a DecisionTransition envelope. - Assert.Empty(repo.SignalsByKind); - } - - [Fact] - public async Task DecisionTransition_taken_non_terminal_no_classifier_writes_only_SessionsByStage() - { - var (handler, repo) = Sut(); - - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "DecisionTransition", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - StepIndex = 3, - FromStage = "AwaitingEsp", - ToStage = "EspInProgress", - Taken = true, - IsTerminal = false, - }); - - Assert.Empty(repo.SessionsByTerminal); - Assert.Single(repo.SessionsByStage); - Assert.Empty(repo.DeadEndsByReason); - Assert.Empty(repo.ClassifierVerdictsByIdLevel); - } - - [Fact] - public async Task DecisionTransition_dead_end_with_classifier_writes_DeadEndsByReason_and_verdict() - { - var (handler, repo) = Sut(); - - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "DecisionTransition", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - StepIndex = 5, - FromStage = "EspInProgress", - ToStage = "EspInProgress", - Taken = false, - IsTerminal = false, - DeadEndReason = "hybrid_reboot_gate_blocking", - ClassifierVerdictId = "whiteglove-sealing", - ClassifierHypothesisLevel = "Weak", - }); - - Assert.Single(repo.DeadEndsByReason); - Assert.Equal("hybrid_reboot_gate_blocking", repo.DeadEndsByReason[0].DeadEndReason); - Assert.Equal("EspInProgress", repo.DeadEndsByReason[0].FromStage); - Assert.Equal("EspInProgress", repo.DeadEndsByReason[0].AttemptedToStage); - - Assert.Single(repo.ClassifierVerdictsByIdLevel); - Assert.Equal("Weak", repo.ClassifierVerdictsByIdLevel[0].HypothesisLevel); - - // Taken=false → SessionsByStage is NOT updated (no stage entry). - Assert.Empty(repo.SessionsByStage); - Assert.Empty(repo.SessionsByTerminal); - } - - [Fact] - public async Task DecisionTransition_dead_end_without_reason_writes_nothing_to_DeadEnds() - { - // Defense in depth: an envelope with Taken=false but no reason should not create a - // DeadEnd row (empty reason would collide into a single PK across unrelated blocks). - var (handler, repo) = Sut(); - - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "DecisionTransition", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - StepIndex = 1, - FromStage = "A", - ToStage = "A", - Taken = false, - DeadEndReason = null, - }); - - Assert.Empty(repo.DeadEndsByReason); - } - - [Fact] - public async Task DecisionTransition_missing_stepIndex_is_dropped_entirely() - { - var (handler, repo) = Sut(); - - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "DecisionTransition", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - StepIndex = null, - ToStage = "WhiteGloveSealed", - Taken = true, - IsTerminal = true, - }); - - Assert.Empty(repo.SessionsByTerminal); - Assert.Empty(repo.SessionsByStage); - Assert.Empty(repo.DeadEndsByReason); - Assert.Empty(repo.ClassifierVerdictsByIdLevel); - } - - // ============================================================ Misc - - [Fact] - public async Task Unknown_SourceKind_is_logged_and_dropped() - { - var (handler, repo) = Sut(); - - await handler.HandleAsync(new IndexReconcileEnvelope - { - SourceKind = "FutureThingWeDontKnowAbout", - TenantId = TenantId, - SessionId = SessionId, - OccurredAtUtc = Occurred, - }); - - Assert.Empty(repo.SignalsByKind); - Assert.Empty(repo.SessionsByTerminal); - Assert.Empty(repo.SessionsByStage); - Assert.Empty(repo.DeadEndsByReason); - Assert.Empty(repo.ClassifierVerdictsByIdLevel); - } - - [Fact] - public async Task Null_envelope_is_logged_and_dropped() - { - var (handler, repo) = Sut(); - - await handler.HandleAsync(null!); - - Assert.Empty(repo.SignalsByKind); - } - - // ============================================================ FakeIndexTableRepository - - private sealed class FakeIndexTableRepository : IIndexTableRepository - { - public List SessionsByTerminal { get; } = new(); - public List SessionsByStage { get; } = new(); - public List DeadEndsByReason { get; } = new(); - public List ClassifierVerdictsByIdLevel { get; } = new(); - public List SignalsByKind { get; } = new(); - - public Task StoreSessionsByTerminalAsync( - IReadOnlyList records, CancellationToken cancellationToken = default) - { - SessionsByTerminal.AddRange(records); - return Task.FromResult(records.Count); - } - - public Task StoreSessionsByStageAsync( - IReadOnlyList records, CancellationToken cancellationToken = default) - { - SessionsByStage.AddRange(records); - return Task.FromResult(records.Count); - } - - public Task StoreDeadEndsByReasonAsync( - IReadOnlyList records, CancellationToken cancellationToken = default) - { - DeadEndsByReason.AddRange(records); - return Task.FromResult(records.Count); - } - - public Task StoreClassifierVerdictsByIdLevelAsync( - IReadOnlyList records, CancellationToken cancellationToken = default) - { - ClassifierVerdictsByIdLevel.AddRange(records); - return Task.FromResult(records.Count); - } - - public Task StoreSignalsByKindAsync( - IReadOnlyList records, CancellationToken cancellationToken = default) - { - SignalsByKind.AddRange(records); - return Task.FromResult(records.Count); - } - } -} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileTimerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileTimerTests.cs deleted file mode 100644 index 61e0e993..00000000 --- a/src/Backend/AutopilotMonitor.Functions.Tests/IndexReconcileTimerTests.cs +++ /dev/null @@ -1,156 +0,0 @@ -using AutopilotMonitor.Functions.Functions.Admin; -using AutopilotMonitor.Functions.Services; -using AutopilotMonitor.Shared.DataAccess; -using AutopilotMonitor.Shared.Models; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; - -namespace AutopilotMonitor.Functions.Tests; - -/// -/// Tests for the 2h safety-net (Plan §2.8, §M5.d.4). -/// Covers the flag-gate short-circuit, the happy path (enqueue every primary row -/// in the window), and the empty-window case. -/// -public class IndexReconcileTimerTests -{ - private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; - private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; - - private static readonly DateTime Cutoff = new(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc); - - // ============================================================ flag-off short-circuit - - [Fact] - public async Task Feature_flag_off_skips_queries_and_enqueue() - { - var adminConfigSvc = BuildAdminConfigService(enableIndexDualWrite: false); - var signalRepo = new Mock(MockBehavior.Strict); - var transitionRepo = new Mock(MockBehavior.Strict); - var producer = new Mock(MockBehavior.Strict); - - var timer = new IndexReconcileTimer( - adminConfigSvc, - signalRepo.Object, - transitionRepo.Object, - producer.Object, - NullLogger.Instance); - - await timer.RunReconcileAsync(Cutoff, default); - - // Strict mocks throw if any unconfigured call happens — so zero setups + no throw - // proves the timer short-circuited before any query / enqueue. - signalRepo.VerifyNoOtherCalls(); - transitionRepo.VerifyNoOtherCalls(); - producer.VerifyNoOtherCalls(); - } - - // ============================================================ happy path - - [Fact] - public async Task Flag_on_with_primary_rows_enqueues_all_envelopes() - { - var adminConfigSvc = BuildAdminConfigService(enableIndexDualWrite: true); - - var signals = new List - { - new() { TenantId = TenantId, SessionId = SessionId, SessionSignalOrdinal = 1, Kind = "EspPhaseChanged", OccurredAtUtc = Cutoff.AddMinutes(5) }, - new() { TenantId = TenantId, SessionId = SessionId, SessionSignalOrdinal = 2, Kind = "DesktopArrived", OccurredAtUtc = Cutoff.AddMinutes(10) }, - }; - var transitions = new List - { - new() { TenantId = TenantId, SessionId = SessionId, StepIndex = 1, Taken = true, FromStage = "A", ToStage = "B", OccurredAtUtc = Cutoff.AddMinutes(7) }, - }; - - var signalRepo = new Mock(); - signalRepo - .Setup(r => r.QueryByTimestampAtOrAfterAsync(Cutoff, It.IsAny(), It.IsAny())) - .ReturnsAsync(signals); - var transitionRepo = new Mock(); - transitionRepo - .Setup(r => r.QueryByTimestampAtOrAfterAsync(Cutoff, It.IsAny(), It.IsAny())) - .ReturnsAsync(transitions); - - List? captured = null; - var producer = new Mock(); - producer - .Setup(p => p.EnqueueBatchAsync(It.IsAny>(), It.IsAny())) - .Callback, CancellationToken>((e, _) => captured = e.ToList()) - .ReturnsAsync(3); - - var timer = new IndexReconcileTimer( - adminConfigSvc, - signalRepo.Object, - transitionRepo.Object, - producer.Object, - NullLogger.Instance); - - await timer.RunReconcileAsync(Cutoff, default); - - Assert.NotNull(captured); - Assert.Equal(3, captured!.Count); - Assert.Equal("Signal", captured[0].SourceKind); - Assert.Equal(1L, captured[0].SessionSignalOrdinal); - Assert.Equal("Signal", captured[1].SourceKind); - Assert.Equal(2L, captured[1].SessionSignalOrdinal); - Assert.Equal("DecisionTransition", captured[2].SourceKind); - Assert.Equal(1, captured[2].StepIndex); - } - - // ============================================================ empty-window: no enqueue - - [Fact] - public async Task Flag_on_empty_window_skips_enqueue() - { - var adminConfigSvc = BuildAdminConfigService(enableIndexDualWrite: true); - - var signalRepo = new Mock(); - signalRepo - .Setup(r => r.QueryByTimestampAtOrAfterAsync(Cutoff, It.IsAny(), It.IsAny())) - .ReturnsAsync(new List()); - var transitionRepo = new Mock(); - transitionRepo - .Setup(r => r.QueryByTimestampAtOrAfterAsync(Cutoff, It.IsAny(), It.IsAny())) - .ReturnsAsync(new List()); - - var producer = new Mock(MockBehavior.Strict); - - var timer = new IndexReconcileTimer( - adminConfigSvc, - signalRepo.Object, - transitionRepo.Object, - producer.Object, - NullLogger.Instance); - - await timer.RunReconcileAsync(Cutoff, default); - - // Empty batch → no producer call (strict mock throws otherwise). - producer.VerifyNoOtherCalls(); - } - - // ============================================================ cutoff window invariant - - [Fact] - public void ReconcileWindow_is_4_hours_and_larger_than_timer_cadence() - { - // The 2h cron fires every 2h but scans a 4h window — so every primary row is - // re-checked at least twice before falling out of range. Protect the invariant. - Assert.Equal(TimeSpan.FromHours(4), IndexReconcileTimer.ReconcileWindow); - } - - // ============================================================ helpers - - private static AdminConfigurationService BuildAdminConfigService(bool enableIndexDualWrite) - { - var mock = new Mock( - Mock.Of(), - NullLogger.Instance, - new MemoryCache(new MemoryCacheOptions())); - - mock.Setup(s => s.GetConfigurationAsync()) - .ReturnsAsync(new AdminConfiguration { EnableIndexDualWrite = enableIndexDualWrite }); - - return mock.Object; - } -} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IndexRowKeysTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IndexRowKeysTests.cs deleted file mode 100644 index 8882b94f..00000000 --- a/src/Backend/AutopilotMonitor.Functions.Tests/IndexRowKeysTests.cs +++ /dev/null @@ -1,153 +0,0 @@ -using AutopilotMonitor.Functions.DataAccess.TableStorage; - -namespace AutopilotMonitor.Functions.Tests; - -/// -/// Pure PK/RK format tests for (Plan §2.8 query matrix, §M5.d). -/// These verify the string shape contract the queue-driven index writer + the reconciliation -/// timer both depend on — any drift here silently breaks cross-session queries. -/// -public class IndexRowKeysTests -{ - private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; - private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; - - private static readonly DateTime Occurred = - new(2026, 4, 21, 14, 7, 33, DateTimeKind.Utc); - - // ============================================================ SessionsByTerminal - - [Fact] - public void SessionsByTerminal_pk_is_tenant_underscore_terminalStage() - { - Assert.Equal( - $"{TenantId}_WhiteGloveSealed", - IndexRowKeys.BuildSessionsByTerminalPk(TenantId, "WhiteGloveSealed")); - } - - [Fact] - public void SessionsByTerminal_rk_is_yyyyMMddHHmmss_underscore_sessionId() - { - Assert.Equal( - $"20260421140733_{SessionId}", - IndexRowKeys.BuildSessionsByTerminalRk(Occurred, SessionId)); - } - - // ============================================================ SessionsByStage - - [Fact] - public void SessionsByStage_pk_is_tenant_underscore_stage() - { - Assert.Equal( - $"{TenantId}_EspInProgress", - IndexRowKeys.BuildSessionsByStagePk(TenantId, "EspInProgress")); - } - - [Fact] - public void SessionsByStage_rk_pads_ticks_to_19_digits_for_lex_ordering() - { - var rk = IndexRowKeys.BuildSessionsByStageRk(Occurred, SessionId); - - // 19-digit zero-padding is the contract — string comparison then matches numeric time order. - var parts = rk.Split('_'); - Assert.Equal(19, parts[0].Length); - Assert.Equal(Occurred.Ticks.ToString("D19"), parts[0]); - Assert.Equal(SessionId, parts[1]); - } - - [Fact] - public void SessionsByStage_rk_orders_lexicographically_matching_chronology() - { - var earlier = IndexRowKeys.BuildSessionsByStageRk(Occurred, SessionId); - var later = IndexRowKeys.BuildSessionsByStageRk(Occurred.AddSeconds(1), SessionId); - - Assert.True(string.CompareOrdinal(earlier, later) < 0); - } - - // ============================================================ DeadEndsByReason - - [Fact] - public void DeadEndsByReason_pk_is_tenant_underscore_reason() - { - Assert.Equal( - $"{TenantId}_hybrid_reboot_gate_blocking", - IndexRowKeys.BuildDeadEndsByReasonPk(TenantId, "hybrid_reboot_gate_blocking")); - } - - [Fact] - public void DeadEndsByReason_rk_combines_timestamp_session_stepIndex_padded_to_6() - { - Assert.Equal( - $"20260421140733_{SessionId}_000042", - IndexRowKeys.BuildDeadEndsByReasonRk(Occurred, SessionId, 42)); - } - - // ============================================================ ClassifierVerdictsByIdLevel - - [Fact] - public void ClassifierVerdictsByIdLevel_pk_is_tenant_underscore_classifierId_underscore_level() - { - Assert.Equal( - $"{TenantId}_whiteglove-sealing_Weak", - IndexRowKeys.BuildClassifierVerdictsByIdLevelPk(TenantId, "whiteglove-sealing", "Weak")); - } - - [Fact] - public void ClassifierVerdictsByIdLevel_rk_combines_timestamp_session_stepIndex_padded_to_6() - { - Assert.Equal( - $"20260421140733_{SessionId}_000007", - IndexRowKeys.BuildClassifierVerdictsByIdLevelRk(Occurred, SessionId, 7)); - } - - // ============================================================ SignalsByKind - - [Fact] - public void SignalsByKind_pk_is_tenant_underscore_signalKind() - { - Assert.Equal( - $"{TenantId}_EspTerminalFailure", - IndexRowKeys.BuildSignalsByKindPk(TenantId, "EspTerminalFailure")); - } - - [Fact] - public void SignalsByKind_rk_combines_timestamp_session_ordinal_padded_to_10() - { - Assert.Equal( - $"20260421140733_{SessionId}_0000000123", - IndexRowKeys.BuildSignalsByKindRk(Occurred, SessionId, 123)); - } - - // ============================================================ UTC coercion + sanitizing - - [Fact] - public void Timestamp_always_normalizes_to_UTC() - { - // If a caller passes Local-kind DateTime, conversion must still produce the UTC wall clock. - var localKind = DateTime.SpecifyKind(Occurred, DateTimeKind.Local); - var rk = IndexRowKeys.BuildSessionsByTerminalRk(localKind, SessionId); - - Assert.Equal($"{localKind.ToUniversalTime():yyyyMMddHHmmss}_{SessionId}", rk); - } - - [Theory] - [InlineData("foo/bar", "foo_bar")] - [InlineData("foo\\bar", "foo_bar")] - [InlineData("foo#bar", "foo_bar")] - [InlineData("foo?bar", "foo_bar")] - [InlineData("clean_name", "clean_name")] - [InlineData("", "")] - public void Sanitize_replaces_forbidden_key_chars_with_underscore(string input, string expected) - { - Assert.Equal(expected, IndexRowKeys.Sanitize(input)); - } - - [Fact] - public void Sanitize_preserves_enum_style_identifiers_unchanged() - { - // Discriminators are overwhelmingly enum names — sanitize must not mangle them. - Assert.Equal("WhiteGloveSealed", IndexRowKeys.Sanitize("WhiteGloveSealed")); - Assert.Equal("EspTerminalFailure", IndexRowKeys.Sanitize("EspTerminalFailure")); - Assert.Equal("hybrid_reboot_gate_blocking", IndexRowKeys.Sanitize("hybrid_reboot_gate_blocking")); - } -} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionGuardTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionGuardTests.cs index 54b0906c..c9628e6d 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionGuardTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionGuardTests.cs @@ -106,10 +106,10 @@ public class SessionDeletionGuardTests .ReturnsAsync(row); var ex = await Assert.ThrowsAsync( - () => guard.EnsureWritableAsync(TenantId, SessionId, callerContext: "IndexReconcile")); + () => guard.EnsureWritableAsync(TenantId, SessionId, callerContext: "AnalyzeQueue")); Assert.Equal(SessionDeletionState.Running, ex.CurrentState); - Assert.Equal("IndexReconcile", ex.CallerContext); + Assert.Equal("AnalyzeQueue", ex.CallerContext); Assert.Equal("MANIFEST-ABC", ex.ManifestId); } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs index 29063dda..287df708 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SessionDeletionHandlerTests.cs @@ -752,15 +752,15 @@ public class SessionDeletionHandlerTests Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_install_failed", Rk = $"6299999999999999_{SessionId}" } } }, new DeletionStep { Order = 10, Table = Constants.TableNames.CveIndex, Class = DeletionStepClass.DiscriminatorPkRkExact, RowCount = 1, Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_CVE-2024-0001", Rk = SessionId } } }, - new DeletionStep { Order = 11, Table = Constants.TableNames.SessionsByTerminal, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, + new DeletionStep { Order = 11, Table = "SessionsByTerminal", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_failed", Rk = "rk-1" } } }, - new DeletionStep { Order = 12, Table = Constants.TableNames.SessionsByStage, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, + new DeletionStep { Order = 12, Table = "SessionsByStage", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_AccountSetup", Rk = "rk-2" } } }, - new DeletionStep { Order = 13, Table = Constants.TableNames.DeadEndsByReason, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0, + new DeletionStep { Order = 13, Table = "DeadEndsByReason", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 0, Rows = new List() }, - new DeletionStep { Order = 14, Table = Constants.TableNames.ClassifierVerdictsByIdLevel, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, + new DeletionStep { Order = 14, Table = "ClassifierVerdictsByIdLevel", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_clf_warn", Rk = "rk-3" } } }, - new DeletionStep { Order = 15, Table = Constants.TableNames.SignalsByKind, Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, + new DeletionStep { Order = 15, Table = "SignalsByKind", Class = DeletionStepClass.DiscriminatorPkProp, RowCount = 1, Rows = new List { new DeletionRowDump { Pk = $"{TenantId}_hello", Rk = "rk-4" } } }, new DeletionStep { diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TableIndexRepositoryMappingTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TableIndexRepositoryMappingTests.cs deleted file mode 100644 index 2339fc12..00000000 --- a/src/Backend/AutopilotMonitor.Functions.Tests/TableIndexRepositoryMappingTests.cs +++ /dev/null @@ -1,167 +0,0 @@ -using AutopilotMonitor.Functions.DataAccess.TableStorage; -using AutopilotMonitor.Shared.Models; - -namespace AutopilotMonitor.Functions.Tests; - -/// -/// Pure mapping tests for 's 5 ToEntity overloads -/// (Plan §2.8, §M5.d). Transaction-batching + Azurite round-trip are out of scope here; -/// this file verifies that every projected column and key lands on the entity as expected. -/// -public class TableIndexRepositoryMappingTests -{ - private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; - private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; - - private static readonly DateTime Occurred = - new(2026, 4, 21, 14, 7, 33, DateTimeKind.Utc); - - // ============================================================ SessionsByTerminal - - [Fact] - public void SessionsByTerminal_ToEntity_projects_keys_and_typed_columns() - { - var record = new SessionsByTerminalRecord - { - TenantId = TenantId, - SessionId = SessionId, - TerminalStage = "WhiteGloveSealed", - OccurredAtUtc = Occurred, - StepIndex = 12, - }; - - var entity = TableIndexRepository.ToEntity(record); - - Assert.Equal($"{TenantId}_WhiteGloveSealed", entity.PartitionKey); - Assert.Equal($"20260421140733_{SessionId}", entity.RowKey); - Assert.Equal(TenantId, entity.GetString("TenantId")); - Assert.Equal(SessionId, entity.GetString("SessionId")); - Assert.Equal("WhiteGloveSealed", entity.GetString("TerminalStage")); - Assert.Equal(Occurred, entity.GetDateTime("OccurredAtUtc")); - Assert.Equal(12, entity.GetInt32("StepIndex")); - } - - // ============================================================ SessionsByStage - - [Fact] - public void SessionsByStage_ToEntity_uses_padded_ticks_rowkey_for_lex_ordering() - { - var record = new SessionsByStageRecord - { - TenantId = TenantId, - SessionId = SessionId, - Stage = "EspInProgress", - LastUpdatedUtc = Occurred, - StepIndex = 4, - }; - - var entity = TableIndexRepository.ToEntity(record); - - Assert.Equal($"{TenantId}_EspInProgress", entity.PartitionKey); - Assert.Equal($"{Occurred.Ticks:D19}_{SessionId}", entity.RowKey); - Assert.Equal("EspInProgress", entity.GetString("Stage")); - Assert.Equal(Occurred, entity.GetDateTime("LastUpdatedUtc")); - Assert.Equal(4, entity.GetInt32("StepIndex")); - } - - // ============================================================ DeadEndsByReason - - [Fact] - public void DeadEndsByReason_ToEntity_includes_attempted_stage_and_from_stage() - { - var record = new DeadEndsByReasonRecord - { - TenantId = TenantId, - SessionId = SessionId, - DeadEndReason = "hybrid_reboot_gate_blocking", - StepIndex = 7, - FromStage = "EspInProgress", - AttemptedToStage = "WhiteGloveSealed", - OccurredAtUtc = Occurred, - }; - - var entity = TableIndexRepository.ToEntity(record); - - Assert.Equal($"{TenantId}_hybrid_reboot_gate_blocking", entity.PartitionKey); - Assert.Equal($"20260421140733_{SessionId}_000007", entity.RowKey); - Assert.Equal("hybrid_reboot_gate_blocking", entity.GetString("DeadEndReason")); - Assert.Equal(7, entity.GetInt32("StepIndex")); - Assert.Equal("EspInProgress", entity.GetString("FromStage")); - Assert.Equal("WhiteGloveSealed", entity.GetString("AttemptedToStage")); - Assert.Equal(Occurred, entity.GetDateTime("OccurredAtUtc")); - } - - // ============================================================ ClassifierVerdictsByIdLevel - - [Fact] - public void ClassifierVerdictsByIdLevel_ToEntity_combines_classifierId_and_level_in_pk() - { - var record = new ClassifierVerdictsByIdLevelRecord - { - TenantId = TenantId, - SessionId = SessionId, - ClassifierId = "whiteglove-sealing", - HypothesisLevel = "Weak", - StepIndex = 5, - OccurredAtUtc = Occurred, - }; - - var entity = TableIndexRepository.ToEntity(record); - - Assert.Equal($"{TenantId}_whiteglove-sealing_Weak", entity.PartitionKey); - Assert.Equal($"20260421140733_{SessionId}_000005", entity.RowKey); - Assert.Equal("whiteglove-sealing", entity.GetString("ClassifierId")); - Assert.Equal("Weak", entity.GetString("HypothesisLevel")); - Assert.Equal(5, entity.GetInt32("StepIndex")); - Assert.Equal(Occurred, entity.GetDateTime("OccurredAtUtc")); - } - - // ============================================================ SignalsByKind - - [Fact] - public void SignalsByKind_ToEntity_carries_ordinal_backref_for_primary_row_hop() - { - var record = new SignalsByKindRecord - { - TenantId = TenantId, - SessionId = SessionId, - SignalKind = "EspTerminalFailure", - SessionSignalOrdinal = 987L, - OccurredAtUtc = Occurred, - SourceOrigin = "EspAndHelloTrackerAdapter", - }; - - var entity = TableIndexRepository.ToEntity(record); - - Assert.Equal($"{TenantId}_EspTerminalFailure", entity.PartitionKey); - Assert.Equal($"20260421140733_{SessionId}_0000000987", entity.RowKey); - Assert.Equal("EspTerminalFailure", entity.GetString("SignalKind")); - Assert.Equal(987L, entity.GetInt64("SessionSignalOrdinal")); - Assert.Equal(Occurred, entity.GetDateTime("OccurredAtUtc")); - Assert.Equal("EspAndHelloTrackerAdapter", entity.GetString("SourceOrigin")); - } - - // ============================================================ Null-tolerance on optional string columns - - [Fact] - public void DeadEndsByReason_ToEntity_tolerates_null_strings_by_defaulting_to_empty() - { - // The repo is called from a queue worker; a malformed upstream message shouldn't NRE the mapper. - var record = new DeadEndsByReasonRecord - { - TenantId = TenantId, - SessionId = SessionId, - DeadEndReason = null!, - StepIndex = 1, - FromStage = null!, - AttemptedToStage = null!, - OccurredAtUtc = Occurred, - }; - - var entity = TableIndexRepository.ToEntity(record); - - Assert.Equal(string.Empty, entity.GetString("DeadEndReason")); - Assert.Equal(string.Empty, entity.GetString("FromStage")); - Assert.Equal(string.Empty, entity.GetString("AttemptedToStage")); - } -} diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/DataAccessServiceExtensions.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/DataAccessServiceExtensions.cs index 69817984..a1a5ae6e 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/DataAccessServiceExtensions.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/DataAccessServiceExtensions.cs @@ -46,7 +46,6 @@ namespace AutopilotMonitor.Functions.DataAccess // V2 Decision Engine repositories (Plan §M5). services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); // Tenant-offboarding audit (Marker / History / ByTenant pointer). services.AddSingleton(); diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/IndexRowKeys.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/IndexRowKeys.cs deleted file mode 100644 index e798c548..00000000 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/IndexRowKeys.cs +++ /dev/null @@ -1,87 +0,0 @@ -using System; -using System.Globalization; - -namespace AutopilotMonitor.Functions.DataAccess.TableStorage -{ - /// - /// Pure PK/RK builders for the 5 V2 Decision Engine index tables - /// (Plan §2.8 query matrix, §M5.d). State-less + zero-IO so every key shape is - /// unit-testable against canned inputs without an Azurite harness. - /// - internal static class IndexRowKeys - { - private const string TimestampFormat = "yyyyMMddHHmmss"; - - // Azure Tables forbids these characters in PK/RK values. Discriminators are typically - // enum names (safe ASCII); the sanitize step is defensive cover for free-form strings - // like DeadEndReason that could in principle include a slash. - private static readonly char[] ForbiddenKeyChars = { '/', '\\', '#', '?' }; - - // ============================================================ SessionsByTerminal - - public static string BuildSessionsByTerminalPk(string tenantId, string terminalStage) - => $"{tenantId}_{Sanitize(terminalStage)}"; - - public static string BuildSessionsByTerminalRk(DateTime occurredAtUtc, string sessionId) - => $"{FormatTimestamp(occurredAtUtc)}_{sessionId}"; - - // ============================================================ SessionsByStage - - public static string BuildSessionsByStagePk(string tenantId, string stage) - => $"{tenantId}_{Sanitize(stage)}"; - - /// - /// Ticks are zero-padded to 19 digits (DateTime.MaxValue has 19-digit ticks) so - /// lexicographic RK ordering matches numeric time order. - /// - public static string BuildSessionsByStageRk(DateTime lastUpdatedUtc, string sessionId) - => $"{lastUpdatedUtc.ToUniversalTime().Ticks.ToString("D19", CultureInfo.InvariantCulture)}_{sessionId}"; - - // ============================================================ DeadEndsByReason - - public static string BuildDeadEndsByReasonPk(string tenantId, string deadEndReason) - => $"{tenantId}_{Sanitize(deadEndReason)}"; - - public static string BuildDeadEndsByReasonRk(DateTime occurredAtUtc, string sessionId, int stepIndex) - => $"{FormatTimestamp(occurredAtUtc)}_{sessionId}_{stepIndex.ToString("D6", CultureInfo.InvariantCulture)}"; - - // ============================================================ ClassifierVerdictsByIdLevel - - public static string BuildClassifierVerdictsByIdLevelPk(string tenantId, string classifierId, string hypothesisLevel) - => $"{tenantId}_{Sanitize(classifierId)}_{Sanitize(hypothesisLevel)}"; - - public static string BuildClassifierVerdictsByIdLevelRk(DateTime occurredAtUtc, string sessionId, int stepIndex) - => $"{FormatTimestamp(occurredAtUtc)}_{sessionId}_{stepIndex.ToString("D6", CultureInfo.InvariantCulture)}"; - - // ============================================================ SignalsByKind - - public static string BuildSignalsByKindPk(string tenantId, string signalKind) - => $"{tenantId}_{Sanitize(signalKind)}"; - - public static string BuildSignalsByKindRk(DateTime occurredAtUtc, string sessionId, long sessionSignalOrdinal) - => $"{FormatTimestamp(occurredAtUtc)}_{sessionId}_{sessionSignalOrdinal.ToString("D10", CultureInfo.InvariantCulture)}"; - - // ============================================================ Helpers - - private static string FormatTimestamp(DateTime utc) - => utc.ToUniversalTime().ToString(TimestampFormat, CultureInfo.InvariantCulture); - - /// - /// Replaces characters Azure Tables disallows in PK/RK values with _. Enum-derived - /// discriminators never trigger this; it's defensive cover for free-form strings. - /// - internal static string Sanitize(string? value) - { - if (string.IsNullOrEmpty(value)) return string.Empty; - if (value.IndexOfAny(ForbiddenKeyChars) < 0) return value; - - var buffer = new char[value.Length]; - for (var i = 0; i < value.Length; i++) - { - var c = value[i]; - buffer[i] = Array.IndexOf(ForbiddenKeyChars, c) >= 0 ? '_' : c; - } - return new string(buffer); - } - } -} diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs index 199d7d6c..85b53c6c 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs @@ -901,7 +901,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage { "WhiteGloveSealingPatternIdsJson", config.WhiteGloveSealingPatternIdsJson ?? string.Empty }, { "NvdApiKey", config.NvdApiKey }, { "VulnerabilityCorrelationEnabled", config.VulnerabilityCorrelationEnabled }, - { "EnableIndexDualWrite", config.EnableIndexDualWrite }, { "AutoApproveNewTenants", config.AutoApproveNewTenants }, { "ImeMsiArchivingEnabled", config.ImeMsiArchivingEnabled }, { "MaxImeMsiDownloadSizeMB", config.MaxImeMsiDownloadSizeMB }, @@ -973,7 +972,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage WhiteGloveSealingPatternIdsJson = entity.GetString("WhiteGloveSealingPatternIdsJson"), NvdApiKey = entity.GetString("NvdApiKey"), VulnerabilityCorrelationEnabled = entity.GetBoolean("VulnerabilityCorrelationEnabled") ?? true, - EnableIndexDualWrite = entity.GetBoolean("EnableIndexDualWrite") ?? false, AutoApproveNewTenants = entity.GetBoolean("AutoApproveNewTenants") ?? false, ImeMsiArchivingEnabled = entity.GetBoolean("ImeMsiArchivingEnabled") ?? true, MaxImeMsiDownloadSizeMB = entity.GetInt32("MaxImeMsiDownloadSizeMB") ?? 250, diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableDecisionTransitionRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableDecisionTransitionRepository.cs index a7501f12..dfe97b33 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableDecisionTransitionRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableDecisionTransitionRepository.cs @@ -107,33 +107,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage return results; } - public async Task> QueryByTimestampAtOrAfterAsync( - DateTime cutoffUtc, int maxResults = 50_000, CancellationToken cancellationToken = default) - { - var table = _storage.GetTableClient(Constants.TableNames.DecisionTransitions); - var filter = $"Timestamp ge datetime'{cutoffUtc.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.fffffffZ}'"; - - var results = new List(capacity: 128); - var pages = table.QueryAsync( - filter: filter, - maxPerPage: 1000, - cancellationToken: cancellationToken); - - await foreach (var entity in pages.ConfigureAwait(false)) - { - if (results.Count >= maxResults) - { - _logger.LogWarning( - "DecisionTransitions: time-range query reached maxResults cap {Cap} at cutoff {Cutoff:o} — narrow window or bump cap", - maxResults, cutoffUtc); - break; - } - results.Add(FromEntity(entity)); - } - - return results; - } - /// /// Projects an Azure back into a , /// reassembling chunked PayloadJson if present. Internal for mapping tests. diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableIndexRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableIndexRepository.cs deleted file mode 100644 index bab7303c..00000000 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableIndexRepository.cs +++ /dev/null @@ -1,175 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Azure.Data.Tables; -using AutopilotMonitor.Functions.Security; -using AutopilotMonitor.Functions.Services; -using AutopilotMonitor.Shared; -using AutopilotMonitor.Shared.DataAccess; -using AutopilotMonitor.Shared.Models; -using Microsoft.Extensions.Logging; - -namespace AutopilotMonitor.Functions.DataAccess.TableStorage -{ - /// - /// Azure Tables implementation of (Plan §2.8, §M5.d). - /// One class covers all 5 index tables — each table's projection lives in a dedicated - /// ToEntity overload, while the shared batched-transaction write path - /// (group-by-PK, 100-op chunk) mirrors . - /// - public sealed class TableIndexRepository : IIndexTableRepository - { - /// Azure Table Storage limit per entity-group-transaction. - internal const int TransactionChunkSize = 100; - - private readonly TableStorageService _storage; - private readonly ILogger _logger; - - public TableIndexRepository(TableStorageService storage, ILogger logger) - { - _storage = storage; - _logger = logger; - } - - // ============================================================ public Store methods - - public Task StoreSessionsByTerminalAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default) - => StoreAsync(Constants.TableNames.SessionsByTerminal, records, ToEntity, cancellationToken); - - public Task StoreSessionsByStageAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default) - => StoreAsync(Constants.TableNames.SessionsByStage, records, ToEntity, cancellationToken); - - public Task StoreDeadEndsByReasonAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default) - => StoreAsync(Constants.TableNames.DeadEndsByReason, records, ToEntity, cancellationToken); - - public Task StoreClassifierVerdictsByIdLevelAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default) - => StoreAsync(Constants.TableNames.ClassifierVerdictsByIdLevel, records, ToEntity, cancellationToken); - - public Task StoreSignalsByKindAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default) - => StoreAsync(Constants.TableNames.SignalsByKind, records, ToEntity, cancellationToken); - - // ============================================================ shared write path - - private async Task StoreAsync( - string tableName, - IReadOnlyList records, - Func toEntity, - CancellationToken cancellationToken) - { - if (records is null || records.Count == 0) return 0; - - var entities = new List(records.Count); - foreach (var r in records) - { - var entity = toEntity(r); - SecurityValidator.EnsureValidGuid(entity.GetString("TenantId"), "TenantId"); - SecurityValidator.EnsureValidGuid(entity.GetString("SessionId"), "SessionId"); - entities.Add(entity); - } - - var table = _storage.GetTableClient(tableName); - var committed = 0; - - foreach (var group in entities.GroupBy(e => e.PartitionKey)) - { - // Collapse duplicate RowKeys before building the transaction — Azure Tables - // rejects the whole batch with InvalidDuplicateRow otherwise. Dedup also yields - // RowKey ordering, replacing the previous explicit OrderBy. - var (deduped, dropped) = TableBatchDedup.ByRowKey(group); - if (dropped > 0) - _logger.LogWarning( - "{Table}: dropped {Dropped} duplicate-RowKey index row(s) (last-wins) for partition {Pk}", - tableName, dropped, group.Key); - - for (var offset = 0; offset < deduped.Count; offset += TransactionChunkSize) - { - cancellationToken.ThrowIfCancellationRequested(); - - var chunk = deduped.Skip(offset).Take(TransactionChunkSize).ToList(); - var actions = chunk - .Select(e => new TableTransactionAction(TableTransactionActionType.UpsertReplace, e)) - .ToList(); - - await table.SubmitTransactionAsync(actions, cancellationToken).ConfigureAwait(false); - committed += chunk.Count; - } - } - - _logger.LogDebug("{Table}: committed {Count} index rows", tableName, committed); - return committed; - } - - // ============================================================ per-record entity mapping - - internal static TableEntity ToEntity(SessionsByTerminalRecord r) => new( - IndexRowKeys.BuildSessionsByTerminalPk(r.TenantId, r.TerminalStage), - IndexRowKeys.BuildSessionsByTerminalRk(r.OccurredAtUtc, r.SessionId)) - { - ["TenantId"] = r.TenantId, - ["SessionId"] = r.SessionId, - ["TerminalStage"] = r.TerminalStage ?? string.Empty, - ["OccurredAtUtc"] = r.OccurredAtUtc, - ["StepIndex"] = r.StepIndex, - }; - - internal static TableEntity ToEntity(SessionsByStageRecord r) => new( - IndexRowKeys.BuildSessionsByStagePk(r.TenantId, r.Stage), - IndexRowKeys.BuildSessionsByStageRk(r.LastUpdatedUtc, r.SessionId)) - { - ["TenantId"] = r.TenantId, - ["SessionId"] = r.SessionId, - ["Stage"] = r.Stage ?? string.Empty, - ["LastUpdatedUtc"] = r.LastUpdatedUtc, - ["StepIndex"] = r.StepIndex, - }; - - internal static TableEntity ToEntity(DeadEndsByReasonRecord r) => new( - IndexRowKeys.BuildDeadEndsByReasonPk(r.TenantId, r.DeadEndReason), - IndexRowKeys.BuildDeadEndsByReasonRk(r.OccurredAtUtc, r.SessionId, r.StepIndex)) - { - ["TenantId"] = r.TenantId, - ["SessionId"] = r.SessionId, - ["DeadEndReason"] = r.DeadEndReason ?? string.Empty, - ["StepIndex"] = r.StepIndex, - ["FromStage"] = r.FromStage ?? string.Empty, - ["AttemptedToStage"] = r.AttemptedToStage ?? string.Empty, - ["OccurredAtUtc"] = r.OccurredAtUtc, - }; - - internal static TableEntity ToEntity(ClassifierVerdictsByIdLevelRecord r) => new( - IndexRowKeys.BuildClassifierVerdictsByIdLevelPk(r.TenantId, r.ClassifierId, r.HypothesisLevel), - IndexRowKeys.BuildClassifierVerdictsByIdLevelRk(r.OccurredAtUtc, r.SessionId, r.StepIndex)) - { - ["TenantId"] = r.TenantId, - ["SessionId"] = r.SessionId, - ["ClassifierId"] = r.ClassifierId ?? string.Empty, - ["HypothesisLevel"] = r.HypothesisLevel ?? string.Empty, - ["StepIndex"] = r.StepIndex, - ["OccurredAtUtc"] = r.OccurredAtUtc, - }; - - internal static TableEntity ToEntity(SignalsByKindRecord r) => new( - IndexRowKeys.BuildSignalsByKindPk(r.TenantId, r.SignalKind), - IndexRowKeys.BuildSignalsByKindRk(r.OccurredAtUtc, r.SessionId, r.SessionSignalOrdinal)) - { - ["TenantId"] = r.TenantId, - ["SessionId"] = r.SessionId, - ["SignalKind"] = r.SignalKind ?? string.Empty, - ["SessionSignalOrdinal"] = r.SessionSignalOrdinal, - ["OccurredAtUtc"] = r.OccurredAtUtc, - ["SourceOrigin"] = r.SourceOrigin ?? string.Empty, - }; - } -} diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSignalRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSignalRepository.cs index e9976bd4..d6494c13 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSignalRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSignalRepository.cs @@ -116,35 +116,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage return results; } - public async Task> QueryByTimestampAtOrAfterAsync( - DateTime cutoffUtc, int maxResults = 50_000, CancellationToken cancellationToken = default) - { - var table = _storage.GetTableClient(Constants.TableNames.Signals); - // Timestamp is the Azure Tables system-managed server-side property. Filter shape - // follows OData datetime literal format. - var filter = $"Timestamp ge datetime'{cutoffUtc.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.fffffffZ}'"; - - var results = new List(capacity: 128); - var pages = table.QueryAsync( - filter: filter, - maxPerPage: 1000, - cancellationToken: cancellationToken); - - await foreach (var entity in pages.ConfigureAwait(false)) - { - if (results.Count >= maxResults) - { - _logger.LogWarning( - "Signals: time-range query reached maxResults cap {Cap} at cutoff {Cutoff:o} — narrow window or bump cap", - maxResults, cutoffUtc); - break; - } - results.Add(FromEntity(entity)); - } - - return results; - } - /// /// Projects an Azure back into a , /// reassembling chunked PayloadJson if present. Internal for mapping tests. diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/IndexReconcileTimer.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/IndexReconcileTimer.cs deleted file mode 100644 index 96af808d..00000000 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/IndexReconcileTimer.cs +++ /dev/null @@ -1,100 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using AutopilotMonitor.Functions.Services; -using AutopilotMonitor.Functions.Services.Indexing; -using AutopilotMonitor.Shared.DataAccess; -using Microsoft.Azure.Functions.Worker; -using Microsoft.Extensions.Logging; - -namespace AutopilotMonitor.Functions.Functions.Admin -{ - /// - /// 2h safety-net timer (Plan §2.8, §M5.d.4) that recovers missed index-table writes. - /// Scans the last 4h of primary Signals + DecisionTransitions rows and - /// re-enqueues envelopes onto the telemetry-index-reconcile queue via the - /// same producer the hot ingest path uses. The queue-triggered handler (M5.d.3) - /// then upserts index rows idempotently. - /// - /// Why 2h / 4h: window is deliberately larger than cadence so every row - /// gets re-checked at least twice before falling out of scan range. Queue failures - /// or pod restarts during ingest are recoverable within the overlap. - /// - /// - /// Feature gate: . - /// When false, the scan is short-circuited — no queries, no enqueues. - /// - /// - public sealed class IndexReconcileTimer - { - // Cron "0 0 */2 * * *" = top of every second hour (00:00, 02:00, 04:00 UTC, …) - // — matches the MaintenanceFunction cadence so ops only has two timer heartbeats - // to babysit. - private const string Cron = "0 0 */2 * * *"; - - /// Safety-net window: scan primaries committed in the last N hours. - internal static readonly TimeSpan ReconcileWindow = TimeSpan.FromHours(4); - - private readonly AdminConfigurationService _adminConfig; - private readonly ISignalRepository _signalRepo; - private readonly IDecisionTransitionRepository _transitionRepo; - private readonly IIndexReconcileProducer _producer; - private readonly ILogger _logger; - - public IndexReconcileTimer( - AdminConfigurationService adminConfig, - ISignalRepository signalRepo, - IDecisionTransitionRepository transitionRepo, - IIndexReconcileProducer producer, - ILogger logger) - { - _adminConfig = adminConfig; - _signalRepo = signalRepo; - _transitionRepo = transitionRepo; - _producer = producer; - _logger = logger; - } - - [Function("IndexReconcileTimer")] - public Task Run([TimerTrigger(Cron)] object timer, CancellationToken cancellationToken) - { - var cutoff = DateTime.UtcNow - ReconcileWindow; - return RunReconcileAsync(cutoff, cancellationToken); - } - - /// - /// Testable core. Separated from the TimerTrigger entry so unit tests can drive it - /// with a deterministic cutoff without an Azure Functions host. - /// - internal async Task RunReconcileAsync(DateTime cutoffUtc, CancellationToken cancellationToken) - { - var config = await _adminConfig.GetConfigurationAsync().ConfigureAwait(false); - if (!config.EnableIndexDualWrite) - { - _logger.LogInformation( - "IndexReconcileTimer: EnableIndexDualWrite=false — skipping scan (cutoff {Cutoff:o})", - cutoffUtc); - return; - } - - var signals = await _signalRepo - .QueryByTimestampAtOrAfterAsync(cutoffUtc, cancellationToken: cancellationToken) - .ConfigureAwait(false); - var transitions = await _transitionRepo - .QueryByTimestampAtOrAfterAsync(cutoffUtc, cancellationToken: cancellationToken) - .ConfigureAwait(false); - - var envelopes = IndexReconcileEnvelopeFactory.BuildBatch(signals, transitions); - - _logger.LogInformation( - "IndexReconcileTimer: re-enqueueing {Count} envelope(s) ({S} signals + {T} transitions) since {Cutoff:o}", - envelopes.Count, signals.Count, transitions.Count, cutoffUtc); - - if (envelopes.Count == 0) return; - - // The producer is already flag-gated and swallows queue-side failures. We trust - // its contract here — timer never throws back into the host. - await _producer.EnqueueBatchAsync(envelopes, cancellationToken).ConfigureAwait(false); - } - } -} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs index 5ec9a17a..5705e638 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs @@ -4,7 +4,6 @@ using AutopilotMonitor.Functions.Helpers; using AutopilotMonitor.Functions.Security; using AutopilotMonitor.Functions.Services; using AutopilotMonitor.Functions.Services.Deletion; -using AutopilotMonitor.Functions.Services.Indexing; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; using Microsoft.Azure.Functions.Worker; @@ -42,7 +41,6 @@ namespace AutopilotMonitor.Functions.Functions.Ingest private readonly ISessionRepository _sessionRepo; private readonly ISignalRepository _signalRepo; private readonly IDecisionTransitionRepository _transitionRepo; - private readonly IIndexReconcileProducer _indexReconcileProducer; private readonly EventIngestProcessor _eventProcessor; private readonly TenantConfigurationService _configService; private readonly AdminConfigurationService _adminConfigService; @@ -62,7 +60,6 @@ namespace AutopilotMonitor.Functions.Functions.Ingest ISessionRepository sessionRepo, ISignalRepository signalRepo, IDecisionTransitionRepository transitionRepo, - IIndexReconcileProducer indexReconcileProducer, EventIngestProcessor eventProcessor, TenantConfigurationService configService, AdminConfigurationService adminConfigService, @@ -81,7 +78,6 @@ namespace AutopilotMonitor.Functions.Functions.Ingest _sessionRepo = sessionRepo; _signalRepo = signalRepo; _transitionRepo = transitionRepo; - _indexReconcileProducer = indexReconcileProducer; _eventProcessor = eventProcessor; _configService = configService; _adminConfigService = adminConfigService; @@ -454,16 +450,6 @@ namespace AutopilotMonitor.Functions.Functions.Ingest var signalCount = await _signalRepo.StoreBatchAsync(signals); var transitionCount = await _transitionRepo.StoreBatchAsync(transitions); - // M5.d.2: after the primary commit, fan out one envelope per row onto the - // telemetry-index-reconcile queue for the index-table writer (M5.d.3). - // The producer is feature-flag-gated (AdminConfiguration.EnableIndexDualWrite, - // default off) and never rethrows — see IIndexReconcileProducer contract. - if (signals.Count > 0 || transitions.Count > 0) - { - var envelopes = IndexReconcileEnvelopeFactory.BuildBatch(signals, transitions); - await _indexReconcileProducer.EnqueueBatchAsync(envelopes); - } - int eventCount; string? adminAction; List? pendingActions; diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs index 3f48a504..3635c3d9 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionDeletionMaintenanceFunction.cs @@ -50,7 +50,7 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance /// public sealed class SessionDeletionMaintenanceFunction { - // Plan §16 R14: every 12h. const per repo convention (see IndexReconcileTimer.cs:33). + // Plan §16 R14: every 12h. const per repo convention (see SessionSweepFunction.cs). private const string Cron = "0 0 */12 * * *"; // Plan §10: Preparing-GC threshold. Producer wall-clock for manifest build of even a diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionSweepFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionSweepFunction.cs index 421e837f..d23ce23a 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionSweepFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/SessionSweepFunction.cs @@ -25,7 +25,7 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance public class SessionSweepFunction { // Minute 30 — deliberately offset from the 2h Maintenance timer (minute 0) so the two - // never fire together. const per repo convention (see IndexReconcileTimer.cs). + // never fire together. const per repo convention (see MaintenanceFunction.cs). private const string Cron = "0 30 * * * *"; private readonly MaintenanceService _maintenanceService; diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Queue/IndexReconcileQueueFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Queue/IndexReconcileQueueFunction.cs deleted file mode 100644 index 6fcc0300..00000000 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Queue/IndexReconcileQueueFunction.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; -using AutopilotMonitor.Functions.Services.Indexing; -using AutopilotMonitor.Functions.Services.Queueing; -using AutopilotMonitor.Shared; -using AutopilotMonitor.Shared.Models; -using Azure.Storage.Queues.Models; -using Microsoft.Azure.Functions.Worker; -using Microsoft.Extensions.Logging; - -namespace AutopilotMonitor.Functions.Functions.Queue -{ - /// - /// Host-managed consumer of the telemetry-index-reconcile queue (Plan §M5.d.3; lease, - /// retry ladder and poison move owned by the Functions host — see - /// ). Index writes live unchanged in - /// ; the 2h IndexReconcileTimer remains the safety net. - /// - public sealed class IndexReconcileQueueFunction - { - private readonly IndexReconcileHandler _handler; - private readonly ILogger _logger; - - public IndexReconcileQueueFunction( - IndexReconcileHandler handler, - ILogger logger) - { - _handler = handler; - _logger = logger; - } - - [Function("IndexReconcileQueue")] - public Task Run( - [QueueTrigger(Constants.QueueNames.TelemetryIndexReconcile, Connection = Constants.QueueNames.TriggerConnection)] - QueueMessage message, - CancellationToken cancellationToken) - { - if (!QueueEnvelopeReader.TryRead(message, _logger, out var envelope)) - return Task.CompletedTask; - - return _handler.HandleAsync(envelope, cancellationToken); - } - } -} diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index 1e7436cf..12f3c2a6 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -311,19 +311,8 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); -// V2 Decision Engine index-table dual-write producer (Plan §2.8, §M5.d). Gated by -// AdminConfiguration.EnableIndexDualWrite (default false) inside the implementation. -builder.Services.AddSingleton< - AutopilotMonitor.Shared.DataAccess.IIndexReconcileProducer, - AutopilotMonitor.Functions.Services.Indexing.AzureQueueIndexReconcileProducer>(); - -// V2 Decision Engine index-table reconcile consumer (Plan §M5.d.3). Plain class, not -// interface-abstracted — Cosmos swap would reshape around IIndexTableRepository, not here. -builder.Services.AddSingleton< - AutopilotMonitor.Functions.Services.Indexing.IndexReconcileHandler>(); - -// Consumer: Functions/Queue/IndexReconcileQueueFunction — a [QueueTrigger] on the -// Constants.QueueNames.TriggerConnection connection (host-resolved app setting +// Queue producers + handlers. Consumers are [QueueTrigger] functions under Functions/Queue/ +// on the Constants.QueueNames.TriggerConnection connection (host-resolved app setting // `DataStorage__queueServiceUri`; see host.json extensions.queues). Stateless queue consumers // are host-managed so the Flex scale controller wakes an instance for a message and renews the // lease; only the cascade workers with lease/CAS state machines stay BackgroundServices. @@ -331,8 +320,8 @@ builder.Services.AddSingleton< // Auto-analyze fan-out at session end. Replaces the previous in-function fire-and-forget // Task.Run that ran the rule engine after enrollment_complete / enrollment_failed / async // vulnerability correlation — Functions scale-in could kill the Task.Run mid-flight, leaving -// rule results un-persisted (manual "Analyze Now" was the only recovery). Same producer + -// handler pattern as IndexReconcile above; consumer = Functions/Queue/AnalyzeOnEnrollmentEndQueueFunction. +// rule results un-persisted (manual "Analyze Now" was the only recovery). +// Consumer = Functions/Queue/AnalyzeOnEnrollmentEndQueueFunction. builder.Services.AddSingleton< AutopilotMonitor.Functions.Services.Analyze.IAnalyzeOnEnrollmentEndProducer, AutopilotMonitor.Functions.Services.Analyze.AzureQueueAnalyzeOnEnrollmentEndProducer>(); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/AdminConfigurationService.cs b/src/Backend/AutopilotMonitor.Functions/Services/AdminConfigurationService.cs index 2a869768..d2b2a913 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/AdminConfigurationService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/AdminConfigurationService.cs @@ -33,7 +33,7 @@ namespace AutopilotMonitor.Functions.Services /// Gets global admin configuration (uses cache with 5-minute TTL). /// /// Virtual so it can be mocked via Moq in consumer tests that need controlled - /// flag values (e.g. IndexReconcileTimer flag-gate tests). + /// flag values (e.g. queue-function flag-gate tests). /// /// public virtual async Task GetConfigurationAsync() diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AzureQueueAnalyzeOnEnrollmentEndProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AzureQueueAnalyzeOnEnrollmentEndProducer.cs index d992ab1f..2aa94266 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AzureQueueAnalyzeOnEnrollmentEndProducer.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AzureQueueAnalyzeOnEnrollmentEndProducer.cs @@ -12,8 +12,8 @@ namespace AutopilotMonitor.Functions.Services.Analyze { /// /// Azure Storage Queue implementation of . - /// Mirrors AzureQueueIndexReconcileProducer: Managed Identity preferred, connection - /// string fallback, Base64 encoding (matches the consumer worker). + /// Managed Identity preferred, connection string fallback, Base64 encoding (matches the + /// consumer function). /// /// Fault tolerance: queue-side exceptions (network, throttling, queue missing) are /// logged but never rethrown. The agent's HTTP 200 must not be blocked. A missed enqueue diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/DeletionManifestBuilder.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/DeletionManifestBuilder.cs index 8c6de92b..19ae48eb 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/DeletionManifestBuilder.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/DeletionManifestBuilder.cs @@ -98,7 +98,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion var safeSessionId = ODataSanitizer.EscapeValue(sessionId); var compositeSessionPk = $"{tenantId}_{sessionId}"; - // ---- Steps 1-15: cascade tables, in the §3 order. ---- + // ---- Steps 1-10: cascade tables, in the §3 order. ---- await AddPkBySessionStepAsync(manifest, order: 1, table: Constants.TableNames.Events, compositeSessionPk, safeTenantId, safeSessionId, cancellationToken); await AddPkBySessionStepAsync(manifest, order: 2, table: Constants.TableNames.RuleResults, compositeSessionPk, safeTenantId, safeSessionId, cancellationToken); await AddPropTenantPkStepAsync(manifest, order: 3, table: Constants.TableNames.AppInstallSummaries, tenantId, sessionId, safeTenantId, safeSessionId, cancellationToken); @@ -109,39 +109,34 @@ namespace AutopilotMonitor.Functions.Services.Deletion await AddPkBySessionStepAsync(manifest, order: 8, table: Constants.TableNames.DecisionTransitions, compositeSessionPk, safeTenantId, safeSessionId, cancellationToken); await AddDiscriminatorPkRkSuffixStepAsync(manifest, order: 9, table: Constants.TableNames.EventTypeIndex, tenantId, sessionId, safeTenantId, cancellationToken); await AddDiscriminatorPkRkExactStepAsync(manifest, order: 10, table: Constants.TableNames.CveIndex, tenantId, sessionId, safeTenantId, cancellationToken); - await AddDiscriminatorPkPropStepAsync(manifest, order: 11, table: Constants.TableNames.SessionsByTerminal, tenantId, sessionId, safeTenantId, safeSessionId, cancellationToken); - await AddDiscriminatorPkPropStepAsync(manifest, order: 12, table: Constants.TableNames.SessionsByStage, tenantId, sessionId, safeTenantId, safeSessionId, cancellationToken); - await AddDiscriminatorPkPropStepAsync(manifest, order: 13, table: Constants.TableNames.DeadEndsByReason, tenantId, sessionId, safeTenantId, safeSessionId, cancellationToken); - await AddDiscriminatorPkPropStepAsync(manifest, order: 14, table: Constants.TableNames.ClassifierVerdictsByIdLevel, tenantId, sessionId, safeTenantId, safeSessionId, cancellationToken); - await AddDiscriminatorPkPropStepAsync(manifest, order: 15, table: Constants.TableNames.SignalsByKind, tenantId, sessionId, safeTenantId, safeSessionId, cancellationToken); - // ---- Step 16: F1 time-attribution breakdown (PK=tenant, RK=sessionId; PR2). ---- - await AddPkRkExactStepAsync(manifest, order: 16, table: Constants.TableNames.SessionTimeBreakdowns, partitionKey: tenantId, rowKey: sessionId, cancellationToken); + // ---- Step 11: F1 time-attribution breakdown (PK=tenant, RK=sessionId; PR2). ---- + await AddPkRkExactStepAsync(manifest, order: 11, table: Constants.TableNames.SessionTimeBreakdowns, partitionKey: tenantId, rowKey: sessionId, cancellationToken); - // ---- Steps 17-19: session annotations (PK=tenant, RK="{sessionId}_{lane}"). ---- + // ---- Steps 12-14: session annotations (PK=tenant, RK="{sessionId}_{lane}"). ---- // Lanes are a closed set of 3, so exact-RK steps cover every possible row. - var annotationOrder = 17; + var annotationOrder = 12; foreach (var lane in AnnotationLanes.All) { await AddPkRkExactStepAsync(manifest, order: annotationOrder++, table: Constants.TableNames.SessionAnnotations, partitionKey: tenantId, rowKey: $"{sessionId}_{lane}", cancellationToken); } - // ---- Steps 20 + 21: SoftwareInventory side-row (omit both for pre-side-row sessions). ---- + // ---- Steps 15 + 16: SoftwareInventory side-row (omit both for pre-side-row sessions). ---- var contributionsRow = await _reader.GetEntityOrNullAsync( Constants.TableNames.SessionInventoryContributions, tenantId, sessionId, cancellationToken); if (contributionsRow != null) { - AddSoftwareInventoryDecrementStep(manifest, order: 20, contributionsRow); - AddContributionsRowStep(manifest, order: 21, contributionsRow); + AddSoftwareInventoryDecrementStep(manifest, order: 15, contributionsRow); + AddContributionsRowStep(manifest, order: 16, contributionsRow); } - // ---- Step 22: sessionId → tenantId lookup row (PK=sessionId, RK="tenant"). ---- + // ---- Step 17: sessionId → tenantId lookup row (PK=sessionId, RK="tenant"). ---- // Deleted just before the tombstone so cross-tenant resolution keeps working for // as long as the session itself exists; absent for legacy sessions (0-row step). - await AddPkRkExactStepAsync(manifest, order: 22, table: Constants.TableNames.SessionTenantLookup, partitionKey: sessionId, rowKey: "tenant", cancellationToken); + await AddPkRkExactStepAsync(manifest, order: 17, table: Constants.TableNames.SessionTenantLookup, partitionKey: sessionId, rowKey: "tenant", cancellationToken); - // ---- Step 23: Tombstone (SessionsIndex first, then Sessions). ---- - AddTombstoneStep(manifest, order: 23, sessionsIndexRow, sessionRow); + // ---- Step 18: Tombstone (SessionsIndex first, then Sessions). ---- + AddTombstoneStep(manifest, order: 18, sessionsIndexRow, sessionRow); // PreflightCounts derive from each step's RowCount, plus the AGGREGATE decrements length. manifest.PreflightCounts = ComputePreflightCounts(manifest); @@ -269,30 +264,6 @@ namespace AutopilotMonitor.Functions.Services.Deletion }); } - private async Task AddDiscriminatorPkPropStepAsync( - DeletionManifest manifest, int order, string table, - string tenantId, string sessionId, string safeTenantId, string safeSessionId, - CancellationToken cancellationToken) - { - // Server-side: PK prefix scan AND SessionId eq '{sessionId}'. Eligible because every - // DISCRIMINATOR_PK_PROP table writes the SessionId property explicitly (see IndexRowKeys - // call sites; the writers project SessionId so the server-side filter is reliable). - var filter = $"PartitionKey ge '{safeTenantId}_' and PartitionKey lt '{safeTenantId}_~' and SessionId eq '{safeSessionId}'"; - var rows = new List(); - await foreach (var entity in _reader.QueryAsync(table, filter, cancellationToken)) - { - rows.Add(TableEntityDumpConverter.MapEntityToDump(entity)); - } - manifest.Steps.Add(new DeletionStep - { - Order = order, - Table = table, - Class = DeletionStepClass.DiscriminatorPkProp, - RowCount = rows.Count, - Rows = rows, - }); - } - private void AddSoftwareInventoryDecrementStep(DeletionManifest manifest, int order, TableEntity contributionsRow) { var decrements = DecodeDecrementKeys(contributionsRow); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/ISessionDeletionEnqueuer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/ISessionDeletionEnqueuer.cs index 2d0aaf39..556c0d95 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Deletion/ISessionDeletionEnqueuer.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Deletion/ISessionDeletionEnqueuer.cs @@ -11,7 +11,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion /// existing producer is the production binding, tests substitute a fake. /// /// Mirrors the sibling queue producers (IAnalyzeOnEnrollmentEndProducer, - /// IVulnerabilityCorrelateProducer, IIndexReconcileProducer): one method + /// IVulnerabilityCorrelateProducer): one method /// returning an outcome enum so callers translate to HTTP statuses without coupling to /// internal CAS / blob mechanics. /// diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Indexing/AzureQueueIndexReconcileProducer.cs b/src/Backend/AutopilotMonitor.Functions/Services/Indexing/AzureQueueIndexReconcileProducer.cs deleted file mode 100644 index 19001b05..00000000 --- a/src/Backend/AutopilotMonitor.Functions/Services/Indexing/AzureQueueIndexReconcileProducer.cs +++ /dev/null @@ -1,113 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Azure.Storage.Queues; -using AutopilotMonitor.Functions.Services.Queueing; -using AutopilotMonitor.Shared; -using AutopilotMonitor.Shared.DataAccess; -using AutopilotMonitor.Shared.Models; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; - -namespace AutopilotMonitor.Functions.Services.Indexing -{ - /// - /// Azure Storage Queue implementation of - /// (Plan §2.8, §M5.d). Writes one Base64-encoded JSON message per envelope onto - /// the telemetry-index-reconcile queue. - /// - /// Gate: feature flag AdminConfiguration.EnableIndexDualWrite is - /// consulted via (5-min memory cache). - /// When false, the producer no-ops — the primary commit has already happened - /// and pre-M5.d behaviour stays bit-exact. - /// - /// - /// Fault tolerance: queue-side exceptions (network, throttling, queue - /// missing) are logged but NOT rethrown. Primary rows are the source of truth; - /// missed envelopes get recovered by the 2h IndexReconcileTimer (M5.d.4). - /// - /// - public sealed class AzureQueueIndexReconcileProducer : IIndexReconcileProducer - { - private readonly QueueClient _queueClient; - private readonly AdminConfigurationService _adminConfig; - private readonly ILogger _logger; - - private int _queueEnsured; // 0 = not yet ensured, 1 = CreateIfNotExistsAsync has run - - public AzureQueueIndexReconcileProducer( - QueueClientFactory queueFactory, - AdminConfigurationService adminConfig, - ILogger logger) - { - _adminConfig = adminConfig; - _logger = logger; - // Base64 matches the Azure Functions QueueTrigger default so the M5.d.3 consumer - // binding can decode our messages without an explicit override. - _queueClient = queueFactory.Create(Constants.QueueNames.TelemetryIndexReconcile); - } - - public async Task EnqueueBatchAsync( - IReadOnlyList envelopes, - CancellationToken cancellationToken = default) - { - if (envelopes is null || envelopes.Count == 0) return 0; - - // Flag check — cached 5min inside AdminConfigurationService, cheap per call. - var config = await _adminConfig.GetConfigurationAsync().ConfigureAwait(false); - if (!config.EnableIndexDualWrite) - { - return 0; - } - - await EnsureQueueExistsAsync(cancellationToken).ConfigureAwait(false); - - var sent = 0; - foreach (var envelope in envelopes) - { - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var body = JsonConvert.SerializeObject(envelope); - await _queueClient.SendMessageAsync(body, cancellationToken).ConfigureAwait(false); - sent++; - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "IndexReconcile enqueue failed (source={Source} tenant={Tenant} session={Session}) — timer reconciliation will recover", - envelope.SourceKind, envelope.TenantId, envelope.SessionId); - } - } - - if (sent > 0) - { - _logger.LogDebug("IndexReconcile: enqueued {Count} envelope(s)", sent); - } - return sent; - } - - private async Task EnsureQueueExistsAsync(CancellationToken cancellationToken) - { - // Cheap double-check: if CreateIfNotExistsAsync has already succeeded once we skip - // the round trip. Interlocked flag keeps concurrent first-callers exclusive without - // introducing a lock on the fast path. - if (_queueEnsured == 1) return; - - try - { - await _queueClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - System.Threading.Interlocked.Exchange(ref _queueEnsured, 1); - } - catch (Exception ex) - { - // Leave _queueEnsured=0 so a later call retries — we'll still attempt SendMessageAsync - // below; if the queue genuinely doesn't exist that call fails + gets logged per-envelope. - _logger.LogWarning(ex, "IndexReconcile queue CreateIfNotExists failed — will retry next batch"); - } - } - } -} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Indexing/IndexReconcileEnvelopeFactory.cs b/src/Backend/AutopilotMonitor.Functions/Services/Indexing/IndexReconcileEnvelopeFactory.cs deleted file mode 100644 index 917eb747..00000000 --- a/src/Backend/AutopilotMonitor.Functions/Services/Indexing/IndexReconcileEnvelopeFactory.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System.Collections.Generic; -using AutopilotMonitor.Shared.Models; - -namespace AutopilotMonitor.Functions.Services.Indexing -{ - /// - /// Pure factory that lifts committed primary rows into s - /// for the telemetry-index-reconcile queue (Plan §2.8, §M5.d). State-less + zero-IO - /// so callers can unit-test mapping behaviour against canned inputs without a live queue. - /// - /// One envelope per primary row. The handler (M5.d.3) decides which of the 0–3 index - /// tables to write, so the decision here is shape-only: copy discriminators + back-refs. - /// - /// - internal static class IndexReconcileEnvelopeFactory - { - public static IndexReconcileEnvelope FromSignal(SignalRecord record) - { - return new IndexReconcileEnvelope - { - EnvelopeVersion = "1", - SourceKind = "Signal", - TenantId = record.TenantId, - SessionId = record.SessionId, - OccurredAtUtc = record.OccurredAtUtc, - SessionSignalOrdinal = record.SessionSignalOrdinal, - SignalKind = record.Kind, - SourceOrigin = record.SourceOrigin, - }; - } - - public static IndexReconcileEnvelope FromDecisionTransition(DecisionTransitionRecord record) - { - return new IndexReconcileEnvelope - { - EnvelopeVersion = "1", - SourceKind = "DecisionTransition", - TenantId = record.TenantId, - SessionId = record.SessionId, - OccurredAtUtc = record.OccurredAtUtc, - StepIndex = record.StepIndex, - FromStage = record.FromStage, - ToStage = record.ToStage, - Taken = record.Taken, - IsTerminal = record.IsTerminal, - DeadEndReason = record.DeadEndReason, - ClassifierVerdictId = record.ClassifierVerdictId, - ClassifierHypothesisLevel = record.ClassifierHypothesisLevel, - }; - } - - /// - /// Builds envelopes for both kinds in one pass, preserving input order. Convenience for - /// , - /// whose per-request batch mixes signals + transitions. - /// - public static List BuildBatch( - IReadOnlyList signals, - IReadOnlyList transitions) - { - var result = new List( - (signals?.Count ?? 0) + (transitions?.Count ?? 0)); - - if (signals != null) - { - foreach (var s in signals) result.Add(FromSignal(s)); - } - if (transitions != null) - { - foreach (var t in transitions) result.Add(FromDecisionTransition(t)); - } - return result; - } - } -} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Indexing/IndexReconcileHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Indexing/IndexReconcileHandler.cs deleted file mode 100644 index 9284827f..00000000 --- a/src/Backend/AutopilotMonitor.Functions/Services/Indexing/IndexReconcileHandler.cs +++ /dev/null @@ -1,203 +0,0 @@ -using System.Threading; -using System.Threading.Tasks; -using AutopilotMonitor.Shared.DataAccess; -using AutopilotMonitor.Shared.Models; -using Microsoft.Extensions.Logging; - -namespace AutopilotMonitor.Functions.Services.Indexing -{ - /// - /// Fan-out logic for s consumed from the - /// telemetry-index-reconcile queue (Plan §2.8, §M5.d.3). Projects a single - /// envelope into the 0–3 applicable index-table rows via . - /// - /// Routing: - /// - /// Signal → SignalsByKind (always, 1 row). - /// DecisionTransition, IsTerminal=true → SessionsByTerminal. - /// DecisionTransition, Taken=true → SessionsByStage - /// (session entered a new stage; older rows for the same SessionId in other stage - /// PKs remain until the 2h reconcile timer prunes or the next stage entry supersedes - /// them in query-time ordering). - /// DecisionTransition, Taken=false + non-null DeadEndReason - /// → DeadEndsByReason. - /// DecisionTransition with non-null ClassifierVerdictId - /// → ClassifierVerdictsByIdLevel. - /// - /// - /// - /// Failure semantics: malformed envelopes (unknown , - /// missing required discriminators) are logged + dropped — retrying them against a poison - /// queue won't help. Transient Table-Storage errors rethrow, so the QueueTrigger platform - /// retry + poison-queue machinery kicks in. - /// - /// - /// Plain class (no I-interface): repo layer is already DAL-abstracted via - /// ; wrapping the handler too would be Cosmos-swap bloat. - /// Testability comes from constructor-injecting a fake repo. - /// - /// - public class IndexReconcileHandler - { - private readonly IIndexTableRepository _indexRepo; - private readonly ILogger _logger; - - public IndexReconcileHandler( - IIndexTableRepository indexRepo, - ILogger logger) - { - _indexRepo = indexRepo; - _logger = logger; - } - - public Task HandleAsync(IndexReconcileEnvelope envelope, CancellationToken cancellationToken = default) - { - if (envelope is null) - { - _logger.LogWarning("IndexReconcile: null envelope — dropping"); - return Task.CompletedTask; - } - - return envelope.SourceKind switch - { - "Signal" => HandleSignalAsync(envelope, cancellationToken), - "DecisionTransition" => HandleDecisionTransitionAsync(envelope, cancellationToken), - _ => LogAndDrop(envelope), - }; - } - - private Task LogAndDrop(IndexReconcileEnvelope envelope) - { - _logger.LogWarning( - "IndexReconcile: unknown SourceKind '{Kind}' (envelopeVersion={Version}) — dropping", - envelope.SourceKind, envelope.EnvelopeVersion); - return Task.CompletedTask; - } - - // ============================================================ Signal path - - private async Task HandleSignalAsync(IndexReconcileEnvelope e, CancellationToken ct) - { - if (e.SessionSignalOrdinal is not long ordinal || string.IsNullOrEmpty(e.SignalKind)) - { - _logger.LogWarning( - "IndexReconcile Signal envelope missing SessionSignalOrdinal or SignalKind (tenant={Tenant} session={Session}) — dropping", - e.TenantId, e.SessionId); - return; - } - - await _indexRepo.StoreSignalsByKindAsync( - new[] - { - new SignalsByKindRecord - { - TenantId = e.TenantId, - SessionId = e.SessionId, - SignalKind = e.SignalKind, - SessionSignalOrdinal = ordinal, - OccurredAtUtc = e.OccurredAtUtc, - SourceOrigin = e.SourceOrigin ?? string.Empty, - } - }, - ct).ConfigureAwait(false); - } - - // ============================================================ DecisionTransition path - - private async Task HandleDecisionTransitionAsync(IndexReconcileEnvelope e, CancellationToken ct) - { - if (e.StepIndex is not int stepIndex) - { - _logger.LogWarning( - "IndexReconcile DecisionTransition envelope missing StepIndex (tenant={Tenant} session={Session}) — dropping", - e.TenantId, e.SessionId); - return; - } - - var writes = 0; - - // SessionsByTerminal — only if a terminal stage was reached. - if (e.IsTerminal == true && !string.IsNullOrEmpty(e.ToStage)) - { - await _indexRepo.StoreSessionsByTerminalAsync( - new[] - { - new SessionsByTerminalRecord - { - TenantId = e.TenantId, - SessionId = e.SessionId, - TerminalStage = e.ToStage!, - OccurredAtUtc = e.OccurredAtUtc, - StepIndex = stepIndex, - } - }, - ct).ConfigureAwait(false); - writes++; - } - - // SessionsByStage — every Taken transition advances the "current stage" row. - if (e.Taken == true && !string.IsNullOrEmpty(e.ToStage)) - { - await _indexRepo.StoreSessionsByStageAsync( - new[] - { - new SessionsByStageRecord - { - TenantId = e.TenantId, - SessionId = e.SessionId, - Stage = e.ToStage!, - LastUpdatedUtc = e.OccurredAtUtc, - StepIndex = stepIndex, - } - }, - ct).ConfigureAwait(false); - writes++; - } - - // DeadEndsByReason — blocked transitions with an explained reason. - if (e.Taken == false && !string.IsNullOrEmpty(e.DeadEndReason)) - { - await _indexRepo.StoreDeadEndsByReasonAsync( - new[] - { - new DeadEndsByReasonRecord - { - TenantId = e.TenantId, - SessionId = e.SessionId, - DeadEndReason = e.DeadEndReason!, - StepIndex = stepIndex, - FromStage = e.FromStage ?? string.Empty, - AttemptedToStage = e.ToStage ?? string.Empty, - OccurredAtUtc = e.OccurredAtUtc, - } - }, - ct).ConfigureAwait(false); - writes++; - } - - // ClassifierVerdictsByIdLevel — any transition step that recorded a classifier verdict. - if (!string.IsNullOrEmpty(e.ClassifierVerdictId)) - { - await _indexRepo.StoreClassifierVerdictsByIdLevelAsync( - new[] - { - new ClassifierVerdictsByIdLevelRecord - { - TenantId = e.TenantId, - SessionId = e.SessionId, - ClassifierId = e.ClassifierVerdictId!, - HypothesisLevel = e.ClassifierHypothesisLevel ?? string.Empty, - StepIndex = stepIndex, - OccurredAtUtc = e.OccurredAtUtc, - } - }, - ct).ConfigureAwait(false); - writes++; - } - - _logger.LogDebug( - "IndexReconcile DecisionTransition handled (tenant={Tenant} session={Session} step={Step}): {Writes} index row(s)", - e.TenantId, e.SessionId, stepIndex, writes); - } - } -} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs index daa6ccb3..37754de7 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Offboarding/TenantOffboardingHandler.cs @@ -127,11 +127,6 @@ namespace AutopilotMonitor.Functions.Services.Offboarding Constants.TableNames.DecisionTransitions, Constants.TableNames.EventTypeIndex, Constants.TableNames.CveIndex, - Constants.TableNames.SessionsByTerminal, - Constants.TableNames.SessionsByStage, - Constants.TableNames.DeadEndsByReason, - Constants.TableNames.ClassifierVerdictsByIdLevel, - Constants.TableNames.SignalsByKind, }; // Plan §6.3.1 — Variant B (Discriminator + TenantId property). diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Deletion.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Deletion.cs index 9fcb6b84..9357957f 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Deletion.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Deletion.cs @@ -379,28 +379,6 @@ namespace AutopilotMonitor.Functions.Services return DeleteByExactKeysInBatchesAsync(Shared.Constants.TableNames.CveIndex, keys, ct); } - // ---------------- DISCRIMINATOR_PK_PROP ----------------------------------------------- - - public Task DeleteSessionsByTerminalAsync( - IReadOnlyList<(string Pk, string Rk)> keys, CancellationToken ct = default) - => DeleteByExactKeysInBatchesAsync(Shared.Constants.TableNames.SessionsByTerminal, keys, ct); - - public Task DeleteSessionsByStageAsync( - IReadOnlyList<(string Pk, string Rk)> keys, CancellationToken ct = default) - => DeleteByExactKeysInBatchesAsync(Shared.Constants.TableNames.SessionsByStage, keys, ct); - - public Task DeleteDeadEndsByReasonAsync( - IReadOnlyList<(string Pk, string Rk)> keys, CancellationToken ct = default) - => DeleteByExactKeysInBatchesAsync(Shared.Constants.TableNames.DeadEndsByReason, keys, ct); - - public Task DeleteClassifierVerdictsByIdLevelAsync( - IReadOnlyList<(string Pk, string Rk)> keys, CancellationToken ct = default) - => DeleteByExactKeysInBatchesAsync(Shared.Constants.TableNames.ClassifierVerdictsByIdLevel, keys, ct); - - public Task DeleteSignalsByKindAsync( - IReadOnlyList<(string Pk, string Rk)> keys, CancellationToken ct = default) - => DeleteByExactKeysInBatchesAsync(Shared.Constants.TableNames.SignalsByKind, keys, ct); - // ---------------- PK_RK_EXACT (single-row) -------------------------------------------- public async Task DeleteDeviceSnapshotAsync(string tenantId, string sessionId, CancellationToken ct = default) diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index cad3e6a8..16e5b66d 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -135,8 +135,6 @@ export interface AdminConfiguration { whiteGloveSealingPatternIdsJson: string; /** Whether vulnerability correlation is globally enabled. When false, agents still collect inventory but backend skips correlation. Default: true */ vulnerabilityCorrelationEnabled: boolean; - /** Feature flag for V2 Decision Engine index-table dual-write (Plan §M5.d). When true, IngestTelemetryFunction enqueues telemetry-index-reconcile envelopes after committing each primary Signals / DecisionTransitions row; a queue-triggered handler (M5.d.3) then writes the 0–3 applicable index rows. Default: false — enables controlled rollout. The 2h reconcile timer (M5.d.4) is the safety-net for queue failures even once the flag is on. */ - enableIndexDualWrite: boolean; /** When true, new tenant signups are activated automatically ~1 minute after first sign-in (tenant-auto-approve queue worker). When false, every signup waits for a manual Global Admin approval in Tenant Management. Default false — the operator opts into auto-activation and can flip it off at any time (e.g. on abuse) to return to manual vetting; messages already in the queue are then dropped, not parked. Round-tripped via the 4-file web chain (memory feedback_admin_config_ui_roundtrip). */ autoApproveNewTenants: boolean; /** When true, the first fleet-wide sighting of a new IME version enqueues an archive job (ime-msi-archive queue) that downloads the installer from the CSP-reported URL into the ime-archive blob container. Roughly one download per Microsoft IME release (~monthly). Default true — flip off to pause archiving (queued messages then wait, they are not dropped). Round-tripped via the 4-file web chain (memory feedback_admin_config_ui_roundtrip). */ diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index b296bcfe..aebbae93 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -1323,16 +1323,6 @@ namespace AutopilotMonitor.Shared public const string Signals = "Signals"; public const string DecisionTransitions = "DecisionTransitions"; - // V2 Decision Engine index tables (Plan §2.8 query matrix + §M5.d). Secondary - // projections written asynchronously via the `telemetry-index-reconcile` queue - // after the primary Signals / DecisionTransitions row has committed. Eventual - // consistency; a 2h timer re-indexes the last 4h as a safety-net on queue failures. - public const string SessionsByTerminal = "SessionsByTerminal"; - public const string SessionsByStage = "SessionsByStage"; - public const string DeadEndsByReason = "DeadEndsByReason"; - public const string ClassifierVerdictsByIdLevel = "ClassifierVerdictsByIdLevel"; - public const string SignalsByKind = "SignalsByKind"; - // Tenant offboarding audit (cascade-worker plan, Rev 9). Houses three // PartitionKey patterns side-by-side: // - "OffboardingMarker" (1 row/tenant, RK=tenantId) active 403-gate marker @@ -1458,11 +1448,6 @@ namespace AutopilotMonitor.Shared SessionTombstones, Signals, DecisionTransitions, - SessionsByTerminal, - SessionsByStage, - DeadEndsByReason, - ClassifierVerdictsByIdLevel, - SignalsByKind, OffboardingAudit, TenantOffboardingCustomsArchive, ScriptNameCache, @@ -1587,14 +1572,6 @@ namespace AutopilotMonitor.Shared /// public const string NotificationDispatch = "notification-dispatch"; - /// - /// V2 Decision Engine index-table fan-out (Plan §2.8, §M5.d). One message per - /// committed primary row (Signal or DecisionTransition); consumer writes the - /// 0–3 applicable index rows. Eventual consistency; the 2h reconcile timer is - /// the safety net on queue failures. - /// - public const string TelemetryIndexReconcile = "telemetry-index-reconcile"; - /// /// Auto-analyze fan-out at session end (enrollment_complete / enrollment_failed) /// and after async vulnerability correlation. Replaces the in-function diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IDecisionTransitionRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IDecisionTransitionRepository.cs index 77a0b766..fade33c4 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IDecisionTransitionRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IDecisionTransitionRepository.cs @@ -28,19 +28,5 @@ namespace AutopilotMonitor.Shared.DataAccess /// Task> QueryBySessionAsync( string tenantId, string sessionId, int maxResults = 1000, CancellationToken cancellationToken = default); - - /// - /// Cross-partition scan of all decision transitions whose Azure Tables server-side - /// Timestamp is at or after . Used by the - /// IndexReconcileTimer (Plan §M5.d.4) to re-enqueue the last 4h of primary - /// rows as a safety-net against queue failures. - /// - /// Perf note: Azure Tables has no PK-bound filter here, so this is a scan; - /// bounds memory. If the cap is reached the caller - /// should log + narrow the window rather than trust completeness. - /// - /// - Task> QueryByTimestampAtOrAfterAsync( - DateTime cutoffUtc, int maxResults = 50_000, CancellationToken cancellationToken = default); } } diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IIndexReconcileProducer.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IIndexReconcileProducer.cs deleted file mode 100644 index 56ea478e..00000000 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IIndexReconcileProducer.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using AutopilotMonitor.Shared.Models; - -namespace AutopilotMonitor.Shared.DataAccess -{ - /// - /// Enqueues s onto the - /// telemetry-index-reconcile queue (Plan §2.8, §M5.d). Implementations MUST - /// no-op when the AdminConfiguration.EnableIndexDualWrite flag is false — - /// this is the rollout gate that keeps pre-M5.d behaviour bit-exact by default. - /// - /// Exceptions from the underlying queue should NOT propagate back to the ingest - /// path. The primary-row write is the source of truth; index rows are eventually - /// consistent and the 2h reconcile timer (M5.d.4) is the safety net for failures. - /// - /// - public interface IIndexReconcileProducer - { - /// - /// Enqueues one queue message per envelope. Returns the number of messages actually - /// sent (0 if the feature flag is off or the batch is empty). - /// - Task EnqueueBatchAsync( - IReadOnlyList envelopes, - CancellationToken cancellationToken = default); - } -} diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IIndexTableRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IIndexTableRepository.cs deleted file mode 100644 index 7b713608..00000000 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IIndexTableRepository.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using AutopilotMonitor.Shared.Models; - -namespace AutopilotMonitor.Shared.DataAccess -{ - /// - /// Write-only repository for the 5 V2 Decision Engine index tables - /// (Plan §2.8 query matrix, §M5.d). Drives async fan-out from the primary - /// Signals / DecisionTransitions rows after they've committed, via the - /// telemetry-index-reconcile queue and the 2h safety-net timer. - /// - /// All writes are idempotent via (PartitionKey, RowKey): a retried reconcile - /// replays without duplicating rows. - /// batches require a shared PartitionKey, so the implementation groups + chunks input - /// per the 100-op transaction limit. Callers should partition by target table before - /// invoking the corresponding Store…Async method. - /// - /// - public interface IIndexTableRepository - { - Task StoreSessionsByTerminalAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default); - - Task StoreSessionsByStageAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default); - - Task StoreDeadEndsByReasonAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default); - - Task StoreClassifierVerdictsByIdLevelAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default); - - Task StoreSignalsByKindAsync( - IReadOnlyList records, - CancellationToken cancellationToken = default); - } -} diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISignalRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISignalRepository.cs index d67daa30..b45c6577 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISignalRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISignalRepository.cs @@ -70,19 +70,5 @@ namespace AutopilotMonitor.Shared.DataAccess int maxResults = 1000, CancellationToken cancellationToken = default, long maxTotalPayloadChars = SignalQueryLimits.DefaultMaxTotalPayloadChars); - - /// - /// Cross-partition scan of all signals whose Azure Tables server-side Timestamp - /// is at or after . Used by the - /// IndexReconcileTimer (Plan §M5.d.4) to re-enqueue the last 4h of primary - /// rows as a safety-net against queue failures. - /// - /// Perf note: Azure Tables has no PK-bound filter here, so this is a scan; - /// bounds memory. If the cap is reached the caller - /// should log + narrow the window rather than trust completeness. - /// - /// - Task> QueryByTimestampAtOrAfterAsync( - DateTime cutoffUtc, int maxResults = 50_000, CancellationToken cancellationToken = default); } } diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs b/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs index 025d2117..b51eb025 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Config/AdminConfiguration.cs @@ -525,16 +525,6 @@ namespace AutopilotMonitor.Shared.Models /// public bool VulnerabilityCorrelationEnabled { get; set; } = true; - /// - /// Feature flag for V2 Decision Engine index-table dual-write (Plan §M5.d). - /// When true, IngestTelemetryFunction enqueues telemetry-index-reconcile - /// envelopes after committing each primary Signals / DecisionTransitions - /// row; a queue-triggered handler (M5.d.3) then writes the 0–3 applicable index rows. - /// Default: false — enables controlled rollout. The 2h reconcile timer (M5.d.4) is - /// the safety-net for queue failures even once the flag is on. - /// - public bool EnableIndexDualWrite { get; set; } = false; - /// /// When true, new tenant signups are activated automatically ~1 minute after first /// sign-in (tenant-auto-approve queue worker). When false, every signup waits for a diff --git a/src/Shared/AutopilotMonitor.Shared/Models/DecisionTransitionRecord.cs b/src/Shared/AutopilotMonitor.Shared/Models/DecisionTransitionRecord.cs index c3518482..c4b763e2 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/DecisionTransitionRecord.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/DecisionTransitionRecord.cs @@ -10,11 +10,9 @@ namespace AutopilotMonitor.Shared.Models /// PK = {TenantId}_{SessionId}, RK = {StepIndex:D10}. /// /// - /// The index-discriminator columns (, , - /// ) are projected eagerly so index-table fan-out - /// (SessionsByTerminal / ClassifierVerdictsByIdLevel / …) can happen without re-parsing - /// . The primary write stores them inline; the queue-driven - /// index writer reads them straight off the primary row. + /// The discriminator columns (, , + /// ) are projected eagerly so readers such as the + /// decision-graph builder never re-parse . /// /// public sealed class DecisionTransitionRecord diff --git a/src/Shared/AutopilotMonitor.Shared/Models/IndexReconcileEnvelope.cs b/src/Shared/AutopilotMonitor.Shared/Models/IndexReconcileEnvelope.cs deleted file mode 100644 index c592871f..00000000 --- a/src/Shared/AutopilotMonitor.Shared/Models/IndexReconcileEnvelope.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; - -namespace AutopilotMonitor.Shared.Models -{ - /// - /// Queue-message envelope for the telemetry-index-reconcile queue - /// (Plan §2.8, §M5.d). Carries all discriminators needed for the queue-triggered - /// handler (M5.d.3) to fan-out one primary row into the 0–3 applicable index tables, - /// without needing to re-read the primary row from storage. - /// - /// Flat shape (nullable fields per ) deliberately avoids - /// polymorphic Newtonsoft deserialization — see M5.c.3 finding that abstract-type - /// round-trips through JSON need bespoke type-handling configuration, which isn't - /// worth the complexity for a two-variant envelope. - /// - /// - public sealed class IndexReconcileEnvelope - { - /// Schema version — bump on breaking envelope changes so consumers can reject or migrate. - public string EnvelopeVersion { get; set; } = "1"; - - /// Either "Signal" or "DecisionTransition"; drives which nullable fields carry data. - public string SourceKind { get; set; } = string.Empty; - - public string TenantId { get; set; } = string.Empty; - public string SessionId { get; set; } = string.Empty; - - /// When the source event happened on the agent clock; primary RK is derived from this. - public DateTime OccurredAtUtc { get; set; } - - // ---------- SourceKind="Signal" fields ---------- - - /// Back-ref to Signals primary row (its RK). - public long? SessionSignalOrdinal { get; set; } - - public string? SignalKind { get; set; } - - public string? SourceOrigin { get; set; } - - // ---------- SourceKind="DecisionTransition" fields ---------- - - /// Back-ref to DecisionTransitions primary row (its RK). - public int? StepIndex { get; set; } - - public string? FromStage { get; set; } - public string? ToStage { get; set; } - - public bool? Taken { get; set; } - public bool? IsTerminal { get; set; } - - public string? DeadEndReason { get; set; } - - public string? ClassifierVerdictId { get; set; } - public string? ClassifierHypothesisLevel { get; set; } - } -} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/IndexRecords.cs b/src/Shared/AutopilotMonitor.Shared/Models/IndexRecords.cs deleted file mode 100644 index 92b15ec8..00000000 --- a/src/Shared/AutopilotMonitor.Shared/Models/IndexRecords.cs +++ /dev/null @@ -1,125 +0,0 @@ -using System; - -namespace AutopilotMonitor.Shared.Models -{ - /// - /// Index-table record projecting a terminal onto the - /// SessionsByTerminal table (Plan §2.8 query matrix, §M5.d). Written async via the - /// telemetry-index-reconcile queue after the primary DecisionTransitions - /// row has committed; enables "which sessions ended in {TerminalStage}?" queries. - /// - /// PK = {TenantId}_{TerminalStage}, RK = {OccurredAtUtc:yyyyMMddHHmmss}_{SessionId}. - /// - /// - public sealed class SessionsByTerminalRecord - { - public string TenantId { get; set; } = string.Empty; - public string SessionId { get; set; } = string.Empty; - - /// The terminal SessionStage name (e.g. Completed, Failed, WhiteGloveSealed). - public string TerminalStage { get; set; } = string.Empty; - - public DateTime OccurredAtUtc { get; set; } - - /// Back-ref into the primary DecisionTransitions row so queries can hop to full detail. - public int StepIndex { get; set; } - } - - /// - /// Index-table record tracking a session's stage history on the SessionsByStage table - /// (Plan §2.8 query matrix, §M5.d). Written per stage-entering transition; older stage rows - /// for the same session remain until queue-side or reconciliation logic removes them. - /// - /// PK = {TenantId}_{Stage}, RK = {LastUpdatedUtcTicks:D19}_{SessionId} — - /// ticks zero-padded to 19 digits so lexicographic RK ordering matches numeric time order. - /// - /// - public sealed class SessionsByStageRecord - { - public string TenantId { get; set; } = string.Empty; - public string SessionId { get; set; } = string.Empty; - - /// The SessionStage name the session entered with this row. - public string Stage { get; set; } = string.Empty; - - public DateTime LastUpdatedUtc { get; set; } - - /// Back-ref into the primary DecisionTransitions row that moved the session into this stage. - public int StepIndex { get; set; } - } - - /// - /// Index-table record for dead-end transitions on the DeadEndsByReason table - /// (Plan §2.8 query matrix, §M5.d). Written for - /// rows with Taken=false and non-null DeadEndReason. - /// - /// PK = {TenantId}_{DeadEndReason}, RK = {OccurredAtUtc:yyyyMMddHHmmss}_{SessionId}_{StepIndex:D6}. - /// - /// - public sealed class DeadEndsByReasonRecord - { - public string TenantId { get; set; } = string.Empty; - public string SessionId { get; set; } = string.Empty; - - public string DeadEndReason { get; set; } = string.Empty; - - /// Back-ref into the primary DecisionTransitions row. - public int StepIndex { get; set; } - - public string FromStage { get; set; } = string.Empty; - - /// The ToStage the transition attempted but was blocked from reaching. - public string AttemptedToStage { get; set; } = string.Empty; - - public DateTime OccurredAtUtc { get; set; } - } - - /// - /// Index-table record projecting classifier verdicts onto the - /// ClassifierVerdictsByIdLevel table (Plan §2.8 query matrix, §M5.d). Written for - /// rows carrying a non-null ClassifierVerdictId. - /// - /// PK = {TenantId}_{ClassifierId}_{HypothesisLevel}, - /// RK = {OccurredAtUtc:yyyyMMddHHmmss}_{SessionId}_{StepIndex:D6}. - /// - /// - public sealed class ClassifierVerdictsByIdLevelRecord - { - public string TenantId { get; set; } = string.Empty; - public string SessionId { get; set; } = string.Empty; - - public string ClassifierId { get; set; } = string.Empty; - - /// Hypothesis level name (e.g. Strong, Weak, None, Inconclusive). - public string HypothesisLevel { get; set; } = string.Empty; - - /// Back-ref into the primary DecisionTransitions row. - public int StepIndex { get; set; } - - public DateTime OccurredAtUtc { get; set; } - } - - /// - /// Index-table record projecting signals onto the SignalsByKind table - /// (Plan §2.8 query matrix, §M5.d). Written per ; enables - /// cross-session "all signals of kind {SignalKind} for tenant {TenantId}" queries. - /// - /// PK = {TenantId}_{SignalKind}, - /// RK = {OccurredAtUtc:yyyyMMddHHmmss}_{SessionId}_{SessionSignalOrdinal:D10}. - /// - /// - public sealed class SignalsByKindRecord - { - public string TenantId { get; set; } = string.Empty; - public string SessionId { get; set; } = string.Empty; - - public string SignalKind { get; set; } = string.Empty; - - /// Back-ref into the primary Signals row. - public long SessionSignalOrdinal { get; set; } - - public DateTime OccurredAtUtc { get; set; } - - public string SourceOrigin { get; set; } = string.Empty; - } -} diff --git a/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx b/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx index 00d1f7bf..e2edbbb3 100644 --- a/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/AdminConfigContext.tsx @@ -58,8 +58,6 @@ interface AdminConfigContextValue { setAllowAgentDowngrade: (value: boolean) => void; modernDeploymentHarmlessEventIds: string; setModernDeploymentHarmlessEventIds: (value: string) => void; - enableIndexDualWrite: boolean; - setEnableIndexDualWrite: (value: boolean) => void; sessionDeletionKillSwitch: boolean; setSessionDeletionKillSwitch: (value: boolean) => void; autoApproveNewTenants: boolean; @@ -167,7 +165,6 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) const [slaNotificationCooldownHours, setSlaNotificationCooldownHours] = useState(24); const [allowAgentDowngrade, setAllowAgentDowngrade] = useState(false); const [modernDeploymentHarmlessEventIds, setModernDeploymentHarmlessEventIds] = useState("100, 1005, 1010"); - const [enableIndexDualWrite, setEnableIndexDualWrite] = useState(false); const [sessionDeletionKillSwitch, setSessionDeletionKillSwitch] = useState(false); const [autoApproveNewTenants, setAutoApproveNewTenants] = useState(false); const [selfServiceAppHomingEnabled, setSelfServiceAppHomingEnabled] = useState(false); @@ -240,7 +237,6 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) setSlaNotificationCooldownHours(data.slaNotificationCooldownHours ?? 24); setAllowAgentDowngrade(data.allowAgentDowngrade ?? false); setModernDeploymentHarmlessEventIds(parseHarmlessEventIdsJson(data.modernDeploymentHarmlessEventIdsJson)); - setEnableIndexDualWrite(data.enableIndexDualWrite ?? false); setSessionDeletionKillSwitch(data.sessionDeletionKillSwitch ?? false); setAutoApproveNewTenants(data.autoApproveNewTenants ?? false); setSelfServiceAppHomingEnabled(data.selfServiceAppHomingEnabled ?? false); @@ -375,7 +371,6 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIdsJson: serializeHarmlessEventIds(modernDeploymentHarmlessEventIds), - enableIndexDualWrite, sessionDeletionKillSwitch, autoApproveNewTenants, selfServiceAppHomingEnabled, @@ -407,7 +402,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) } finally { setSavingConfig(false); } - }, [isGlobalAdmin, adminConfig, globalRateLimit, userRateLimit, globalAdminRateLimit, platformStatsBlobSasUrl, agentMigrateApiBaseUrl, agentMigrateTenantOverridesJson, collectorIdleTimeoutMinutes, desktopDetectorNoCandidateTimeoutMinutes, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, enableIndexDualWrite, sessionDeletionKillSwitch, autoApproveNewTenants, selfServiceAppHomingEnabled, imeMsiArchivingEnabled, maxImeMsiDownloadSizeMB, getAccessToken]); + }, [isGlobalAdmin, adminConfig, globalRateLimit, userRateLimit, globalAdminRateLimit, platformStatsBlobSasUrl, agentMigrateApiBaseUrl, agentMigrateTenantOverridesJson, collectorIdleTimeoutMinutes, desktopDetectorNoCandidateTimeoutMinutes, opsEventRetentionDays, slaNotificationCooldownHours, allowAgentDowngrade, modernDeploymentHarmlessEventIds, sessionDeletionKillSwitch, autoApproveNewTenants, selfServiceAppHomingEnabled, imeMsiArchivingEnabled, maxImeMsiDownloadSizeMB, getAccessToken]); // Reset admin config const handleResetAdminConfig = useCallback(() => { @@ -424,7 +419,6 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) setSlaNotificationCooldownHours(adminConfig.slaNotificationCooldownHours ?? 24); setAllowAgentDowngrade(adminConfig.allowAgentDowngrade ?? false); setModernDeploymentHarmlessEventIds(parseHarmlessEventIdsJson(adminConfig.modernDeploymentHarmlessEventIdsJson)); - setEnableIndexDualWrite(adminConfig.enableIndexDualWrite ?? false); setSessionDeletionKillSwitch(adminConfig.sessionDeletionKillSwitch ?? false); setAutoApproveNewTenants(adminConfig.autoApproveNewTenants ?? false); setSelfServiceAppHomingEnabled(adminConfig.selfServiceAppHomingEnabled ?? false); @@ -608,7 +602,6 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode }) slaNotificationCooldownHours, setSlaNotificationCooldownHours, allowAgentDowngrade, setAllowAgentDowngrade, modernDeploymentHarmlessEventIds, setModernDeploymentHarmlessEventIds, - enableIndexDualWrite, setEnableIndexDualWrite, sessionDeletionKillSwitch, setSessionDeletionKillSwitch, autoApproveNewTenants, setAutoApproveNewTenants, selfServiceAppHomingEnabled, setSelfServiceAppHomingEnabled, diff --git a/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx index 6bfe365a..473387e1 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/AdminConfigSettingsSection.tsx @@ -29,8 +29,6 @@ interface AdminConfigSettingsSectionProps { setAllowAgentDowngrade: (value: boolean) => void; modernDeploymentHarmlessEventIds: string; setModernDeploymentHarmlessEventIds: (value: string) => void; - enableIndexDualWrite: boolean; - setEnableIndexDualWrite: (value: boolean) => void; sessionDeletionKillSwitch: boolean; setSessionDeletionKillSwitch: (value: boolean) => void; autoApproveNewTenants: boolean; @@ -71,8 +69,6 @@ export function AdminConfigSettingsSection({ setAllowAgentDowngrade, modernDeploymentHarmlessEventIds, setModernDeploymentHarmlessEventIds, - enableIndexDualWrite, - setEnableIndexDualWrite, sessionDeletionKillSwitch, setSessionDeletionKillSwitch, autoApproveNewTenants, @@ -306,34 +302,6 @@ export function AdminConfigSettingsSection({ -
- -
-