v2 codex-01 phase 3: re-arm persisted deadlines after recovery replay

Wires IDeadlineScheduler.RehydrateFromSnapshot (long-present, zero production callers) into EnrollmentOrchestrator.Start so deadlines persisted inside DecisionState survive a restart. Without this any safety-net armed pre-crash (HelloSafety, Part2Safety, FinalizingGrace, ClassifierTick) silently vanishes and a resumed session can hang until the max-lifetime watchdog trips. Placement matters: the rehydrate call MUST run after the Fired event is subscribed (step 12 in Start) AND after _ingress.Start (step 13) because (a) past-due deadlines fire immediately via ThreadPool and would otherwise fall into the void without a subscriber, (b) OnDeadlineFired returns early when _ingress is null and would drop the synthesised DeadlineFired signal. Put at step 13.5 just before the WG Part-2 bridge so rehydrated fires precede fresh collector signals. Rehydrate failures are logged at error but do NOT throw — this path will get a synthetic EffectInfrastructureFailure transition once Codex follow-up #2 lands, at which point the session can terminate cleanly instead of relying on the max-lifetime watchdog. Tests (2 new in RecoveryPathTests): - Snapshot_with_past_due_deadline_fires_DeadlineFired_signal_on_start — end-to-end: scheduler fires via ThreadPool, bridge posts signal, signal reaches the SignalLog within 3s. - Snapshot_with_future_deadline_is_scheduled_but_not_yet_fired — future-due deadline reports IsScheduled=true, no premature fire. All suites green: DecisionCore 146/146, Agent.V2.Core 640/640 (+2), Functions 727/727. Build 0 warnings. Plan: .claude/plans/codex-01-recovery-signallog-replay.md §3 Phase 3. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-10-24 20:53:06 +00:00 · 2026-04-24 10:45:31 +02:00
parent 216967e51c
commit 5029bdd4c3
2 changed files with 128 additions and 0 deletions
@@ -362,6 +362,94 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
            sut.Stop();
        }

+        // ============================================================ Codex #1 Phase 3 — deadline re-arm
+
+        [Fact]
+        public void Snapshot_with_past_due_deadline_fires_DeadlineFired_signal_on_start()
+        {
+            // Past-due re-arm: a deadline whose DueAtUtc lies before clock.UtcNow must fire
+            // immediately on rehydrate. The scheduler's past-due path queues to ThreadPool,
+            // the Fired bridge synthesises a DeadlineFired signal, and it lands on the log.
+            using var rig = new Rig();
+            Directory.CreateDirectory(rig.StateDir);
+
+            var pastDue = new ActiveDeadline(
+                name: "hello_safety",
+                dueAtUtc: At.AddMinutes(-5),
+                firesSignalKind: DecisionSignalKind.DeadlineFired,
+                firesPayload: new Dictionary<string, string>(StringComparer.Ordinal)
+                {
+                    ["deadlineName"] = "hello_safety",
+                });
+            var seed = DecisionState.CreateInitial("S1", "T1")
+                .ToBuilder()
+                .WithStage(SessionStage.AwaitingHello)
+                .WithStepIndex(3)
+                .WithLastAppliedSignalOrdinal(2)
+                .AddDeadline(pastDue)
+                .Build();
+            new SnapshotPersistence(
+                Path.Combine(rig.StateDir, "snapshot.json"),
+                () => rig.Clock.UtcNow).Save(seed);
+
+            var sut = rig.Build();
+            sut.Start();
+
+            var signalLog = GetSignalLog(sut);
+            Assert.True(
+                SpinWait.SpinUntil(
+                    () => signalLog.ReadAll().Any(s =>
+                        s.Kind == DecisionSignalKind.DeadlineFired &&
+                        s.Payload != null &&
+                        s.Payload.TryGetValue("deadlineName", out var n) &&
+                        n == "hello_safety"),
+                    3000),
+                "Expected DeadlineFired signal for re-armed past-due deadline.");
+
+            sut.Stop();
+        }
+
+        [Fact]
+        public void Snapshot_with_future_deadline_is_scheduled_but_not_yet_fired()
+        {
+            // Future re-arm: the rehydrated deadline is live on the scheduler (IsScheduled=true)
+            // but no DeadlineFired signal has been emitted because its wall-clock due time
+            // has not arrived.
+            using var rig = new Rig();
+            Directory.CreateDirectory(rig.StateDir);
+
+            var future = new ActiveDeadline(
+                name: "part2_safety",
+                dueAtUtc: DateTime.UtcNow.AddHours(2), // real wall-clock future (scheduler uses wall clock)
+                firesSignalKind: DecisionSignalKind.DeadlineFired);
+            var seed = DecisionState.CreateInitial("S1", "T1")
+                .ToBuilder()
+                .WithStage(SessionStage.WhiteGloveSealed)
+                .WithStepIndex(5)
+                .WithLastAppliedSignalOrdinal(4)
+                .AddDeadline(future)
+                .Build();
+            new SnapshotPersistence(
+                Path.Combine(rig.StateDir, "snapshot.json"),
+                () => rig.Clock.UtcNow).Save(seed);
+
+            var sut = rig.Build();
+            sut.Start();
+
+            // Scheduler must know about the re-armed deadline.
+            var scheduler = GetScheduler(sut);
+            Assert.True(scheduler.IsScheduled("part2_safety"),
+                "Future-due persisted deadline was not re-armed on the scheduler.");
+
+            // Give the ThreadPool a moment to prove we don't fire it prematurely.
+            Thread.Sleep(100);
+            var signalLog = GetSignalLog(sut);
+            Assert.DoesNotContain(signalLog.ReadAll(),
+                s => s.Kind == DecisionSignalKind.DeadlineFired);
+
+            sut.Stop();
+        }
+
        // ================================================================= Sonderfall 3: Transport-Resume

        [Fact]
@@ -413,6 +501,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
            return (ISignalLogWriter)field!.GetValue(sut)!;
        }

+        private static IDeadlineScheduler GetScheduler(EnrollmentOrchestrator sut)
+        {
+            var field = typeof(EnrollmentOrchestrator).GetField(
+                "_scheduler",
+                System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic);
+            return (IDeadlineScheduler)field!.GetValue(sut)!;
+        }
+
        private static DrainResult InvokeDrain(EnrollmentOrchestrator sut)
        {
            var method = typeof(EnrollmentOrchestrator).GetMethod(
@@ -468,6 +468,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
            // 13) Ingress-Worker starten (vor Collectors — sonst race-prone).
            _ingress.Start();

+            // 13.5) Codex follow-up #1 / plan §3 Phase 3 — re-arm persisted deadlines.
+            //       Without this any deadline armed pre-crash (HelloSafety, Part2Safety,
+            //       FinalizingGrace, ClassifierTick, …) would silently disappear on restart
+            //       and the session could hang forever. Must run AFTER _scheduler.Fired
+            //       subscription (step 12) AND AFTER _ingress.Start (step 13) because
+            //       past-due deadlines fire immediately via ThreadPool and the bridge
+            //       handler posts synthetic DeadlineFired signals through _ingress. Running
+            //       it before either one would either lose the fire (no subscriber) or drop
+            //       the signal (OnDeadlineFired returns early when _ingress is null).
+            //       Sits before the WG Part-2 bridge and before collectors start so
+            //       rehydrated past-due fires precede any fresh adapter-generated signals.
+            if (initialState.Deadlines.Count > 0)
+            {
+                _logger.Info(
+                    $"EnrollmentOrchestrator: re-arming {initialState.Deadlines.Count} " +
+                    $"persisted deadline(s) post-recovery.");
+                try
+                {
+                    _scheduler.RehydrateFromSnapshot(initialState.Deadlines);
+                }
+                catch (Exception ex)
+                {
+                    // Codex follow-up #2 will convert this into a synthetic
+                    // EffectInfrastructureFailure → enrollment_failed transition. For now
+                    // we log at error so operators see the symptom — the max-lifetime
+                    // watchdog remains the last-resort termination path.
+                    _logger.Error(
+                        "EnrollmentOrchestrator: deadline rehydration failed; session running without some deadlines.",
+                        ex);
+                }
+            }
+
            // 13a) Recovery Sonderfall 1: WG Part-1 → Part-2 bridge zünden. Das Signal muss
            //      nach _ingress.Start laufen, aber VOR Collectors/Adapters — so sieht der
            //      Reducer-Worker zuerst die Bridge-Transition (WhiteGloveSealed → AwaitingUserSignIn)