From 5029bdd4c3ef0211f91e6191d2c5d2b25d87f068 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Fri, 24 Apr 2026 10:45:31 +0200 Subject: [PATCH] v2 codex-01 phase 3: re-arm persisted deadlines after recovery replay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires IDeadlineScheduler.RehydrateFromSnapshot (long-present, zero production callers) into EnrollmentOrchestrator.Start so deadlines persisted inside DecisionState survive a restart. Without this any safety-net armed pre-crash (HelloSafety, Part2Safety, FinalizingGrace, ClassifierTick) silently vanishes and a resumed session can hang until the max-lifetime watchdog trips. Placement matters: the rehydrate call MUST run after the Fired event is subscribed (step 12 in Start) AND after _ingress.Start (step 13) because (a) past-due deadlines fire immediately via ThreadPool and would otherwise fall into the void without a subscriber, (b) OnDeadlineFired returns early when _ingress is null and would drop the synthesised DeadlineFired signal. Put at step 13.5 just before the WG Part-2 bridge so rehydrated fires precede fresh collector signals. Rehydrate failures are logged at error but do NOT throw — this path will get a synthetic EffectInfrastructureFailure transition once Codex follow-up #2 lands, at which point the session can terminate cleanly instead of relying on the max-lifetime watchdog. Tests (2 new in RecoveryPathTests): - Snapshot_with_past_due_deadline_fires_DeadlineFired_signal_on_start — end-to-end: scheduler fires via ThreadPool, bridge posts signal, signal reaches the SignalLog within 3s. - Snapshot_with_future_deadline_is_scheduled_but_not_yet_fired — future-due deadline reports IsScheduled=true, no premature fire. All suites green: DecisionCore 146/146, Agent.V2.Core 640/640 (+2), Functions 727/727. Build 0 warnings. Plan: .claude/plans/codex-01-recovery-signallog-replay.md §3 Phase 3. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Orchestration/RecoveryPathTests.cs | 96 +++++++++++++++++++ .../Orchestration/EnrollmentOrchestrator.cs | 32 +++++++ 2 files changed, 128 insertions(+) diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/RecoveryPathTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/RecoveryPathTests.cs index cfdf9b4c..c104159a 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/RecoveryPathTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/RecoveryPathTests.cs @@ -362,6 +362,94 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration sut.Stop(); } + // ============================================================ Codex #1 Phase 3 — deadline re-arm + + [Fact] + public void Snapshot_with_past_due_deadline_fires_DeadlineFired_signal_on_start() + { + // Past-due re-arm: a deadline whose DueAtUtc lies before clock.UtcNow must fire + // immediately on rehydrate. The scheduler's past-due path queues to ThreadPool, + // the Fired bridge synthesises a DeadlineFired signal, and it lands on the log. + using var rig = new Rig(); + Directory.CreateDirectory(rig.StateDir); + + var pastDue = new ActiveDeadline( + name: "hello_safety", + dueAtUtc: At.AddMinutes(-5), + firesSignalKind: DecisionSignalKind.DeadlineFired, + firesPayload: new Dictionary(StringComparer.Ordinal) + { + ["deadlineName"] = "hello_safety", + }); + var seed = DecisionState.CreateInitial("S1", "T1") + .ToBuilder() + .WithStage(SessionStage.AwaitingHello) + .WithStepIndex(3) + .WithLastAppliedSignalOrdinal(2) + .AddDeadline(pastDue) + .Build(); + new SnapshotPersistence( + Path.Combine(rig.StateDir, "snapshot.json"), + () => rig.Clock.UtcNow).Save(seed); + + var sut = rig.Build(); + sut.Start(); + + var signalLog = GetSignalLog(sut); + Assert.True( + SpinWait.SpinUntil( + () => signalLog.ReadAll().Any(s => + s.Kind == DecisionSignalKind.DeadlineFired && + s.Payload != null && + s.Payload.TryGetValue("deadlineName", out var n) && + n == "hello_safety"), + 3000), + "Expected DeadlineFired signal for re-armed past-due deadline."); + + sut.Stop(); + } + + [Fact] + public void Snapshot_with_future_deadline_is_scheduled_but_not_yet_fired() + { + // Future re-arm: the rehydrated deadline is live on the scheduler (IsScheduled=true) + // but no DeadlineFired signal has been emitted because its wall-clock due time + // has not arrived. + using var rig = new Rig(); + Directory.CreateDirectory(rig.StateDir); + + var future = new ActiveDeadline( + name: "part2_safety", + dueAtUtc: DateTime.UtcNow.AddHours(2), // real wall-clock future (scheduler uses wall clock) + firesSignalKind: DecisionSignalKind.DeadlineFired); + var seed = DecisionState.CreateInitial("S1", "T1") + .ToBuilder() + .WithStage(SessionStage.WhiteGloveSealed) + .WithStepIndex(5) + .WithLastAppliedSignalOrdinal(4) + .AddDeadline(future) + .Build(); + new SnapshotPersistence( + Path.Combine(rig.StateDir, "snapshot.json"), + () => rig.Clock.UtcNow).Save(seed); + + var sut = rig.Build(); + sut.Start(); + + // Scheduler must know about the re-armed deadline. + var scheduler = GetScheduler(sut); + Assert.True(scheduler.IsScheduled("part2_safety"), + "Future-due persisted deadline was not re-armed on the scheduler."); + + // Give the ThreadPool a moment to prove we don't fire it prematurely. + Thread.Sleep(100); + var signalLog = GetSignalLog(sut); + Assert.DoesNotContain(signalLog.ReadAll(), + s => s.Kind == DecisionSignalKind.DeadlineFired); + + sut.Stop(); + } + // ================================================================= Sonderfall 3: Transport-Resume [Fact] @@ -413,6 +501,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration return (ISignalLogWriter)field!.GetValue(sut)!; } + private static IDeadlineScheduler GetScheduler(EnrollmentOrchestrator sut) + { + var field = typeof(EnrollmentOrchestrator).GetField( + "_scheduler", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + return (IDeadlineScheduler)field!.GetValue(sut)!; + } + private static DrainResult InvokeDrain(EnrollmentOrchestrator sut) { var method = typeof(EnrollmentOrchestrator).GetMethod( diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs index bcc9dc5a..eedd0633 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs @@ -468,6 +468,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration // 13) Ingress-Worker starten (vor Collectors — sonst race-prone). _ingress.Start(); + // 13.5) Codex follow-up #1 / plan §3 Phase 3 — re-arm persisted deadlines. + // Without this any deadline armed pre-crash (HelloSafety, Part2Safety, + // FinalizingGrace, ClassifierTick, …) would silently disappear on restart + // and the session could hang forever. Must run AFTER _scheduler.Fired + // subscription (step 12) AND AFTER _ingress.Start (step 13) because + // past-due deadlines fire immediately via ThreadPool and the bridge + // handler posts synthetic DeadlineFired signals through _ingress. Running + // it before either one would either lose the fire (no subscriber) or drop + // the signal (OnDeadlineFired returns early when _ingress is null). + // Sits before the WG Part-2 bridge and before collectors start so + // rehydrated past-due fires precede any fresh adapter-generated signals. + if (initialState.Deadlines.Count > 0) + { + _logger.Info( + $"EnrollmentOrchestrator: re-arming {initialState.Deadlines.Count} " + + $"persisted deadline(s) post-recovery."); + try + { + _scheduler.RehydrateFromSnapshot(initialState.Deadlines); + } + catch (Exception ex) + { + // Codex follow-up #2 will convert this into a synthetic + // EffectInfrastructureFailure → enrollment_failed transition. For now + // we log at error so operators see the symptom — the max-lifetime + // watchdog remains the last-resort termination path. + _logger.Error( + "EnrollmentOrchestrator: deadline rehydration failed; session running without some deadlines.", + ex); + } + } + // 13a) Recovery Sonderfall 1: WG Part-1 → Part-2 bridge zünden. Das Signal muss // nach _ingress.Start laufen, aber VOR Collectors/Adapters — so sieht der // Reducer-Worker zuerst die Bridge-Transition (WhiteGloveSealed → AwaitingUserSignIn)