mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 04:32:32 +00:00
v2 codex-06: ReducerVerifier semantic replay — reducer-agreement CI gate
Closes Codex review follow-up #6. Until now GetSessionReducerVerificationFunction only ran structural checks (ordinal contiguity, step-index contiguity, ReducerVersion drift, orphaned SignalOrdinalRef). That catches corruption but never verifies the backend's reducer agrees with the agent-produced journal under the same ReducerVersion — exactly the invariant the whole (newState, transition, effects) architecture rests on. Unblocked by Codex #1 phase 1: ReducerReplay.Replay exists in DecisionCore, and the backend already references DecisionCore. Flow: 1. Skip preconditions: drift, ordinal gaps, step-index gaps, empty session, or head PayloadJson deserialization failure → record SemanticReplaySkipReason + optionally emit an Info 'replay_skipped' issue. Never throws. 2. Deserialise each SignalRecord.PayloadJson via SignalSerializer. 3. Fold through a transient DecisionEngine: state = Reduce(state, sig). 4. Compare each produced transition against the stored one on (StepIndex, Trigger, FromStage, ToStage, Taken, DeadEndReason). Discrepancies surface as 'replay_divergence' issues (capped at 20 individual entries; overflow message follows the same pattern as the structural gap reports). 5. Compare replayed final state's Stage to the last stored transition's ToStage — mismatch surfaces as 'replay_final_stage_mismatch' at Error severity. Report additions (backward-compatible): - SemanticReplayPerformed - SemanticReplaySkipReason - SemanticReplayFinalStageMatches - ReplayedFinalStage - TransitionDivergenceCount Side change: GetSessionReducerVerificationFunction XML-doc no longer claims "structural only" — now documents both layers and the skip rules. Tests (+7, ReducerVerifierSemanticReplayTests): - Skip on ReducerVersion drift. - Skip on signal-ordinal gap. - Skip on step-index gap. - Perfect agreement (reducer-seeded journal round-trips). - Divergence: tampered Trigger → one replay_divergence issue. - Divergence: tampered last-ToStage → replay_final_stage_mismatch Error. - Mid-stream deserialization failure tolerated without aborting. Existing structural ReducerVerifierTests unchanged — their stub PayloadJson='{}' deserialization failures now silently set SemanticReplaySkipReason='deserialization_failure' without polluting Issues, preserving the happy-path empty-issues assertion. Suites green: DecisionCore 151/151, Agent.V2.Core 643/643, Functions 734/734 (+7). Build 0 warnings. Plan: .claude/plans/codex-06-reducer-verifier-semantic-replay.md Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,291 @@
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Serialization;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Codex follow-up #6 — semantic replay coverage for <see cref="ReducerVerifier"/>.
|
||||
/// Structural-only cases live in <see cref="ReducerVerifierTests"/>; this file pins the
|
||||
/// contract that when preconditions hold (contiguous ordinals / matching ReducerVersion)
|
||||
/// the verifier folds the real signal stream through a live <see cref="DecisionEngine"/>
|
||||
/// and reports divergence.
|
||||
/// </summary>
|
||||
public class ReducerVerifierSemanticReplayTests
|
||||
{
|
||||
private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890";
|
||||
private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901";
|
||||
|
||||
/// <summary>The live backend reducer version — matches what Verify uses for drift checks.</summary>
|
||||
private static string CurrentReducerVersion { get; } = new DecisionEngine().ReducerVersion;
|
||||
|
||||
// ---- Seed helpers ---------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Build a canonical 3-signal stream + a matching journal by folding the live reducer
|
||||
/// over them. The helper is the "truth" data for agreement tests — anything that differs
|
||||
/// from this output in a test fixture is by definition divergence.
|
||||
/// </summary>
|
||||
private static (SignalRecord[] signals, DecisionTransitionRecord[] transitions, DecisionState finalState)
|
||||
BuildAgreedStream()
|
||||
{
|
||||
var sigs = new[]
|
||||
{
|
||||
MakeSignal(0, DecisionSignalKind.SessionStarted),
|
||||
MakeSignal(1, DecisionSignalKind.AppInstallCompleted),
|
||||
MakeSignal(2, DecisionSignalKind.AppInstallCompleted),
|
||||
};
|
||||
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial(TenantId, SessionId);
|
||||
var transitionRecords = new DecisionTransitionRecord[sigs.Length];
|
||||
|
||||
for (var i = 0; i < sigs.Length; i++)
|
||||
{
|
||||
var step = engine.Reduce(state, sigs[i]);
|
||||
state = step.NewState;
|
||||
|
||||
transitionRecords[i] = new DecisionTransitionRecord
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
StepIndex = step.Transition.StepIndex,
|
||||
SessionTraceOrdinal = step.Transition.SessionTraceOrdinal,
|
||||
SignalOrdinalRef = step.Transition.SignalOrdinalRef,
|
||||
OccurredAtUtc = step.Transition.OccurredAtUtc,
|
||||
Trigger = step.Transition.Trigger,
|
||||
FromStage = step.Transition.FromStage.ToString(),
|
||||
ToStage = step.Transition.ToStage.ToString(),
|
||||
Taken = step.Transition.Taken,
|
||||
DeadEndReason = step.Transition.DeadEndReason,
|
||||
ReducerVersion = step.Transition.ReducerVersion,
|
||||
PayloadJson = TransitionSerializer.Serialize(step.Transition),
|
||||
};
|
||||
}
|
||||
|
||||
var signalRecords = new SignalRecord[sigs.Length];
|
||||
for (var i = 0; i < sigs.Length; i++)
|
||||
{
|
||||
signalRecords[i] = new SignalRecord
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
SessionSignalOrdinal = sigs[i].SessionSignalOrdinal,
|
||||
SessionTraceOrdinal = sigs[i].SessionTraceOrdinal,
|
||||
Kind = sigs[i].Kind.ToString(),
|
||||
KindSchemaVersion = sigs[i].KindSchemaVersion,
|
||||
OccurredAtUtc = sigs[i].OccurredAtUtc,
|
||||
SourceOrigin = sigs[i].SourceOrigin,
|
||||
PayloadJson = SignalSerializer.Serialize(sigs[i]),
|
||||
};
|
||||
}
|
||||
|
||||
return (signalRecords, transitionRecords, state);
|
||||
}
|
||||
|
||||
private static DecisionSignal MakeSignal(long ordinal, DecisionSignalKind kind) =>
|
||||
new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: kind,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: new DateTime(2026, 4, 24, 10, 0, 0, DateTimeKind.Utc).AddSeconds(ordinal),
|
||||
sourceOrigin: "replay-test",
|
||||
evidence: new Evidence(EvidenceKind.Synthetic, $"replay:ord-{ordinal}", "test"));
|
||||
|
||||
// ---- Skip conditions ------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Replay_skipped_when_stored_reducer_version_drifts()
|
||||
{
|
||||
var (signals, transitions, _) = BuildAgreedStream();
|
||||
// Override ReducerVersion on the first transition to simulate drift.
|
||||
transitions[0] = CloneWithReducerVersion(transitions[0], "0.9.0.0");
|
||||
|
||||
var report = ReducerVerifier.Verify(TenantId, SessionId, signals, transitions, CurrentReducerVersion);
|
||||
|
||||
Assert.False(report.SemanticReplayPerformed);
|
||||
Assert.Equal("reducer_version_drift", report.SemanticReplaySkipReason);
|
||||
Assert.Contains(report.Issues, i => i.Kind == "replay_skipped");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Replay_skipped_when_signal_stream_has_ordinal_gap()
|
||||
{
|
||||
var (signals, transitions, _) = BuildAgreedStream();
|
||||
// Drop the middle signal → ordinal gap.
|
||||
var gapped = new[] { signals[0], signals[2] };
|
||||
|
||||
var report = ReducerVerifier.Verify(TenantId, SessionId, gapped, transitions, CurrentReducerVersion);
|
||||
|
||||
Assert.False(report.SemanticReplayPerformed);
|
||||
Assert.Equal("non_contiguous_signal_ordinals", report.SemanticReplaySkipReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Replay_skipped_when_transition_journal_has_step_index_gap()
|
||||
{
|
||||
var (signals, transitions, _) = BuildAgreedStream();
|
||||
var gapped = new[] { transitions[0], transitions[2] }; // StepIndex gap
|
||||
|
||||
var report = ReducerVerifier.Verify(TenantId, SessionId, signals, gapped, CurrentReducerVersion);
|
||||
|
||||
Assert.False(report.SemanticReplayPerformed);
|
||||
Assert.Equal("non_contiguous_step_indices", report.SemanticReplaySkipReason);
|
||||
}
|
||||
|
||||
// ---- Agreement + divergence ----------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Replay_perfect_agreement_marks_final_stage_matches_and_zero_divergence()
|
||||
{
|
||||
var (signals, transitions, finalState) = BuildAgreedStream();
|
||||
|
||||
var report = ReducerVerifier.Verify(TenantId, SessionId, signals, transitions, CurrentReducerVersion);
|
||||
|
||||
Assert.True(report.SemanticReplayPerformed);
|
||||
Assert.Null(report.SemanticReplaySkipReason);
|
||||
Assert.True(report.SemanticReplayFinalStageMatches);
|
||||
Assert.Equal(finalState.Stage.ToString(), report.ReplayedFinalStage);
|
||||
Assert.Equal(0, report.TransitionDivergenceCount);
|
||||
Assert.DoesNotContain(report.Issues, i => i.Kind == "replay_divergence");
|
||||
Assert.DoesNotContain(report.Issues, i => i.Kind == "replay_final_stage_mismatch");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Replay_divergence_reports_issue_when_stored_trigger_was_tampered()
|
||||
{
|
||||
var (signals, transitions, _) = BuildAgreedStream();
|
||||
// Tamper with a stored transition: swap Trigger. Replay will produce the
|
||||
// authentic trigger and divergence is reported.
|
||||
transitions[1] = CloneWithTrigger(transitions[1], "TamperedTrigger");
|
||||
|
||||
var report = ReducerVerifier.Verify(TenantId, SessionId, signals, transitions, CurrentReducerVersion);
|
||||
|
||||
Assert.True(report.SemanticReplayPerformed);
|
||||
Assert.Equal(1, report.TransitionDivergenceCount);
|
||||
var issue = Assert.Single(report.Issues, i => i.Kind == "replay_divergence");
|
||||
Assert.Contains("TamperedTrigger", issue.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Replay_final_stage_mismatch_raises_error_issue_when_last_stored_ToStage_differs()
|
||||
{
|
||||
var (signals, transitions, _) = BuildAgreedStream();
|
||||
transitions[^1] = CloneWithToStage(transitions[^1], "Completed"); // pretend the last stored landed in Completed
|
||||
|
||||
var report = ReducerVerifier.Verify(TenantId, SessionId, signals, transitions, CurrentReducerVersion);
|
||||
|
||||
Assert.True(report.SemanticReplayPerformed);
|
||||
Assert.False(report.SemanticReplayFinalStageMatches);
|
||||
Assert.Contains(report.Issues,
|
||||
i => i.Kind == "replay_final_stage_mismatch" && i.Severity == "Error");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Replay_handles_mid_stream_signal_deserialization_failure_without_aborting()
|
||||
{
|
||||
var (signals, transitions, _) = BuildAgreedStream();
|
||||
// Corrupt the second signal's PayloadJson so deserialization fails there.
|
||||
signals[1].PayloadJson = "this is not json";
|
||||
|
||||
var report = ReducerVerifier.Verify(TenantId, SessionId, signals, transitions, CurrentReducerVersion);
|
||||
|
||||
// Replay ran on the valid prefix (signal 0); deserialization error reported for the
|
||||
// truncated tail rather than aborting the whole verification.
|
||||
Assert.True(report.SemanticReplayPerformed);
|
||||
Assert.Contains(report.Issues, i => i.Kind == "replay_deserialization_error");
|
||||
}
|
||||
|
||||
// ---- Clone helpers (records are POCO, so tweaks need full copies) --------------
|
||||
|
||||
private static DecisionTransitionRecord CloneWithReducerVersion(
|
||||
DecisionTransitionRecord source, string reducerVersion) =>
|
||||
new DecisionTransitionRecord
|
||||
{
|
||||
TenantId = source.TenantId,
|
||||
SessionId = source.SessionId,
|
||||
StepIndex = source.StepIndex,
|
||||
SessionTraceOrdinal = source.SessionTraceOrdinal,
|
||||
SignalOrdinalRef = source.SignalOrdinalRef,
|
||||
OccurredAtUtc = source.OccurredAtUtc,
|
||||
Trigger = source.Trigger,
|
||||
FromStage = source.FromStage,
|
||||
ToStage = source.ToStage,
|
||||
Taken = source.Taken,
|
||||
DeadEndReason = source.DeadEndReason,
|
||||
ReducerVersion = reducerVersion,
|
||||
PayloadJson = source.PayloadJson,
|
||||
};
|
||||
|
||||
private static DecisionTransitionRecord CloneWithTrigger(
|
||||
DecisionTransitionRecord source, string trigger)
|
||||
{
|
||||
// We rewrite PayloadJson too so TransitionSerializer sees the tampered trigger
|
||||
// and the verifier's comparison picks it up.
|
||||
var transition = TransitionSerializer.Deserialize(source.PayloadJson);
|
||||
var tampered = new DecisionTransition(
|
||||
stepIndex: transition.StepIndex,
|
||||
sessionTraceOrdinal: transition.SessionTraceOrdinal,
|
||||
signalOrdinalRef: transition.SignalOrdinalRef,
|
||||
occurredAtUtc: transition.OccurredAtUtc,
|
||||
trigger: trigger,
|
||||
fromStage: transition.FromStage,
|
||||
toStage: transition.ToStage,
|
||||
taken: transition.Taken,
|
||||
deadEndReason: transition.DeadEndReason,
|
||||
reducerVersion: transition.ReducerVersion);
|
||||
return new DecisionTransitionRecord
|
||||
{
|
||||
TenantId = source.TenantId,
|
||||
SessionId = source.SessionId,
|
||||
StepIndex = source.StepIndex,
|
||||
SessionTraceOrdinal = source.SessionTraceOrdinal,
|
||||
SignalOrdinalRef = source.SignalOrdinalRef,
|
||||
OccurredAtUtc = source.OccurredAtUtc,
|
||||
Trigger = trigger,
|
||||
FromStage = source.FromStage,
|
||||
ToStage = source.ToStage,
|
||||
Taken = source.Taken,
|
||||
DeadEndReason = source.DeadEndReason,
|
||||
ReducerVersion = source.ReducerVersion,
|
||||
PayloadJson = TransitionSerializer.Serialize(tampered),
|
||||
};
|
||||
}
|
||||
|
||||
private static DecisionTransitionRecord CloneWithToStage(
|
||||
DecisionTransitionRecord source, string toStage)
|
||||
{
|
||||
var transition = TransitionSerializer.Deserialize(source.PayloadJson);
|
||||
var tampered = new DecisionTransition(
|
||||
stepIndex: transition.StepIndex,
|
||||
sessionTraceOrdinal: transition.SessionTraceOrdinal,
|
||||
signalOrdinalRef: transition.SignalOrdinalRef,
|
||||
occurredAtUtc: transition.OccurredAtUtc,
|
||||
trigger: transition.Trigger,
|
||||
fromStage: transition.FromStage,
|
||||
toStage: (SessionStage)Enum.Parse(typeof(SessionStage), toStage),
|
||||
taken: transition.Taken,
|
||||
deadEndReason: transition.DeadEndReason,
|
||||
reducerVersion: transition.ReducerVersion);
|
||||
return new DecisionTransitionRecord
|
||||
{
|
||||
TenantId = source.TenantId,
|
||||
SessionId = source.SessionId,
|
||||
StepIndex = source.StepIndex,
|
||||
SessionTraceOrdinal = source.SessionTraceOrdinal,
|
||||
SignalOrdinalRef = source.SignalOrdinalRef,
|
||||
OccurredAtUtc = source.OccurredAtUtc,
|
||||
Trigger = source.Trigger,
|
||||
FromStage = source.FromStage,
|
||||
ToStage = toStage,
|
||||
Taken = source.Taken,
|
||||
DeadEndReason = source.DeadEndReason,
|
||||
ReducerVersion = source.ReducerVersion,
|
||||
PayloadJson = TransitionSerializer.Serialize(tampered),
|
||||
};
|
||||
}
|
||||
}
|
||||
+14
-9
@@ -11,16 +11,21 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
{
|
||||
/// <summary>
|
||||
/// <c>GET /api/sessions/{sessionId}/reducer-verification</c> — admin/ops endpoint that
|
||||
/// runs structural verification on a session's persisted SignalLog + DecisionTransitions
|
||||
/// journal (Plan §M5). Not tenant-exposed: gated by <c>GlobalAdminOnly</c> in the
|
||||
/// <c>EndpointAccessPolicyCatalog</c>.
|
||||
/// runs structural + semantic verification on a session's persisted SignalLog +
|
||||
/// DecisionTransitions journal (Plan §M5). Not tenant-exposed: gated by
|
||||
/// <c>GlobalAdminOnly</c> in the <c>EndpointAccessPolicyCatalog</c>.
|
||||
/// <para>
|
||||
/// <b>Why structural only (not full engine replay)?</b> A full replay requires deserialising
|
||||
/// the polymorphic <c>DecisionSignal.Evidence</c> from <c>SignalRecord.PayloadJson</c>,
|
||||
/// which is a non-trivial refactor beyond the scope of this commit. The structural checks
|
||||
/// catch the realistic release-gate failure modes (corruption, ReducerVersion drift, orphan
|
||||
/// references) and give ops a usable tool today; a dedicated follow-up can add engine
|
||||
/// replay once the Evidence deserialisation contract is pinned down.
|
||||
/// <b>Structural checks</b> — always run: ordinal contiguity, step-index contiguity,
|
||||
/// ReducerVersion drift, orphaned <c>SignalOrdinalRef</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Semantic replay</b> (Codex follow-up #6) — folds the persisted signal stream
|
||||
/// through the live backend <c>DecisionEngine</c> via <c>SignalSerializer.Deserialize</c>
|
||||
/// and compares the produced transitions to the stored journal on the semantic fields
|
||||
/// (Trigger / FromStage / ToStage / Taken / DeadEndReason / StepIndex). Skipped
|
||||
/// automatically when the stored ReducerVersion differs, the signal stream has gaps,
|
||||
/// or deserialisation fails at the head — those are structural conditions where a
|
||||
/// replay would not be meaningful.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class GetSessionReducerVerificationFunction
|
||||
|
||||
@@ -1,16 +1,37 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Serialization;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Structural verification of a session's persisted SignalLog + DecisionTransitions
|
||||
/// journal. Pure function — takes loaded records, produces a report. No IO, no DI.
|
||||
/// See <see cref="ReducerVerificationReport"/> for scope boundary vs. full engine replay.
|
||||
/// Structural + semantic verification of a session's persisted SignalLog +
|
||||
/// DecisionTransitions journal. Pure function — takes loaded records, produces a
|
||||
/// report. No IO, no DI.
|
||||
/// <para>
|
||||
/// <b>Structural checks</b> (always run): ordinal contiguity, step-index contiguity,
|
||||
/// ReducerVersion drift, orphaned <c>SignalOrdinalRef</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Semantic replay</b> (Codex follow-up #6, added when preconditions hold):
|
||||
/// deserialise <see cref="SignalRecord.PayloadJson"/> into <see cref="DecisionSignal"/>
|
||||
/// instances, fold them through the live backend <see cref="DecisionEngine"/>, and
|
||||
/// compare the produced transitions to the stored journal on the semantic fields
|
||||
/// (Trigger / FromStage / ToStage / Taken / DeadEndReason / StepIndex). The agent-side
|
||||
/// journal is the agent's reducer-chain; this replay verifies the backend's reducer
|
||||
/// agrees — divergence under the same ReducerVersion is a hard release-gate bug.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static class ReducerVerifier
|
||||
{
|
||||
/// <summary>Cap for individual replay-divergence issues, mirrors the gap caps.</summary>
|
||||
private const int MaxReplayDivergenceIssues = 20;
|
||||
|
||||
public static ReducerVerificationReport Verify(
|
||||
string tenantId,
|
||||
string sessionId,
|
||||
@@ -36,6 +57,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
Kind = "empty_session",
|
||||
Message = "Session has no signals or transitions persisted.",
|
||||
});
|
||||
report.SemanticReplaySkipReason = "empty_session";
|
||||
return report;
|
||||
}
|
||||
|
||||
@@ -168,7 +190,212 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Semantic replay (Codex follow-up #6) ------------------------------------
|
||||
RunSemanticReplay(signals, transitions, report);
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Folds the persisted <paramref name="signals"/> stream through a transient
|
||||
/// <see cref="DecisionEngine"/> instance and compares the produced transitions to
|
||||
/// the journal. Skips when preconditions (contiguity, ReducerVersion agreement) are
|
||||
/// not met — divergence under those conditions is not a reducer bug but expected
|
||||
/// drift.
|
||||
/// </summary>
|
||||
private static void RunSemanticReplay(
|
||||
IReadOnlyList<SignalRecord>? signals,
|
||||
IReadOnlyList<DecisionTransitionRecord>? transitions,
|
||||
ReducerVerificationReport report)
|
||||
{
|
||||
if (signals == null || signals.Count == 0 || transitions == null || transitions.Count == 0)
|
||||
{
|
||||
// Empty-session skip already recorded above.
|
||||
if (string.IsNullOrEmpty(report.SemanticReplaySkipReason))
|
||||
{
|
||||
report.SemanticReplaySkipReason = "empty_session";
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (report.ReducerVersionDrift)
|
||||
{
|
||||
report.SemanticReplaySkipReason = "reducer_version_drift";
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Info",
|
||||
Kind = "replay_skipped",
|
||||
Message = "Semantic replay skipped because stored ReducerVersion differs from current — replay against a different reducer build is not meaningful.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!report.SignalOrdinalsContiguous)
|
||||
{
|
||||
report.SemanticReplaySkipReason = "non_contiguous_signal_ordinals";
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Info",
|
||||
Kind = "replay_skipped",
|
||||
Message = "Semantic replay skipped because the signal stream has ordinal gaps; the structural warnings above are the actionable signal.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!report.StepIndicesContiguous)
|
||||
{
|
||||
report.SemanticReplaySkipReason = "non_contiguous_step_indices";
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Info",
|
||||
Kind = "replay_skipped",
|
||||
Message = "Semantic replay skipped because the transition journal has StepIndex gaps.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
var orderedSignals = signals.OrderBy(s => s.SessionSignalOrdinal).ToList();
|
||||
var orderedTransitions = transitions.OrderBy(t => t.StepIndex).ToList();
|
||||
|
||||
// Deserialise signals up-front — a single malformed blob at the head means we
|
||||
// cannot start the replay at all.
|
||||
var decodedSignals = new List<DecisionSignal>(orderedSignals.Count);
|
||||
for (var i = 0; i < orderedSignals.Count; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
decodedSignals.Add(SignalSerializer.Deserialize(orderedSignals[i].PayloadJson));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (i == 0)
|
||||
{
|
||||
// Silent skip at head — PayloadJson stubs (common in structural tests)
|
||||
// should not pollute the issue list; the SemanticReplaySkipReason field
|
||||
// carries the diagnostic for real operators.
|
||||
report.SemanticReplaySkipReason = "deserialization_failure";
|
||||
return;
|
||||
}
|
||||
|
||||
// Mid-stream failure: truncate here, continue what we have. The divergence
|
||||
// tail will surface as transition-count mismatch naturally.
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Warning",
|
||||
Kind = "replay_deserialization_error",
|
||||
Message = $"Signal at ordinal {orderedSignals[i].SessionSignalOrdinal} PayloadJson failed to deserialise; replay truncated here: {ex.GetType().Name}: {ex.Message}",
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Same for the journal — we need the semantic fields directly, and PayloadJson
|
||||
// round-trips them via TransitionSerializer. The typed columns on the record are
|
||||
// projections; the JSON is authoritative.
|
||||
var decodedTransitions = new List<DecisionTransition>(orderedTransitions.Count);
|
||||
for (var i = 0; i < orderedTransitions.Count; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
decodedTransitions.Add(TransitionSerializer.Deserialize(orderedTransitions[i].PayloadJson));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Warning",
|
||||
Kind = "replay_deserialization_error",
|
||||
Message = $"Transition at StepIndex {orderedTransitions[i].StepIndex} PayloadJson failed to deserialise — excluded from comparison: {ex.GetType().Name}: {ex.Message}",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedSignals.Count == 0 || decodedTransitions.Count == 0)
|
||||
{
|
||||
report.SemanticReplaySkipReason = "deserialization_failure";
|
||||
return;
|
||||
}
|
||||
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial(report.TenantId, report.SessionId);
|
||||
var divergences = 0;
|
||||
|
||||
var stepsToCompare = Math.Min(decodedSignals.Count, decodedTransitions.Count);
|
||||
for (var i = 0; i < stepsToCompare; i++)
|
||||
{
|
||||
var step = engine.Reduce(state, decodedSignals[i]);
|
||||
state = step.NewState;
|
||||
var replayed = step.Transition;
|
||||
var stored = decodedTransitions[i];
|
||||
|
||||
if (!TransitionsSemanticallyEqual(replayed, stored))
|
||||
{
|
||||
divergences++;
|
||||
if (divergences <= MaxReplayDivergenceIssues)
|
||||
{
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Warning",
|
||||
Kind = "replay_divergence",
|
||||
Message =
|
||||
$"Replay vs. stored diverge at StepIndex={stored.StepIndex}: " +
|
||||
$"stored=[Trigger={stored.Trigger}, {stored.FromStage}→{stored.ToStage}, Taken={stored.Taken}, DeadEnd={stored.DeadEndReason ?? "<null>"}] " +
|
||||
$"replayed=[Trigger={replayed.Trigger}, {replayed.FromStage}→{replayed.ToStage}, Taken={replayed.Taken}, DeadEnd={replayed.DeadEndReason ?? "<null>"}].",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (divergences > MaxReplayDivergenceIssues)
|
||||
{
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Warning",
|
||||
Kind = "replay_divergence",
|
||||
Message = $"... {divergences - MaxReplayDivergenceIssues} additional replay divergences not listed.",
|
||||
});
|
||||
}
|
||||
|
||||
// Count imbalance between signals and stored transitions is itself a divergence:
|
||||
// every signal should produce exactly one transition (taken or dead-end).
|
||||
if (decodedSignals.Count != decodedTransitions.Count)
|
||||
{
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Warning",
|
||||
Kind = "replay_divergence",
|
||||
Message = $"Signal count ({decodedSignals.Count}) != transition count ({decodedTransitions.Count}); reducer should produce one transition per signal.",
|
||||
});
|
||||
}
|
||||
|
||||
// Final-stage comparison against the last stored transition's ToStage.
|
||||
var lastStoredToStage = orderedTransitions[orderedTransitions.Count - 1].ToStage;
|
||||
var replayedStageName = state.Stage.ToString();
|
||||
report.ReplayedFinalStage = replayedStageName;
|
||||
report.SemanticReplayFinalStageMatches = string.Equals(
|
||||
replayedStageName, lastStoredToStage, StringComparison.Ordinal);
|
||||
if (!report.SemanticReplayFinalStageMatches)
|
||||
{
|
||||
report.Issues.Add(new VerificationIssue
|
||||
{
|
||||
Severity = "Error",
|
||||
Kind = "replay_final_stage_mismatch",
|
||||
Message = $"Replay arrived at final stage '{replayedStageName}' but the last stored transition's ToStage was '{lastStoredToStage}'.",
|
||||
});
|
||||
}
|
||||
|
||||
report.SemanticReplayPerformed = true;
|
||||
report.TransitionDivergenceCount = divergences;
|
||||
}
|
||||
|
||||
private static bool TransitionsSemanticallyEqual(DecisionTransition a, DecisionTransition b)
|
||||
{
|
||||
if (a.StepIndex != b.StepIndex) return false;
|
||||
if (!string.Equals(a.Trigger, b.Trigger, StringComparison.Ordinal)) return false;
|
||||
if (a.FromStage != b.FromStage) return false;
|
||||
if (a.ToStage != b.ToStage) return false;
|
||||
if (a.Taken != b.Taken) return false;
|
||||
if (!string.Equals(a.DeadEndReason ?? string.Empty, b.DeadEndReason ?? string.Empty, StringComparison.Ordinal)) return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,40 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public int OrphanedTransitionCount { get; set; }
|
||||
|
||||
// ---- Semantic replay (Codex follow-up #6) ----------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// True when the verifier re-played the persisted signal stream through the live
|
||||
/// backend <c>DecisionEngine</c> and compared the produced transitions to the stored
|
||||
/// journal. False when the replay was skipped — see <see cref="SemanticReplaySkipReason"/>.
|
||||
/// </summary>
|
||||
public bool SemanticReplayPerformed { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Discriminator when <see cref="SemanticReplayPerformed"/> is false. Known values:
|
||||
/// <c>"empty_session"</c>, <c>"reducer_version_drift"</c>, <c>"non_contiguous_signal_ordinals"</c>,
|
||||
/// <c>"non_contiguous_step_indices"</c>, <c>"deserialization_failure"</c>.
|
||||
/// </summary>
|
||||
public string? SemanticReplaySkipReason { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// True when the replayed <see cref="DecisionCore.State.DecisionState.Stage"/> matches
|
||||
/// the <c>ToStage</c> of the last stored transition. Only meaningful when
|
||||
/// <see cref="SemanticReplayPerformed"/> is true.
|
||||
/// </summary>
|
||||
public bool SemanticReplayFinalStageMatches { get; set; }
|
||||
|
||||
/// <summary>The stage the replay arrived at (stringified <c>SessionStage</c>); null on skip.</summary>
|
||||
public string? ReplayedFinalStage { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of positions where the replayed transition diverged from the stored one on
|
||||
/// the compared fields (Trigger, FromStage, ToStage, Taken, DeadEndReason, StepIndex).
|
||||
/// 0 means perfect agreement. Individual divergences are emitted as
|
||||
/// <c>replay_divergence</c> <see cref="VerificationIssue"/>s up to a cap of 20.
|
||||
/// </summary>
|
||||
public int TransitionDivergenceCount { get; set; }
|
||||
|
||||
/// <summary>Human-readable issue stream for the Inspector's verification panel.</summary>
|
||||
public List<VerificationIssue> Issues { get; set; } = new List<VerificationIssue>();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user