diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/SignalAdapters/ImeLogTrackerAdapterHistoricReplayTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/SignalAdapters/ImeLogTrackerAdapterHistoricReplayTests.cs new file mode 100644 index 00000000..1e1ba3cd --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/SignalAdapters/ImeLogTrackerAdapterHistoricReplayTests.cs @@ -0,0 +1,257 @@ +using System; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime; +using AutopilotMonitor.Agent.V2.Core.SignalAdapters; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters +{ + /// + /// Historic-replay handling (session eaf3d8c4): a previous enrollment's IME log surviving + /// on disk made the agent replay week-old script activity — 156 phantom script_completed + /// events with ~170 h durations (clock-clamped completion minus RAW stale start) and an + /// immediate-upload flood. Script events whose source line is > 24 h stale are now + /// suppressed entirely (one-shot historic_script_replay_detected summary instead), + /// and durations are computed from a timeline-consistent timestamp pair with a 24 h + /// plausibility backstop. + /// + public sealed class ImeLogTrackerAdapterHistoricReplayTests + { + private static readonly DateTime ClockNow = new DateTime(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc); + + // --------------------------------------------------------------------- + // Suppression: stale (> 24 h past) source lines are a previous enrollment + // --------------------------------------------------------------------- + + [Fact] + public void Stale_completion_is_suppressed_and_emits_oneshot_summary() + { + using var f = new ImeLogTrackerAdapterFixture(ClockNow); + using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock); + + var ancient = ClockNow.AddDays(-7); + f.Tracker.LastMatchedLogTimestamp = ancient; + + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "d78c1822", + ScriptType = "platform", + ExitCode = 0, + Result = "Failed", + StartedAtUtc = ancient.AddMinutes(-30), + // Bootstrap marker in stdout — week-old evidence must not drive the + // bootstrap-detected trace either. + Stdout = "Bootstrap script version: v2.0", + }); + + Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted)); + Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptFailed)); + Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected)); + Assert.Empty(f.InfoEvents(SharedEventTypes.AgentTrace)); + + var summary = f.InfoEvent(SharedEventTypes.HistoricScriptReplayDetected); + Assert.Equal(ancient.ToString("o"), summary.Payload!["earliestRejectedSourceTimestamp"]); + } + + [Fact] + public void Summary_is_oneshot_across_started_and_completed_suppressions() + { + using var f = new ImeLogTrackerAdapterFixture(ClockNow); + using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock); + + f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7); + adapter.TriggerScriptStartedFromTest(new ScriptStartedInfo { PolicyId = "aaa11111", ScriptType = "platform" }); + + f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7).AddMinutes(1); + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "aaa11111", + ScriptType = "platform", + ExitCode = 0, + Result = "Success", + }); + + f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7).AddMinutes(2); + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "bbb22222", + ScriptType = "remediation", + ScriptPart = "detection", + ExitCode = 0, + ComplianceResult = "True", + }); + + Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptStarted)); + Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted)); + // Exactly ONE summary regardless of how many replayed lines followed — and it dates + // the replay window from the FIRST (earliest) suppressed line. + var summary = Assert.Single(f.InfoEvents(SharedEventTypes.HistoricScriptReplayDetected)); + Assert.Equal(ClockNow.AddDays(-7).ToString("o"), summary.Payload!["earliestRejectedSourceTimestamp"]); + } + + [Fact] + public void Fresh_events_after_suppression_emit_normally() + { + using var f = new ImeLogTrackerAdapterFixture(ClockNow); + using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock); + + f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7); + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "aaa11111", + ScriptType = "platform", + ExitCode = 0, + Result = "Success", + }); + Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted)); + + // The agent catches up to current log content — normal emission resumes. + f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-2); + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "ccc33333", + ScriptType = "platform", + ExitCode = 0, + Result = "Success", + StartedAtUtc = ClockNow.AddMinutes(-3), + }); + + var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted)); + Assert.Equal("ccc33333", info.Payload!["policyId"]); + Assert.Equal("60.00", info.Payload["durationSeconds"]); + } + + [Fact] + public void Clock_fallback_without_source_timestamp_still_emits() + { + // derivedFromClock with NO rejected source ts (synthetic/callback path) is not a + // replay — must emit, and must not trigger the summary. + using var f = new ImeLogTrackerAdapterFixture(ClockNow); + using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock); + + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "ddd44444", + ScriptType = "platform", + ExitCode = 0, + Result = "Success", + }); + + var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted)); + Assert.Equal("true", info.Payload!["derivedTimestamp"]); + Assert.False(info.Payload.ContainsKey("rejectedSourceTimestamp")); + Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricScriptReplayDetected)); + } + + // --------------------------------------------------------------------- + // Duration: timeline-consistent pair + plausibility backstop + // --------------------------------------------------------------------- + + [Fact] + public void FutureSkew_completion_emits_with_clamped_stamp_and_raw_pair_duration() + { + // Mid-enrollment clock jump (WhiteGlove +1h CMTrace skew case): the source ts is + // rejected as future-skewed — NOT suppressed — and the duration must come from the + // raw source pair (both ends on the same skewed timeline), not clamped-now − raw-start. + using var f = new ImeLogTrackerAdapterFixture(ClockNow); + using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock); + + var skewed = ClockNow.AddHours(2); + f.Tracker.LastMatchedLogTimestamp = skewed; + + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "eee55555", + ScriptType = "platform", + ExitCode = 0, + Result = "Success", + StartedAtUtc = skewed.AddSeconds(-360), + }); + + var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted)); + Assert.Equal(ClockNow, info.OccurredAtUtc); // stamp: clamped to clock + Assert.Equal("true", info.Payload!["derivedTimestamp"]); + Assert.Equal(skewed.ToString("o"), info.Payload["rejectedSourceTimestamp"]); + Assert.Equal("360.00", info.Payload["durationSeconds"]); // duration: raw pair + Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricScriptReplayDetected)); + } + + [Fact] + public void FutureSkew_failed_platform_script_timeout_heuristic_uses_raw_pair() + { + using var f = new ImeLogTrackerAdapterFixture(ClockNow); + using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock); + + var skewed = ClockNow.AddHours(2); + f.Tracker.LastMatchedLogTimestamp = skewed; + + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "fff66666", + ScriptType = "platform", + ExitCode = 0, + Result = "Failed", + StartedAtUtc = skewed.AddMinutes(-26), // past the 25-min suspicion threshold + }); + + var suspected = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected)); + Assert.Equal("1560.00", suspected.Payload!["durationSeconds"]); + Assert.Equal(ClockNow, suspected.OccurredAtUtc); + } + + [Fact] + public void Implausible_duration_above_24h_is_omitted() + { + // Fresh (accepted) completion paired with an ancient start — e.g. a stale pending + // slot a legacy path left behind. The duration would be a cross-run lie; omit it. + using var f = new ImeLogTrackerAdapterFixture(ClockNow); + using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock); + + f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-2); + + adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState + { + PolicyId = "abc12345", + ScriptType = "platform", + ExitCode = 0, + Result = "Failed", + StartedAtUtc = ClockNow.AddHours(-30), + }); + + var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptFailed)); + Assert.False(info.Payload!.ContainsKey("durationSeconds")); + Assert.False(info.Payload.ContainsKey("durationBasis")); + // The 30 h "duration" must not read as a hung script either. + Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected)); + } + + // --------------------------------------------------------------------- + // Tracker slot hardening: stale pending start must not pair with a fresh run + // --------------------------------------------------------------------- + + [Fact] + public void Fresh_start_discards_stale_pending_slot_from_previous_enrollment() + { + using var f = new ImeLogTrackerAdapterFixture(ClockNow); + using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock); + + // Previous enrollment: replayed start line whose matching result never appeared + // (the script was mid-run when that enrollment ended). + f.Tracker.SeedPendingPlatformScriptForTesting("beef7777", exitCode: null, exitObservedAtUtc: null, + startedAtUtc: ClockNow.AddDays(-7)); + + // Same policy genuinely re-runs now — the fresh start must begin a NEW execution. + f.Tracker.LastMatchedLogTimestamp = ClockNow.AddSeconds(-90); + f.Tracker.HandlePlatformScriptStarted("beef7777"); + + f.Tracker.LastMatchedLogTimestamp = ClockNow; + f.Tracker.CompletePlatformScriptFromImeResultForTesting("beef7777", "Success", ClockNow); + + var info = f.InfoEvent(SharedEventTypes.ScriptCompleted); + // Without the hardening the slot kept the 7-day-old start: the duration would have + // been dropped by the 24 h backstop (or, before that, shown as ~168 h). + Assert.Equal("90.00", info.Payload!["durationSeconds"]); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.Handlers.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.Handlers.cs index 7e1e24eb..2f84be00 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.Handlers.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.Handlers.cs @@ -208,39 +208,60 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime } else { - // Platform script: AgentExecutor.log entries create/enrich, IME log entries also create - if (!_pendingPlatformScripts.ContainsKey(id)) - { - // A fresh start for this policy (no pending entry) begins a NEW execution. - // Clear any emitted-marker from a prior run so IME re-evaluations / retries of - // the same platform-script policy within one agent lifetime are not silently - // deduped away. Within a single run the start always precedes exit/result, so - // this never clears the current run's own marker. - _platformScriptResultEmitted.Remove(id); - _pendingPlatformScripts[id] = new ScriptExecutionState - { - PolicyId = id, - ScriptType = "platform", - // Prefer the source CMTrace timestamp so a script that started before the - // agent launched (replayed log content) is dated to its real start, not now. - // Set only at slot creation: the start line fires twice (agentexecutor + ime - // source) but the earliest wins. - StartedAtUtc = LastMatchedLogTimestamp ?? DateTime.UtcNow, - }; - // Live "running" indicator — same signal health scripts emit. Gated on slot - // creation so the duplicate start line (agentexecutor + ime source) doesn't - // double-emit for the same execution. - try { OnScriptStarted?.Invoke(new ScriptStartedInfo { PolicyId = id, ScriptType = "platform" }); } - catch (Exception ex) { _logger.Warning($"ImeLogTracker: OnScriptStarted handler threw: {ex.Message}"); } - } - _lastPlatformScriptPolicyId = id; - // Started lines fire twice per script (agentexecutor + ime source) and carry no - // outcome — the matching `platform script completed` line below carries result+exit - // and stays on Info. Keep starts on Debug so Info reflects script outcomes only. - _logger.Debug($"ImeLogTracker: platform script started: {id} (source: {source ?? "ime"})"); + HandlePlatformScriptStarted(id, source); } } + // Platform script: AgentExecutor.log entries create/enrich, IME log entries also create. + // Internal so the stale-slot hardening is directly testable (the public path arrives via + // a regex Match). + internal void HandlePlatformScriptStarted(string id, string source = null) + { + // Stale-slot hardening (session eaf3d8c4): a replayed start line from a previous + // enrollment whose matching result never appeared (script was mid-run when that + // enrollment ended) leaves a pending slot with a days-old StartedAtUtc. When the + // same policy genuinely re-runs now, the fresh start must begin a NEW execution — + // otherwise the ancient start pairs with the fresh completion and the duration + // lies. > 24 h mirrors the adapter's source-timestamp staleness clamp. + var currentStartTs = LastMatchedLogTimestamp ?? DateTime.UtcNow; + if (_pendingPlatformScripts.TryGetValue(id, out var pendingSlot) + && pendingSlot.StartedAtUtc.HasValue + && currentStartTs - pendingSlot.StartedAtUtc.Value > TimeSpan.FromHours(24)) + { + _logger.Debug($"ImeLogTracker: discarding stale pending platform-script slot for {id} (started {pendingSlot.StartedAtUtc:o})"); + _pendingPlatformScripts.Remove(id); + } + if (!_pendingPlatformScripts.ContainsKey(id)) + { + // A fresh start for this policy (no pending entry) begins a NEW execution. + // Clear any emitted-marker from a prior run so IME re-evaluations / retries of + // the same platform-script policy within one agent lifetime are not silently + // deduped away. Within a single run the start always precedes exit/result, so + // this never clears the current run's own marker. + _platformScriptResultEmitted.Remove(id); + _pendingPlatformScripts[id] = new ScriptExecutionState + { + PolicyId = id, + ScriptType = "platform", + // Prefer the source CMTrace timestamp so a script that started before the + // agent launched (replayed log content) is dated to its real start, not now. + // Set only at slot creation: the start line fires twice (agentexecutor + ime + // source) but the earliest wins. + StartedAtUtc = currentStartTs, + }; + // Live "running" indicator — same signal health scripts emit. Gated on slot + // creation so the duplicate start line (agentexecutor + ime source) doesn't + // double-emit for the same execution. + try { OnScriptStarted?.Invoke(new ScriptStartedInfo { PolicyId = id, ScriptType = "platform" }); } + catch (Exception ex) { _logger.Warning($"ImeLogTracker: OnScriptStarted handler threw: {ex.Message}"); } + } + _lastPlatformScriptPolicyId = id; + // Started lines fire twice per script (agentexecutor + ime source) and carry no + // outcome — the matching `platform script completed` line below carries result+exit + // and stays on Info. Keep starts on Debug so Info reflects script outcomes only. + _logger.Debug($"ImeLogTracker: platform script started: {id} (source: {source ?? "ime"})"); + } + // Routes line-by-line script-data handlers (context / exitCode / output) to the right // accumulator based on the pattern's scriptType parameter. Health-script lines feed the // single-slot _pendingHealthScript; platform-script lines feed _pendingPlatformScripts. diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs index 4eed9e46..cf2bdd32 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs @@ -97,12 +97,23 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters // bootstrap marker line. Lets MCP report "which device runs which bootstrap version". private bool _bootstrapDetectedPosted; + // One-shot: historic script replay detected (a previous enrollment's IME log content + // surviving on disk) — its script events are suppressed, and a single + // historic_script_replay_detected Info event explains the gap in the timeline. + private bool _historicReplayNoted; + private int _suppressedHistoricScriptEvents; // forensic count for the log line + // A platform script whose observed run duration reaches this is treated as having hit the // IME script-execution timeout (default ~30 min). 25 min leaves headroom below 30 so clock // jitter / the agent's late replay window can't miss it, while staying well above any // legitimately long install wrapper that completes on its own. private static readonly TimeSpan ScriptTimeoutSuspectedThreshold = TimeSpan.FromMinutes(25); + // Source log lines older than this relative to the clock are pathological — shared by + // ResolveOccurredAt's staleness clamp, the historic-replay suppression and the duration + // plausibility backstop, one constant so the three can never drift apart. + private static readonly TimeSpan StaleSourceThreshold = TimeSpan.FromHours(24); + // Matches the deterministic marker line the bootstrap script writes via Write-Log, // e.g. "Bootstrap script version: v2.0" — see scripts/Bootstrap/Install-AutopilotMonitor.ps1. // Same shape as the web-side regex in utils/bootstrapVersion.ts; kept in sync intentionally. @@ -289,17 +300,11 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters rawSourceTimestamp = sourceTsOverride ?? _tracker.LastMatchedLogTimestamp; if (rawSourceTimestamp.HasValue) { - // CMTrace log timestamps from IME are emitted in UTC, but DateTimeKind may - // arrive as Unspecified depending on the parser path. Normalize via - // SpecifyKind=Utc rather than ToUniversalTime() to avoid double-conversion - // when Kind is already UTC. - var asUtc = rawSourceTimestamp.Value.Kind == DateTimeKind.Utc - ? rawSourceTimestamp.Value - : DateTime.SpecifyKind(rawSourceTimestamp.Value, DateTimeKind.Utc); + var asUtc = NormalizeUtc(rawSourceTimestamp.Value); var clockNow = _clock.UtcNow; var ageHours = (clockNow - asUtc).TotalHours; - if (ageHours > 24.0 || ageHours < -1.0) + if (ageHours > StaleSourceThreshold.TotalHours || ageHours < -1.0) { // Pathologically stale (>24h old) or in the future (>1h skew) — log, // fall back, and let the caller flag the event. @@ -325,6 +330,68 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters data["rejectedSourceTimestamp"] = rawSourceTs.Value.ToString("o", System.Globalization.CultureInfo.InvariantCulture); } + /// + /// CMTrace log timestamps from IME are emitted in UTC, but DateTimeKind may arrive as + /// Unspecified depending on the parser path. Normalize via SpecifyKind=Utc rather than + /// ToUniversalTime() to avoid double-conversion when Kind is already UTC. + /// + private static DateTime NormalizeUtc(DateTime value) => + value.Kind == DateTimeKind.Utc ? value : DateTime.SpecifyKind(value, DateTimeKind.Utc); + + /// + /// True when a script event's source log line is a historic replay: the source timestamp + /// was rejected as stale (> in the PAST). Those lines + /// are runs from a previous enrollment whose IME log survived on disk (session eaf3d8c4: + /// 156 replayed script completions shown as current executions, immediate-upload flood → + /// ingress_backpressure). Future-skew rejections (negative age — mid-enrollment clock + /// jump) and clock-only fallbacks (no source timestamp at all) are NOT replay and must + /// still emit. Emits the one-shot historic_script_replay_detected summary on the + /// first suppression; replay is chronological, so that first suppressed line carries the + /// earliest rejected source timestamp. + /// + private bool SuppressIfHistoricScriptReplay( + bool derivedFromClock, DateTime? rawSourceTs, string eventLabel, string policyId) + { + if (!derivedFromClock || !rawSourceTs.HasValue) return false; + var asUtc = NormalizeUtc(rawSourceTs.Value); + var clockNow = _clock.UtcNow; + var age = clockNow - asUtc; + if (age <= StaleSourceThreshold) return false; // future-skew or fresh → emit + + bool emitSummary; + int suppressedSoFar; + lock (_lock) + { + suppressedSoFar = ++_suppressedHistoricScriptEvents; + emitSummary = !_historicReplayNoted; + _historicReplayNoted = true; + } + + if (emitSummary) + { + var culture = System.Globalization.CultureInfo.InvariantCulture; + // Standard batch upload on purpose — no immediateUpload; the whole point of the + // suppression is to stop the replayed batch from flooding the ingest. + _post.Emit( + eventType: SharedEventTypes.HistoricScriptReplayDetected, + source: SourceLabel, + message: "Historic script executions from a previous enrollment detected in the IME log " + + $"(earliest source timestamp {asUtc:o}) — their script events are suppressed for this session.", + severity: EventSeverity.Info, + data: new Dictionary(StringComparer.Ordinal) + { + ["decision"] = "historic_script_replay_suppressed", + ["earliestRejectedSourceTimestamp"] = asUtc.ToString("o", culture), + ["staleThresholdHours"] = StaleSourceThreshold.TotalHours.ToString("F0", culture), + }); + } + + _logger?.Debug( + $"ImeAdapter: suppressed historic {eventLabel} for policy {policyId} " + + $"(source ts {asUtc:o}, age {age.TotalHours:F1}h, suppressed so far: {suppressedSoFar})"); + return true; + } + private void EmitEspPhase(string phase) { if (string.IsNullOrEmpty(phase)) return; @@ -910,13 +977,27 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters now = ResolveOccurredAt(script.ExitObservedAtUtc, out derivedFromClock, out rawSourceTs); else now = ResolveOccurredAt(out derivedFromClock, out rawSourceTs); + + // Historic replay (a previous enrollment's log) — do not emit at all. The early + // return also skips MaybeEmitBootstrapDetected / MaybeEmitScriptTimeoutSuspected, + // which would otherwise fire on week-old evidence. + if (SuppressIfHistoricScriptReplay(derivedFromClock, rawSourceTs, eventType, script.PolicyId)) + return; + TagDerivedTimestamp(data, derivedFromClock, rawSourceTs); - // Per-script run duration (start line → this completion, `now`). Surfaced on every + // Per-script run duration (start line → this completion). Surfaced on every // script completion so the UI can flag slow / inefficient scripts, not just timed-out // ones. Omitted when the start line was never observed (StartedAtUtc null); clamped at 0 // against clock skew between the (possibly clock-derived) completion stamp and start. // + // When the completion timestamp was clock-derived because the source line's timestamp + // was rejected (future-skew CMTrace clock jump — the stale case never reaches here), + // `now` lives on the clock's timeline while StartedAtUtc was captured RAW from the + // source log. Mixing the two produced absurd durations (session eaf3d8c4: 170 h), so + // the duration uses the raw source pair — both ends on the same timeline, the + // difference is the true runtime. The event stamp stays `now` (timeline position). + // // L19 (delta review 2026-07-02): the same field name carries two different semantics — // start→own-end-signal (≈ script runtime) vs HS-SCRIPT-START→HS-NEW-RESULT (the whole // cycle INCLUDING IME's 30 s–3 min batched reporting latency, i.e. systematically @@ -924,10 +1005,16 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters // consumers never compare the two as equals. The emitting handler stamps the basis on // the state (health-script early-signal completions are script_runtime too); the // scriptType fallback covers states from paths that predate the field. + var completionForDuration = derivedFromClock && rawSourceTs.HasValue + ? NormalizeUtc(rawSourceTs.Value) + : now; if (script.StartedAtUtc.HasValue) { - var durationSeconds = (now - script.StartedAtUtc.Value).TotalSeconds; - if (durationSeconds >= 0) + var durationSeconds = (completionForDuration - script.StartedAtUtc.Value).TotalSeconds; + // Plausibility backstop: IME's script execution timeout is ~30 min; anything above + // 24 h is a cross-run artifact (e.g. a stale pending start slot from a previous + // enrollment paired with a fresh completion) — omit rather than lie. + if (durationSeconds >= 0 && durationSeconds <= StaleSourceThreshold.TotalSeconds) { data["durationSeconds"] = durationSeconds.ToString("F2", culture); data["durationBasis"] = script.DurationBasis @@ -979,7 +1066,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters $"ImeAdapter: script completed policyId={shortId} type={script.ScriptType ?? "?"} result={script.Result ?? "?"} exit={(script.ExitCode.HasValue ? script.ExitCode.Value.ToString() : "n/a")}"); MaybeEmitBootstrapDetected(script, now); - MaybeEmitScriptTimeoutSuspected(script, now); + MaybeEmitScriptTimeoutSuspected(script, now, completionForDuration); } /// @@ -991,15 +1078,19 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters /// late start / pre-failed ESP. Advisory only: no state mutation, the underlying /// script_failed is still emitted separately. Health (remediation) scripts are /// excluded — they re-run on their own cadence and have a different timeout regime. + /// is the timeline-consistent completion for duration + /// math (raw source ts on rejected-timestamp emissions); + /// stamps the emitted event. /// - private void MaybeEmitScriptTimeoutSuspected(ScriptExecutionState script, DateTime completionUtc) + private void MaybeEmitScriptTimeoutSuspected(ScriptExecutionState script, DateTime occurredAtUtc, DateTime durationEndUtc) { if (IsRemediation(script.ScriptType)) return; // platform scripts only if (!script.StartedAtUtc.HasValue) return; // no start ⇒ no duration if (!string.Equals(script.Result, "Failed", StringComparison.OrdinalIgnoreCase)) return; - var duration = completionUtc - script.StartedAtUtc.Value; + var duration = durationEndUtc - script.StartedAtUtc.Value; if (duration < ScriptTimeoutSuspectedThreshold) return; + if (duration > StaleSourceThreshold) return; // implausible — cross-run artifact, not a hung script if (!_tracker.TryClaimScriptTimeoutSuspected(script.PolicyId)) return; // restart-safe dedup per policyId (persisted tracker state) @@ -1026,7 +1117,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters message: $"Platform script {shortId} ran {durationMinutes:F0} min and was marked Failed — suspected IME script timeout blocking the enrollment.", severity: EventSeverity.Warning, data: data, - occurredAtUtc: completionUtc); + occurredAtUtc: occurredAtUtc); _logger?.Info( $"ImeAdapter: script_timeout_suspected policyId={shortId} duration={durationMinutes:F1}min result={script.Result} espPhase={espPhase ?? "(none)"}"); @@ -1081,6 +1172,12 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters if (!string.IsNullOrEmpty(patternId)) data["patternId"] = patternId!; var now = ResolveOccurredAt(out var derivedFromClock, out var rawSourceTs); + + // Historic replay — suppressing here also kills the immediateUpload flood the + // replayed batch caused (session eaf3d8c4: 3× ingress_backpressure). + if (SuppressIfHistoricScriptReplay(derivedFromClock, rawSourceTs, SharedEventTypes.ScriptStarted, info.PolicyId)) + return; + TagDerivedTimestamp(data, derivedFromClock, rawSourceTs); var label = IsRemediation(info.ScriptType) ? "Health script" : "Platform script"; diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IngestHistoricScriptReplayTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IngestHistoricScriptReplayTests.cs new file mode 100644 index 00000000..467ff4b9 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/IngestHistoricScriptReplayTests.cs @@ -0,0 +1,89 @@ +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Session eaf3d8c4 (2026-07-23): a previous enrollment's IME log surviving on disk made the +/// agent replay a week-old script history — 156 replayed script_completed events +/// inflated the session's RemediationScriptCount. Newer agents suppress the replay at the +/// source; this predicate covers the fleet of not-yet-updated agents by skipping the script +/// counter increment for events whose rejectedSourceTimestamp (the source log line's +/// own time, preserved by the agent's staleness clamp) is more than 24 h older than the +/// event's clock-clamped stamp. +/// +public class IngestHistoricScriptReplayTests +{ + private static readonly DateTime EventStamp = new(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc); + + private static EnrollmentEvent ScriptEvent(object? rejectedSourceTimestamp, bool includeKey = true) + { + var evt = new EnrollmentEvent + { + EventType = "script_completed", + Timestamp = EventStamp, + Data = new Dictionary { ["scriptType"] = "remediation" }, + }; + if (includeKey) + evt.Data["rejectedSourceTimestamp"] = rejectedSourceTimestamp!; + return evt; + } + + [Fact] + public void StaleRejectedSourceTimestamp_IsHistoricReplay() + { + var evt = ScriptEvent(EventStamp.AddDays(-7).ToString("o")); + Assert.True(EventIngestProcessor.IsHistoricScriptReplay(evt)); + } + + [Fact] + public void JustOver24h_IsHistoricReplay() + { + var evt = ScriptEvent(EventStamp.AddHours(-25).ToString("o")); + Assert.True(EventIngestProcessor.IsHistoricScriptReplay(evt)); + } + + [Fact] + public void Within24h_IsNotHistoricReplay() + { + var evt = ScriptEvent(EventStamp.AddHours(-23).ToString("o")); + Assert.False(EventIngestProcessor.IsHistoricScriptReplay(evt)); + } + + [Fact] + public void FutureSkewRejection_IsNotHistoricReplay() + { + // A rejected source timestamp AHEAD of the event stamp is a mid-enrollment clock + // jump (WhiteGlove +1h CMTrace skew), not replayed history. + var evt = ScriptEvent(EventStamp.AddHours(2).ToString("o")); + Assert.False(EventIngestProcessor.IsHistoricScriptReplay(evt)); + } + + [Fact] + public void MissingKey_IsNotHistoricReplay() + { + Assert.False(EventIngestProcessor.IsHistoricScriptReplay(ScriptEvent(null, includeKey: false))); + } + + [Fact] + public void NullData_IsNotHistoricReplay() + { + var evt = new EnrollmentEvent { EventType = "script_completed", Timestamp = EventStamp, Data = null! }; + Assert.False(EventIngestProcessor.IsHistoricScriptReplay(evt)); + } + + [Theory] + [InlineData("")] + [InlineData("not-a-date")] + public void UnparseableValue_IsNotHistoricReplay(string raw) + { + Assert.False(EventIngestProcessor.IsHistoricScriptReplay(ScriptEvent(raw))); + } + + [Fact] + public void NullEvent_IsNotHistoricReplay() + { + Assert.False(EventIngestProcessor.IsHistoricScriptReplay(null)); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs index 84e5a9a1..8524bfce 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs @@ -82,6 +82,13 @@ namespace AutopilotMonitor.Functions.Services break; case "script_completed": case "script_failed": + // Legacy-agent stale-replay guard: events flagged with a + // rejectedSourceTimestamp > 24 h older than the event stamp are replayed + // history from a previous enrollment (session eaf3d8c4 inflated + // RemediationScriptCount to 156 in a 7-min session). The fixed agent + // suppresses them at the source; this covers agents not yet rolled out. + if (IsHistoricScriptReplay(evt)) + break; var scriptType = evt.Data?.ContainsKey("scriptType") == true ? evt.Data["scriptType"]?.ToString() : null; if (string.Equals(scriptType, "platform", StringComparison.OrdinalIgnoreCase)) @@ -119,6 +126,28 @@ namespace AutopilotMonitor.Functions.Services return string.Equals(reason, "max_lifetime", StringComparison.OrdinalIgnoreCase); } + /// + /// True for a script event whose Data.rejectedSourceTimestamp is more than 24 h + /// older than the event's own (clock-clamped) timestamp: the agent replayed IME log + /// content from a previous enrollment that survived on disk. Such runs happened days ago + /// and must not count as this session's script executions. A rejected source timestamp + /// in the FUTURE (negative difference) is a mid-enrollment clock jump, not a replay. + /// The 24 h bound mirrors the agent's source-timestamp staleness clamp. + /// + internal static bool IsHistoricScriptReplay(EnrollmentEvent? evt) + { + if (evt?.Data == null || !evt.Data.ContainsKey("rejectedSourceTimestamp")) + return false; + var raw = evt.Data["rejectedSourceTimestamp"]?.ToString(); + if (string.IsNullOrEmpty(raw)) + return false; + if (!DateTime.TryParse(raw, System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.AssumeUniversal | System.Globalization.DateTimeStyles.AdjustToUniversal, + out var rejected)) + return false; + return evt.Timestamp - rejected > TimeSpan.FromHours(24); + } + /// /// True for an enrollment_failed event whose Data.failureType is /// agent_timeout — the V1-parity companion of the max-lifetime watchdog shutdown diff --git a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts index 2bfc1038..9fcc130a 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts @@ -80,7 +80,7 @@ export const EVENT_TYPES_CATALOG = { 'office_install_failed', 'office_preinstalled_detected', ], - script_events: ['script_started', 'script_completed', 'script_failed', 'script_timeout_suspected'], + script_events: ['script_started', 'script_completed', 'script_failed', 'script_timeout_suspected', 'historic_script_replay_detected'], network_events: [ 'network_state_change', 'network_connectivity_check', diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts index 18817f84..ee7a7be7 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts @@ -28,7 +28,8 @@ export const KEY_EVENT_TYPES = new Set([ 'enrollment_complete', 'enrollment_failed', 'completion_check', 'desktop_arrived', 'hello_policy_detected', 'waiting_for_hello', 'hello_completion_timeout', 'agent_started', 'agent_shutdown', 'agent_shutting_down', 'agent_trace', - 'script_started', 'script_completed', 'script_failed', 'vulnerability_report', + 'script_started', 'script_completed', 'script_failed', 'historic_script_replay_detected', + 'vulnerability_report', ]); // Phase-defining events promoted to the top of the triage timeline. Module-level // (not handler-local) so the same drift test can validate it. diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 4f0f0f76..5ac0c54c 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -340,6 +340,12 @@ namespace AutopilotMonitor.Shared // start→completion duration (works on replay since start carries the source log // timestamp). Warning, advisory ("suspected"); no state mutation. public const string ScriptTimeoutSuspected = "script_timeout_suspected"; + // One-shot per agent run: the IME adapter detected replayed log content from a + // previous enrollment (script source lines > 24 h stale, e.g. IME logs surviving a + // re-enrollment) and suppressed the historic script_started/script_completed/ + // script_failed events. Data.earliestRejectedSourceTimestamp dates the replayed + // content so operators know why script events are missing (session eaf3d8c4). + public const string HistoricScriptReplayDetected = "historic_script_replay_detected"; public const string EspProvisioningStatus = "esp_provisioning_status"; public const string SoftwareInventoryAnalysis = "software_inventory_analysis"; public const string VulnerabilityReport = "vulnerability_report"; diff --git a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts index 7414e1b0..3f6dbfea 100644 --- a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts +++ b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts @@ -142,6 +142,8 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [ description: "A platform script crashed (exit code != 0), or the remediation phase of a health script crashed. Health-script detection / post-detection phases use exit code as a compliance verdict (non-zero = non-compliant) and emit script_completed even on non-zero exit; only a true script crash in those phases would surface here." }, { value: "script_timeout_suspected", label: "script_timeout_suspected", category: "script", description: "A platform script ran to the IME script-execution timeout (~30 min; threshold 25 min) and was marked Failed while the enrollment was in progress. IME runs platform scripts serially, so a hung script starves app installs and the Autopilot-Monitor bootstrap — the prime suspect behind a late agent start / pre-failed ESP. Advisory (one per policyId); carries durationSeconds, exitCode and espPhase." }, + { value: "historic_script_replay_detected", label: "historic_script_replay_detected", category: "script", + description: "One-shot per agent run: the IME log contained replayed script activity from a previous enrollment (source lines more than 24 h stale, e.g. IME logs surviving a re-enrollment). The historic script_started/script_completed/script_failed events were suppressed for this session; carries earliestRejectedSourceTimestamp so the replayed window is datable." }, // -------- Diagnostics / misc -------- { value: "error_detected", label: "error_detected", category: "diagnostics", diff --git a/src/Web/autopilot-monitor-web/components/ScriptExecutions.tsx b/src/Web/autopilot-monitor-web/components/ScriptExecutions.tsx index c5dc5afb..51a39608 100644 --- a/src/Web/autopilot-monitor-web/components/ScriptExecutions.tsx +++ b/src/Web/autopilot-monitor-web/components/ScriptExecutions.tsx @@ -10,6 +10,7 @@ import { isDetectOnlyRow, isNonCompliantReport, mapRemediationStatus, + partitionHistoricScriptEvents, reduceScriptEvents, scriptCardKey, scriptItemKey, @@ -38,11 +39,15 @@ interface ScriptExecutionsProps { } export default function ScriptExecutions({ events, showScriptOutput, latestBootstrapVersion, displayNamesByRefKey }: ScriptExecutionsProps) { - const cards = useMemo(() => groupScriptItems(reduceScriptEvents(events)), [events]); + // Legacy-agent guard: split off script events replayed from a previous enrollment's IME + // log (newer agents suppress them at the source) so week-old runs never render as current + // executions. The muted note below keeps the gap explainable. + const { current, historicCount } = useMemo(() => partitionHistoricScriptEvents(events), [events]); + const cards = useMemo(() => groupScriptItems(reduceScriptEvents(current)), [current]); const [expanded, setExpanded] = useState(true); - if (cards.length === 0) return null; + if (cards.length === 0 && historicCount === 0) return null; // Header counters work on cards (one per policy), so a remediated cycle reads as // "1 succeeded" rather than "1 succeeded + 2 non-compliant" when its detection / @@ -114,6 +119,14 @@ export default function ScriptExecutions({ events, showScriptOutput, latestBoots displayNamesByRefKey={displayNamesByRefKey} /> ))} + {historicCount > 0 && ( +
+ {historicCount} historic script execution{historicCount === 1 ? "" : "s"} from a previous enrollment hidden +
+ )} )} diff --git a/src/Web/autopilot-monitor-web/lib/__tests__/scriptExecutions.test.ts b/src/Web/autopilot-monitor-web/lib/__tests__/scriptExecutions.test.ts index 04fb7521..ab79d59b 100644 --- a/src/Web/autopilot-monitor-web/lib/__tests__/scriptExecutions.test.ts +++ b/src/Web/autopilot-monitor-web/lib/__tests__/scriptExecutions.test.ts @@ -7,6 +7,7 @@ import { isDetectOnlyRow, isNonCompliantReport, mapRemediationStatus, + partitionHistoricScriptEvents, reduceScriptEvents, scriptItemKey, toNumber, @@ -905,3 +906,82 @@ describe("groupScriptItems — card durationSeconds is the max phase run time", expect(cards[0].durationSeconds).toBeUndefined(); }); }); + +describe("partitionHistoricScriptEvents", () => { + // Agent-side clamp evidence: rejectedSourceTimestamp = the source log line's own time, + // timestamp = the clock-clamped event stamp. > 24 h older ⇒ replayed history from a + // previous enrollment (session eaf3d8c4). + const DAY_MS = 24 * 60 * 60 * 1000; + + it("drops events whose rejectedSourceTimestamp is > 24 h older than the event stamp and counts finals", () => { + const historicFinal = finalEvent({ + ts: 0, + data: { policyId: "p1", scriptType: "remediation", rejectedSourceTimestamp: new Date(1700000000000 - 7 * DAY_MS).toISOString() }, + }); + const historicStart = finalEvent({ + eventType: "script_started", + ts: 1, + data: { policyId: "p1", scriptType: "remediation", rejectedSourceTimestamp: new Date(1700000000000 - 7 * DAY_MS).toISOString() }, + }); + const fresh = finalEvent({ ts: 2, data: { policyId: "p2", scriptType: "platform" } }); + + const { current, historicCount } = partitionHistoricScriptEvents([historicFinal, historicStart, fresh]); + expect(current).toEqual([fresh]); + // Only finals count toward the "hidden executions" note — dropped starts would + // double-count the same run. + expect(historicCount).toBe(1); + }); + + it("keeps future-skew rejections (source timestamp ahead of the event stamp)", () => { + const skewed = finalEvent({ + ts: 0, + data: { policyId: "p1", scriptType: "platform", rejectedSourceTimestamp: new Date(1700000000000 + 2 * 60 * 60 * 1000).toISOString() }, + }); + const { current, historicCount } = partitionHistoricScriptEvents([skewed]); + expect(current).toEqual([skewed]); + expect(historicCount).toBe(0); + }); + + it("keeps within-24h rejections and events without or with malformed rejectedSourceTimestamp", () => { + const within = finalEvent({ + ts: 0, + data: { policyId: "p1", scriptType: "platform", rejectedSourceTimestamp: new Date(1700000000000 - 23 * 60 * 60 * 1000).toISOString() }, + }); + const none = finalEvent({ ts: 1, data: { policyId: "p2", scriptType: "platform" } }); + const malformed = finalEvent({ ts: 2, data: { policyId: "p3", scriptType: "platform", rejectedSourceTimestamp: "not-a-date" } }); + + const { current, historicCount } = partitionHistoricScriptEvents([within, none, malformed]); + expect(current).toHaveLength(3); + expect(historicCount).toBe(0); + }); + + it("reads the snake_case wire variant too", () => { + const historic = finalEvent({ + ts: 0, + data: { policyId: "p1", scriptType: "platform", rejected_source_timestamp: new Date(1700000000000 - 7 * DAY_MS).toISOString() }, + }); + const { current, historicCount } = partitionHistoricScriptEvents([historic]); + expect(current).toEqual([]); + expect(historicCount).toBe(1); + }); +}); + +describe("reduceScriptEvents — implausible duration blanking", () => { + it("blanks durations above 24 h (mixed-timeline artifacts from legacy agents)", () => { + // Session eaf3d8c4: clock-clamped completion minus raw week-old start = 612225 s ≈ 170 h. + const items = reduceScriptEvents([ + finalEvent({ data: { policyId: "p1", scriptType: "remediation", scriptPart: "detection", exitCode: "0", complianceResult: "True", durationSeconds: "612225.00", durationBasis: "script_runtime" } }), + ]); + expect(items[0].durationSeconds).toBeUndefined(); + expect(items[0].reportedAfterSeconds).toBeUndefined(); + }); + + it("keeps the 24 h boundary value and blanks cycle-basis values the same way", () => { + const items = reduceScriptEvents([ + finalEvent({ data: { policyId: "p1", scriptType: "platform", exitCode: "0", result: "Success", durationSeconds: "86400", durationBasis: "script_runtime" } }), + finalEvent({ ts: 1, data: { policyId: "p2", scriptType: "remediation", scriptPart: "detection", exitCode: "0", complianceResult: "True", durationSeconds: "90000", durationBasis: "cycle_including_reporting_latency" } }), + ]); + expect(items[0].durationSeconds).toBe(86400); + expect(items[1].reportedAfterSeconds).toBeUndefined(); + }); +}); diff --git a/src/Web/autopilot-monitor-web/lib/scriptExecutions.ts b/src/Web/autopilot-monitor-web/lib/scriptExecutions.ts index 531c6cd9..e254d54f 100644 --- a/src/Web/autopilot-monitor-web/lib/scriptExecutions.ts +++ b/src/Web/autopilot-monitor-web/lib/scriptExecutions.ts @@ -119,6 +119,51 @@ export function scriptCardKey(card: Pick 0) { + const rej = Date.parse(rejected); + const ts = Date.parse(evt.timestamp); + if (Number.isFinite(rej) && Number.isFinite(ts) && ts - rej > HISTORIC_REPLAY_THRESHOLD_MS) { + if (evt.eventType === "script_completed" || evt.eventType === "script_failed") historicCount++; + continue; + } + } + current.push(evt); + } + return { current, historicCount }; +} + /** * Score how complete a script item's data is (higher = better). Used by the reducer to * pick the best entry when re-emissions of the same script collapse into one row. @@ -261,7 +306,13 @@ export function reduceScriptEvents(events: ScriptInputEvent[]): ScriptItem[] { // the script's own result line (runtime), remediation durations to HS-NEW-RESULT // (cycle) — infer accordingly so old data is never displayed as run time when it // isn't one. - const rawDuration = toNumber(d.durationSeconds ?? d.duration_seconds); + // Scripts cannot run > 24 h inside an enrollment (IME's execution timeout is ~30 min); + // larger values are mixed-timeline artifacts from legacy agents (raw stale start paired + // with a clock-clamped completion — session eaf3d8c4 showed 170 h). Hide, don't lie. + const rawDurationUnchecked = toNumber(d.durationSeconds ?? d.duration_seconds); + const rawDuration = rawDurationUnchecked != null && rawDurationUnchecked <= HISTORIC_REPLAY_THRESHOLD_MS / 1000 + ? rawDurationUnchecked + : undefined; const rawBasis = d.durationBasis ?? d.duration_basis; const isCycleBasis = typeof rawBasis === "string" ? rawBasis === "cycle_including_reporting_latency"