mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 05:00:07 +00:00
fix(agent-v2): two-phase StartupEventGate, restart-safe disk_space_low, prefetch fixes (M3-M6, L17)
Delta-review 2026-07-02 findings: - M4: StartupEventGate persisted the new fingerprint INSIDE ShouldEmit, before the caller emitted — a process death between the save and the spool append suppressed that event for the rest of the enrollment (permanent for static payloads like tpm_info / console_prefetch_detected). ShouldEmit now claims in-memory only; the new MarkEmitted(key) commits to disk AFTER the emission, and SaveSafe writes the committed view only, so other keys` uncommitted claims never ride along. All five consumers updated. - M3: disk_space_low hysteresis latch was an in-memory bool → the Warning (ImmediateUpload) re-fired after every stall restart on a disk-starved device. Latch is now seeded from + persisted via the StartupEventGate (new read-only HasFingerprint peek); the re-arm transition persists too. - M5: ConsolePrefetchScanner compared timestamps from two clock epochs (.pf last-write vs now-minus-uptime); an OOBE NTP correction could swallow a real Shift+F10 hit. 10-min skew tolerance on the after-boot test. - M6: prefetch Warning is once-per-enrollment now — the old fingerprint contained prefetchLastRunUtc, which every ESP `cmd /c` installer wrapper refreshes, so multi-reboot sessions re-warned per restart. (Also moots L11.) - L17: a geo re-lookup that only feeds the pending timezone retry no longer emits a device_location failure after an earlier success; outbound_ip got its own latch instead of riding the device_location one. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+36
@@ -113,6 +113,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Analyzers
|
||||
Assert.Null(rig.PrefetchEvent());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Artifact_within_clock_skew_tolerance_before_boot_is_still_flagged()
|
||||
{
|
||||
// M5: an NTP correction between the .pf write and the scan can push the derived boot
|
||||
// time past a REAL console's artifact timestamp. Within the tolerance the artifact
|
||||
// must still count as after-boot.
|
||||
using var rig = new Rig();
|
||||
rig.WritePf("CMD.EXE-0BD30981.pf", BootUtc.AddMinutes(-5)); // inside the 10-min tolerance
|
||||
|
||||
rig.NewScanner().AnalyzeAtStartup();
|
||||
|
||||
Assert.NotNull(rig.PrefetchEvent());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_suppresses_even_when_installer_cmd_refreshed_the_artifact()
|
||||
{
|
||||
// M6: ESP `cmd /c` installer wrappers refresh CMD.EXE-*.pf between reboots. The
|
||||
// scanner cannot attribute the newer timestamp to a human console, so the Warning is
|
||||
// once-per-enrollment — a refreshed artifact must NOT re-emit on the next restart.
|
||||
using var rig = new Rig();
|
||||
rig.WritePf("CMD.EXE-0BD30981.pf", BootUtc.AddMinutes(4));
|
||||
var gate = rig.NewGate();
|
||||
|
||||
rig.NewScanner(gate: gate).AnalyzeAtStartup();
|
||||
|
||||
rig.WritePf("CMD.EXE-0BD30981.pf", BootUtc.AddMinutes(45)); // installer churn
|
||||
rig.NewScanner(gate: gate).AnalyzeAtStartup();
|
||||
|
||||
var emitted = rig.Sink.Posted.Count(p =>
|
||||
p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == "console_prefetch_detected");
|
||||
Assert.Equal(1, emitted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_cmd_artifact_stays_silent()
|
||||
{
|
||||
|
||||
+64
-1
@@ -33,13 +33,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Persistence
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldEmit_state_survives_an_agent_restart()
|
||||
public void ShouldEmit_state_survives_an_agent_restart_once_committed()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
|
||||
var firstRun = new StartupEventGate(tmp.Path, logger);
|
||||
Assert.True(firstRun.ShouldEmit("aad_join_status", "not-joined"));
|
||||
firstRun.MarkEmitted("aad_join_status"); // event went out → commit
|
||||
|
||||
// New instance over the same state directory = agent restarted after a reboot.
|
||||
var secondRun = new StartupEventGate(tmp.Path, logger);
|
||||
@@ -47,6 +48,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Persistence
|
||||
Assert.True(secondRun.ShouldEmit("aad_join_status", "joined")); // late join → emits
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Uncommitted_claim_does_not_survive_a_restart()
|
||||
{
|
||||
// M4 (delta review 2026-07-02): a crash between ShouldEmit and the actual emission
|
||||
// must NOT suppress the event for the rest of the enrollment — only MarkEmitted
|
||||
// (called after the emission) persists the claim.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
|
||||
var firstRun = new StartupEventGate(tmp.Path, logger);
|
||||
Assert.True(firstRun.ShouldEmit("tpm_info", "fp-static"));
|
||||
// process dies here — no MarkEmitted
|
||||
|
||||
var secondRun = new StartupEventGate(tmp.Path, logger);
|
||||
Assert.True(secondRun.ShouldEmit("tpm_info", "fp-static")); // re-emits after the crash
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MarkEmitted_commits_only_its_own_key()
|
||||
{
|
||||
// Two events claim; only A's emission goes out before the crash. B must re-emit.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
|
||||
var firstRun = new StartupEventGate(tmp.Path, logger);
|
||||
Assert.True(firstRun.ShouldEmit("event_a", "fp-a"));
|
||||
Assert.True(firstRun.ShouldEmit("event_b", "fp-b"));
|
||||
firstRun.MarkEmitted("event_a"); // only A emitted before the crash
|
||||
|
||||
var secondRun = new StartupEventGate(tmp.Path, logger);
|
||||
Assert.False(secondRun.ShouldEmit("event_a", "fp-a"));
|
||||
Assert.True(secondRun.ShouldEmit("event_b", "fp-b"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MarkEmitted_without_prior_claim_is_a_noop()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var gate = new StartupEventGate(tmp.Path, NewLogger(tmp.Path));
|
||||
|
||||
gate.MarkEmitted("gate_exempt_type"); // never claimed → must not throw or persist
|
||||
Assert.True(gate.ShouldEmit("gate_exempt_type", "fp"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HasFingerprint_peeks_without_claiming()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp.Path);
|
||||
|
||||
var firstRun = new StartupEventGate(tmp.Path, logger);
|
||||
Assert.False(firstRun.HasFingerprint("disk_space_low", "low"));
|
||||
Assert.True(firstRun.ShouldEmit("disk_space_low", "low"));
|
||||
firstRun.MarkEmitted("disk_space_low");
|
||||
|
||||
var secondRun = new StartupEventGate(tmp.Path, logger);
|
||||
Assert.True(secondRun.HasFingerprint("disk_space_low", "low"));
|
||||
Assert.False(secondRun.HasFingerprint("disk_space_low", "rearmed"));
|
||||
// The peek made no claim: an actual state change still goes through normally.
|
||||
Assert.True(secondRun.ShouldEmit("disk_space_low", "rearmed"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Keys_are_independent()
|
||||
{
|
||||
|
||||
+27
-10
@@ -7,7 +7,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
/// MON-B10 — covers the state-change-only transition logic behind the one-shot
|
||||
/// <c>disk_space_low</c> warning. The warning must fire exactly once on crossing below 2 GB,
|
||||
/// stay silent while the drive remains low (no heartbeat), and only re-arm after free space
|
||||
/// recovers past the 3 GB hysteresis mark.
|
||||
/// recovers past the 3 GB hysteresis mark. The <c>rearmed</c> out-flag (M3) tells the caller
|
||||
/// when to persist the re-arm so the latch survives agent restarts.
|
||||
/// </summary>
|
||||
public class PerformanceCollectorDiskLowTests
|
||||
{
|
||||
@@ -15,24 +16,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
public void AboveThreshold_DoesNotWarn_AndStaysDisarmed()
|
||||
{
|
||||
bool warned = false;
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(50.0, ref warned));
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(50.0, ref warned, out var rearmed));
|
||||
Assert.False(warned);
|
||||
Assert.False(rearmed); // was never latched → nothing to re-arm/persist
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CrossingBelowThreshold_WarnsOnce_AndLatches()
|
||||
{
|
||||
bool warned = false;
|
||||
Assert.True(PerformanceCollector.EvaluateDiskLowTransition(1.9, ref warned));
|
||||
Assert.True(PerformanceCollector.EvaluateDiskLowTransition(1.9, ref warned, out var rearmed));
|
||||
Assert.True(warned);
|
||||
Assert.False(rearmed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StillLowAfterWarning_DoesNotReWarn()
|
||||
{
|
||||
bool warned = true; // already warned on a previous low read
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(1.5, ref warned));
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(1.5, ref warned, out var rearmed));
|
||||
Assert.True(warned);
|
||||
Assert.False(rearmed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -41,16 +45,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
// Between 2 GB (threshold) and 3 GB (recovery): not low enough to re-warn,
|
||||
// not recovered enough to re-arm. Must remain latched and silent.
|
||||
bool warned = true;
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(2.5, ref warned));
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(2.5, ref warned, out var rearmed));
|
||||
Assert.True(warned);
|
||||
Assert.False(rearmed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecoveryPastHysteresisMark_ReArms()
|
||||
{
|
||||
bool warned = true;
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(3.0, ref warned));
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(3.0, ref warned, out var rearmed));
|
||||
Assert.False(warned);
|
||||
Assert.True(rearmed); // caller persists this transition (M3)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -59,15 +65,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
bool warned = false;
|
||||
|
||||
// First drop → warns and latches.
|
||||
Assert.True(PerformanceCollector.EvaluateDiskLowTransition(1.0, ref warned));
|
||||
Assert.True(PerformanceCollector.EvaluateDiskLowTransition(1.0, ref warned, out _));
|
||||
Assert.True(warned);
|
||||
|
||||
// Recovers past the hysteresis mark → re-arms.
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(4.0, ref warned));
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(4.0, ref warned, out var rearmed));
|
||||
Assert.False(warned);
|
||||
Assert.True(rearmed);
|
||||
|
||||
// Drops again → warns a second time.
|
||||
Assert.True(PerformanceCollector.EvaluateDiskLowTransition(1.8, ref warned));
|
||||
Assert.True(PerformanceCollector.EvaluateDiskLowTransition(1.8, ref warned, out _));
|
||||
Assert.True(warned);
|
||||
}
|
||||
|
||||
@@ -76,8 +83,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
{
|
||||
// 2.0 GB is the boundary; the predicate is strict "below", so it must not warn.
|
||||
bool warned = false;
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(2.0, ref warned));
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(2.0, ref warned, out var rearmed));
|
||||
Assert.False(warned);
|
||||
Assert.False(rearmed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AboveRecovery_WithoutPriorWarning_DoesNotReportRearm()
|
||||
{
|
||||
// A healthy disk must not spam the persisted gate state on every sample.
|
||||
bool warned = false;
|
||||
Assert.False(PerformanceCollector.EvaluateDiskLowTransition(10.0, ref warned, out var rearmed));
|
||||
Assert.False(rearmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+29
-8
@@ -37,6 +37,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
internal const string CmdPrefetchPattern = "CMD.EXE-*.pf";
|
||||
internal const string ConhostPrefetchPattern = "CONHOST.EXE-*.pf";
|
||||
|
||||
/// <summary>
|
||||
/// M5: absorbs the clock-epoch mismatch between the .pf NTFS last-write (stamped under
|
||||
/// the clock at write time) and the boot time derived at scan time as now-minus-uptime
|
||||
/// (shifts with every NTP correction). OOBE clock corrections are typically seconds to a
|
||||
/// few minutes; image/sysprep artifacts predate boot by far more than this.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan BootClockSkewTolerance = TimeSpan.FromMinutes(10);
|
||||
|
||||
private const string Attribution =
|
||||
"coarse: a console executed on this device; cannot distinguish a human Shift+F10 from a " +
|
||||
"legitimate install-launched cmd once ESP is running (cmd.exe shares one prefetch artifact)";
|
||||
@@ -107,12 +115,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
return;
|
||||
}
|
||||
|
||||
bool ranAfterBoot = cmdArtifact.Value.lastRunUtc > bootTimeUtc.Value;
|
||||
// M5 (delta review 2026-07-02): the two timestamps come from different clock epochs.
|
||||
// The .pf last-write was stamped under the clock at write time; boot time is derived
|
||||
// at SCAN time as now-minus-uptime, so an NTP correction in between (routine during
|
||||
// OOBE) shifts it by the correction amount. A forward correction made a real
|
||||
// Shift+F10 console look pre-boot ("stale image artifact"). The tolerance absorbs
|
||||
// typical OOBE clock corrections; image/sysprep artifacts are hours-to-months older
|
||||
// than boot and stay firmly below it.
|
||||
bool ranAfterBoot = cmdArtifact.Value.lastRunUtc > bootTimeUtc.Value - BootClockSkewTolerance;
|
||||
if (!ranAfterBoot)
|
||||
{
|
||||
// Stale artifact from the image build / sysprep — not this boot's console.
|
||||
_logger.Debug($"{Name}: cmd prefetch last-run {cmdArtifact.Value.lastRunUtc:o} predates boot " +
|
||||
$"{bootTimeUtc.Value:o} — stale image artifact, not flagged");
|
||||
$"{bootTimeUtc.Value:o} (incl. {BootClockSkewTolerance.TotalMinutes:F0} min skew tolerance) — stale image artifact, not flagged");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -132,14 +147,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
{ "coverageComplete", false },
|
||||
};
|
||||
|
||||
// Restart dedup: the same artifact + last-run was already reported by a previous agent run
|
||||
// in this enrollment. A NEW cmd run (newer last-run) changes the fingerprint and re-emits.
|
||||
// Restart dedup — once per enrollment (M6, delta review 2026-07-02). The fingerprint
|
||||
// used to include prefetchLastRunUtc + corroboratingArtifacts, but every ESP `cmd /c`
|
||||
// installer wrapper refreshes CMD.EXE-*.pf between reboots, so multi-reboot sessions
|
||||
// re-warned per restart. The scanner cannot distinguish a new human console from
|
||||
// routine installer cmd churn via the artifact timestamp — one Warning per enrollment
|
||||
// is the honest signal; the live ConsoleBypassWatcher covers consoles opened later.
|
||||
if (_startupGate != null
|
||||
&& !_startupGate.ShouldEmit(
|
||||
Constants.EventTypes.ConsolePrefetchDetected,
|
||||
Core.Persistence.StartupEventGate.ComputeFingerprint(data, excludedKeys: new[] { "bootTimeUtc" })))
|
||||
&& !_startupGate.ShouldEmit(Constants.EventTypes.ConsolePrefetchDetected, "detected"))
|
||||
{
|
||||
_logger.Debug($"{Name}: same cmd prefetch artifact already reported — suppressed (restart dedup)");
|
||||
_logger.Debug($"{Name}: console prefetch already reported this enrollment — suppressed (restart dedup)");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -158,6 +175,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
ImmediateUpload = true,
|
||||
Data = data,
|
||||
});
|
||||
|
||||
// M4: commit the gate claim only after the Warning went out — committing first let a
|
||||
// crash in between suppress this security event for the rest of the enrollment.
|
||||
_startupGate?.MarkEmitted(Constants.EventTypes.ConsolePrefetchDetected);
|
||||
}
|
||||
|
||||
/// <summary>Newest (max last-write) artifact matching <paramref name="pattern"/>, or null.</summary>
|
||||
|
||||
+4
@@ -249,6 +249,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
Message = message,
|
||||
Data = data
|
||||
});
|
||||
|
||||
// M4: commit the gate claim only after the emission went out — committing first
|
||||
// let a crash in between suppress the findings for the rest of the enrollment.
|
||||
_startupGate?.MarkEmitted(Constants.EventTypes.IntegrityBypassAnalysis);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
+5
@@ -234,6 +234,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
Message = message,
|
||||
Data = data
|
||||
});
|
||||
|
||||
// M4: commit the gate claim only after the emission went out — committing first
|
||||
// let a crash in between suppress the findings for the rest of the enrollment.
|
||||
if (trigger == "startup")
|
||||
_startupGate?.MarkEmitted(Constants.EventTypes.LocalAdminAnalysis);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
+3
@@ -111,6 +111,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
}
|
||||
|
||||
EmitInventoryEvents("startup", EnrollmentPhase.Unknown, _startupInventory, null);
|
||||
|
||||
// M4: commit the gate claim only after the (chunked) emission went out.
|
||||
_startupGate?.MarkEmitted(Constants.EventTypes.SoftwareInventoryAnalysis + ":startup");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
+4
@@ -757,6 +757,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo
|
||||
Message = message,
|
||||
Data = data
|
||||
});
|
||||
|
||||
// M4: commit the gate claim only after the emission went out (no-op for gate-exempt
|
||||
// event types — they never claimed).
|
||||
_startupGate?.MarkEmitted(eventType);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+36
-4
@@ -26,9 +26,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
// had no actionable signal. Emit a single disk_space_low Warning on the transition below the
|
||||
// threshold; re-arm only after free space recovers past a higher mark (hysteresis) so the
|
||||
// event stays state-change-only and never turns into a heartbeat.
|
||||
//
|
||||
// M3 (delta review 2026-07-02): the latch is seeded from + persisted via the
|
||||
// StartupEventGate — a memory-only flag re-warned (ImmediateUpload) after every agent
|
||||
// restart on a persistently disk-starved device (stall restarts: up to ~88×/session).
|
||||
private const double DiskLowThresholdGb = 2.0;
|
||||
private const double DiskRecoveryThresholdGb = 3.0;
|
||||
private const string DiskLowFingerprint = "low";
|
||||
private const string DiskRearmedFingerprint = "rearmed";
|
||||
private bool _diskLowWarned;
|
||||
private readonly Core.Persistence.StartupEventGate _startupGate;
|
||||
|
||||
// Network throughput tracking
|
||||
private string _activeNicId;
|
||||
@@ -39,9 +46,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
private bool _networkInitialized;
|
||||
|
||||
public PerformanceCollector(string sessionId, string tenantId, InformationalEventPost post,
|
||||
AgentLogger logger, int intervalSeconds = 60)
|
||||
AgentLogger logger, int intervalSeconds = 60,
|
||||
Core.Persistence.StartupEventGate startupGate = null)
|
||||
: base(sessionId, tenantId, post, logger, intervalSeconds)
|
||||
{
|
||||
_startupGate = startupGate;
|
||||
// Seed the hysteresis latch from persisted state so a restart on a still-starved
|
||||
// disk does not re-warn (read-only peek — no gate claim).
|
||||
_diskLowWarned = _startupGate?.HasFingerprint(Constants.EventTypes.DiskSpaceLow, DiskLowFingerprint) ?? false;
|
||||
}
|
||||
|
||||
/// <summary>Use a 5-second warm-up delay so counters are primed before first read.</summary>
|
||||
@@ -157,9 +169,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
|
||||
// MON-B10 — surface a dedicated Warning the moment the drive crosses below the
|
||||
// threshold, instead of leaving the only signal buried in this Debug snapshot.
|
||||
if (EvaluateDiskLowTransition(freeGb, ref _diskLowWarned))
|
||||
if (EvaluateDiskLowTransition(freeGb, ref _diskLowWarned, out var rearmed))
|
||||
{
|
||||
EmitDiskLowWarning(freeGb, totalGb, systemDrive);
|
||||
PersistDiskLowState(DiskLowFingerprint);
|
||||
}
|
||||
else if (rearmed)
|
||||
{
|
||||
PersistDiskLowState(DiskRearmedFingerprint);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -289,8 +306,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
/// <param name="freeGb">Current free space on the system drive, in GB.</param>
|
||||
/// <param name="alreadyWarned">Per-collector latch: <c>true</c> while a low-disk warning is in
|
||||
/// effect. Updated in place to reflect the new armed state.</param>
|
||||
internal static bool EvaluateDiskLowTransition(double freeGb, ref bool alreadyWarned)
|
||||
internal static bool EvaluateDiskLowTransition(double freeGb, ref bool alreadyWarned, out bool rearmed)
|
||||
{
|
||||
rearmed = false;
|
||||
if (freeGb < DiskLowThresholdGb)
|
||||
{
|
||||
if (alreadyWarned) return false; // still low, already warned → stay quiet
|
||||
@@ -298,14 +316,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
return true;
|
||||
}
|
||||
|
||||
if (freeGb >= DiskRecoveryThresholdGb)
|
||||
if (freeGb >= DiskRecoveryThresholdGb && alreadyWarned)
|
||||
{
|
||||
alreadyWarned = false; // recovered with margin → re-arm for a future drop
|
||||
rearmed = true; // caller persists the re-arm so it survives restarts (M3)
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// M3: records the disk-low latch state in the StartupEventGate so it survives agent
|
||||
/// restarts. Uses the gate's claim+commit pair as a persisted state cell — there is no
|
||||
/// separate event emission here (the Warning already went out for the "low" transition;
|
||||
/// the re-arm transition is state-only by design).
|
||||
/// </summary>
|
||||
private void PersistDiskLowState(string fingerprint)
|
||||
{
|
||||
if (_startupGate == null) return;
|
||||
if (_startupGate.ShouldEmit(Constants.EventTypes.DiskSpaceLow, fingerprint))
|
||||
_startupGate.MarkEmitted(Constants.EventTypes.DiskSpaceLow);
|
||||
}
|
||||
|
||||
private void EmitDiskLowWarning(double freeGb, double totalGb, string drive)
|
||||
{
|
||||
var data = new Dictionary<string, object>(capacity: 4, StringComparer.Ordinal)
|
||||
|
||||
@@ -313,7 +313,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
idleTimeoutMinutes: collectors.CollectorIdleTimeoutMinutes,
|
||||
networkMetrics: _networkMetrics,
|
||||
agentVersion: _agentVersion,
|
||||
telemetrySpool: telemetrySpool));
|
||||
telemetrySpool: telemetrySpool,
|
||||
startupGate: _startupEventGate)); // M3 — disk_space_low latch survives restarts
|
||||
}
|
||||
|
||||
// V1 parity (CollectorCoordinator.StartOptionalCollectors:375-382) — wire the
|
||||
|
||||
+6
-2
@@ -54,6 +54,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
private readonly NetworkMetrics? _networkMetrics;
|
||||
private readonly string _agentVersion;
|
||||
private readonly ITelemetrySpool? _telemetrySpool;
|
||||
private readonly Persistence.StartupEventGate? _startupGate;
|
||||
|
||||
// Codex Finding 4 — reference to the concrete SignalIngress (when available) so we
|
||||
// can subscribe to its SignalPosted event. This lets us observe activity from
|
||||
@@ -86,7 +87,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
int idleTimeoutMinutes,
|
||||
NetworkMetrics? networkMetrics,
|
||||
string agentVersion,
|
||||
ITelemetrySpool? telemetrySpool = null)
|
||||
ITelemetrySpool? telemetrySpool = null,
|
||||
Persistence.StartupEventGate? startupGate = null)
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
@@ -101,6 +103,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
_networkMetrics = networkMetrics;
|
||||
_agentVersion = agentVersion;
|
||||
_telemetrySpool = telemetrySpool;
|
||||
_startupGate = startupGate;
|
||||
_lastRealEventTimeUtc = DateTime.UtcNow;
|
||||
|
||||
// Post goes to the raw ingress — no per-host wrapping. Activity observation
|
||||
@@ -184,7 +187,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
if (_perfEnabled && _performanceCollector == null)
|
||||
{
|
||||
_performanceCollector = new PerformanceCollector(
|
||||
_sessionId, _tenantId, _post, _logger, _perfIntervalSeconds);
|
||||
_sessionId, _tenantId, _post, _logger, _perfIntervalSeconds,
|
||||
startupGate: _startupGate); // M3 — disk_space_low latch survives restarts
|
||||
_performanceCollector.Start();
|
||||
}
|
||||
if (_selfMetricsEnabled && _selfMetricsCollector == null && _networkMetrics != null)
|
||||
|
||||
@@ -40,23 +40,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Persistence
|
||||
private readonly string _stateFilePath;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly object _lock = new object();
|
||||
|
||||
// M4 (delta review 2026-07-02): two views. `_entries` is the in-process truth that
|
||||
// ShouldEmit claims against (immediate, so concurrent/repeated in-process checks stay
|
||||
// deduped). `_persisted` is what actually reaches disk — a key is copied over only by
|
||||
// MarkEmitted/MarkSucceeded, i.e. AFTER the caller's emission went through. Persisting
|
||||
// inside ShouldEmit meant a process death between the fingerprint save and the spool
|
||||
// append suppressed that event for the rest of the enrollment (fail-closed on telemetry,
|
||||
// permanent for static payloads like tpm_info or console_prefetch_detected).
|
||||
private readonly Dictionary<string, GateEntry> _entries;
|
||||
private readonly Dictionary<string, GateEntry> _persisted;
|
||||
|
||||
public StartupEventGate(string stateDirectory, AgentLogger logger)
|
||||
{
|
||||
_stateDirectory = Environment.ExpandEnvironmentVariables(stateDirectory);
|
||||
_stateFilePath = Path.Combine(_stateDirectory, "startup-event-state.json");
|
||||
_logger = logger;
|
||||
_entries = LoadSafe();
|
||||
_persisted = LoadSafe();
|
||||
_entries = new Dictionary<string, GateEntry>(_persisted, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public string StateFilePath => _stateFilePath;
|
||||
|
||||
/// <summary>
|
||||
/// Emit-on-change: true when no fingerprint is recorded for <paramref name="key"/> or the
|
||||
/// recorded one differs — the new fingerprint is persisted in the same call, so the caller
|
||||
/// emits exactly when this returns true. False means "identical payload already emitted in
|
||||
/// this session" (possibly by a previous agent run).
|
||||
/// recorded one differs. The new fingerprint is claimed IN MEMORY only — the caller must
|
||||
/// invoke <see cref="MarkEmitted"/> after the emission actually went out, which is what
|
||||
/// persists the claim across restarts. A crash (or an emit failure) between the two calls
|
||||
/// therefore re-emits on the next run instead of silently suppressing forever.
|
||||
/// False means "identical payload already emitted in this session" (possibly by a
|
||||
/// previous agent run).
|
||||
/// </summary>
|
||||
public bool ShouldEmit(string key, string fingerprint)
|
||||
{
|
||||
@@ -75,11 +88,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Persistence
|
||||
Succeeded = existing?.Succeeded ?? false,
|
||||
UpdatedUtc = DateTime.UtcNow,
|
||||
};
|
||||
SaveSafe();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits the in-memory claim made by the preceding <see cref="ShouldEmit"/> to disk.
|
||||
/// Call after the event emission succeeded. No-op when the key was never claimed
|
||||
/// (e.g. gate-exempt event types).
|
||||
/// </summary>
|
||||
public void MarkEmitted(string key)
|
||||
{
|
||||
if (string.IsNullOrEmpty(key)) return;
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_entries.TryGetValue(key, out var entry)) return;
|
||||
_persisted[key] = entry;
|
||||
SaveSafe();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Read-only peek: true when the key's current fingerprint (possibly restored from a
|
||||
/// previous agent run) equals <paramref name="fingerprint"/>. Unlike
|
||||
/// <see cref="ShouldEmit"/> this makes NO claim — use it to seed in-memory latches
|
||||
/// (e.g. the disk_space_low hysteresis) from persisted state at construction time.
|
||||
/// </summary>
|
||||
public bool HasFingerprint(string key, string fingerprint)
|
||||
{
|
||||
if (string.IsNullOrEmpty(key)) return false;
|
||||
lock (_lock)
|
||||
{
|
||||
return _entries.TryGetValue(key, out var entry)
|
||||
&& string.Equals(entry.Fingerprint, fingerprint, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Retry-until-success: true when <see cref="MarkSucceeded"/> latched this key in any run.</summary>
|
||||
public bool AlreadySucceeded(string key)
|
||||
{
|
||||
@@ -90,19 +134,25 @@ namespace AutopilotMonitor.Agent.V2.Core.Persistence
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Latches a retry-until-success key; subsequent runs skip the probe entirely.</summary>
|
||||
/// <summary>
|
||||
/// Latches a retry-until-success key; subsequent runs skip the probe entirely.
|
||||
/// Persists immediately — success latches happen AFTER the probe's emission by contract
|
||||
/// (see StartupEnvironmentProbes), so there is no suppression window here.
|
||||
/// </summary>
|
||||
public void MarkSucceeded(string key)
|
||||
{
|
||||
if (string.IsNullOrEmpty(key)) return;
|
||||
lock (_lock)
|
||||
{
|
||||
_entries.TryGetValue(key, out var existing);
|
||||
_entries[key] = new GateEntry
|
||||
var entry = new GateEntry
|
||||
{
|
||||
Fingerprint = existing?.Fingerprint,
|
||||
Succeeded = true,
|
||||
UpdatedUtc = DateTime.UtcNow,
|
||||
};
|
||||
_entries[key] = entry;
|
||||
_persisted[key] = entry;
|
||||
SaveSafe();
|
||||
}
|
||||
}
|
||||
@@ -170,7 +220,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Persistence
|
||||
{
|
||||
Directory.CreateDirectory(_stateDirectory);
|
||||
var tempPath = _stateFilePath + ".tmp";
|
||||
File.WriteAllText(tempPath, JsonConvert.SerializeObject(_entries));
|
||||
// Persist the committed view only — uncommitted ShouldEmit claims of OTHER keys
|
||||
// must not ride along, or their emissions could be suppressed by a crash too.
|
||||
File.WriteAllText(tempPath, JsonConvert.SerializeObject(_persisted));
|
||||
if (File.Exists(_stateFilePath))
|
||||
{
|
||||
File.Replace(tempPath, _stateFilePath, destinationBackupFileName: null, ignoreMetadataErrors: true);
|
||||
|
||||
@@ -116,24 +116,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Runtime
|
||||
if (gate == null || !gate.AlreadySucceeded(Constants.EventTypes.DeviceLocation))
|
||||
{
|
||||
SafeEmit(post, logger, BuildGeoEvent(configuration, attempt.Location));
|
||||
|
||||
// Outbound (public egress) IP — Trace severity so it stays out of the
|
||||
// timeline by default; persisted/queryable for network correlation.
|
||||
if (!string.IsNullOrEmpty(attempt.Location.Ip))
|
||||
SafeEmit(post, logger, BuildOutboundIpEvent(configuration, attempt.Location));
|
||||
|
||||
gate?.MarkSucceeded(Constants.EventTypes.DeviceLocation);
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.Debug("StartupEnvironmentProbes: location re-resolved for the pending timezone retry — device_location already emitted in a previous run.");
|
||||
}
|
||||
|
||||
// Outbound (public egress) IP — Trace severity so it stays out of the timeline by
|
||||
// default; persisted/queryable for network correlation. Own latch (L17): the first
|
||||
// successful provider response can lack the ip field, so it must not ride the
|
||||
// device_location latch — any later lookup (e.g. a timezone retry) that does carry
|
||||
// it still emits the trace once.
|
||||
if (!string.IsNullOrEmpty(attempt.Location.Ip)
|
||||
&& (gate == null || !gate.AlreadySucceeded(Constants.EventTypes.OutboundIp)))
|
||||
{
|
||||
SafeEmit(post, logger, BuildOutboundIpEvent(configuration, attempt.Location));
|
||||
gate?.MarkSucceeded(Constants.EventTypes.OutboundIp);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Failure event each failing start is intentional — it documents the retry trail,
|
||||
// and the unset gate key makes the next agent run try again.
|
||||
SafeEmit(post, logger, BuildGeoFailureEvent(configuration, attempt));
|
||||
// Failure event each failing start is intentional (retry trail) — but only while
|
||||
// the location itself is still outstanding. When a prior run already emitted
|
||||
// device_location and this lookup ran solely to feed the pending timezone retry,
|
||||
// a failure event here would put a device_location failure AFTER its success on
|
||||
// the session timeline (L17) — misleading noise, suppressed.
|
||||
if (gate == null || !gate.AlreadySucceeded(Constants.EventTypes.DeviceLocation))
|
||||
SafeEmit(post, logger, BuildGeoFailureEvent(configuration, attempt));
|
||||
else
|
||||
logger.Debug("StartupEnvironmentProbes: geo re-lookup for the pending timezone retry failed — failure event suppressed (location already reported).");
|
||||
}
|
||||
|
||||
if (configuration.EnableTimezoneAutoSet && !string.IsNullOrEmpty(attempt?.Location?.Timezone))
|
||||
|
||||
Reference in New Issue
Block a user