From 665a9a14cc6f6ea32bd4dffc703d0fed5c40d585 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Tue, 14 Apr 2026 09:33:44 +0200 Subject: [PATCH] better agent self-update observability --- .../Core/VersionCheckEventBuilderTests.cs | 250 ++++++++++++++ .../Monitoring/Core/MonitoringService.cs | 179 ++-------- .../Core/VersionCheckEventBuilder.cs | 317 ++++++++++++++++++ .../AutopilotMonitor.Agent/SelfUpdater.cs | 63 +++- .../AutopilotMonitor.Agent/buildcounter.txt | 2 +- .../AutopilotMonitor.Shared/Constants.cs | 25 +- .../docs/sections/SectionAgentInternals.tsx | 4 +- 7 files changed, 671 insertions(+), 169 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.Core.Tests/Core/VersionCheckEventBuilderTests.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/VersionCheckEventBuilder.cs diff --git a/src/Agent/AutopilotMonitor.Agent.Core.Tests/Core/VersionCheckEventBuilderTests.cs b/src/Agent/AutopilotMonitor.Agent.Core.Tests/Core/VersionCheckEventBuilderTests.cs new file mode 100644 index 00000000..3099f7ec --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.Core.Tests/Core/VersionCheckEventBuilderTests.cs @@ -0,0 +1,250 @@ +using System; +using System.IO; +using AutopilotMonitor.Agent.Core.Monitoring.Core; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace AutopilotMonitor.Agent.Core.Tests.Core +{ + /// + /// Covers the three outcomes (updated / skipped / check_failed / up_to_date), the dedup rules, + /// and the defensive behaviors (no marker, corrupt JSON, stale markers). + /// + public class VersionCheckEventBuilderTests : IDisposable + { + private readonly string _tempDir; + private readonly VersionCheckEventBuilder.Paths _paths; + + private const string SessionA = "11111111-1111-1111-1111-111111111111"; + private const string SessionB = "22222222-2222-2222-2222-222222222222"; + private const string TenantId = "tenant-xyz"; + + public VersionCheckEventBuilderTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), "vceb-test-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + _paths = new VersionCheckEventBuilder.Paths + { + UpdatedMarker = Path.Combine(_tempDir, "self-update-info.json"), + SkippedMarker = Path.Combine(_tempDir, "self-update-skipped.json"), + CheckedMarker = Path.Combine(_tempDir, "self-update-checked.json"), + LastEmit = Path.Combine(_tempDir, "last-version-check.json"), + }; + } + + public void Dispose() + { + try { if (Directory.Exists(_tempDir)) Directory.Delete(_tempDir, recursive: true); } catch { } + } + + private void WriteChecked(string current = "1.0.949", string latest = "1.0.949", long versionCheckMs = 123) + { + var json = new JObject + { + ["outcome"] = "up_to_date", + ["currentVersion"] = current, + ["latestVersion"] = latest, + ["versionCheckMs"] = versionCheckMs, + ["checkedAtUtc"] = DateTime.UtcNow.ToString("O"), + }; + File.WriteAllText(_paths.CheckedMarker, json.ToString()); + } + + private void WriteUpdated(string previous = "1.0.949", string newVer = "1.0.950", string trigger = "startup") + { + var now = DateTime.UtcNow; + var json = new JObject + { + ["outcome"] = "updated", + ["previousVersion"] = previous, + ["newVersion"] = newVer, + ["triggerReason"] = trigger, + ["updatedAtUtc"] = now.ToString("O"), + ["exitAtUtc"] = now.AddSeconds(-2).ToString("O"), + ["versionCheckMs"] = 100L, + ["downloadMs"] = 2000L, + ["zipSizeBytes"] = 1234567L, + ["verifyMs"] = 40L, + ["extractMs"] = 200L, + ["swapMs"] = 50L, + ["totalUpdateMs"] = 2400L, + }; + File.WriteAllText(_paths.UpdatedMarker, json.ToString()); + } + + private void WriteSkipped(string reason = "download_failed", string latest = "1.0.950", string outcome = null) + { + var json = new JObject + { + ["reason"] = reason, + ["currentVersion"] = "1.0.949", + ["latestVersion"] = latest, + ["skippedAtUtc"] = DateTime.UtcNow.ToString("O"), + ["errorDetail"] = "timeout after 10000ms", + }; + if (outcome != null) json["outcome"] = outcome; + File.WriteAllText(_paths.SkippedMarker, json.ToString()); + } + + private void WriteLastEmit(string sessionId, string latestVersion, string outcome) + { + var json = new JObject + { + ["sessionId"] = sessionId, + ["latestVersion"] = latestVersion, + ["outcome"] = outcome, + ["emittedAtUtc"] = DateTime.UtcNow.ToString("O"), + }; + File.WriteAllText(_paths.LastEmit, json.ToString()); + } + + [Fact] + public void NoMarker_ReturnsNoEvent() + { + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.Null(result.Event); + Assert.False(result.Deduped); + Assert.Null(result.Outcome); + Assert.Null(result.ParseError); + } + + [Fact] + public void UpToDateMarker_FirstTime_EmitsEventAndPersistsLastEmit() + { + WriteChecked(latest: "1.0.949"); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.NotNull(result.Event); + Assert.Equal(Constants.EventTypes.AgentVersionCheck, result.Event.EventType); + Assert.Equal(EventSeverity.Info, result.Event.Severity); + Assert.Equal("up_to_date", result.Event.Data["outcome"]); + Assert.Equal("1.0.949", result.Event.Data["latestVersion"]); + Assert.False(File.Exists(_paths.CheckedMarker)); // marker deleted + Assert.True(File.Exists(_paths.LastEmit)); // last-emit persisted + } + + [Fact] + public void UpToDateMarker_SameSessionSameVersion_IsDeduped() + { + WriteLastEmit(SessionA, "1.0.949", "up_to_date"); + WriteChecked(latest: "1.0.949"); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.Null(result.Event); + Assert.True(result.Deduped); + Assert.Equal("up_to_date", result.Outcome); + Assert.False(File.Exists(_paths.CheckedMarker)); + } + + [Fact] + public void UpToDateMarker_DifferentSession_EmitsEvent() + { + WriteLastEmit(SessionA, "1.0.949", "up_to_date"); + WriteChecked(latest: "1.0.949"); + + var result = VersionCheckEventBuilder.TryBuild(SessionB, TenantId, DateTime.UtcNow, _paths); + + Assert.NotNull(result.Event); + Assert.False(result.Deduped); + } + + [Fact] + public void UpToDateMarker_SameSessionButNewerLatestVersion_EmitsEvent() + { + // White-glove scenario: after a pause a new release appeared. Dedup must break. + WriteLastEmit(SessionA, "1.0.949", "up_to_date"); + WriteChecked(latest: "1.0.950"); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.NotNull(result.Event); + Assert.False(result.Deduped); + Assert.Equal("1.0.950", result.Event.Data["latestVersion"]); + } + + [Fact] + public void UpdatedMarker_AlwaysEmits_IgnoresLastEmit() + { + WriteLastEmit(SessionA, "1.0.950", "updated"); + WriteUpdated(previous: "1.0.949", newVer: "1.0.950", trigger: "startup"); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.NotNull(result.Event); + Assert.Equal("updated", result.Event.Data["outcome"]); + Assert.Equal(EventSeverity.Info, result.Event.Severity); + Assert.Equal("1.0.949", result.Event.Data["previousVersion"]); + Assert.Equal("1.0.950", result.Event.Data["newVersion"]); + Assert.True(result.Event.Data.ContainsKey("downtimeMs")); + Assert.False(File.Exists(_paths.UpdatedMarker)); + } + + [Fact] + public void UpdatedMarker_RuntimeTrigger_IsWarningSeverity() + { + WriteUpdated(trigger: "runtime_hash_mismatch"); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.Equal(EventSeverity.Warning, result.Event.Severity); + } + + [Fact] + public void SkippedMarker_DownloadFailed_EmitsWithSkippedOutcome() + { + WriteSkipped(reason: "download_failed"); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.NotNull(result.Event); + Assert.Equal("skipped", result.Event.Data["outcome"]); + Assert.Equal("download_failed", result.Event.Data["reason"]); + Assert.Equal(EventSeverity.Warning, result.Event.Severity); + Assert.False(File.Exists(_paths.SkippedMarker)); + } + + [Fact] + public void SkippedMarker_VersionCheckFailed_DerivesCheckFailedOutcome() + { + // Backward-compat: old marker without outcome field — must still map correctly. + WriteSkipped(reason: "version_check_failed", latest: null, outcome: null); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.Equal("check_failed", result.Event.Data["outcome"]); + } + + [Fact] + public void Priority_UpdatedBeatsSkippedBeatsChecked() + { + // All three present (edge case after a crash). Updated wins, the others are cleaned up. + WriteUpdated(); + WriteSkipped(); + WriteChecked(); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.Equal("updated", result.Event.Data["outcome"]); + Assert.False(File.Exists(_paths.UpdatedMarker)); + Assert.False(File.Exists(_paths.SkippedMarker)); + Assert.False(File.Exists(_paths.CheckedMarker)); + } + + [Fact] + public void CorruptMarker_DoesNotThrow_ReturnsParseError() + { + File.WriteAllText(_paths.CheckedMarker, "{ not valid json"); + + var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths); + + Assert.Null(result.Event); + Assert.NotNull(result.ParseError); + Assert.False(File.Exists(_paths.CheckedMarker)); // still cleaned up + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs index f89ece9e..afbf14ed 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs @@ -267,12 +267,10 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core }); } - // If the agent was self-updated on the previous run, emit a timeline event - EmitSelfUpdateEventIfMarkerExists(); - - // If the startup self-update was skipped (network timeout etc.), emit telemetry - // so we can measure how often this happens across the fleet. - EmitSelfUpdateSkippedEventIfMarkerExists(); + // Consume self-update markers (updated / skipped / checked) left by SelfUpdater on the + // previous startup and emit a single agent_version_check event. Dedup suppresses + // repeated up_to_date events for the same latestVersion within the same session. + EmitVersionCheckEventFromMarkers(); // Emit deferred security_audit (initial UnrestrictedMode) now that agent_started has its sequence if (_deferredSecurityAuditConfigVersion.HasValue) @@ -711,159 +709,40 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core } /// - /// Reads the self-update marker file left by SelfUpdater, emits an agent_self_updated event, - /// and deletes the marker. Best-effort — failures are logged but never block startup. - /// Includes phase timings, trigger reason, and a computed downtime (new process startup - /// time minus old process exit time) so we can aggregate update duration and event-loss - /// window across the fleet. + /// Consumes any self-update markers left by SelfUpdater and emits a single + /// agent_version_check event. Three marker variants map to the outcomes: + /// + /// self-update-info.json → outcome=updated + /// self-update-skipped.json → outcome=skipped or check_failed + /// self-update-checked.json → outcome=up_to_date (subject to session-scoped dedup) + /// + /// Best-effort — failures are logged but never block startup. See + /// for the pure logic + dedup rules. /// - private void EmitSelfUpdateEventIfMarkerExists() + private void EmitVersionCheckEventFromMarkers() { - try + var result = VersionCheckEventBuilder.TryBuild( + _configuration.SessionId, + _configuration.TenantId, + _agentStartTimeUtc); + + if (!string.IsNullOrEmpty(result.ParseError)) { - var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateMarkerFile); - if (!File.Exists(markerPath)) - return; - - var json = File.ReadAllText(markerPath); - var marker = Newtonsoft.Json.Linq.JObject.Parse(json); - - var previousVersion = (string)marker["previousVersion"] ?? "unknown"; - var newVersion = (string)marker["newVersion"] ?? "unknown"; - var updatedAtUtc = (string)marker["updatedAtUtc"] ?? "unknown"; - var triggerReason = (string)marker["triggerReason"] ?? "startup"; - var exitAtUtc = (string)marker["exitAtUtc"]; - - long? ReadLong(string key) - { - var token = marker[key]; - if (token == null || token.Type == Newtonsoft.Json.Linq.JTokenType.Null) - return null; - try { return token.ToObject(); } - catch { return null; } - } - - var versionCheckMs = ReadLong("versionCheckMs"); - var downloadMs = ReadLong("downloadMs"); - var zipSizeBytes = ReadLong("zipSizeBytes"); - var verifyMs = ReadLong("verifyMs"); - var extractMs = ReadLong("extractMs"); - var swapMs = ReadLong("swapMs"); - var totalUpdateMs = ReadLong("totalUpdateMs"); - - // Downtime = new process start (_agentStartTimeUtc) minus old process exit. - // Measured across the PowerShell Wait-Process gap + new agent startup up to - // MonitoringService init. Closest proxy for "events potentially missed". - long? downtimeMs = null; - if (!string.IsNullOrEmpty(exitAtUtc) && - DateTime.TryParse(exitAtUtc, System.Globalization.CultureInfo.InvariantCulture, - System.Globalization.DateTimeStyles.RoundtripKind, out var exitDt)) - { - downtimeMs = (long)(_agentStartTimeUtc - exitDt).TotalMilliseconds; - } - - _logger.Info($"Self-update detected: {previousVersion} → {newVersion} (trigger={triggerReason}, downtimeMs={downtimeMs?.ToString() ?? "n/a"})"); - - var data = new Dictionary - { - { "previousVersion", previousVersion }, - { "newVersion", newVersion }, - { "updatedAtUtc", updatedAtUtc }, - { "triggerReason", triggerReason } - }; - if (exitAtUtc != null) data["exitAtUtc"] = exitAtUtc; - if (versionCheckMs.HasValue) data["versionCheckMs"] = versionCheckMs.Value; - if (downloadMs.HasValue) data["downloadMs"] = downloadMs.Value; - if (zipSizeBytes.HasValue) data["zipSizeBytes"] = zipSizeBytes.Value; - if (verifyMs.HasValue) data["verifyMs"] = verifyMs.Value; - if (extractMs.HasValue) data["extractMs"] = extractMs.Value; - if (swapMs.HasValue) data["swapMs"] = swapMs.Value; - if (totalUpdateMs.HasValue) data["totalUpdateMs"] = totalUpdateMs.Value; - if (downtimeMs.HasValue) data["downtimeMs"] = downtimeMs.Value; - - // Runtime-triggered updates are bumped to Warning so they stand out in the UI — - // startup updates stay Info because they are the normal rollout path. - var severity = string.Equals(triggerReason, "runtime_hash_mismatch", StringComparison.Ordinal) - ? EventSeverity.Warning - : EventSeverity.Info; - - EmitEvent(new EnrollmentEvent - { - SessionId = _configuration.SessionId, - TenantId = _configuration.TenantId, - EventType = Constants.EventTypes.AgentSelfUpdated, - Severity = severity, - Source = "Agent", - Message = $"Agent self-updated from {previousVersion} to {newVersion} (trigger={triggerReason})", - Data = data, - ImmediateUpload = true - }); - - TryDeleteMarker(markerPath); + _logger.Warning($"Could not process version-check marker: {result.ParseError}"); + return; } - catch (Exception ex) + + if (result.Deduped) { - _logger.Warning($"Could not process self-update marker: {ex.Message}"); + _logger.Verbose($"agent_version_check skipped (dedup: up_to_date, same session, same latestVersion)"); + return; } - } - /// - /// Reads the self-update-skipped marker file left by SelfUpdater (startup trigger path only), - /// emits an agent_self_update_skipped event, and deletes the marker. Best-effort — failures - /// are logged but never block startup. - /// - private void EmitSelfUpdateSkippedEventIfMarkerExists() - { - try + if (result.Event != null) { - var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateSkippedMarkerFile); - if (!File.Exists(markerPath)) - return; - - var json = File.ReadAllText(markerPath); - var marker = Newtonsoft.Json.Linq.JObject.Parse(json); - - var reason = (string)marker["reason"] ?? "unknown"; - var currentVersion = (string)marker["currentVersion"] ?? "unknown"; - var latestVersion = (string)marker["latestVersion"]; - var skippedAtUtc = (string)marker["skippedAtUtc"] ?? "unknown"; - var errorDetail = (string)marker["errorDetail"] ?? string.Empty; - - _logger.Warning($"Startup self-update skipped: reason={reason}, current={currentVersion}, latest={latestVersion ?? "n/a"}"); - - var data = new Dictionary - { - { "reason", reason }, - { "currentVersion", currentVersion }, - { "skippedAtUtc", skippedAtUtc }, - { "errorDetail", errorDetail } - }; - if (latestVersion != null) - data["latestVersion"] = latestVersion; - - EmitEvent(new EnrollmentEvent - { - SessionId = _configuration.SessionId, - TenantId = _configuration.TenantId, - EventType = Constants.EventTypes.AgentSelfUpdateSkipped, - Severity = EventSeverity.Warning, - Source = "Agent", - Message = $"Startup self-update skipped (reason={reason})", - Data = data, - ImmediateUpload = true - }); - - TryDeleteMarker(markerPath); + _logger.Info($"Agent version check: outcome={result.Outcome}"); + EmitEvent(result.Event); } - catch (Exception ex) - { - _logger.Warning($"Could not process self-update-skipped marker: {ex.Message}"); - } - } - - private static void TryDeleteMarker(string path) - { - try { File.Delete(path); } catch { } } // ────────────────────────────────────────────────────────────────── diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/VersionCheckEventBuilder.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/VersionCheckEventBuilder.cs new file mode 100644 index 00000000..1bcad5aa --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/VersionCheckEventBuilder.cs @@ -0,0 +1,317 @@ +using System; +using System.Collections.Generic; +using System.IO; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; +using Newtonsoft.Json.Linq; + +namespace AutopilotMonitor.Agent.Core.Monitoring.Core +{ + /// + /// Consumes the three self-update marker files written by SelfUpdater on every agent startup + /// (updated / skipped / checked) and converts them into a single agent_version_check + /// event with an outcome discriminator. Applies session-scoped dedup for the + /// up_to_date outcome so repeated reboots in the same session don't spam the timeline. + /// + /// All filesystem paths are injectable so this can be exercised in unit tests without touching + /// %ProgramData%. + /// + public static class VersionCheckEventBuilder + { + public sealed class Paths + { + public string UpdatedMarker { get; set; } + public string SkippedMarker { get; set; } + public string CheckedMarker { get; set; } + public string LastEmit { get; set; } + + public static Paths Default() => new Paths + { + UpdatedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateMarkerFile), + SkippedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateSkippedMarkerFile), + CheckedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateCheckedMarkerFile), + LastEmit = Environment.ExpandEnvironmentVariables(Constants.LastVersionCheckEmitFile) + }; + } + + public sealed class BuildResult + { + /// Event to emit; null when nothing to report (no marker) or when deduped. + public EnrollmentEvent Event { get; set; } + + /// True when a marker was read but the event was suppressed by dedup. + public bool Deduped { get; set; } + + /// Outcome that was detected (even if deduped), for logging. Null when no marker. + public string Outcome { get; set; } + + /// Human-readable message for warning logs when a marker file was malformed. + public string ParseError { get; set; } + } + + /// + /// Reads available markers, applies dedup, updates the last-emit file and deletes consumed markers. + /// Returns the event to emit (or null). Never throws — errors are returned via ParseError. + /// + /// Priority: Updated > Skipped > Checked. If more than one marker exists (rare — stale files + /// from a crashed run), the highest-priority one wins and the others are deleted defensively. + /// + public static BuildResult TryBuild( + string sessionId, + string tenantId, + DateTime agentStartTimeUtc, + Paths paths = null) + { + paths = paths ?? Paths.Default(); + var result = new BuildResult(); + + try + { + // Priority order — once consumed, all three are deleted. + if (File.Exists(paths.UpdatedMarker)) + { + result.Outcome = "updated"; + result.Event = BuildUpdatedEvent(paths.UpdatedMarker, sessionId, tenantId, agentStartTimeUtc); + } + else if (File.Exists(paths.SkippedMarker)) + { + result.Event = BuildSkippedEvent(paths.SkippedMarker, sessionId, tenantId, out var skipOutcome); + result.Outcome = skipOutcome; + } + else if (File.Exists(paths.CheckedMarker)) + { + result.Outcome = "up_to_date"; + var marker = ReadMarker(paths.CheckedMarker); + var latestVersion = (string)marker["latestVersion"] ?? "unknown"; + + // Session-scoped dedup: suppress repeated up_to_date events for the same + // latestVersion within the same session. Update/Skip/Failure are never deduped. + var lastEmit = TryReadLastEmit(paths.LastEmit); + if (lastEmit != null && + string.Equals(lastEmit.SessionId, sessionId, StringComparison.Ordinal) && + string.Equals(lastEmit.LatestVersion, latestVersion, StringComparison.Ordinal) && + string.Equals(lastEmit.Outcome, "up_to_date", StringComparison.Ordinal)) + { + result.Deduped = true; + } + else + { + result.Event = BuildUpToDateEvent(marker, sessionId, tenantId); + } + } + else + { + return result; // no marker — nothing to do + } + } + catch (Exception ex) + { + result.ParseError = ex.Message; + } + + // Persist the last-emit state before cleanup so a crash right after emit doesn't re-emit. + if (result.Event != null) + { + var latestVersion = ExtractLatestVersion(result.Event); + TryWriteLastEmit(paths.LastEmit, sessionId, latestVersion, result.Outcome); + } + + // Always delete markers once consumed (or attempted) — stale markers would cause loops. + TryDelete(paths.UpdatedMarker); + TryDelete(paths.SkippedMarker); + TryDelete(paths.CheckedMarker); + + return result; + } + + // ── Individual builders ───────────────────────────────────────────── + + private static EnrollmentEvent BuildUpdatedEvent(string path, string sessionId, string tenantId, DateTime agentStartTimeUtc) + { + var marker = ReadMarker(path); + + var previousVersion = (string)marker["previousVersion"] ?? "unknown"; + var newVersion = (string)marker["newVersion"] ?? "unknown"; + var updatedAtUtc = (string)marker["updatedAtUtc"] ?? "unknown"; + var triggerReason = (string)marker["triggerReason"] ?? "startup"; + var exitAtUtc = (string)marker["exitAtUtc"]; + + long? downtimeMs = null; + if (!string.IsNullOrEmpty(exitAtUtc) && + DateTime.TryParse(exitAtUtc, System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.RoundtripKind, out var exitDt)) + { + downtimeMs = (long)(agentStartTimeUtc - exitDt).TotalMilliseconds; + } + + var data = new Dictionary + { + { "outcome", "updated" }, + { "previousVersion", previousVersion }, + { "newVersion", newVersion }, + { "latestVersion", newVersion }, + { "updatedAtUtc", updatedAtUtc }, + { "triggerReason", triggerReason } + }; + if (exitAtUtc != null) data["exitAtUtc"] = exitAtUtc; + CopyLong(marker, "versionCheckMs", data); + CopyLong(marker, "downloadMs", data); + CopyLong(marker, "zipSizeBytes", data); + CopyLong(marker, "verifyMs", data); + CopyLong(marker, "extractMs", data); + CopyLong(marker, "swapMs", data); + CopyLong(marker, "totalUpdateMs", data); + if (downtimeMs.HasValue) data["downtimeMs"] = downtimeMs.Value; + + var severity = string.Equals(triggerReason, "runtime_hash_mismatch", StringComparison.Ordinal) + ? EventSeverity.Warning + : EventSeverity.Info; + + return new EnrollmentEvent + { + SessionId = sessionId, + TenantId = tenantId, + EventType = Constants.EventTypes.AgentVersionCheck, + Severity = severity, + Source = "Agent", + Message = $"Agent self-updated from {previousVersion} to {newVersion} (trigger={triggerReason})", + Data = data, + ImmediateUpload = true + }; + } + + private static EnrollmentEvent BuildSkippedEvent(string path, string sessionId, string tenantId, out string outcome) + { + var marker = ReadMarker(path); + + var reason = (string)marker["reason"] ?? "unknown"; + var currentVersion = (string)marker["currentVersion"] ?? "unknown"; + var latestVersion = (string)marker["latestVersion"]; + var skippedAtUtc = (string)marker["skippedAtUtc"] ?? "unknown"; + var errorDetail = (string)marker["errorDetail"] ?? string.Empty; + + // Marker may already carry outcome (new SelfUpdater). For forward-compat with old markers + // lying around across an upgrade, derive it from the reason. + outcome = (string)marker["outcome"] + ?? (string.Equals(reason, "version_check_failed", StringComparison.Ordinal) ? "check_failed" : "skipped"); + + var data = new Dictionary + { + { "outcome", outcome }, + { "reason", reason }, + { "currentVersion", currentVersion }, + { "skippedAtUtc", skippedAtUtc }, + { "errorDetail", errorDetail } + }; + if (latestVersion != null) + data["latestVersion"] = latestVersion; + + return new EnrollmentEvent + { + SessionId = sessionId, + TenantId = tenantId, + EventType = Constants.EventTypes.AgentVersionCheck, + Severity = EventSeverity.Warning, + Source = "Agent", + Message = outcome == "check_failed" + ? $"Agent version check failed (reason={reason})" + : $"Agent self-update skipped (reason={reason})", + Data = data, + ImmediateUpload = true + }; + } + + private static EnrollmentEvent BuildUpToDateEvent(JObject marker, string sessionId, string tenantId) + { + var currentVersion = (string)marker["currentVersion"] ?? "unknown"; + var latestVersion = (string)marker["latestVersion"] ?? "unknown"; + var checkedAtUtc = (string)marker["checkedAtUtc"] ?? "unknown"; + + var data = new Dictionary + { + { "outcome", "up_to_date" }, + { "currentVersion", currentVersion }, + { "latestVersion", latestVersion }, + { "checkedAtUtc", checkedAtUtc } + }; + CopyLong(marker, "versionCheckMs", data); + + return new EnrollmentEvent + { + SessionId = sessionId, + TenantId = tenantId, + EventType = Constants.EventTypes.AgentVersionCheck, + Severity = EventSeverity.Info, + Source = "Agent", + Message = $"Agent version check: up to date ({currentVersion})", + Data = data, + ImmediateUpload = true + }; + } + + // ── Helpers ───────────────────────────────────────────────────────── + + private sealed class LastEmit + { + public string SessionId { get; set; } + public string LatestVersion { get; set; } + public string Outcome { get; set; } + } + + private static JObject ReadMarker(string path) => JObject.Parse(File.ReadAllText(path)); + + private static void CopyLong(JObject marker, string key, Dictionary target) + { + var token = marker[key]; + if (token == null || token.Type == JTokenType.Null) return; + try { target[key] = token.ToObject(); } catch { /* skip malformed field */ } + } + + private static string ExtractLatestVersion(EnrollmentEvent evt) + { + if (evt?.Data == null) return "unknown"; + return evt.Data.TryGetValue("latestVersion", out var v) && v != null ? v.ToString() : "unknown"; + } + + private static LastEmit TryReadLastEmit(string path) + { + try + { + if (!File.Exists(path)) return null; + var obj = JObject.Parse(File.ReadAllText(path)); + return new LastEmit + { + SessionId = (string)obj["sessionId"], + LatestVersion = (string)obj["latestVersion"], + Outcome = (string)obj["outcome"] + }; + } + catch { return null; } + } + + private static void TryWriteLastEmit(string path, string sessionId, string latestVersion, string outcome) + { + try + { + var dir = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + Directory.CreateDirectory(dir); + + var payload = new JObject + { + ["sessionId"] = sessionId ?? string.Empty, + ["latestVersion"] = latestVersion ?? "unknown", + ["outcome"] = outcome ?? "unknown", + ["emittedAtUtc"] = DateTime.UtcNow.ToString("O") + }; + File.WriteAllText(path, payload.ToString(Newtonsoft.Json.Formatting.None)); + } + catch { /* best-effort */ } + } + + private static void TryDelete(string path) + { + try { if (File.Exists(path)) File.Delete(path); } catch { } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent/SelfUpdater.cs b/src/Agent/AutopilotMonitor.Agent/SelfUpdater.cs index 177eb94f..c75708da 100644 --- a/src/Agent/AutopilotMonitor.Agent/SelfUpdater.cs +++ b/src/Agent/AutopilotMonitor.Agent/SelfUpdater.cs @@ -97,7 +97,7 @@ namespace AutopilotMonitor.Agent /// /// /// "startup" (default) or "runtime_hash_mismatch". Recorded in the marker file so the - /// agent_self_updated event can distinguish the two paths. + /// agent_version_check event (outcome=updated) can distinguish the two paths. /// /// /// Overrides the ZIP download timeout. Startup path leaves null (fast-fail, 10s) to preserve @@ -145,7 +145,9 @@ namespace AutopilotMonitor.Agent if (!forceUpdate && !IsNewerVersion(currentVersion, latestVersion)) { log($"Self-update: current version {currentVersion} is up to date (latest: {latestVersion})"); - return; // happy path — no skip marker + if (isStartupTrigger) + WriteCheckedMarker(currentVersion, latestVersion, versionCheckMs, log); + return; } if (forceUpdate) @@ -551,10 +553,10 @@ namespace AutopilotMonitor.Agent } /// - /// Writes a small JSON marker so the next agent startup can emit an agent_self_updated event. - /// Best-effort — failure here must not block the update. - /// Includes phase timings, trigger reason, and exit timestamp so the downstream event - /// can carry aggregable telemetry (downtime, total update duration, etc.). + /// Writes a small JSON marker so the next agent startup can emit an agent_version_check event + /// with outcome=updated. Best-effort — failure here must not block the update. Includes phase + /// timings, trigger reason, and exit timestamp so the downstream event can carry aggregable + /// telemetry (downtime, total update duration, etc.). /// private static void WriteSelfUpdateMarker( string previousVersion, @@ -579,6 +581,7 @@ namespace AutopilotMonitor.Agent var now = DateTime.UtcNow; var payload = new JObject { + ["outcome"] = "updated", ["previousVersion"] = previousVersion ?? "unknown", ["newVersion"] = newVersion ?? "unknown", ["triggerReason"] = triggerReason ?? "startup", @@ -605,9 +608,10 @@ namespace AutopilotMonitor.Agent } /// - /// Writes a small JSON marker so the next agent startup can emit an agent_self_update_skipped - /// event. Only called on the startup trigger path (runtime-triggered failures are logged - /// normally because the full logger is already up). Best-effort — never throws. + /// Writes a small JSON marker so the next agent startup can emit an agent_version_check event + /// with outcome=skipped (or check_failed if the version.json fetch itself failed). Only called + /// on the startup trigger path (runtime-triggered failures are logged normally because the full + /// logger is already up). Best-effort — never throws. /// private static void WriteSkipMarker(string reason, string currentVersion, string latestVersion, string errorDetail, Action log) { @@ -621,8 +625,15 @@ namespace AutopilotMonitor.Agent var detail = errorDetail ?? string.Empty; if (detail.Length > 200) detail = detail.Substring(0, 200); + // version_check_failed = we never got a latestVersion; everything else = check succeeded + // but the download/verify/extract/swap step failed. + var outcome = string.Equals(reason, "version_check_failed", StringComparison.Ordinal) + ? "check_failed" + : "skipped"; + var payload = new JObject { + ["outcome"] = outcome, ["reason"] = reason ?? "unknown", ["currentVersion"] = currentVersion ?? "unknown", ["latestVersion"] = latestVersion, // may be null if version check failed @@ -631,7 +642,7 @@ namespace AutopilotMonitor.Agent }; File.WriteAllText(markerPath, payload.ToString(Newtonsoft.Json.Formatting.None)); - log($"Self-update: skip marker written (reason={reason})"); + log($"Self-update: skip marker written (outcome={outcome}, reason={reason})"); } catch (Exception ex) { @@ -639,6 +650,38 @@ namespace AutopilotMonitor.Agent } } + /// + /// Writes a small JSON marker on the happy path (current version is already up to date) so + /// the next agent startup can emit an agent_version_check event with outcome=up_to_date. + /// Subject to session-scoped dedup by the MonitoringService emitter. Best-effort — never throws. + /// + private static void WriteCheckedMarker(string currentVersion, string latestVersion, long versionCheckMs, Action log) + { + try + { + var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateCheckedMarkerFile); + var dir = Path.GetDirectoryName(markerPath); + if (!Directory.Exists(dir)) + Directory.CreateDirectory(dir); + + var payload = new JObject + { + ["outcome"] = "up_to_date", + ["currentVersion"] = currentVersion ?? "unknown", + ["latestVersion"] = latestVersion ?? "unknown", + ["versionCheckMs"] = versionCheckMs, + ["checkedAtUtc"] = DateTime.UtcNow.ToString("O") + }; + + File.WriteAllText(markerPath, payload.ToString(Newtonsoft.Json.Formatting.None)); + log("Self-update: checked marker written (up_to_date) for next-startup event"); + } + catch (Exception ex) + { + log($"Self-update: could not write checked marker — {ex.Message} (non-critical)"); + } + } + /// /// Cleans up staging directory and temp ZIP file. /// diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt index a0394c20..56f639a0 100644 --- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt @@ -1 +1 @@ -949 +951 diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 61fa7cf9..fb734fcf 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -30,19 +30,33 @@ namespace AutopilotMonitor.Shared public const string StateDirectory = @"%ProgramData%\AutopilotMonitor\State"; /// - /// Marker file written by SelfUpdater before restart; read by MonitoringService on next startup - /// to emit an agent_self_updated event. + /// Marker file written by SelfUpdater after a successful update + restart; read by MonitoringService + /// on next startup to emit an agent_version_check event with outcome=updated. /// public const string SelfUpdateMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-info.json"; /// /// Marker file written by SelfUpdater when the startup update path is skipped (network timeout, /// download failure, integrity mismatch, etc.); read by MonitoringService on next startup to emit - /// an agent_self_update_skipped event. Only written on the startup trigger path — runtime-triggered - /// failures are logged normally because the full logger is already up. + /// an agent_version_check event with outcome=skipped or check_failed. Only written on the startup + /// trigger path — runtime-triggered failures are logged normally because the full logger is already up. /// public const string SelfUpdateSkippedMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-skipped.json"; + /// + /// Marker file written by SelfUpdater on the happy path (current version already up to date); + /// read by MonitoringService on next startup to emit an agent_version_check event with + /// outcome=up_to_date. Subject to session-scoped dedup via LastVersionCheckEmitFile. + /// + public const string SelfUpdateCheckedMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-checked.json"; + + /// + /// Persists the last emitted agent_version_check event across agent restarts. Used by + /// MonitoringService to dedup up_to_date events within the same session when the latestVersion + /// has not changed. Updated every time an event is emitted. + /// + public const string LastVersionCheckEmitFile = @"%ProgramData%\AutopilotMonitor\State\last-version-check.json"; + /// /// Default path for the IME pattern match log file (debugging/diagnostics) /// @@ -175,8 +189,7 @@ namespace AutopilotMonitor.Shared public const string EspProvisioningStatus = "esp_provisioning_status"; public const string SoftwareInventoryAnalysis = "software_inventory_analysis"; public const string VulnerabilityReport = "vulnerability_report"; - public const string AgentSelfUpdated = "agent_self_updated"; - public const string AgentSelfUpdateSkipped = "agent_self_update_skipped"; + public const string AgentVersionCheck = "agent_version_check"; // Stall detection (Ebene 2 — StallProbeCollector) public const string StallProbeCheck = "stall_probe_check"; // Trace heartbeat from Probe 2 (15 min) when no anomaly found diff --git a/src/Web/autopilot-monitor-web/app/docs/sections/SectionAgentInternals.tsx b/src/Web/autopilot-monitor-web/app/docs/sections/SectionAgentInternals.tsx index ea446e27..de354cff 100644 --- a/src/Web/autopilot-monitor-web/app/docs/sections/SectionAgentInternals.tsx +++ b/src/Web/autopilot-monitor-web/app/docs/sections/SectionAgentInternals.tsx @@ -662,7 +662,7 @@ export function SectionAgentInternals() { Rename trick for locked binaries (.old suffix). Copies staged EXE to Agent directory. - Writes self-update marker, restarts via PowerShell Wait-Process. Next startup emits agent_self_updated event. + Writes self-update marker, restarts via PowerShell Wait-Process. Next startup emits agent_version_check event (outcome=updated). @@ -885,7 +885,7 @@ export function SectionAgentInternals() { ["agent_started", "MonitoringService", "Agent started (version, config, previous exit type)"], ["agent_shutdown", "MonitoringService", "Graceful shutdown"], ["agent_trace", "Various", "Decision tracing (verbose diagnostics)"], - ["agent_self_updated", "MonitoringService", "Post-update event (old -> new version)"], + ["agent_version_check", "MonitoringService", "Version check outcome (up_to_date / updated / skipped / check_failed) - emitted every startup, session-scoped dedup for up_to_date"], ["system_reboot_detected", "MonitoringService", "Reboot detected between sessions"], ["security_audit", "MonitoringService", "Security flag status at startup"], ]}