better agent self-update observability

This commit is contained in:
Oliver Kieselbach
2026-04-14 09:33:44 +02:00
parent 000f7e390d
commit 665a9a14cc
7 changed files with 671 additions and 169 deletions
@@ -0,0 +1,250 @@
using System;
using System.IO;
using AutopilotMonitor.Agent.Core.Monitoring.Core;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json.Linq;
using Xunit;
namespace AutopilotMonitor.Agent.Core.Tests.Core
{
/// <summary>
/// Covers the three outcomes (updated / skipped / check_failed / up_to_date), the dedup rules,
/// and the defensive behaviors (no marker, corrupt JSON, stale markers).
/// </summary>
public class VersionCheckEventBuilderTests : IDisposable
{
private readonly string _tempDir;
private readonly VersionCheckEventBuilder.Paths _paths;
private const string SessionA = "11111111-1111-1111-1111-111111111111";
private const string SessionB = "22222222-2222-2222-2222-222222222222";
private const string TenantId = "tenant-xyz";
public VersionCheckEventBuilderTests()
{
_tempDir = Path.Combine(Path.GetTempPath(), "vceb-test-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(_tempDir);
_paths = new VersionCheckEventBuilder.Paths
{
UpdatedMarker = Path.Combine(_tempDir, "self-update-info.json"),
SkippedMarker = Path.Combine(_tempDir, "self-update-skipped.json"),
CheckedMarker = Path.Combine(_tempDir, "self-update-checked.json"),
LastEmit = Path.Combine(_tempDir, "last-version-check.json"),
};
}
public void Dispose()
{
try { if (Directory.Exists(_tempDir)) Directory.Delete(_tempDir, recursive: true); } catch { }
}
private void WriteChecked(string current = "1.0.949", string latest = "1.0.949", long versionCheckMs = 123)
{
var json = new JObject
{
["outcome"] = "up_to_date",
["currentVersion"] = current,
["latestVersion"] = latest,
["versionCheckMs"] = versionCheckMs,
["checkedAtUtc"] = DateTime.UtcNow.ToString("O"),
};
File.WriteAllText(_paths.CheckedMarker, json.ToString());
}
private void WriteUpdated(string previous = "1.0.949", string newVer = "1.0.950", string trigger = "startup")
{
var now = DateTime.UtcNow;
var json = new JObject
{
["outcome"] = "updated",
["previousVersion"] = previous,
["newVersion"] = newVer,
["triggerReason"] = trigger,
["updatedAtUtc"] = now.ToString("O"),
["exitAtUtc"] = now.AddSeconds(-2).ToString("O"),
["versionCheckMs"] = 100L,
["downloadMs"] = 2000L,
["zipSizeBytes"] = 1234567L,
["verifyMs"] = 40L,
["extractMs"] = 200L,
["swapMs"] = 50L,
["totalUpdateMs"] = 2400L,
};
File.WriteAllText(_paths.UpdatedMarker, json.ToString());
}
private void WriteSkipped(string reason = "download_failed", string latest = "1.0.950", string outcome = null)
{
var json = new JObject
{
["reason"] = reason,
["currentVersion"] = "1.0.949",
["latestVersion"] = latest,
["skippedAtUtc"] = DateTime.UtcNow.ToString("O"),
["errorDetail"] = "timeout after 10000ms",
};
if (outcome != null) json["outcome"] = outcome;
File.WriteAllText(_paths.SkippedMarker, json.ToString());
}
private void WriteLastEmit(string sessionId, string latestVersion, string outcome)
{
var json = new JObject
{
["sessionId"] = sessionId,
["latestVersion"] = latestVersion,
["outcome"] = outcome,
["emittedAtUtc"] = DateTime.UtcNow.ToString("O"),
};
File.WriteAllText(_paths.LastEmit, json.ToString());
}
[Fact]
public void NoMarker_ReturnsNoEvent()
{
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.Null(result.Event);
Assert.False(result.Deduped);
Assert.Null(result.Outcome);
Assert.Null(result.ParseError);
}
[Fact]
public void UpToDateMarker_FirstTime_EmitsEventAndPersistsLastEmit()
{
WriteChecked(latest: "1.0.949");
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.NotNull(result.Event);
Assert.Equal(Constants.EventTypes.AgentVersionCheck, result.Event.EventType);
Assert.Equal(EventSeverity.Info, result.Event.Severity);
Assert.Equal("up_to_date", result.Event.Data["outcome"]);
Assert.Equal("1.0.949", result.Event.Data["latestVersion"]);
Assert.False(File.Exists(_paths.CheckedMarker)); // marker deleted
Assert.True(File.Exists(_paths.LastEmit)); // last-emit persisted
}
[Fact]
public void UpToDateMarker_SameSessionSameVersion_IsDeduped()
{
WriteLastEmit(SessionA, "1.0.949", "up_to_date");
WriteChecked(latest: "1.0.949");
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.Null(result.Event);
Assert.True(result.Deduped);
Assert.Equal("up_to_date", result.Outcome);
Assert.False(File.Exists(_paths.CheckedMarker));
}
[Fact]
public void UpToDateMarker_DifferentSession_EmitsEvent()
{
WriteLastEmit(SessionA, "1.0.949", "up_to_date");
WriteChecked(latest: "1.0.949");
var result = VersionCheckEventBuilder.TryBuild(SessionB, TenantId, DateTime.UtcNow, _paths);
Assert.NotNull(result.Event);
Assert.False(result.Deduped);
}
[Fact]
public void UpToDateMarker_SameSessionButNewerLatestVersion_EmitsEvent()
{
// White-glove scenario: after a pause a new release appeared. Dedup must break.
WriteLastEmit(SessionA, "1.0.949", "up_to_date");
WriteChecked(latest: "1.0.950");
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.NotNull(result.Event);
Assert.False(result.Deduped);
Assert.Equal("1.0.950", result.Event.Data["latestVersion"]);
}
[Fact]
public void UpdatedMarker_AlwaysEmits_IgnoresLastEmit()
{
WriteLastEmit(SessionA, "1.0.950", "updated");
WriteUpdated(previous: "1.0.949", newVer: "1.0.950", trigger: "startup");
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.NotNull(result.Event);
Assert.Equal("updated", result.Event.Data["outcome"]);
Assert.Equal(EventSeverity.Info, result.Event.Severity);
Assert.Equal("1.0.949", result.Event.Data["previousVersion"]);
Assert.Equal("1.0.950", result.Event.Data["newVersion"]);
Assert.True(result.Event.Data.ContainsKey("downtimeMs"));
Assert.False(File.Exists(_paths.UpdatedMarker));
}
[Fact]
public void UpdatedMarker_RuntimeTrigger_IsWarningSeverity()
{
WriteUpdated(trigger: "runtime_hash_mismatch");
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.Equal(EventSeverity.Warning, result.Event.Severity);
}
[Fact]
public void SkippedMarker_DownloadFailed_EmitsWithSkippedOutcome()
{
WriteSkipped(reason: "download_failed");
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.NotNull(result.Event);
Assert.Equal("skipped", result.Event.Data["outcome"]);
Assert.Equal("download_failed", result.Event.Data["reason"]);
Assert.Equal(EventSeverity.Warning, result.Event.Severity);
Assert.False(File.Exists(_paths.SkippedMarker));
}
[Fact]
public void SkippedMarker_VersionCheckFailed_DerivesCheckFailedOutcome()
{
// Backward-compat: old marker without outcome field — must still map correctly.
WriteSkipped(reason: "version_check_failed", latest: null, outcome: null);
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.Equal("check_failed", result.Event.Data["outcome"]);
}
[Fact]
public void Priority_UpdatedBeatsSkippedBeatsChecked()
{
// All three present (edge case after a crash). Updated wins, the others are cleaned up.
WriteUpdated();
WriteSkipped();
WriteChecked();
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.Equal("updated", result.Event.Data["outcome"]);
Assert.False(File.Exists(_paths.UpdatedMarker));
Assert.False(File.Exists(_paths.SkippedMarker));
Assert.False(File.Exists(_paths.CheckedMarker));
}
[Fact]
public void CorruptMarker_DoesNotThrow_ReturnsParseError()
{
File.WriteAllText(_paths.CheckedMarker, "{ not valid json");
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
Assert.Null(result.Event);
Assert.NotNull(result.ParseError);
Assert.False(File.Exists(_paths.CheckedMarker)); // still cleaned up
}
}
}
@@ -267,12 +267,10 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
});
}
// If the agent was self-updated on the previous run, emit a timeline event
EmitSelfUpdateEventIfMarkerExists();
// If the startup self-update was skipped (network timeout etc.), emit telemetry
// so we can measure how often this happens across the fleet.
EmitSelfUpdateSkippedEventIfMarkerExists();
// Consume self-update markers (updated / skipped / checked) left by SelfUpdater on the
// previous startup and emit a single agent_version_check event. Dedup suppresses
// repeated up_to_date events for the same latestVersion within the same session.
EmitVersionCheckEventFromMarkers();
// Emit deferred security_audit (initial UnrestrictedMode) now that agent_started has its sequence
if (_deferredSecurityAuditConfigVersion.HasValue)
@@ -711,159 +709,40 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
}
/// <summary>
/// Reads the self-update marker file left by SelfUpdater, emits an agent_self_updated event,
/// and deletes the marker. Best-effort — failures are logged but never block startup.
/// Includes phase timings, trigger reason, and a computed downtime (new process startup
/// time minus old process exit time) so we can aggregate update duration and event-loss
/// window across the fleet.
/// Consumes any self-update markers left by SelfUpdater and emits a single
/// <c>agent_version_check</c> event. Three marker variants map to the outcomes:
/// <list type="bullet">
/// <item><description>self-update-info.json → outcome=updated</description></item>
/// <item><description>self-update-skipped.json → outcome=skipped or check_failed</description></item>
/// <item><description>self-update-checked.json → outcome=up_to_date (subject to session-scoped dedup)</description></item>
/// </list>
/// Best-effort — failures are logged but never block startup. See
/// <see cref="VersionCheckEventBuilder"/> for the pure logic + dedup rules.
/// </summary>
private void EmitSelfUpdateEventIfMarkerExists()
private void EmitVersionCheckEventFromMarkers()
{
try
var result = VersionCheckEventBuilder.TryBuild(
_configuration.SessionId,
_configuration.TenantId,
_agentStartTimeUtc);
if (!string.IsNullOrEmpty(result.ParseError))
{
var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateMarkerFile);
if (!File.Exists(markerPath))
return;
var json = File.ReadAllText(markerPath);
var marker = Newtonsoft.Json.Linq.JObject.Parse(json);
var previousVersion = (string)marker["previousVersion"] ?? "unknown";
var newVersion = (string)marker["newVersion"] ?? "unknown";
var updatedAtUtc = (string)marker["updatedAtUtc"] ?? "unknown";
var triggerReason = (string)marker["triggerReason"] ?? "startup";
var exitAtUtc = (string)marker["exitAtUtc"];
long? ReadLong(string key)
{
var token = marker[key];
if (token == null || token.Type == Newtonsoft.Json.Linq.JTokenType.Null)
return null;
try { return token.ToObject<long>(); }
catch { return null; }
}
var versionCheckMs = ReadLong("versionCheckMs");
var downloadMs = ReadLong("downloadMs");
var zipSizeBytes = ReadLong("zipSizeBytes");
var verifyMs = ReadLong("verifyMs");
var extractMs = ReadLong("extractMs");
var swapMs = ReadLong("swapMs");
var totalUpdateMs = ReadLong("totalUpdateMs");
// Downtime = new process start (_agentStartTimeUtc) minus old process exit.
// Measured across the PowerShell Wait-Process gap + new agent startup up to
// MonitoringService init. Closest proxy for "events potentially missed".
long? downtimeMs = null;
if (!string.IsNullOrEmpty(exitAtUtc) &&
DateTime.TryParse(exitAtUtc, System.Globalization.CultureInfo.InvariantCulture,
System.Globalization.DateTimeStyles.RoundtripKind, out var exitDt))
{
downtimeMs = (long)(_agentStartTimeUtc - exitDt).TotalMilliseconds;
}
_logger.Info($"Self-update detected: {previousVersion} → {newVersion} (trigger={triggerReason}, downtimeMs={downtimeMs?.ToString() ?? "n/a"})");
var data = new Dictionary<string, object>
{
{ "previousVersion", previousVersion },
{ "newVersion", newVersion },
{ "updatedAtUtc", updatedAtUtc },
{ "triggerReason", triggerReason }
};
if (exitAtUtc != null) data["exitAtUtc"] = exitAtUtc;
if (versionCheckMs.HasValue) data["versionCheckMs"] = versionCheckMs.Value;
if (downloadMs.HasValue) data["downloadMs"] = downloadMs.Value;
if (zipSizeBytes.HasValue) data["zipSizeBytes"] = zipSizeBytes.Value;
if (verifyMs.HasValue) data["verifyMs"] = verifyMs.Value;
if (extractMs.HasValue) data["extractMs"] = extractMs.Value;
if (swapMs.HasValue) data["swapMs"] = swapMs.Value;
if (totalUpdateMs.HasValue) data["totalUpdateMs"] = totalUpdateMs.Value;
if (downtimeMs.HasValue) data["downtimeMs"] = downtimeMs.Value;
// Runtime-triggered updates are bumped to Warning so they stand out in the UI —
// startup updates stay Info because they are the normal rollout path.
var severity = string.Equals(triggerReason, "runtime_hash_mismatch", StringComparison.Ordinal)
? EventSeverity.Warning
: EventSeverity.Info;
EmitEvent(new EnrollmentEvent
{
SessionId = _configuration.SessionId,
TenantId = _configuration.TenantId,
EventType = Constants.EventTypes.AgentSelfUpdated,
Severity = severity,
Source = "Agent",
Message = $"Agent self-updated from {previousVersion} to {newVersion} (trigger={triggerReason})",
Data = data,
ImmediateUpload = true
});
TryDeleteMarker(markerPath);
_logger.Warning($"Could not process version-check marker: {result.ParseError}");
return;
}
catch (Exception ex)
if (result.Deduped)
{
_logger.Warning($"Could not process self-update marker: {ex.Message}");
_logger.Verbose($"agent_version_check skipped (dedup: up_to_date, same session, same latestVersion)");
return;
}
}
/// <summary>
/// Reads the self-update-skipped marker file left by SelfUpdater (startup trigger path only),
/// emits an agent_self_update_skipped event, and deletes the marker. Best-effort — failures
/// are logged but never block startup.
/// </summary>
private void EmitSelfUpdateSkippedEventIfMarkerExists()
{
try
if (result.Event != null)
{
var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateSkippedMarkerFile);
if (!File.Exists(markerPath))
return;
var json = File.ReadAllText(markerPath);
var marker = Newtonsoft.Json.Linq.JObject.Parse(json);
var reason = (string)marker["reason"] ?? "unknown";
var currentVersion = (string)marker["currentVersion"] ?? "unknown";
var latestVersion = (string)marker["latestVersion"];
var skippedAtUtc = (string)marker["skippedAtUtc"] ?? "unknown";
var errorDetail = (string)marker["errorDetail"] ?? string.Empty;
_logger.Warning($"Startup self-update skipped: reason={reason}, current={currentVersion}, latest={latestVersion ?? "n/a"}");
var data = new Dictionary<string, object>
{
{ "reason", reason },
{ "currentVersion", currentVersion },
{ "skippedAtUtc", skippedAtUtc },
{ "errorDetail", errorDetail }
};
if (latestVersion != null)
data["latestVersion"] = latestVersion;
EmitEvent(new EnrollmentEvent
{
SessionId = _configuration.SessionId,
TenantId = _configuration.TenantId,
EventType = Constants.EventTypes.AgentSelfUpdateSkipped,
Severity = EventSeverity.Warning,
Source = "Agent",
Message = $"Startup self-update skipped (reason={reason})",
Data = data,
ImmediateUpload = true
});
TryDeleteMarker(markerPath);
_logger.Info($"Agent version check: outcome={result.Outcome}");
EmitEvent(result.Event);
}
catch (Exception ex)
{
_logger.Warning($"Could not process self-update-skipped marker: {ex.Message}");
}
}
private static void TryDeleteMarker(string path)
{
try { File.Delete(path); } catch { }
}
// ──────────────────────────────────────────────────────────────────
@@ -0,0 +1,317 @@
using System;
using System.Collections.Generic;
using System.IO;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json.Linq;
namespace AutopilotMonitor.Agent.Core.Monitoring.Core
{
/// <summary>
/// Consumes the three self-update marker files written by SelfUpdater on every agent startup
/// (updated / skipped / checked) and converts them into a single <c>agent_version_check</c>
/// event with an <c>outcome</c> discriminator. Applies session-scoped dedup for the
/// up_to_date outcome so repeated reboots in the same session don't spam the timeline.
///
/// All filesystem paths are injectable so this can be exercised in unit tests without touching
/// %ProgramData%.
/// </summary>
public static class VersionCheckEventBuilder
{
public sealed class Paths
{
public string UpdatedMarker { get; set; }
public string SkippedMarker { get; set; }
public string CheckedMarker { get; set; }
public string LastEmit { get; set; }
public static Paths Default() => new Paths
{
UpdatedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateMarkerFile),
SkippedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateSkippedMarkerFile),
CheckedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateCheckedMarkerFile),
LastEmit = Environment.ExpandEnvironmentVariables(Constants.LastVersionCheckEmitFile)
};
}
public sealed class BuildResult
{
/// <summary>Event to emit; null when nothing to report (no marker) or when deduped.</summary>
public EnrollmentEvent Event { get; set; }
/// <summary>True when a marker was read but the event was suppressed by dedup.</summary>
public bool Deduped { get; set; }
/// <summary>Outcome that was detected (even if deduped), for logging. Null when no marker.</summary>
public string Outcome { get; set; }
/// <summary>Human-readable message for warning logs when a marker file was malformed.</summary>
public string ParseError { get; set; }
}
/// <summary>
/// Reads available markers, applies dedup, updates the last-emit file and deletes consumed markers.
/// Returns the event to emit (or null). Never throws — errors are returned via ParseError.
///
/// Priority: Updated > Skipped > Checked. If more than one marker exists (rare — stale files
/// from a crashed run), the highest-priority one wins and the others are deleted defensively.
/// </summary>
public static BuildResult TryBuild(
string sessionId,
string tenantId,
DateTime agentStartTimeUtc,
Paths paths = null)
{
paths = paths ?? Paths.Default();
var result = new BuildResult();
try
{
// Priority order — once consumed, all three are deleted.
if (File.Exists(paths.UpdatedMarker))
{
result.Outcome = "updated";
result.Event = BuildUpdatedEvent(paths.UpdatedMarker, sessionId, tenantId, agentStartTimeUtc);
}
else if (File.Exists(paths.SkippedMarker))
{
result.Event = BuildSkippedEvent(paths.SkippedMarker, sessionId, tenantId, out var skipOutcome);
result.Outcome = skipOutcome;
}
else if (File.Exists(paths.CheckedMarker))
{
result.Outcome = "up_to_date";
var marker = ReadMarker(paths.CheckedMarker);
var latestVersion = (string)marker["latestVersion"] ?? "unknown";
// Session-scoped dedup: suppress repeated up_to_date events for the same
// latestVersion within the same session. Update/Skip/Failure are never deduped.
var lastEmit = TryReadLastEmit(paths.LastEmit);
if (lastEmit != null &&
string.Equals(lastEmit.SessionId, sessionId, StringComparison.Ordinal) &&
string.Equals(lastEmit.LatestVersion, latestVersion, StringComparison.Ordinal) &&
string.Equals(lastEmit.Outcome, "up_to_date", StringComparison.Ordinal))
{
result.Deduped = true;
}
else
{
result.Event = BuildUpToDateEvent(marker, sessionId, tenantId);
}
}
else
{
return result; // no marker — nothing to do
}
}
catch (Exception ex)
{
result.ParseError = ex.Message;
}
// Persist the last-emit state before cleanup so a crash right after emit doesn't re-emit.
if (result.Event != null)
{
var latestVersion = ExtractLatestVersion(result.Event);
TryWriteLastEmit(paths.LastEmit, sessionId, latestVersion, result.Outcome);
}
// Always delete markers once consumed (or attempted) — stale markers would cause loops.
TryDelete(paths.UpdatedMarker);
TryDelete(paths.SkippedMarker);
TryDelete(paths.CheckedMarker);
return result;
}
// ── Individual builders ─────────────────────────────────────────────
private static EnrollmentEvent BuildUpdatedEvent(string path, string sessionId, string tenantId, DateTime agentStartTimeUtc)
{
var marker = ReadMarker(path);
var previousVersion = (string)marker["previousVersion"] ?? "unknown";
var newVersion = (string)marker["newVersion"] ?? "unknown";
var updatedAtUtc = (string)marker["updatedAtUtc"] ?? "unknown";
var triggerReason = (string)marker["triggerReason"] ?? "startup";
var exitAtUtc = (string)marker["exitAtUtc"];
long? downtimeMs = null;
if (!string.IsNullOrEmpty(exitAtUtc) &&
DateTime.TryParse(exitAtUtc, System.Globalization.CultureInfo.InvariantCulture,
System.Globalization.DateTimeStyles.RoundtripKind, out var exitDt))
{
downtimeMs = (long)(agentStartTimeUtc - exitDt).TotalMilliseconds;
}
var data = new Dictionary<string, object>
{
{ "outcome", "updated" },
{ "previousVersion", previousVersion },
{ "newVersion", newVersion },
{ "latestVersion", newVersion },
{ "updatedAtUtc", updatedAtUtc },
{ "triggerReason", triggerReason }
};
if (exitAtUtc != null) data["exitAtUtc"] = exitAtUtc;
CopyLong(marker, "versionCheckMs", data);
CopyLong(marker, "downloadMs", data);
CopyLong(marker, "zipSizeBytes", data);
CopyLong(marker, "verifyMs", data);
CopyLong(marker, "extractMs", data);
CopyLong(marker, "swapMs", data);
CopyLong(marker, "totalUpdateMs", data);
if (downtimeMs.HasValue) data["downtimeMs"] = downtimeMs.Value;
var severity = string.Equals(triggerReason, "runtime_hash_mismatch", StringComparison.Ordinal)
? EventSeverity.Warning
: EventSeverity.Info;
return new EnrollmentEvent
{
SessionId = sessionId,
TenantId = tenantId,
EventType = Constants.EventTypes.AgentVersionCheck,
Severity = severity,
Source = "Agent",
Message = $"Agent self-updated from {previousVersion} to {newVersion} (trigger={triggerReason})",
Data = data,
ImmediateUpload = true
};
}
private static EnrollmentEvent BuildSkippedEvent(string path, string sessionId, string tenantId, out string outcome)
{
var marker = ReadMarker(path);
var reason = (string)marker["reason"] ?? "unknown";
var currentVersion = (string)marker["currentVersion"] ?? "unknown";
var latestVersion = (string)marker["latestVersion"];
var skippedAtUtc = (string)marker["skippedAtUtc"] ?? "unknown";
var errorDetail = (string)marker["errorDetail"] ?? string.Empty;
// Marker may already carry outcome (new SelfUpdater). For forward-compat with old markers
// lying around across an upgrade, derive it from the reason.
outcome = (string)marker["outcome"]
?? (string.Equals(reason, "version_check_failed", StringComparison.Ordinal) ? "check_failed" : "skipped");
var data = new Dictionary<string, object>
{
{ "outcome", outcome },
{ "reason", reason },
{ "currentVersion", currentVersion },
{ "skippedAtUtc", skippedAtUtc },
{ "errorDetail", errorDetail }
};
if (latestVersion != null)
data["latestVersion"] = latestVersion;
return new EnrollmentEvent
{
SessionId = sessionId,
TenantId = tenantId,
EventType = Constants.EventTypes.AgentVersionCheck,
Severity = EventSeverity.Warning,
Source = "Agent",
Message = outcome == "check_failed"
? $"Agent version check failed (reason={reason})"
: $"Agent self-update skipped (reason={reason})",
Data = data,
ImmediateUpload = true
};
}
private static EnrollmentEvent BuildUpToDateEvent(JObject marker, string sessionId, string tenantId)
{
var currentVersion = (string)marker["currentVersion"] ?? "unknown";
var latestVersion = (string)marker["latestVersion"] ?? "unknown";
var checkedAtUtc = (string)marker["checkedAtUtc"] ?? "unknown";
var data = new Dictionary<string, object>
{
{ "outcome", "up_to_date" },
{ "currentVersion", currentVersion },
{ "latestVersion", latestVersion },
{ "checkedAtUtc", checkedAtUtc }
};
CopyLong(marker, "versionCheckMs", data);
return new EnrollmentEvent
{
SessionId = sessionId,
TenantId = tenantId,
EventType = Constants.EventTypes.AgentVersionCheck,
Severity = EventSeverity.Info,
Source = "Agent",
Message = $"Agent version check: up to date ({currentVersion})",
Data = data,
ImmediateUpload = true
};
}
// ── Helpers ─────────────────────────────────────────────────────────
private sealed class LastEmit
{
public string SessionId { get; set; }
public string LatestVersion { get; set; }
public string Outcome { get; set; }
}
private static JObject ReadMarker(string path) => JObject.Parse(File.ReadAllText(path));
private static void CopyLong(JObject marker, string key, Dictionary<string, object> target)
{
var token = marker[key];
if (token == null || token.Type == JTokenType.Null) return;
try { target[key] = token.ToObject<long>(); } catch { /* skip malformed field */ }
}
private static string ExtractLatestVersion(EnrollmentEvent evt)
{
if (evt?.Data == null) return "unknown";
return evt.Data.TryGetValue("latestVersion", out var v) && v != null ? v.ToString() : "unknown";
}
private static LastEmit TryReadLastEmit(string path)
{
try
{
if (!File.Exists(path)) return null;
var obj = JObject.Parse(File.ReadAllText(path));
return new LastEmit
{
SessionId = (string)obj["sessionId"],
LatestVersion = (string)obj["latestVersion"],
Outcome = (string)obj["outcome"]
};
}
catch { return null; }
}
private static void TryWriteLastEmit(string path, string sessionId, string latestVersion, string outcome)
{
try
{
var dir = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
Directory.CreateDirectory(dir);
var payload = new JObject
{
["sessionId"] = sessionId ?? string.Empty,
["latestVersion"] = latestVersion ?? "unknown",
["outcome"] = outcome ?? "unknown",
["emittedAtUtc"] = DateTime.UtcNow.ToString("O")
};
File.WriteAllText(path, payload.ToString(Newtonsoft.Json.Formatting.None));
}
catch { /* best-effort */ }
}
private static void TryDelete(string path)
{
try { if (File.Exists(path)) File.Delete(path); } catch { }
}
}
}
+53 -10
View File
@@ -97,7 +97,7 @@ namespace AutopilotMonitor.Agent
/// </param>
/// <param name="triggerReason">
/// "startup" (default) or "runtime_hash_mismatch". Recorded in the marker file so the
/// agent_self_updated event can distinguish the two paths.
/// agent_version_check event (outcome=updated) can distinguish the two paths.
/// </param>
/// <param name="downloadTimeoutMsOverride">
/// Overrides the ZIP download timeout. Startup path leaves null (fast-fail, 10s) to preserve
@@ -145,7 +145,9 @@ namespace AutopilotMonitor.Agent
if (!forceUpdate && !IsNewerVersion(currentVersion, latestVersion))
{
log($"Self-update: current version {currentVersion} is up to date (latest: {latestVersion})");
return; // happy path — no skip marker
if (isStartupTrigger)
WriteCheckedMarker(currentVersion, latestVersion, versionCheckMs, log);
return;
}
if (forceUpdate)
@@ -551,10 +553,10 @@ namespace AutopilotMonitor.Agent
}
/// <summary>
/// Writes a small JSON marker so the next agent startup can emit an agent_self_updated event.
/// Best-effort — failure here must not block the update.
/// Includes phase timings, trigger reason, and exit timestamp so the downstream event
/// can carry aggregable telemetry (downtime, total update duration, etc.).
/// Writes a small JSON marker so the next agent startup can emit an agent_version_check event
/// with outcome=updated. Best-effort — failure here must not block the update. Includes phase
/// timings, trigger reason, and exit timestamp so the downstream event can carry aggregable
/// telemetry (downtime, total update duration, etc.).
/// </summary>
private static void WriteSelfUpdateMarker(
string previousVersion,
@@ -579,6 +581,7 @@ namespace AutopilotMonitor.Agent
var now = DateTime.UtcNow;
var payload = new JObject
{
["outcome"] = "updated",
["previousVersion"] = previousVersion ?? "unknown",
["newVersion"] = newVersion ?? "unknown",
["triggerReason"] = triggerReason ?? "startup",
@@ -605,9 +608,10 @@ namespace AutopilotMonitor.Agent
}
/// <summary>
/// Writes a small JSON marker so the next agent startup can emit an agent_self_update_skipped
/// event. Only called on the startup trigger path (runtime-triggered failures are logged
/// normally because the full logger is already up). Best-effort — never throws.
/// Writes a small JSON marker so the next agent startup can emit an agent_version_check event
/// with outcome=skipped (or check_failed if the version.json fetch itself failed). Only called
/// on the startup trigger path (runtime-triggered failures are logged normally because the full
/// logger is already up). Best-effort — never throws.
/// </summary>
private static void WriteSkipMarker(string reason, string currentVersion, string latestVersion, string errorDetail, Action<string> log)
{
@@ -621,8 +625,15 @@ namespace AutopilotMonitor.Agent
var detail = errorDetail ?? string.Empty;
if (detail.Length > 200) detail = detail.Substring(0, 200);
// version_check_failed = we never got a latestVersion; everything else = check succeeded
// but the download/verify/extract/swap step failed.
var outcome = string.Equals(reason, "version_check_failed", StringComparison.Ordinal)
? "check_failed"
: "skipped";
var payload = new JObject
{
["outcome"] = outcome,
["reason"] = reason ?? "unknown",
["currentVersion"] = currentVersion ?? "unknown",
["latestVersion"] = latestVersion, // may be null if version check failed
@@ -631,7 +642,7 @@ namespace AutopilotMonitor.Agent
};
File.WriteAllText(markerPath, payload.ToString(Newtonsoft.Json.Formatting.None));
log($"Self-update: skip marker written (reason={reason})");
log($"Self-update: skip marker written (outcome={outcome}, reason={reason})");
}
catch (Exception ex)
{
@@ -639,6 +650,38 @@ namespace AutopilotMonitor.Agent
}
}
/// <summary>
/// Writes a small JSON marker on the happy path (current version is already up to date) so
/// the next agent startup can emit an agent_version_check event with outcome=up_to_date.
/// Subject to session-scoped dedup by the MonitoringService emitter. Best-effort — never throws.
/// </summary>
private static void WriteCheckedMarker(string currentVersion, string latestVersion, long versionCheckMs, Action<string> log)
{
try
{
var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateCheckedMarkerFile);
var dir = Path.GetDirectoryName(markerPath);
if (!Directory.Exists(dir))
Directory.CreateDirectory(dir);
var payload = new JObject
{
["outcome"] = "up_to_date",
["currentVersion"] = currentVersion ?? "unknown",
["latestVersion"] = latestVersion ?? "unknown",
["versionCheckMs"] = versionCheckMs,
["checkedAtUtc"] = DateTime.UtcNow.ToString("O")
};
File.WriteAllText(markerPath, payload.ToString(Newtonsoft.Json.Formatting.None));
log("Self-update: checked marker written (up_to_date) for next-startup event");
}
catch (Exception ex)
{
log($"Self-update: could not write checked marker — {ex.Message} (non-critical)");
}
}
/// <summary>
/// Cleans up staging directory and temp ZIP file.
/// </summary>
@@ -1 +1 @@
949
951
@@ -30,19 +30,33 @@ namespace AutopilotMonitor.Shared
public const string StateDirectory = @"%ProgramData%\AutopilotMonitor\State";
/// <summary>
/// Marker file written by SelfUpdater before restart; read by MonitoringService on next startup
/// to emit an agent_self_updated event.
/// Marker file written by SelfUpdater after a successful update + restart; read by MonitoringService
/// on next startup to emit an agent_version_check event with outcome=updated.
/// </summary>
public const string SelfUpdateMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-info.json";
/// <summary>
/// Marker file written by SelfUpdater when the startup update path is skipped (network timeout,
/// download failure, integrity mismatch, etc.); read by MonitoringService on next startup to emit
/// an agent_self_update_skipped event. Only written on the startup trigger path — runtime-triggered
/// failures are logged normally because the full logger is already up.
/// an agent_version_check event with outcome=skipped or check_failed. Only written on the startup
/// trigger path — runtime-triggered failures are logged normally because the full logger is already up.
/// </summary>
public const string SelfUpdateSkippedMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-skipped.json";
/// <summary>
/// Marker file written by SelfUpdater on the happy path (current version already up to date);
/// read by MonitoringService on next startup to emit an agent_version_check event with
/// outcome=up_to_date. Subject to session-scoped dedup via LastVersionCheckEmitFile.
/// </summary>
public const string SelfUpdateCheckedMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-checked.json";
/// <summary>
/// Persists the last emitted agent_version_check event across agent restarts. Used by
/// MonitoringService to dedup up_to_date events within the same session when the latestVersion
/// has not changed. Updated every time an event is emitted.
/// </summary>
public const string LastVersionCheckEmitFile = @"%ProgramData%\AutopilotMonitor\State\last-version-check.json";
/// <summary>
/// Default path for the IME pattern match log file (debugging/diagnostics)
/// </summary>
@@ -175,8 +189,7 @@ namespace AutopilotMonitor.Shared
public const string EspProvisioningStatus = "esp_provisioning_status";
public const string SoftwareInventoryAnalysis = "software_inventory_analysis";
public const string VulnerabilityReport = "vulnerability_report";
public const string AgentSelfUpdated = "agent_self_updated";
public const string AgentSelfUpdateSkipped = "agent_self_update_skipped";
public const string AgentVersionCheck = "agent_version_check";
// Stall detection (Ebene 2 — StallProbeCollector)
public const string StallProbeCheck = "stall_probe_check"; // Trace heartbeat from Probe 2 (15 min) when no anomaly found
@@ -662,7 +662,7 @@ export function SectionAgentInternals() {
Rename trick for locked binaries (.old suffix). Copies staged EXE to Agent directory.
</Step>
<Step n={4} color="sky" title="Restart:">
Writes self-update marker, restarts via PowerShell Wait-Process. Next startup emits agent_self_updated event.
Writes self-update marker, restarts via PowerShell Wait-Process. Next startup emits agent_version_check event (outcome=updated).
</Step>
</div>
<InfoBox color="sky">
@@ -885,7 +885,7 @@ export function SectionAgentInternals() {
["agent_started", "MonitoringService", "Agent started (version, config, previous exit type)"],
["agent_shutdown", "MonitoringService", "Graceful shutdown"],
["agent_trace", "Various", "Decision tracing (verbose diagnostics)"],
["agent_self_updated", "MonitoringService", "Post-update event (old -> new version)"],
["agent_version_check", "MonitoringService", "Version check outcome (up_to_date / updated / skipped / check_failed) - emitted every startup, session-scoped dedup for up_to_date"],
["system_reboot_detected", "MonitoringService", "Reboot detected between sessions"],
["security_audit", "MonitoringService", "Security flag status at startup"],
]}