mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
better agent self-update observability
This commit is contained in:
@@ -0,0 +1,250 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using AutopilotMonitor.Agent.Core.Monitoring.Core;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.Core.Tests.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Covers the three outcomes (updated / skipped / check_failed / up_to_date), the dedup rules,
|
||||
/// and the defensive behaviors (no marker, corrupt JSON, stale markers).
|
||||
/// </summary>
|
||||
public class VersionCheckEventBuilderTests : IDisposable
|
||||
{
|
||||
private readonly string _tempDir;
|
||||
private readonly VersionCheckEventBuilder.Paths _paths;
|
||||
|
||||
private const string SessionA = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionB = "22222222-2222-2222-2222-222222222222";
|
||||
private const string TenantId = "tenant-xyz";
|
||||
|
||||
public VersionCheckEventBuilderTests()
|
||||
{
|
||||
_tempDir = Path.Combine(Path.GetTempPath(), "vceb-test-" + Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(_tempDir);
|
||||
_paths = new VersionCheckEventBuilder.Paths
|
||||
{
|
||||
UpdatedMarker = Path.Combine(_tempDir, "self-update-info.json"),
|
||||
SkippedMarker = Path.Combine(_tempDir, "self-update-skipped.json"),
|
||||
CheckedMarker = Path.Combine(_tempDir, "self-update-checked.json"),
|
||||
LastEmit = Path.Combine(_tempDir, "last-version-check.json"),
|
||||
};
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try { if (Directory.Exists(_tempDir)) Directory.Delete(_tempDir, recursive: true); } catch { }
|
||||
}
|
||||
|
||||
private void WriteChecked(string current = "1.0.949", string latest = "1.0.949", long versionCheckMs = 123)
|
||||
{
|
||||
var json = new JObject
|
||||
{
|
||||
["outcome"] = "up_to_date",
|
||||
["currentVersion"] = current,
|
||||
["latestVersion"] = latest,
|
||||
["versionCheckMs"] = versionCheckMs,
|
||||
["checkedAtUtc"] = DateTime.UtcNow.ToString("O"),
|
||||
};
|
||||
File.WriteAllText(_paths.CheckedMarker, json.ToString());
|
||||
}
|
||||
|
||||
private void WriteUpdated(string previous = "1.0.949", string newVer = "1.0.950", string trigger = "startup")
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var json = new JObject
|
||||
{
|
||||
["outcome"] = "updated",
|
||||
["previousVersion"] = previous,
|
||||
["newVersion"] = newVer,
|
||||
["triggerReason"] = trigger,
|
||||
["updatedAtUtc"] = now.ToString("O"),
|
||||
["exitAtUtc"] = now.AddSeconds(-2).ToString("O"),
|
||||
["versionCheckMs"] = 100L,
|
||||
["downloadMs"] = 2000L,
|
||||
["zipSizeBytes"] = 1234567L,
|
||||
["verifyMs"] = 40L,
|
||||
["extractMs"] = 200L,
|
||||
["swapMs"] = 50L,
|
||||
["totalUpdateMs"] = 2400L,
|
||||
};
|
||||
File.WriteAllText(_paths.UpdatedMarker, json.ToString());
|
||||
}
|
||||
|
||||
private void WriteSkipped(string reason = "download_failed", string latest = "1.0.950", string outcome = null)
|
||||
{
|
||||
var json = new JObject
|
||||
{
|
||||
["reason"] = reason,
|
||||
["currentVersion"] = "1.0.949",
|
||||
["latestVersion"] = latest,
|
||||
["skippedAtUtc"] = DateTime.UtcNow.ToString("O"),
|
||||
["errorDetail"] = "timeout after 10000ms",
|
||||
};
|
||||
if (outcome != null) json["outcome"] = outcome;
|
||||
File.WriteAllText(_paths.SkippedMarker, json.ToString());
|
||||
}
|
||||
|
||||
private void WriteLastEmit(string sessionId, string latestVersion, string outcome)
|
||||
{
|
||||
var json = new JObject
|
||||
{
|
||||
["sessionId"] = sessionId,
|
||||
["latestVersion"] = latestVersion,
|
||||
["outcome"] = outcome,
|
||||
["emittedAtUtc"] = DateTime.UtcNow.ToString("O"),
|
||||
};
|
||||
File.WriteAllText(_paths.LastEmit, json.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NoMarker_ReturnsNoEvent()
|
||||
{
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.Null(result.Event);
|
||||
Assert.False(result.Deduped);
|
||||
Assert.Null(result.Outcome);
|
||||
Assert.Null(result.ParseError);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpToDateMarker_FirstTime_EmitsEventAndPersistsLastEmit()
|
||||
{
|
||||
WriteChecked(latest: "1.0.949");
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.NotNull(result.Event);
|
||||
Assert.Equal(Constants.EventTypes.AgentVersionCheck, result.Event.EventType);
|
||||
Assert.Equal(EventSeverity.Info, result.Event.Severity);
|
||||
Assert.Equal("up_to_date", result.Event.Data["outcome"]);
|
||||
Assert.Equal("1.0.949", result.Event.Data["latestVersion"]);
|
||||
Assert.False(File.Exists(_paths.CheckedMarker)); // marker deleted
|
||||
Assert.True(File.Exists(_paths.LastEmit)); // last-emit persisted
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpToDateMarker_SameSessionSameVersion_IsDeduped()
|
||||
{
|
||||
WriteLastEmit(SessionA, "1.0.949", "up_to_date");
|
||||
WriteChecked(latest: "1.0.949");
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.Null(result.Event);
|
||||
Assert.True(result.Deduped);
|
||||
Assert.Equal("up_to_date", result.Outcome);
|
||||
Assert.False(File.Exists(_paths.CheckedMarker));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpToDateMarker_DifferentSession_EmitsEvent()
|
||||
{
|
||||
WriteLastEmit(SessionA, "1.0.949", "up_to_date");
|
||||
WriteChecked(latest: "1.0.949");
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionB, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.NotNull(result.Event);
|
||||
Assert.False(result.Deduped);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpToDateMarker_SameSessionButNewerLatestVersion_EmitsEvent()
|
||||
{
|
||||
// White-glove scenario: after a pause a new release appeared. Dedup must break.
|
||||
WriteLastEmit(SessionA, "1.0.949", "up_to_date");
|
||||
WriteChecked(latest: "1.0.950");
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.NotNull(result.Event);
|
||||
Assert.False(result.Deduped);
|
||||
Assert.Equal("1.0.950", result.Event.Data["latestVersion"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpdatedMarker_AlwaysEmits_IgnoresLastEmit()
|
||||
{
|
||||
WriteLastEmit(SessionA, "1.0.950", "updated");
|
||||
WriteUpdated(previous: "1.0.949", newVer: "1.0.950", trigger: "startup");
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.NotNull(result.Event);
|
||||
Assert.Equal("updated", result.Event.Data["outcome"]);
|
||||
Assert.Equal(EventSeverity.Info, result.Event.Severity);
|
||||
Assert.Equal("1.0.949", result.Event.Data["previousVersion"]);
|
||||
Assert.Equal("1.0.950", result.Event.Data["newVersion"]);
|
||||
Assert.True(result.Event.Data.ContainsKey("downtimeMs"));
|
||||
Assert.False(File.Exists(_paths.UpdatedMarker));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UpdatedMarker_RuntimeTrigger_IsWarningSeverity()
|
||||
{
|
||||
WriteUpdated(trigger: "runtime_hash_mismatch");
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.Equal(EventSeverity.Warning, result.Event.Severity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SkippedMarker_DownloadFailed_EmitsWithSkippedOutcome()
|
||||
{
|
||||
WriteSkipped(reason: "download_failed");
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.NotNull(result.Event);
|
||||
Assert.Equal("skipped", result.Event.Data["outcome"]);
|
||||
Assert.Equal("download_failed", result.Event.Data["reason"]);
|
||||
Assert.Equal(EventSeverity.Warning, result.Event.Severity);
|
||||
Assert.False(File.Exists(_paths.SkippedMarker));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SkippedMarker_VersionCheckFailed_DerivesCheckFailedOutcome()
|
||||
{
|
||||
// Backward-compat: old marker without outcome field — must still map correctly.
|
||||
WriteSkipped(reason: "version_check_failed", latest: null, outcome: null);
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.Equal("check_failed", result.Event.Data["outcome"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Priority_UpdatedBeatsSkippedBeatsChecked()
|
||||
{
|
||||
// All three present (edge case after a crash). Updated wins, the others are cleaned up.
|
||||
WriteUpdated();
|
||||
WriteSkipped();
|
||||
WriteChecked();
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.Equal("updated", result.Event.Data["outcome"]);
|
||||
Assert.False(File.Exists(_paths.UpdatedMarker));
|
||||
Assert.False(File.Exists(_paths.SkippedMarker));
|
||||
Assert.False(File.Exists(_paths.CheckedMarker));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CorruptMarker_DoesNotThrow_ReturnsParseError()
|
||||
{
|
||||
File.WriteAllText(_paths.CheckedMarker, "{ not valid json");
|
||||
|
||||
var result = VersionCheckEventBuilder.TryBuild(SessionA, TenantId, DateTime.UtcNow, _paths);
|
||||
|
||||
Assert.Null(result.Event);
|
||||
Assert.NotNull(result.ParseError);
|
||||
Assert.False(File.Exists(_paths.CheckedMarker)); // still cleaned up
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -267,12 +267,10 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
|
||||
});
|
||||
}
|
||||
|
||||
// If the agent was self-updated on the previous run, emit a timeline event
|
||||
EmitSelfUpdateEventIfMarkerExists();
|
||||
|
||||
// If the startup self-update was skipped (network timeout etc.), emit telemetry
|
||||
// so we can measure how often this happens across the fleet.
|
||||
EmitSelfUpdateSkippedEventIfMarkerExists();
|
||||
// Consume self-update markers (updated / skipped / checked) left by SelfUpdater on the
|
||||
// previous startup and emit a single agent_version_check event. Dedup suppresses
|
||||
// repeated up_to_date events for the same latestVersion within the same session.
|
||||
EmitVersionCheckEventFromMarkers();
|
||||
|
||||
// Emit deferred security_audit (initial UnrestrictedMode) now that agent_started has its sequence
|
||||
if (_deferredSecurityAuditConfigVersion.HasValue)
|
||||
@@ -711,159 +709,40 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the self-update marker file left by SelfUpdater, emits an agent_self_updated event,
|
||||
/// and deletes the marker. Best-effort — failures are logged but never block startup.
|
||||
/// Includes phase timings, trigger reason, and a computed downtime (new process startup
|
||||
/// time minus old process exit time) so we can aggregate update duration and event-loss
|
||||
/// window across the fleet.
|
||||
/// Consumes any self-update markers left by SelfUpdater and emits a single
|
||||
/// <c>agent_version_check</c> event. Three marker variants map to the outcomes:
|
||||
/// <list type="bullet">
|
||||
/// <item><description>self-update-info.json → outcome=updated</description></item>
|
||||
/// <item><description>self-update-skipped.json → outcome=skipped or check_failed</description></item>
|
||||
/// <item><description>self-update-checked.json → outcome=up_to_date (subject to session-scoped dedup)</description></item>
|
||||
/// </list>
|
||||
/// Best-effort — failures are logged but never block startup. See
|
||||
/// <see cref="VersionCheckEventBuilder"/> for the pure logic + dedup rules.
|
||||
/// </summary>
|
||||
private void EmitSelfUpdateEventIfMarkerExists()
|
||||
private void EmitVersionCheckEventFromMarkers()
|
||||
{
|
||||
try
|
||||
var result = VersionCheckEventBuilder.TryBuild(
|
||||
_configuration.SessionId,
|
||||
_configuration.TenantId,
|
||||
_agentStartTimeUtc);
|
||||
|
||||
if (!string.IsNullOrEmpty(result.ParseError))
|
||||
{
|
||||
var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateMarkerFile);
|
||||
if (!File.Exists(markerPath))
|
||||
return;
|
||||
|
||||
var json = File.ReadAllText(markerPath);
|
||||
var marker = Newtonsoft.Json.Linq.JObject.Parse(json);
|
||||
|
||||
var previousVersion = (string)marker["previousVersion"] ?? "unknown";
|
||||
var newVersion = (string)marker["newVersion"] ?? "unknown";
|
||||
var updatedAtUtc = (string)marker["updatedAtUtc"] ?? "unknown";
|
||||
var triggerReason = (string)marker["triggerReason"] ?? "startup";
|
||||
var exitAtUtc = (string)marker["exitAtUtc"];
|
||||
|
||||
long? ReadLong(string key)
|
||||
{
|
||||
var token = marker[key];
|
||||
if (token == null || token.Type == Newtonsoft.Json.Linq.JTokenType.Null)
|
||||
return null;
|
||||
try { return token.ToObject<long>(); }
|
||||
catch { return null; }
|
||||
}
|
||||
|
||||
var versionCheckMs = ReadLong("versionCheckMs");
|
||||
var downloadMs = ReadLong("downloadMs");
|
||||
var zipSizeBytes = ReadLong("zipSizeBytes");
|
||||
var verifyMs = ReadLong("verifyMs");
|
||||
var extractMs = ReadLong("extractMs");
|
||||
var swapMs = ReadLong("swapMs");
|
||||
var totalUpdateMs = ReadLong("totalUpdateMs");
|
||||
|
||||
// Downtime = new process start (_agentStartTimeUtc) minus old process exit.
|
||||
// Measured across the PowerShell Wait-Process gap + new agent startup up to
|
||||
// MonitoringService init. Closest proxy for "events potentially missed".
|
||||
long? downtimeMs = null;
|
||||
if (!string.IsNullOrEmpty(exitAtUtc) &&
|
||||
DateTime.TryParse(exitAtUtc, System.Globalization.CultureInfo.InvariantCulture,
|
||||
System.Globalization.DateTimeStyles.RoundtripKind, out var exitDt))
|
||||
{
|
||||
downtimeMs = (long)(_agentStartTimeUtc - exitDt).TotalMilliseconds;
|
||||
}
|
||||
|
||||
_logger.Info($"Self-update detected: {previousVersion} → {newVersion} (trigger={triggerReason}, downtimeMs={downtimeMs?.ToString() ?? "n/a"})");
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "previousVersion", previousVersion },
|
||||
{ "newVersion", newVersion },
|
||||
{ "updatedAtUtc", updatedAtUtc },
|
||||
{ "triggerReason", triggerReason }
|
||||
};
|
||||
if (exitAtUtc != null) data["exitAtUtc"] = exitAtUtc;
|
||||
if (versionCheckMs.HasValue) data["versionCheckMs"] = versionCheckMs.Value;
|
||||
if (downloadMs.HasValue) data["downloadMs"] = downloadMs.Value;
|
||||
if (zipSizeBytes.HasValue) data["zipSizeBytes"] = zipSizeBytes.Value;
|
||||
if (verifyMs.HasValue) data["verifyMs"] = verifyMs.Value;
|
||||
if (extractMs.HasValue) data["extractMs"] = extractMs.Value;
|
||||
if (swapMs.HasValue) data["swapMs"] = swapMs.Value;
|
||||
if (totalUpdateMs.HasValue) data["totalUpdateMs"] = totalUpdateMs.Value;
|
||||
if (downtimeMs.HasValue) data["downtimeMs"] = downtimeMs.Value;
|
||||
|
||||
// Runtime-triggered updates are bumped to Warning so they stand out in the UI —
|
||||
// startup updates stay Info because they are the normal rollout path.
|
||||
var severity = string.Equals(triggerReason, "runtime_hash_mismatch", StringComparison.Ordinal)
|
||||
? EventSeverity.Warning
|
||||
: EventSeverity.Info;
|
||||
|
||||
EmitEvent(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _configuration.SessionId,
|
||||
TenantId = _configuration.TenantId,
|
||||
EventType = Constants.EventTypes.AgentSelfUpdated,
|
||||
Severity = severity,
|
||||
Source = "Agent",
|
||||
Message = $"Agent self-updated from {previousVersion} to {newVersion} (trigger={triggerReason})",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
});
|
||||
|
||||
TryDeleteMarker(markerPath);
|
||||
_logger.Warning($"Could not process version-check marker: {result.ParseError}");
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
if (result.Deduped)
|
||||
{
|
||||
_logger.Warning($"Could not process self-update marker: {ex.Message}");
|
||||
_logger.Verbose($"agent_version_check skipped (dedup: up_to_date, same session, same latestVersion)");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the self-update-skipped marker file left by SelfUpdater (startup trigger path only),
|
||||
/// emits an agent_self_update_skipped event, and deletes the marker. Best-effort — failures
|
||||
/// are logged but never block startup.
|
||||
/// </summary>
|
||||
private void EmitSelfUpdateSkippedEventIfMarkerExists()
|
||||
{
|
||||
try
|
||||
if (result.Event != null)
|
||||
{
|
||||
var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateSkippedMarkerFile);
|
||||
if (!File.Exists(markerPath))
|
||||
return;
|
||||
|
||||
var json = File.ReadAllText(markerPath);
|
||||
var marker = Newtonsoft.Json.Linq.JObject.Parse(json);
|
||||
|
||||
var reason = (string)marker["reason"] ?? "unknown";
|
||||
var currentVersion = (string)marker["currentVersion"] ?? "unknown";
|
||||
var latestVersion = (string)marker["latestVersion"];
|
||||
var skippedAtUtc = (string)marker["skippedAtUtc"] ?? "unknown";
|
||||
var errorDetail = (string)marker["errorDetail"] ?? string.Empty;
|
||||
|
||||
_logger.Warning($"Startup self-update skipped: reason={reason}, current={currentVersion}, latest={latestVersion ?? "n/a"}");
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "reason", reason },
|
||||
{ "currentVersion", currentVersion },
|
||||
{ "skippedAtUtc", skippedAtUtc },
|
||||
{ "errorDetail", errorDetail }
|
||||
};
|
||||
if (latestVersion != null)
|
||||
data["latestVersion"] = latestVersion;
|
||||
|
||||
EmitEvent(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _configuration.SessionId,
|
||||
TenantId = _configuration.TenantId,
|
||||
EventType = Constants.EventTypes.AgentSelfUpdateSkipped,
|
||||
Severity = EventSeverity.Warning,
|
||||
Source = "Agent",
|
||||
Message = $"Startup self-update skipped (reason={reason})",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
});
|
||||
|
||||
TryDeleteMarker(markerPath);
|
||||
_logger.Info($"Agent version check: outcome={result.Outcome}");
|
||||
EmitEvent(result.Event);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"Could not process self-update-skipped marker: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDeleteMarker(string path)
|
||||
{
|
||||
try { File.Delete(path); } catch { }
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace AutopilotMonitor.Agent.Core.Monitoring.Core
|
||||
{
|
||||
/// <summary>
|
||||
/// Consumes the three self-update marker files written by SelfUpdater on every agent startup
|
||||
/// (updated / skipped / checked) and converts them into a single <c>agent_version_check</c>
|
||||
/// event with an <c>outcome</c> discriminator. Applies session-scoped dedup for the
|
||||
/// up_to_date outcome so repeated reboots in the same session don't spam the timeline.
|
||||
///
|
||||
/// All filesystem paths are injectable so this can be exercised in unit tests without touching
|
||||
/// %ProgramData%.
|
||||
/// </summary>
|
||||
public static class VersionCheckEventBuilder
|
||||
{
|
||||
public sealed class Paths
|
||||
{
|
||||
public string UpdatedMarker { get; set; }
|
||||
public string SkippedMarker { get; set; }
|
||||
public string CheckedMarker { get; set; }
|
||||
public string LastEmit { get; set; }
|
||||
|
||||
public static Paths Default() => new Paths
|
||||
{
|
||||
UpdatedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateMarkerFile),
|
||||
SkippedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateSkippedMarkerFile),
|
||||
CheckedMarker = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateCheckedMarkerFile),
|
||||
LastEmit = Environment.ExpandEnvironmentVariables(Constants.LastVersionCheckEmitFile)
|
||||
};
|
||||
}
|
||||
|
||||
public sealed class BuildResult
|
||||
{
|
||||
/// <summary>Event to emit; null when nothing to report (no marker) or when deduped.</summary>
|
||||
public EnrollmentEvent Event { get; set; }
|
||||
|
||||
/// <summary>True when a marker was read but the event was suppressed by dedup.</summary>
|
||||
public bool Deduped { get; set; }
|
||||
|
||||
/// <summary>Outcome that was detected (even if deduped), for logging. Null when no marker.</summary>
|
||||
public string Outcome { get; set; }
|
||||
|
||||
/// <summary>Human-readable message for warning logs when a marker file was malformed.</summary>
|
||||
public string ParseError { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads available markers, applies dedup, updates the last-emit file and deletes consumed markers.
|
||||
/// Returns the event to emit (or null). Never throws — errors are returned via ParseError.
|
||||
///
|
||||
/// Priority: Updated > Skipped > Checked. If more than one marker exists (rare — stale files
|
||||
/// from a crashed run), the highest-priority one wins and the others are deleted defensively.
|
||||
/// </summary>
|
||||
public static BuildResult TryBuild(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
DateTime agentStartTimeUtc,
|
||||
Paths paths = null)
|
||||
{
|
||||
paths = paths ?? Paths.Default();
|
||||
var result = new BuildResult();
|
||||
|
||||
try
|
||||
{
|
||||
// Priority order — once consumed, all three are deleted.
|
||||
if (File.Exists(paths.UpdatedMarker))
|
||||
{
|
||||
result.Outcome = "updated";
|
||||
result.Event = BuildUpdatedEvent(paths.UpdatedMarker, sessionId, tenantId, agentStartTimeUtc);
|
||||
}
|
||||
else if (File.Exists(paths.SkippedMarker))
|
||||
{
|
||||
result.Event = BuildSkippedEvent(paths.SkippedMarker, sessionId, tenantId, out var skipOutcome);
|
||||
result.Outcome = skipOutcome;
|
||||
}
|
||||
else if (File.Exists(paths.CheckedMarker))
|
||||
{
|
||||
result.Outcome = "up_to_date";
|
||||
var marker = ReadMarker(paths.CheckedMarker);
|
||||
var latestVersion = (string)marker["latestVersion"] ?? "unknown";
|
||||
|
||||
// Session-scoped dedup: suppress repeated up_to_date events for the same
|
||||
// latestVersion within the same session. Update/Skip/Failure are never deduped.
|
||||
var lastEmit = TryReadLastEmit(paths.LastEmit);
|
||||
if (lastEmit != null &&
|
||||
string.Equals(lastEmit.SessionId, sessionId, StringComparison.Ordinal) &&
|
||||
string.Equals(lastEmit.LatestVersion, latestVersion, StringComparison.Ordinal) &&
|
||||
string.Equals(lastEmit.Outcome, "up_to_date", StringComparison.Ordinal))
|
||||
{
|
||||
result.Deduped = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
result.Event = BuildUpToDateEvent(marker, sessionId, tenantId);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return result; // no marker — nothing to do
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.ParseError = ex.Message;
|
||||
}
|
||||
|
||||
// Persist the last-emit state before cleanup so a crash right after emit doesn't re-emit.
|
||||
if (result.Event != null)
|
||||
{
|
||||
var latestVersion = ExtractLatestVersion(result.Event);
|
||||
TryWriteLastEmit(paths.LastEmit, sessionId, latestVersion, result.Outcome);
|
||||
}
|
||||
|
||||
// Always delete markers once consumed (or attempted) — stale markers would cause loops.
|
||||
TryDelete(paths.UpdatedMarker);
|
||||
TryDelete(paths.SkippedMarker);
|
||||
TryDelete(paths.CheckedMarker);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Individual builders ─────────────────────────────────────────────
|
||||
|
||||
private static EnrollmentEvent BuildUpdatedEvent(string path, string sessionId, string tenantId, DateTime agentStartTimeUtc)
|
||||
{
|
||||
var marker = ReadMarker(path);
|
||||
|
||||
var previousVersion = (string)marker["previousVersion"] ?? "unknown";
|
||||
var newVersion = (string)marker["newVersion"] ?? "unknown";
|
||||
var updatedAtUtc = (string)marker["updatedAtUtc"] ?? "unknown";
|
||||
var triggerReason = (string)marker["triggerReason"] ?? "startup";
|
||||
var exitAtUtc = (string)marker["exitAtUtc"];
|
||||
|
||||
long? downtimeMs = null;
|
||||
if (!string.IsNullOrEmpty(exitAtUtc) &&
|
||||
DateTime.TryParse(exitAtUtc, System.Globalization.CultureInfo.InvariantCulture,
|
||||
System.Globalization.DateTimeStyles.RoundtripKind, out var exitDt))
|
||||
{
|
||||
downtimeMs = (long)(agentStartTimeUtc - exitDt).TotalMilliseconds;
|
||||
}
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "outcome", "updated" },
|
||||
{ "previousVersion", previousVersion },
|
||||
{ "newVersion", newVersion },
|
||||
{ "latestVersion", newVersion },
|
||||
{ "updatedAtUtc", updatedAtUtc },
|
||||
{ "triggerReason", triggerReason }
|
||||
};
|
||||
if (exitAtUtc != null) data["exitAtUtc"] = exitAtUtc;
|
||||
CopyLong(marker, "versionCheckMs", data);
|
||||
CopyLong(marker, "downloadMs", data);
|
||||
CopyLong(marker, "zipSizeBytes", data);
|
||||
CopyLong(marker, "verifyMs", data);
|
||||
CopyLong(marker, "extractMs", data);
|
||||
CopyLong(marker, "swapMs", data);
|
||||
CopyLong(marker, "totalUpdateMs", data);
|
||||
if (downtimeMs.HasValue) data["downtimeMs"] = downtimeMs.Value;
|
||||
|
||||
var severity = string.Equals(triggerReason, "runtime_hash_mismatch", StringComparison.Ordinal)
|
||||
? EventSeverity.Warning
|
||||
: EventSeverity.Info;
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
SessionId = sessionId,
|
||||
TenantId = tenantId,
|
||||
EventType = Constants.EventTypes.AgentVersionCheck,
|
||||
Severity = severity,
|
||||
Source = "Agent",
|
||||
Message = $"Agent self-updated from {previousVersion} to {newVersion} (trigger={triggerReason})",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
};
|
||||
}
|
||||
|
||||
private static EnrollmentEvent BuildSkippedEvent(string path, string sessionId, string tenantId, out string outcome)
|
||||
{
|
||||
var marker = ReadMarker(path);
|
||||
|
||||
var reason = (string)marker["reason"] ?? "unknown";
|
||||
var currentVersion = (string)marker["currentVersion"] ?? "unknown";
|
||||
var latestVersion = (string)marker["latestVersion"];
|
||||
var skippedAtUtc = (string)marker["skippedAtUtc"] ?? "unknown";
|
||||
var errorDetail = (string)marker["errorDetail"] ?? string.Empty;
|
||||
|
||||
// Marker may already carry outcome (new SelfUpdater). For forward-compat with old markers
|
||||
// lying around across an upgrade, derive it from the reason.
|
||||
outcome = (string)marker["outcome"]
|
||||
?? (string.Equals(reason, "version_check_failed", StringComparison.Ordinal) ? "check_failed" : "skipped");
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "outcome", outcome },
|
||||
{ "reason", reason },
|
||||
{ "currentVersion", currentVersion },
|
||||
{ "skippedAtUtc", skippedAtUtc },
|
||||
{ "errorDetail", errorDetail }
|
||||
};
|
||||
if (latestVersion != null)
|
||||
data["latestVersion"] = latestVersion;
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
SessionId = sessionId,
|
||||
TenantId = tenantId,
|
||||
EventType = Constants.EventTypes.AgentVersionCheck,
|
||||
Severity = EventSeverity.Warning,
|
||||
Source = "Agent",
|
||||
Message = outcome == "check_failed"
|
||||
? $"Agent version check failed (reason={reason})"
|
||||
: $"Agent self-update skipped (reason={reason})",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
};
|
||||
}
|
||||
|
||||
private static EnrollmentEvent BuildUpToDateEvent(JObject marker, string sessionId, string tenantId)
|
||||
{
|
||||
var currentVersion = (string)marker["currentVersion"] ?? "unknown";
|
||||
var latestVersion = (string)marker["latestVersion"] ?? "unknown";
|
||||
var checkedAtUtc = (string)marker["checkedAtUtc"] ?? "unknown";
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "outcome", "up_to_date" },
|
||||
{ "currentVersion", currentVersion },
|
||||
{ "latestVersion", latestVersion },
|
||||
{ "checkedAtUtc", checkedAtUtc }
|
||||
};
|
||||
CopyLong(marker, "versionCheckMs", data);
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
SessionId = sessionId,
|
||||
TenantId = tenantId,
|
||||
EventType = Constants.EventTypes.AgentVersionCheck,
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "Agent",
|
||||
Message = $"Agent version check: up to date ({currentVersion})",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
};
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class LastEmit
|
||||
{
|
||||
public string SessionId { get; set; }
|
||||
public string LatestVersion { get; set; }
|
||||
public string Outcome { get; set; }
|
||||
}
|
||||
|
||||
private static JObject ReadMarker(string path) => JObject.Parse(File.ReadAllText(path));
|
||||
|
||||
private static void CopyLong(JObject marker, string key, Dictionary<string, object> target)
|
||||
{
|
||||
var token = marker[key];
|
||||
if (token == null || token.Type == JTokenType.Null) return;
|
||||
try { target[key] = token.ToObject<long>(); } catch { /* skip malformed field */ }
|
||||
}
|
||||
|
||||
private static string ExtractLatestVersion(EnrollmentEvent evt)
|
||||
{
|
||||
if (evt?.Data == null) return "unknown";
|
||||
return evt.Data.TryGetValue("latestVersion", out var v) && v != null ? v.ToString() : "unknown";
|
||||
}
|
||||
|
||||
private static LastEmit TryReadLastEmit(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(path)) return null;
|
||||
var obj = JObject.Parse(File.ReadAllText(path));
|
||||
return new LastEmit
|
||||
{
|
||||
SessionId = (string)obj["sessionId"],
|
||||
LatestVersion = (string)obj["latestVersion"],
|
||||
Outcome = (string)obj["outcome"]
|
||||
};
|
||||
}
|
||||
catch { return null; }
|
||||
}
|
||||
|
||||
private static void TryWriteLastEmit(string path, string sessionId, string latestVersion, string outcome)
|
||||
{
|
||||
try
|
||||
{
|
||||
var dir = Path.GetDirectoryName(path);
|
||||
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
|
||||
Directory.CreateDirectory(dir);
|
||||
|
||||
var payload = new JObject
|
||||
{
|
||||
["sessionId"] = sessionId ?? string.Empty,
|
||||
["latestVersion"] = latestVersion ?? "unknown",
|
||||
["outcome"] = outcome ?? "unknown",
|
||||
["emittedAtUtc"] = DateTime.UtcNow.ToString("O")
|
||||
};
|
||||
File.WriteAllText(path, payload.ToString(Newtonsoft.Json.Formatting.None));
|
||||
}
|
||||
catch { /* best-effort */ }
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try { if (File.Exists(path)) File.Delete(path); } catch { }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -97,7 +97,7 @@ namespace AutopilotMonitor.Agent
|
||||
/// </param>
|
||||
/// <param name="triggerReason">
|
||||
/// "startup" (default) or "runtime_hash_mismatch". Recorded in the marker file so the
|
||||
/// agent_self_updated event can distinguish the two paths.
|
||||
/// agent_version_check event (outcome=updated) can distinguish the two paths.
|
||||
/// </param>
|
||||
/// <param name="downloadTimeoutMsOverride">
|
||||
/// Overrides the ZIP download timeout. Startup path leaves null (fast-fail, 10s) to preserve
|
||||
@@ -145,7 +145,9 @@ namespace AutopilotMonitor.Agent
|
||||
if (!forceUpdate && !IsNewerVersion(currentVersion, latestVersion))
|
||||
{
|
||||
log($"Self-update: current version {currentVersion} is up to date (latest: {latestVersion})");
|
||||
return; // happy path — no skip marker
|
||||
if (isStartupTrigger)
|
||||
WriteCheckedMarker(currentVersion, latestVersion, versionCheckMs, log);
|
||||
return;
|
||||
}
|
||||
|
||||
if (forceUpdate)
|
||||
@@ -551,10 +553,10 @@ namespace AutopilotMonitor.Agent
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a small JSON marker so the next agent startup can emit an agent_self_updated event.
|
||||
/// Best-effort — failure here must not block the update.
|
||||
/// Includes phase timings, trigger reason, and exit timestamp so the downstream event
|
||||
/// can carry aggregable telemetry (downtime, total update duration, etc.).
|
||||
/// Writes a small JSON marker so the next agent startup can emit an agent_version_check event
|
||||
/// with outcome=updated. Best-effort — failure here must not block the update. Includes phase
|
||||
/// timings, trigger reason, and exit timestamp so the downstream event can carry aggregable
|
||||
/// telemetry (downtime, total update duration, etc.).
|
||||
/// </summary>
|
||||
private static void WriteSelfUpdateMarker(
|
||||
string previousVersion,
|
||||
@@ -579,6 +581,7 @@ namespace AutopilotMonitor.Agent
|
||||
var now = DateTime.UtcNow;
|
||||
var payload = new JObject
|
||||
{
|
||||
["outcome"] = "updated",
|
||||
["previousVersion"] = previousVersion ?? "unknown",
|
||||
["newVersion"] = newVersion ?? "unknown",
|
||||
["triggerReason"] = triggerReason ?? "startup",
|
||||
@@ -605,9 +608,10 @@ namespace AutopilotMonitor.Agent
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a small JSON marker so the next agent startup can emit an agent_self_update_skipped
|
||||
/// event. Only called on the startup trigger path (runtime-triggered failures are logged
|
||||
/// normally because the full logger is already up). Best-effort — never throws.
|
||||
/// Writes a small JSON marker so the next agent startup can emit an agent_version_check event
|
||||
/// with outcome=skipped (or check_failed if the version.json fetch itself failed). Only called
|
||||
/// on the startup trigger path (runtime-triggered failures are logged normally because the full
|
||||
/// logger is already up). Best-effort — never throws.
|
||||
/// </summary>
|
||||
private static void WriteSkipMarker(string reason, string currentVersion, string latestVersion, string errorDetail, Action<string> log)
|
||||
{
|
||||
@@ -621,8 +625,15 @@ namespace AutopilotMonitor.Agent
|
||||
var detail = errorDetail ?? string.Empty;
|
||||
if (detail.Length > 200) detail = detail.Substring(0, 200);
|
||||
|
||||
// version_check_failed = we never got a latestVersion; everything else = check succeeded
|
||||
// but the download/verify/extract/swap step failed.
|
||||
var outcome = string.Equals(reason, "version_check_failed", StringComparison.Ordinal)
|
||||
? "check_failed"
|
||||
: "skipped";
|
||||
|
||||
var payload = new JObject
|
||||
{
|
||||
["outcome"] = outcome,
|
||||
["reason"] = reason ?? "unknown",
|
||||
["currentVersion"] = currentVersion ?? "unknown",
|
||||
["latestVersion"] = latestVersion, // may be null if version check failed
|
||||
@@ -631,7 +642,7 @@ namespace AutopilotMonitor.Agent
|
||||
};
|
||||
|
||||
File.WriteAllText(markerPath, payload.ToString(Newtonsoft.Json.Formatting.None));
|
||||
log($"Self-update: skip marker written (reason={reason})");
|
||||
log($"Self-update: skip marker written (outcome={outcome}, reason={reason})");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -639,6 +650,38 @@ namespace AutopilotMonitor.Agent
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a small JSON marker on the happy path (current version is already up to date) so
|
||||
/// the next agent startup can emit an agent_version_check event with outcome=up_to_date.
|
||||
/// Subject to session-scoped dedup by the MonitoringService emitter. Best-effort — never throws.
|
||||
/// </summary>
|
||||
private static void WriteCheckedMarker(string currentVersion, string latestVersion, long versionCheckMs, Action<string> log)
|
||||
{
|
||||
try
|
||||
{
|
||||
var markerPath = Environment.ExpandEnvironmentVariables(Constants.SelfUpdateCheckedMarkerFile);
|
||||
var dir = Path.GetDirectoryName(markerPath);
|
||||
if (!Directory.Exists(dir))
|
||||
Directory.CreateDirectory(dir);
|
||||
|
||||
var payload = new JObject
|
||||
{
|
||||
["outcome"] = "up_to_date",
|
||||
["currentVersion"] = currentVersion ?? "unknown",
|
||||
["latestVersion"] = latestVersion ?? "unknown",
|
||||
["versionCheckMs"] = versionCheckMs,
|
||||
["checkedAtUtc"] = DateTime.UtcNow.ToString("O")
|
||||
};
|
||||
|
||||
File.WriteAllText(markerPath, payload.ToString(Newtonsoft.Json.Formatting.None));
|
||||
log("Self-update: checked marker written (up_to_date) for next-startup event");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
log($"Self-update: could not write checked marker — {ex.Message} (non-critical)");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cleans up staging directory and temp ZIP file.
|
||||
/// </summary>
|
||||
|
||||
@@ -1 +1 @@
|
||||
949
|
||||
951
|
||||
|
||||
@@ -30,19 +30,33 @@ namespace AutopilotMonitor.Shared
|
||||
public const string StateDirectory = @"%ProgramData%\AutopilotMonitor\State";
|
||||
|
||||
/// <summary>
|
||||
/// Marker file written by SelfUpdater before restart; read by MonitoringService on next startup
|
||||
/// to emit an agent_self_updated event.
|
||||
/// Marker file written by SelfUpdater after a successful update + restart; read by MonitoringService
|
||||
/// on next startup to emit an agent_version_check event with outcome=updated.
|
||||
/// </summary>
|
||||
public const string SelfUpdateMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-info.json";
|
||||
|
||||
/// <summary>
|
||||
/// Marker file written by SelfUpdater when the startup update path is skipped (network timeout,
|
||||
/// download failure, integrity mismatch, etc.); read by MonitoringService on next startup to emit
|
||||
/// an agent_self_update_skipped event. Only written on the startup trigger path — runtime-triggered
|
||||
/// failures are logged normally because the full logger is already up.
|
||||
/// an agent_version_check event with outcome=skipped or check_failed. Only written on the startup
|
||||
/// trigger path — runtime-triggered failures are logged normally because the full logger is already up.
|
||||
/// </summary>
|
||||
public const string SelfUpdateSkippedMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-skipped.json";
|
||||
|
||||
/// <summary>
|
||||
/// Marker file written by SelfUpdater on the happy path (current version already up to date);
|
||||
/// read by MonitoringService on next startup to emit an agent_version_check event with
|
||||
/// outcome=up_to_date. Subject to session-scoped dedup via LastVersionCheckEmitFile.
|
||||
/// </summary>
|
||||
public const string SelfUpdateCheckedMarkerFile = @"%ProgramData%\AutopilotMonitor\State\self-update-checked.json";
|
||||
|
||||
/// <summary>
|
||||
/// Persists the last emitted agent_version_check event across agent restarts. Used by
|
||||
/// MonitoringService to dedup up_to_date events within the same session when the latestVersion
|
||||
/// has not changed. Updated every time an event is emitted.
|
||||
/// </summary>
|
||||
public const string LastVersionCheckEmitFile = @"%ProgramData%\AutopilotMonitor\State\last-version-check.json";
|
||||
|
||||
/// <summary>
|
||||
/// Default path for the IME pattern match log file (debugging/diagnostics)
|
||||
/// </summary>
|
||||
@@ -175,8 +189,7 @@ namespace AutopilotMonitor.Shared
|
||||
public const string EspProvisioningStatus = "esp_provisioning_status";
|
||||
public const string SoftwareInventoryAnalysis = "software_inventory_analysis";
|
||||
public const string VulnerabilityReport = "vulnerability_report";
|
||||
public const string AgentSelfUpdated = "agent_self_updated";
|
||||
public const string AgentSelfUpdateSkipped = "agent_self_update_skipped";
|
||||
public const string AgentVersionCheck = "agent_version_check";
|
||||
|
||||
// Stall detection (Ebene 2 — StallProbeCollector)
|
||||
public const string StallProbeCheck = "stall_probe_check"; // Trace heartbeat from Probe 2 (15 min) when no anomaly found
|
||||
|
||||
@@ -662,7 +662,7 @@ export function SectionAgentInternals() {
|
||||
Rename trick for locked binaries (.old suffix). Copies staged EXE to Agent directory.
|
||||
</Step>
|
||||
<Step n={4} color="sky" title="Restart:">
|
||||
Writes self-update marker, restarts via PowerShell Wait-Process. Next startup emits agent_self_updated event.
|
||||
Writes self-update marker, restarts via PowerShell Wait-Process. Next startup emits agent_version_check event (outcome=updated).
|
||||
</Step>
|
||||
</div>
|
||||
<InfoBox color="sky">
|
||||
@@ -885,7 +885,7 @@ export function SectionAgentInternals() {
|
||||
["agent_started", "MonitoringService", "Agent started (version, config, previous exit type)"],
|
||||
["agent_shutdown", "MonitoringService", "Graceful shutdown"],
|
||||
["agent_trace", "Various", "Decision tracing (verbose diagnostics)"],
|
||||
["agent_self_updated", "MonitoringService", "Post-update event (old -> new version)"],
|
||||
["agent_version_check", "MonitoringService", "Version check outcome (up_to_date / updated / skipped / check_failed) - emitted every startup, session-scoped dedup for up_to_date"],
|
||||
["system_reboot_detected", "MonitoringService", "Reboot detected between sessions"],
|
||||
["security_audit", "MonitoringService", "Security flag status at startup"],
|
||||
]}
|
||||
|
||||
Reference in New Issue
Block a user