mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat(agent-v2): capture mini-dumps + emit previous_crash_detected event
When the V2 agent process dies via AppDomain.UnhandledException or TaskScheduler.UnobservedTaskException, synchronously write a compact MiniDump (data-segs + handle + thread-info, ~5-20 MB) and a CrashRecord JSON sibling to %ProgramData%\AutopilotMonitor\Crashes\. Next agent start scans the directory, drains pending records into the timeline as previous_crash_detected events (Warning, ImmediateUpload), and applies retention (max 5 dumps, max 7 days). Dumps stay on disk so the existing customer-opt-in diagnostics-upload can collect them. Handlers register in Program.Main before SelfUpdater so any downstream crash is captured; AgentRuntimeHost re-registers once session/tenant are known so the CrashRecord carries those labels. 6 builder/retention tests; 1163/1163 V2.Core tests green. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+185
@@ -0,0 +1,185 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.IO;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Diagnostics;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Diagnostics
|
||||
{
|
||||
/// <summary>
|
||||
/// Event-shape tests for the previous_crash_detected builder. The dump-writing path is
|
||||
/// integration-only (P/Invoke into dbghelp.dll, hard to unit-test) and covered by the
|
||||
/// V2-agent test-VM smoke run.
|
||||
/// </summary>
|
||||
public sealed class PendingCrashReporterTests
|
||||
{
|
||||
private static AgentConfiguration Config() => new AgentConfiguration
|
||||
{
|
||||
SessionId = "S1",
|
||||
TenantId = "T1",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void BuildEvent_emits_warning_with_full_metadata()
|
||||
{
|
||||
var record = new CrashRecord
|
||||
{
|
||||
CrashedAt = new DateTime(2026, 5, 28, 10, 15, 30, DateTimeKind.Utc),
|
||||
SessionId = "previous-session-id",
|
||||
TenantId = "previous-tenant",
|
||||
AgentVersion = "2.0.999",
|
||||
Trigger = "AppDomain.UnhandledException",
|
||||
ExceptionType = "System.NullReferenceException",
|
||||
ExceptionMessage = "Object reference not set",
|
||||
StackTrace = "at FooBar()",
|
||||
DumpFilePath = "20260528_101530_abc.dmp",
|
||||
DumpFileSizeBytes = 12345,
|
||||
DumpWriteSucceeded = true,
|
||||
};
|
||||
|
||||
var evt = PendingCrashReporter.BuildEvent(Config(), record);
|
||||
|
||||
Assert.Equal("previous_crash_detected", evt.EventType);
|
||||
Assert.Equal(EventSeverity.Warning, evt.Severity);
|
||||
Assert.Equal("PendingCrashReporter", evt.Source);
|
||||
Assert.Equal(EnrollmentPhase.Unknown, evt.Phase);
|
||||
Assert.True(evt.ImmediateUpload);
|
||||
Assert.Contains("System.NullReferenceException", evt.Message);
|
||||
Assert.Contains("dump captured", evt.Message);
|
||||
|
||||
Assert.Equal("previous-session-id", evt.Data["previousSessionId"]);
|
||||
Assert.Equal("previous-tenant", evt.Data["previousTenantId"]);
|
||||
Assert.Equal("2.0.999", evt.Data["previousAgentVersion"]);
|
||||
Assert.Equal("AppDomain.UnhandledException", evt.Data["trigger"]);
|
||||
Assert.Equal("System.NullReferenceException", evt.Data["exceptionType"]);
|
||||
Assert.Equal("20260528_101530_abc.dmp", evt.Data["dumpFilePath"]);
|
||||
Assert.Equal((long)12345, evt.Data["dumpFileSizeBytes"]);
|
||||
Assert.Equal(true, evt.Data["dumpWriteSucceeded"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildEvent_handles_missing_dump_gracefully()
|
||||
{
|
||||
var record = new CrashRecord
|
||||
{
|
||||
CrashedAt = DateTime.UtcNow,
|
||||
Trigger = "TaskScheduler.UnobservedTaskException",
|
||||
ExceptionType = "System.AggregateException",
|
||||
ExceptionMessage = "outer",
|
||||
DumpWriteSucceeded = false,
|
||||
};
|
||||
|
||||
var evt = PendingCrashReporter.BuildEvent(Config(), record);
|
||||
|
||||
Assert.Equal(EventSeverity.Warning, evt.Severity);
|
||||
Assert.Contains("no dump", evt.Message);
|
||||
Assert.Equal("(no dump)", evt.Data["dumpFilePath"]);
|
||||
Assert.Equal(false, evt.Data["dumpWriteSucceeded"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildEvent_handles_null_optional_fields()
|
||||
{
|
||||
var record = new CrashRecord
|
||||
{
|
||||
CrashedAt = DateTime.UtcNow,
|
||||
Trigger = "AppDomain.UnhandledException",
|
||||
ExceptionType = null,
|
||||
ExceptionMessage = null,
|
||||
StackTrace = null,
|
||||
SessionId = null,
|
||||
TenantId = null,
|
||||
AgentVersion = null,
|
||||
};
|
||||
|
||||
var evt = PendingCrashReporter.BuildEvent(Config(), record);
|
||||
|
||||
Assert.Equal("unknown", evt.Data["previousSessionId"]);
|
||||
Assert.Equal("unknown", evt.Data["previousTenantId"]);
|
||||
Assert.Equal("unknown", evt.Data["previousAgentVersion"]);
|
||||
Assert.Equal("unknown", evt.Data["exceptionType"]);
|
||||
Assert.Equal("", evt.Data["exceptionMessage"]);
|
||||
Assert.Equal("", evt.Data["stackTrace"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CrashRecord_roundtrips_through_newtonsoft()
|
||||
{
|
||||
var record = new CrashRecord
|
||||
{
|
||||
CrashedAt = new DateTime(2026, 5, 28, 9, 0, 0, DateTimeKind.Utc),
|
||||
SessionId = "sid",
|
||||
Trigger = "AppDomain.UnhandledException",
|
||||
ExceptionType = "X",
|
||||
ExceptionMessage = "m",
|
||||
DumpWriteSucceeded = true,
|
||||
};
|
||||
|
||||
var json = JsonConvert.SerializeObject(record);
|
||||
var back = JsonConvert.DeserializeObject<CrashRecord>(json);
|
||||
|
||||
Assert.NotNull(back);
|
||||
Assert.Equal(record.SessionId, back!.SessionId);
|
||||
Assert.Equal(record.Trigger, back.Trigger);
|
||||
Assert.Equal(record.ExceptionType, back.ExceptionType);
|
||||
Assert.True(back.DumpWriteSucceeded);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Retention
|
||||
|
||||
[Fact]
|
||||
public void ApplyRetention_deletes_files_older_than_max_age()
|
||||
{
|
||||
var tmp = Path.Combine(Path.GetTempPath(), "PCR-Retention-" + Guid.NewGuid().ToString("N").Substring(0, 8));
|
||||
Directory.CreateDirectory(tmp);
|
||||
try
|
||||
{
|
||||
var oldFile = Path.Combine(tmp, "old.dmp");
|
||||
File.WriteAllText(oldFile, "x");
|
||||
File.SetCreationTimeUtc(oldFile, DateTime.UtcNow - TimeSpan.FromDays(30));
|
||||
|
||||
var newFile = Path.Combine(tmp, "new.dmp");
|
||||
File.WriteAllText(newFile, "y");
|
||||
|
||||
CrashDumpCapture.ApplyRetention(tmp);
|
||||
|
||||
Assert.False(File.Exists(oldFile));
|
||||
Assert.True(File.Exists(newFile));
|
||||
}
|
||||
finally
|
||||
{
|
||||
try { Directory.Delete(tmp, recursive: true); } catch { }
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApplyRetention_keeps_only_max_retained_dumps()
|
||||
{
|
||||
var tmp = Path.Combine(Path.GetTempPath(), "PCR-Retention-" + Guid.NewGuid().ToString("N").Substring(0, 8));
|
||||
Directory.CreateDirectory(tmp);
|
||||
try
|
||||
{
|
||||
// Create MaxRetainedCrashes + 3 dumps, each older than the next
|
||||
var baseTime = DateTime.UtcNow - TimeSpan.FromHours(8);
|
||||
for (int i = 0; i < CrashDumpCapture.MaxRetainedCrashes + 3; i++)
|
||||
{
|
||||
var p = Path.Combine(tmp, $"dump-{i}.dmp");
|
||||
File.WriteAllText(p, "x");
|
||||
File.SetCreationTimeUtc(p, baseTime.AddMinutes(i));
|
||||
}
|
||||
|
||||
CrashDumpCapture.ApplyRetention(tmp);
|
||||
|
||||
var remaining = Directory.GetFiles(tmp, "*.dmp");
|
||||
Assert.Equal(CrashDumpCapture.MaxRetainedCrashes, remaining.Length);
|
||||
}
|
||||
finally
|
||||
{
|
||||
try { Directory.Delete(tmp, recursive: true); } catch { }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Diagnostics
|
||||
{
|
||||
/// <summary>
|
||||
/// Registers handlers for <see cref="AppDomain.UnhandledException"/> and
|
||||
/// <see cref="TaskScheduler.UnobservedTaskException"/>. When fired, synchronously writes
|
||||
/// a MiniDump (via <see cref="MiniDumpWriter"/>) and a <see cref="CrashRecord"/> JSON sibling
|
||||
/// to <see cref="CrashesDirectory"/>. <see cref="PendingCrashReporter"/> picks them up on the
|
||||
/// next agent start and emits a <c>previous_crash_detected</c> timeline event.
|
||||
/// <para>
|
||||
/// All work is synchronous and fail-soft — the process is about to die so there's no
|
||||
/// async window. <see cref="StackOverflowException"/>, <c>SIGKILL</c> and OS force-terminate
|
||||
/// are accepted limitations (no dump possible).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class CrashDumpCapture
|
||||
{
|
||||
public const string CrashesDirectoryName = "Crashes";
|
||||
public const int MaxRetainedCrashes = 5;
|
||||
public static readonly TimeSpan MaxCrashAge = TimeSpan.FromDays(7);
|
||||
|
||||
private static int _registered;
|
||||
private static string _crashesDir = string.Empty;
|
||||
private static string _sessionId = string.Empty;
|
||||
private static string _tenantId = string.Empty;
|
||||
private static string _agentVersion = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Idempotent. Registers the global exception handlers. Subsequent calls update the
|
||||
/// captured session/tenant labels but do not re-register handlers.
|
||||
/// </summary>
|
||||
public static void RegisterHandlers(
|
||||
string programDataDirectory,
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
string agentVersion)
|
||||
{
|
||||
_crashesDir = Path.Combine(programDataDirectory, CrashesDirectoryName);
|
||||
_sessionId = sessionId ?? string.Empty;
|
||||
_tenantId = tenantId ?? string.Empty;
|
||||
_agentVersion = agentVersion ?? string.Empty;
|
||||
|
||||
if (Interlocked.Exchange(ref _registered, 1) == 1) return;
|
||||
|
||||
AppDomain.CurrentDomain.UnhandledException += OnAppDomainUnhandledException;
|
||||
TaskScheduler.UnobservedTaskException += OnUnobservedTaskException;
|
||||
}
|
||||
|
||||
/// <summary>Folder where crash artefacts are written. Empty until <see cref="RegisterHandlers"/> runs.</summary>
|
||||
public static string CrashesDirectory => _crashesDir;
|
||||
|
||||
private static void OnAppDomainUnhandledException(object sender, UnhandledExceptionEventArgs e)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (e?.ExceptionObject is Exception ex)
|
||||
{
|
||||
WriteCrashArtefacts(ex, trigger: "AppDomain.UnhandledException");
|
||||
}
|
||||
}
|
||||
catch { /* nowhere left to log */ }
|
||||
}
|
||||
|
||||
private static void OnUnobservedTaskException(object? sender, UnobservedTaskExceptionEventArgs e)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (e?.Exception != null)
|
||||
{
|
||||
WriteCrashArtefacts(e.Exception, trigger: "TaskScheduler.UnobservedTaskException");
|
||||
e.SetObserved();
|
||||
}
|
||||
}
|
||||
catch { /* nowhere left to log */ }
|
||||
}
|
||||
|
||||
private static void WriteCrashArtefacts(Exception ex, string trigger)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(_crashesDir);
|
||||
}
|
||||
catch { return; }
|
||||
|
||||
var timestamp = DateTime.UtcNow;
|
||||
var stem = $"{timestamp:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}".Substring(0, 23);
|
||||
var dumpPath = Path.Combine(_crashesDir, stem + ".dmp");
|
||||
var jsonPath = Path.Combine(_crashesDir, stem + ".json");
|
||||
|
||||
bool dumpOk = MiniDumpWriter.TryWriteDump(dumpPath);
|
||||
long dumpSize = 0;
|
||||
if (dumpOk)
|
||||
{
|
||||
try { dumpSize = new FileInfo(dumpPath).Length; } catch { }
|
||||
}
|
||||
|
||||
var record = new CrashRecord
|
||||
{
|
||||
CrashedAt = timestamp,
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
AgentVersion = _agentVersion,
|
||||
Trigger = trigger,
|
||||
ExceptionType = ex.GetType().FullName,
|
||||
ExceptionMessage = Truncate(ex.Message, 2000),
|
||||
StackTrace = Truncate(ex.ToString(), 8000),
|
||||
DumpFilePath = dumpOk ? Path.GetFileName(dumpPath) : null,
|
||||
DumpFileSizeBytes = dumpOk ? dumpSize : (long?)null,
|
||||
DumpWriteSucceeded = dumpOk,
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
File.WriteAllText(jsonPath, JsonConvert.SerializeObject(record, Formatting.Indented));
|
||||
}
|
||||
catch { /* nowhere left to log */ }
|
||||
}
|
||||
|
||||
private static string? Truncate(string? s, int maxLen) =>
|
||||
s == null ? null : (s.Length <= maxLen ? s : s.Substring(0, maxLen) + "…[truncated]");
|
||||
|
||||
/// <summary>
|
||||
/// Apply retention policy: keep at most <see cref="MaxRetainedCrashes"/> dumps and delete
|
||||
/// anything older than <see cref="MaxCrashAge"/>. Called from <see cref="PendingCrashReporter"/>
|
||||
/// after it has emitted pending records.
|
||||
/// </summary>
|
||||
internal static void ApplyRetention(string crashesDir)
|
||||
{
|
||||
if (!Directory.Exists(crashesDir)) return;
|
||||
|
||||
try
|
||||
{
|
||||
var cutoff = DateTime.UtcNow - MaxCrashAge;
|
||||
foreach (var f in Directory.GetFiles(crashesDir))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.GetCreationTimeUtc(f) < cutoff) File.Delete(f);
|
||||
}
|
||||
catch { /* keep going */ }
|
||||
}
|
||||
|
||||
var dumps = Directory.GetFiles(crashesDir, "*.dmp");
|
||||
if (dumps.Length > MaxRetainedCrashes)
|
||||
{
|
||||
Array.Sort(dumps, (a, b) => File.GetCreationTimeUtc(a).CompareTo(File.GetCreationTimeUtc(b)));
|
||||
var toDelete = dumps.Length - MaxRetainedCrashes;
|
||||
for (int i = 0; i < toDelete; i++)
|
||||
{
|
||||
try { File.Delete(dumps[i]); } catch { }
|
||||
try { File.Delete(Path.ChangeExtension(dumps[i], ".json")); } catch { }
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { /* retention is best-effort */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Diagnostics
|
||||
{
|
||||
/// <summary>
|
||||
/// Serializable record of a crash captured by <see cref="CrashDumpCapture"/>.
|
||||
/// One JSON file per crash sits next to the dump in <c>%ProgramData%\AutopilotMonitor\Crashes\</c>
|
||||
/// until <see cref="PendingCrashReporter"/> picks it up on the next agent start and emits the
|
||||
/// <c>previous_crash_detected</c> event.
|
||||
/// </summary>
|
||||
public sealed class CrashRecord
|
||||
{
|
||||
[JsonProperty("crashedAt")]
|
||||
public DateTime CrashedAt { get; set; }
|
||||
|
||||
[JsonProperty("sessionId")]
|
||||
public string? SessionId { get; set; }
|
||||
|
||||
[JsonProperty("tenantId")]
|
||||
public string? TenantId { get; set; }
|
||||
|
||||
[JsonProperty("agentVersion")]
|
||||
public string? AgentVersion { get; set; }
|
||||
|
||||
[JsonProperty("trigger")]
|
||||
public string Trigger { get; set; } = string.Empty;
|
||||
|
||||
[JsonProperty("exceptionType")]
|
||||
public string? ExceptionType { get; set; }
|
||||
|
||||
[JsonProperty("exceptionMessage")]
|
||||
public string? ExceptionMessage { get; set; }
|
||||
|
||||
[JsonProperty("stackTrace")]
|
||||
public string? StackTrace { get; set; }
|
||||
|
||||
[JsonProperty("dumpFilePath")]
|
||||
public string? DumpFilePath { get; set; }
|
||||
|
||||
[JsonProperty("dumpFileSizeBytes")]
|
||||
public long? DumpFileSizeBytes { get; set; }
|
||||
|
||||
[JsonProperty("dumpWriteSucceeded")]
|
||||
public bool DumpWriteSucceeded { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Diagnostics
|
||||
{
|
||||
/// <summary>
|
||||
/// Thin P/Invoke wrapper around <c>MiniDumpWriteDump</c> from <c>dbghelp.dll</c>.
|
||||
/// Writes a compact MiniDump of the current process so the managed stack and key
|
||||
/// metadata can be recovered post-mortem.
|
||||
/// <para>
|
||||
/// Designed to run from inside an <see cref="AppDomain.UnhandledException"/> handler —
|
||||
/// fully synchronous, swallows all exceptions, returns <c>false</c> on any failure.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class MiniDumpWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Dump flags chosen for a small-but-useful dump (~5-20 MB for the agent):
|
||||
/// data-segs (globals), handles (loaded modules), thread-info (state/priority/affinity),
|
||||
/// unloaded-modules (for module-load failures). No full memory — that would push
|
||||
/// the dump to 100+ MB and break diagnostics-upload size budgets.
|
||||
/// </summary>
|
||||
private const uint MiniDumpFlags =
|
||||
0x00000001 | // MiniDumpWithDataSegs
|
||||
0x00000004 | // MiniDumpWithHandleData
|
||||
0x00001000 | // MiniDumpWithThreadInfo
|
||||
0x00000020; // MiniDumpWithUnloadedModules
|
||||
|
||||
public static bool TryWriteDump(string dumpFilePath)
|
||||
{
|
||||
if (string.IsNullOrEmpty(dumpFilePath)) return false;
|
||||
|
||||
FileStream? fs = null;
|
||||
try
|
||||
{
|
||||
var dir = Path.GetDirectoryName(dumpFilePath);
|
||||
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
|
||||
|
||||
fs = new FileStream(dumpFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
var process = Process.GetCurrentProcess();
|
||||
return MiniDumpWriteDump(
|
||||
process.Handle,
|
||||
(uint)process.Id,
|
||||
fs.SafeFileHandle.DangerousGetHandle(),
|
||||
MiniDumpFlags,
|
||||
IntPtr.Zero,
|
||||
IntPtr.Zero,
|
||||
IntPtr.Zero);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
try { fs?.Dispose(); } catch { }
|
||||
}
|
||||
}
|
||||
|
||||
[DllImport("dbghelp.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
private static extern bool MiniDumpWriteDump(
|
||||
IntPtr hProcess,
|
||||
uint processId,
|
||||
IntPtr hFile,
|
||||
uint dumpType,
|
||||
IntPtr expParam,
|
||||
IntPtr userStreamParam,
|
||||
IntPtr callbackParam);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Diagnostics
|
||||
{
|
||||
/// <summary>
|
||||
/// Scans <see cref="CrashDumpCapture.CrashesDirectory"/> at agent start, emits one
|
||||
/// <c>previous_crash_detected</c> event per pending <see cref="CrashRecord"/>, then
|
||||
/// applies the retention policy. The dump files themselves stay on disk so that a
|
||||
/// diagnostics-upload (customer opt-in) can collect them via the Crashes/ folder.
|
||||
/// </summary>
|
||||
public static class PendingCrashReporter
|
||||
{
|
||||
public static void ScanAndEmit(
|
||||
AgentConfiguration configuration,
|
||||
string programDataDirectory,
|
||||
AgentLogger logger,
|
||||
InformationalEventPost post)
|
||||
{
|
||||
if (configuration == null) throw new ArgumentNullException(nameof(configuration));
|
||||
if (logger == null) throw new ArgumentNullException(nameof(logger));
|
||||
if (post == null) throw new ArgumentNullException(nameof(post));
|
||||
|
||||
var crashesDir = Path.Combine(programDataDirectory, CrashDumpCapture.CrashesDirectoryName);
|
||||
if (!Directory.Exists(crashesDir)) return;
|
||||
|
||||
string[] recordFiles;
|
||||
try { recordFiles = Directory.GetFiles(crashesDir, "*.json"); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"PendingCrashReporter: enumerate crashes dir threw: {ex.Message}");
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var recordFile in recordFiles)
|
||||
{
|
||||
CrashRecord? record = TryReadRecord(recordFile, logger);
|
||||
if (record == null)
|
||||
{
|
||||
TryDelete(recordFile);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
post.Emit(BuildEvent(configuration, record));
|
||||
// Mark JSON consumed so we don't re-emit on next start. The .dmp stays for upload.
|
||||
TryDelete(recordFile);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"PendingCrashReporter: emit failed for '{recordFile}': {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
CrashDumpCapture.ApplyRetention(crashesDir);
|
||||
}
|
||||
|
||||
private static CrashRecord? TryReadRecord(string path, AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
var json = File.ReadAllText(path);
|
||||
return JsonConvert.DeserializeObject<CrashRecord>(json);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"PendingCrashReporter: parse failed for '{path}': {ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try { File.Delete(path); } catch { }
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------- Event builder (internal for tests)
|
||||
|
||||
internal static EnrollmentEvent BuildEvent(AgentConfiguration configuration, CrashRecord record) =>
|
||||
new EnrollmentEvent
|
||||
{
|
||||
SessionId = configuration.SessionId,
|
||||
TenantId = configuration.TenantId,
|
||||
EventType = "previous_crash_detected",
|
||||
Severity = EventSeverity.Warning,
|
||||
Source = "PendingCrashReporter",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Timestamp = DateTime.UtcNow,
|
||||
Message = BuildMessage(record),
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
{ "crashedAt", record.CrashedAt.ToString("o") },
|
||||
{ "previousSessionId", (object?)record.SessionId ?? "unknown" },
|
||||
{ "previousTenantId", (object?)record.TenantId ?? "unknown" },
|
||||
{ "previousAgentVersion", (object?)record.AgentVersion ?? "unknown" },
|
||||
{ "trigger", record.Trigger ?? "unknown" },
|
||||
{ "exceptionType", (object?)record.ExceptionType ?? "unknown" },
|
||||
{ "exceptionMessage", (object?)record.ExceptionMessage ?? "" },
|
||||
{ "stackTrace", (object?)record.StackTrace ?? "" },
|
||||
{ "dumpFilePath", (object?)record.DumpFilePath ?? "(no dump)" },
|
||||
{ "dumpFileSizeBytes", (object?)record.DumpFileSizeBytes ?? 0 },
|
||||
{ "dumpWriteSucceeded", record.DumpWriteSucceeded },
|
||||
},
|
||||
ImmediateUpload = true,
|
||||
};
|
||||
|
||||
private static string BuildMessage(CrashRecord record)
|
||||
{
|
||||
var type = record.ExceptionType ?? "unknown";
|
||||
var dumpInfo = record.DumpWriteSucceeded
|
||||
? $"dump captured ({record.DumpFileSizeBytes ?? 0} bytes)"
|
||||
: "no dump";
|
||||
return $"Previous agent process crashed: {type} via {record.Trigger}; {dumpInfo}";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,16 @@ namespace AutopilotMonitor.Agent.V2
|
||||
// shutdown during self-update still produces a "clean" classification.
|
||||
RegisterCleanExitMarker(dataDirectory);
|
||||
|
||||
// Register crash-dump handlers as early as possible — before SelfUpdate, before
|
||||
// any other startup work — so an unhandled exception anywhere downstream still
|
||||
// produces a MiniDump + CrashRecord. SessionId/TenantId are unknown at this point;
|
||||
// they're filled in later by AgentRuntimeHost via a second RegisterHandlers call.
|
||||
AutopilotMonitor.Agent.V2.Core.Diagnostics.CrashDumpCapture.RegisterHandlers(
|
||||
programDataDirectory: dataDirectory,
|
||||
sessionId: "(pre-runtime)",
|
||||
tenantId: "(pre-runtime)",
|
||||
agentVersion: GetAgentVersion());
|
||||
|
||||
// Startup self-update. Legacy parity: cleanup leftover .old files, load cached
|
||||
// backend hash / downgrade policy, then attempt the update. Failures here never
|
||||
// abort startup — we prefer to run the current version than delay.
|
||||
|
||||
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Diagnostics;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
@@ -558,6 +559,25 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{
|
||||
logger.Warning($"StartupEnvironmentProbes: outer exception: {ex.Message}");
|
||||
}
|
||||
|
||||
// PR2 — Crash-dump capture: refresh the handler labels with the now-known
|
||||
// session/tenant, then drain pending crash records from previous runs
|
||||
// into the timeline as previous_crash_detected events.
|
||||
try
|
||||
{
|
||||
var programData = Environment.ExpandEnvironmentVariables(@"%ProgramData%\AutopilotMonitor");
|
||||
CrashDumpCapture.RegisterHandlers(
|
||||
programDataDirectory: programData,
|
||||
sessionId: agentConfig.SessionId ?? string.Empty,
|
||||
tenantId: agentConfig.TenantId ?? string.Empty,
|
||||
agentVersion: System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown");
|
||||
|
||||
PendingCrashReporter.ScanAndEmit(agentConfig, programData, logger, lifecyclePost);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"PendingCrashReporter: outer exception: {ex.Message}");
|
||||
}
|
||||
});
|
||||
|
||||
logger.Info($"V2 agent runtime ready (session={agentConfig.SessionId}, tenant={agentConfig.TenantId}).");
|
||||
|
||||
@@ -1 +1 @@
|
||||
894
|
||||
898
|
||||
|
||||
Reference in New Issue
Block a user