feat(agent-v2): capture mini-dumps + emit previous_crash_detected event

When the V2 agent process dies via AppDomain.UnhandledException or
TaskScheduler.UnobservedTaskException, synchronously write a compact
MiniDump (data-segs + handle + thread-info, ~5-20 MB) and a CrashRecord
JSON sibling to %ProgramData%\AutopilotMonitor\Crashes\. Next agent
start scans the directory, drains pending records into the timeline as
previous_crash_detected events (Warning, ImmediateUpload), and applies
retention (max 5 dumps, max 7 days). Dumps stay on disk so the existing
customer-opt-in diagnostics-upload can collect them.

Handlers register in Program.Main before SelfUpdater so any downstream
crash is captured; AgentRuntimeHost re-registers once session/tenant
are known so the CrashRecord carries those labels. 6 builder/retention
tests; 1163/1163 V2.Core tests green.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-05-28 12:25:38 +02:00
parent d4acac9b11
commit e0d4094a25
8 changed files with 627 additions and 1 deletions
@@ -0,0 +1,185 @@
#nullable enable
using System;
using System.IO;
using AutopilotMonitor.Agent.V2.Core.Configuration;
using AutopilotMonitor.Agent.V2.Core.Diagnostics;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Diagnostics
{
/// <summary>
/// Event-shape tests for the previous_crash_detected builder. The dump-writing path is
/// integration-only (P/Invoke into dbghelp.dll, hard to unit-test) and covered by the
/// V2-agent test-VM smoke run.
/// </summary>
public sealed class PendingCrashReporterTests
{
private static AgentConfiguration Config() => new AgentConfiguration
{
SessionId = "S1",
TenantId = "T1",
};
[Fact]
public void BuildEvent_emits_warning_with_full_metadata()
{
var record = new CrashRecord
{
CrashedAt = new DateTime(2026, 5, 28, 10, 15, 30, DateTimeKind.Utc),
SessionId = "previous-session-id",
TenantId = "previous-tenant",
AgentVersion = "2.0.999",
Trigger = "AppDomain.UnhandledException",
ExceptionType = "System.NullReferenceException",
ExceptionMessage = "Object reference not set",
StackTrace = "at FooBar()",
DumpFilePath = "20260528_101530_abc.dmp",
DumpFileSizeBytes = 12345,
DumpWriteSucceeded = true,
};
var evt = PendingCrashReporter.BuildEvent(Config(), record);
Assert.Equal("previous_crash_detected", evt.EventType);
Assert.Equal(EventSeverity.Warning, evt.Severity);
Assert.Equal("PendingCrashReporter", evt.Source);
Assert.Equal(EnrollmentPhase.Unknown, evt.Phase);
Assert.True(evt.ImmediateUpload);
Assert.Contains("System.NullReferenceException", evt.Message);
Assert.Contains("dump captured", evt.Message);
Assert.Equal("previous-session-id", evt.Data["previousSessionId"]);
Assert.Equal("previous-tenant", evt.Data["previousTenantId"]);
Assert.Equal("2.0.999", evt.Data["previousAgentVersion"]);
Assert.Equal("AppDomain.UnhandledException", evt.Data["trigger"]);
Assert.Equal("System.NullReferenceException", evt.Data["exceptionType"]);
Assert.Equal("20260528_101530_abc.dmp", evt.Data["dumpFilePath"]);
Assert.Equal((long)12345, evt.Data["dumpFileSizeBytes"]);
Assert.Equal(true, evt.Data["dumpWriteSucceeded"]);
}
[Fact]
public void BuildEvent_handles_missing_dump_gracefully()
{
var record = new CrashRecord
{
CrashedAt = DateTime.UtcNow,
Trigger = "TaskScheduler.UnobservedTaskException",
ExceptionType = "System.AggregateException",
ExceptionMessage = "outer",
DumpWriteSucceeded = false,
};
var evt = PendingCrashReporter.BuildEvent(Config(), record);
Assert.Equal(EventSeverity.Warning, evt.Severity);
Assert.Contains("no dump", evt.Message);
Assert.Equal("(no dump)", evt.Data["dumpFilePath"]);
Assert.Equal(false, evt.Data["dumpWriteSucceeded"]);
}
[Fact]
public void BuildEvent_handles_null_optional_fields()
{
var record = new CrashRecord
{
CrashedAt = DateTime.UtcNow,
Trigger = "AppDomain.UnhandledException",
ExceptionType = null,
ExceptionMessage = null,
StackTrace = null,
SessionId = null,
TenantId = null,
AgentVersion = null,
};
var evt = PendingCrashReporter.BuildEvent(Config(), record);
Assert.Equal("unknown", evt.Data["previousSessionId"]);
Assert.Equal("unknown", evt.Data["previousTenantId"]);
Assert.Equal("unknown", evt.Data["previousAgentVersion"]);
Assert.Equal("unknown", evt.Data["exceptionType"]);
Assert.Equal("", evt.Data["exceptionMessage"]);
Assert.Equal("", evt.Data["stackTrace"]);
}
[Fact]
public void CrashRecord_roundtrips_through_newtonsoft()
{
var record = new CrashRecord
{
CrashedAt = new DateTime(2026, 5, 28, 9, 0, 0, DateTimeKind.Utc),
SessionId = "sid",
Trigger = "AppDomain.UnhandledException",
ExceptionType = "X",
ExceptionMessage = "m",
DumpWriteSucceeded = true,
};
var json = JsonConvert.SerializeObject(record);
var back = JsonConvert.DeserializeObject<CrashRecord>(json);
Assert.NotNull(back);
Assert.Equal(record.SessionId, back!.SessionId);
Assert.Equal(record.Trigger, back.Trigger);
Assert.Equal(record.ExceptionType, back.ExceptionType);
Assert.True(back.DumpWriteSucceeded);
}
// ---------------------------------------------------------------- Retention
[Fact]
public void ApplyRetention_deletes_files_older_than_max_age()
{
var tmp = Path.Combine(Path.GetTempPath(), "PCR-Retention-" + Guid.NewGuid().ToString("N").Substring(0, 8));
Directory.CreateDirectory(tmp);
try
{
var oldFile = Path.Combine(tmp, "old.dmp");
File.WriteAllText(oldFile, "x");
File.SetCreationTimeUtc(oldFile, DateTime.UtcNow - TimeSpan.FromDays(30));
var newFile = Path.Combine(tmp, "new.dmp");
File.WriteAllText(newFile, "y");
CrashDumpCapture.ApplyRetention(tmp);
Assert.False(File.Exists(oldFile));
Assert.True(File.Exists(newFile));
}
finally
{
try { Directory.Delete(tmp, recursive: true); } catch { }
}
}
[Fact]
public void ApplyRetention_keeps_only_max_retained_dumps()
{
var tmp = Path.Combine(Path.GetTempPath(), "PCR-Retention-" + Guid.NewGuid().ToString("N").Substring(0, 8));
Directory.CreateDirectory(tmp);
try
{
// Create MaxRetainedCrashes + 3 dumps, each older than the next
var baseTime = DateTime.UtcNow - TimeSpan.FromHours(8);
for (int i = 0; i < CrashDumpCapture.MaxRetainedCrashes + 3; i++)
{
var p = Path.Combine(tmp, $"dump-{i}.dmp");
File.WriteAllText(p, "x");
File.SetCreationTimeUtc(p, baseTime.AddMinutes(i));
}
CrashDumpCapture.ApplyRetention(tmp);
var remaining = Directory.GetFiles(tmp, "*.dmp");
Assert.Equal(CrashDumpCapture.MaxRetainedCrashes, remaining.Length);
}
finally
{
try { Directory.Delete(tmp, recursive: true); } catch { }
}
}
}
}
@@ -0,0 +1,165 @@
#nullable enable
using System;
using System.IO;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
namespace AutopilotMonitor.Agent.V2.Core.Diagnostics
{
/// <summary>
/// Registers handlers for <see cref="AppDomain.UnhandledException"/> and
/// <see cref="TaskScheduler.UnobservedTaskException"/>. When fired, synchronously writes
/// a MiniDump (via <see cref="MiniDumpWriter"/>) and a <see cref="CrashRecord"/> JSON sibling
/// to <see cref="CrashesDirectory"/>. <see cref="PendingCrashReporter"/> picks them up on the
/// next agent start and emits a <c>previous_crash_detected</c> timeline event.
/// <para>
/// All work is synchronous and fail-soft — the process is about to die so there's no
/// async window. <see cref="StackOverflowException"/>, <c>SIGKILL</c> and OS force-terminate
/// are accepted limitations (no dump possible).
/// </para>
/// </summary>
public static class CrashDumpCapture
{
public const string CrashesDirectoryName = "Crashes";
public const int MaxRetainedCrashes = 5;
public static readonly TimeSpan MaxCrashAge = TimeSpan.FromDays(7);
private static int _registered;
private static string _crashesDir = string.Empty;
private static string _sessionId = string.Empty;
private static string _tenantId = string.Empty;
private static string _agentVersion = string.Empty;
/// <summary>
/// Idempotent. Registers the global exception handlers. Subsequent calls update the
/// captured session/tenant labels but do not re-register handlers.
/// </summary>
public static void RegisterHandlers(
string programDataDirectory,
string sessionId,
string tenantId,
string agentVersion)
{
_crashesDir = Path.Combine(programDataDirectory, CrashesDirectoryName);
_sessionId = sessionId ?? string.Empty;
_tenantId = tenantId ?? string.Empty;
_agentVersion = agentVersion ?? string.Empty;
if (Interlocked.Exchange(ref _registered, 1) == 1) return;
AppDomain.CurrentDomain.UnhandledException += OnAppDomainUnhandledException;
TaskScheduler.UnobservedTaskException += OnUnobservedTaskException;
}
/// <summary>Folder where crash artefacts are written. Empty until <see cref="RegisterHandlers"/> runs.</summary>
public static string CrashesDirectory => _crashesDir;
private static void OnAppDomainUnhandledException(object sender, UnhandledExceptionEventArgs e)
{
try
{
if (e?.ExceptionObject is Exception ex)
{
WriteCrashArtefacts(ex, trigger: "AppDomain.UnhandledException");
}
}
catch { /* nowhere left to log */ }
}
private static void OnUnobservedTaskException(object? sender, UnobservedTaskExceptionEventArgs e)
{
try
{
if (e?.Exception != null)
{
WriteCrashArtefacts(e.Exception, trigger: "TaskScheduler.UnobservedTaskException");
e.SetObserved();
}
}
catch { /* nowhere left to log */ }
}
private static void WriteCrashArtefacts(Exception ex, string trigger)
{
try
{
Directory.CreateDirectory(_crashesDir);
}
catch { return; }
var timestamp = DateTime.UtcNow;
var stem = $"{timestamp:yyyyMMdd_HHmmss}_{Guid.NewGuid():N}".Substring(0, 23);
var dumpPath = Path.Combine(_crashesDir, stem + ".dmp");
var jsonPath = Path.Combine(_crashesDir, stem + ".json");
bool dumpOk = MiniDumpWriter.TryWriteDump(dumpPath);
long dumpSize = 0;
if (dumpOk)
{
try { dumpSize = new FileInfo(dumpPath).Length; } catch { }
}
var record = new CrashRecord
{
CrashedAt = timestamp,
SessionId = _sessionId,
TenantId = _tenantId,
AgentVersion = _agentVersion,
Trigger = trigger,
ExceptionType = ex.GetType().FullName,
ExceptionMessage = Truncate(ex.Message, 2000),
StackTrace = Truncate(ex.ToString(), 8000),
DumpFilePath = dumpOk ? Path.GetFileName(dumpPath) : null,
DumpFileSizeBytes = dumpOk ? dumpSize : (long?)null,
DumpWriteSucceeded = dumpOk,
};
try
{
File.WriteAllText(jsonPath, JsonConvert.SerializeObject(record, Formatting.Indented));
}
catch { /* nowhere left to log */ }
}
private static string? Truncate(string? s, int maxLen) =>
s == null ? null : (s.Length <= maxLen ? s : s.Substring(0, maxLen) + "…[truncated]");
/// <summary>
/// Apply retention policy: keep at most <see cref="MaxRetainedCrashes"/> dumps and delete
/// anything older than <see cref="MaxCrashAge"/>. Called from <see cref="PendingCrashReporter"/>
/// after it has emitted pending records.
/// </summary>
internal static void ApplyRetention(string crashesDir)
{
if (!Directory.Exists(crashesDir)) return;
try
{
var cutoff = DateTime.UtcNow - MaxCrashAge;
foreach (var f in Directory.GetFiles(crashesDir))
{
try
{
if (File.GetCreationTimeUtc(f) < cutoff) File.Delete(f);
}
catch { /* keep going */ }
}
var dumps = Directory.GetFiles(crashesDir, "*.dmp");
if (dumps.Length > MaxRetainedCrashes)
{
Array.Sort(dumps, (a, b) => File.GetCreationTimeUtc(a).CompareTo(File.GetCreationTimeUtc(b)));
var toDelete = dumps.Length - MaxRetainedCrashes;
for (int i = 0; i < toDelete; i++)
{
try { File.Delete(dumps[i]); } catch { }
try { File.Delete(Path.ChangeExtension(dumps[i], ".json")); } catch { }
}
}
}
catch { /* retention is best-effort */ }
}
}
}
@@ -0,0 +1,48 @@
#nullable enable
using System;
using Newtonsoft.Json;
namespace AutopilotMonitor.Agent.V2.Core.Diagnostics
{
/// <summary>
/// Serializable record of a crash captured by <see cref="CrashDumpCapture"/>.
/// One JSON file per crash sits next to the dump in <c>%ProgramData%\AutopilotMonitor\Crashes\</c>
/// until <see cref="PendingCrashReporter"/> picks it up on the next agent start and emits the
/// <c>previous_crash_detected</c> event.
/// </summary>
public sealed class CrashRecord
{
[JsonProperty("crashedAt")]
public DateTime CrashedAt { get; set; }
[JsonProperty("sessionId")]
public string? SessionId { get; set; }
[JsonProperty("tenantId")]
public string? TenantId { get; set; }
[JsonProperty("agentVersion")]
public string? AgentVersion { get; set; }
[JsonProperty("trigger")]
public string Trigger { get; set; } = string.Empty;
[JsonProperty("exceptionType")]
public string? ExceptionType { get; set; }
[JsonProperty("exceptionMessage")]
public string? ExceptionMessage { get; set; }
[JsonProperty("stackTrace")]
public string? StackTrace { get; set; }
[JsonProperty("dumpFilePath")]
public string? DumpFilePath { get; set; }
[JsonProperty("dumpFileSizeBytes")]
public long? DumpFileSizeBytes { get; set; }
[JsonProperty("dumpWriteSucceeded")]
public bool DumpWriteSucceeded { get; set; }
}
}
@@ -0,0 +1,74 @@
#nullable enable
using System;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
namespace AutopilotMonitor.Agent.V2.Core.Diagnostics
{
/// <summary>
/// Thin P/Invoke wrapper around <c>MiniDumpWriteDump</c> from <c>dbghelp.dll</c>.
/// Writes a compact MiniDump of the current process so the managed stack and key
/// metadata can be recovered post-mortem.
/// <para>
/// Designed to run from inside an <see cref="AppDomain.UnhandledException"/> handler —
/// fully synchronous, swallows all exceptions, returns <c>false</c> on any failure.
/// </para>
/// </summary>
public static class MiniDumpWriter
{
/// <summary>
/// Dump flags chosen for a small-but-useful dump (~5-20 MB for the agent):
/// data-segs (globals), handles (loaded modules), thread-info (state/priority/affinity),
/// unloaded-modules (for module-load failures). No full memory — that would push
/// the dump to 100+ MB and break diagnostics-upload size budgets.
/// </summary>
private const uint MiniDumpFlags =
0x00000001 | // MiniDumpWithDataSegs
0x00000004 | // MiniDumpWithHandleData
0x00001000 | // MiniDumpWithThreadInfo
0x00000020; // MiniDumpWithUnloadedModules
public static bool TryWriteDump(string dumpFilePath)
{
if (string.IsNullOrEmpty(dumpFilePath)) return false;
FileStream? fs = null;
try
{
var dir = Path.GetDirectoryName(dumpFilePath);
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
fs = new FileStream(dumpFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
var process = Process.GetCurrentProcess();
return MiniDumpWriteDump(
process.Handle,
(uint)process.Id,
fs.SafeFileHandle.DangerousGetHandle(),
MiniDumpFlags,
IntPtr.Zero,
IntPtr.Zero,
IntPtr.Zero);
}
catch
{
return false;
}
finally
{
try { fs?.Dispose(); } catch { }
}
}
[DllImport("dbghelp.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool MiniDumpWriteDump(
IntPtr hProcess,
uint processId,
IntPtr hFile,
uint dumpType,
IntPtr expParam,
IntPtr userStreamParam,
IntPtr callbackParam);
}
}
@@ -0,0 +1,124 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using AutopilotMonitor.Agent.V2.Core.Configuration;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json;
namespace AutopilotMonitor.Agent.V2.Core.Diagnostics
{
/// <summary>
/// Scans <see cref="CrashDumpCapture.CrashesDirectory"/> at agent start, emits one
/// <c>previous_crash_detected</c> event per pending <see cref="CrashRecord"/>, then
/// applies the retention policy. The dump files themselves stay on disk so that a
/// diagnostics-upload (customer opt-in) can collect them via the Crashes/ folder.
/// </summary>
public static class PendingCrashReporter
{
public static void ScanAndEmit(
AgentConfiguration configuration,
string programDataDirectory,
AgentLogger logger,
InformationalEventPost post)
{
if (configuration == null) throw new ArgumentNullException(nameof(configuration));
if (logger == null) throw new ArgumentNullException(nameof(logger));
if (post == null) throw new ArgumentNullException(nameof(post));
var crashesDir = Path.Combine(programDataDirectory, CrashDumpCapture.CrashesDirectoryName);
if (!Directory.Exists(crashesDir)) return;
string[] recordFiles;
try { recordFiles = Directory.GetFiles(crashesDir, "*.json"); }
catch (Exception ex)
{
logger.Warning($"PendingCrashReporter: enumerate crashes dir threw: {ex.Message}");
return;
}
foreach (var recordFile in recordFiles)
{
CrashRecord? record = TryReadRecord(recordFile, logger);
if (record == null)
{
TryDelete(recordFile);
continue;
}
try
{
post.Emit(BuildEvent(configuration, record));
// Mark JSON consumed so we don't re-emit on next start. The .dmp stays for upload.
TryDelete(recordFile);
}
catch (Exception ex)
{
logger.Warning($"PendingCrashReporter: emit failed for '{recordFile}': {ex.Message}");
}
}
CrashDumpCapture.ApplyRetention(crashesDir);
}
private static CrashRecord? TryReadRecord(string path, AgentLogger logger)
{
try
{
var json = File.ReadAllText(path);
return JsonConvert.DeserializeObject<CrashRecord>(json);
}
catch (Exception ex)
{
logger.Warning($"PendingCrashReporter: parse failed for '{path}': {ex.Message}");
return null;
}
}
private static void TryDelete(string path)
{
try { File.Delete(path); } catch { }
}
// ----------------------------------------------------------- Event builder (internal for tests)
internal static EnrollmentEvent BuildEvent(AgentConfiguration configuration, CrashRecord record) =>
new EnrollmentEvent
{
SessionId = configuration.SessionId,
TenantId = configuration.TenantId,
EventType = "previous_crash_detected",
Severity = EventSeverity.Warning,
Source = "PendingCrashReporter",
Phase = EnrollmentPhase.Unknown,
Timestamp = DateTime.UtcNow,
Message = BuildMessage(record),
Data = new Dictionary<string, object>
{
{ "crashedAt", record.CrashedAt.ToString("o") },
{ "previousSessionId", (object?)record.SessionId ?? "unknown" },
{ "previousTenantId", (object?)record.TenantId ?? "unknown" },
{ "previousAgentVersion", (object?)record.AgentVersion ?? "unknown" },
{ "trigger", record.Trigger ?? "unknown" },
{ "exceptionType", (object?)record.ExceptionType ?? "unknown" },
{ "exceptionMessage", (object?)record.ExceptionMessage ?? "" },
{ "stackTrace", (object?)record.StackTrace ?? "" },
{ "dumpFilePath", (object?)record.DumpFilePath ?? "(no dump)" },
{ "dumpFileSizeBytes", (object?)record.DumpFileSizeBytes ?? 0 },
{ "dumpWriteSucceeded", record.DumpWriteSucceeded },
},
ImmediateUpload = true,
};
private static string BuildMessage(CrashRecord record)
{
var type = record.ExceptionType ?? "unknown";
var dumpInfo = record.DumpWriteSucceeded
? $"dump captured ({record.DumpFileSizeBytes ?? 0} bytes)"
: "no dump";
return $"Previous agent process crashed: {type} via {record.Trigger}; {dumpInfo}";
}
}
}
@@ -122,6 +122,16 @@ namespace AutopilotMonitor.Agent.V2
// shutdown during self-update still produces a "clean" classification.
RegisterCleanExitMarker(dataDirectory);
// Register crash-dump handlers as early as possible — before SelfUpdate, before
// any other startup work — so an unhandled exception anywhere downstream still
// produces a MiniDump + CrashRecord. SessionId/TenantId are unknown at this point;
// they're filled in later by AgentRuntimeHost via a second RegisterHandlers call.
AutopilotMonitor.Agent.V2.Core.Diagnostics.CrashDumpCapture.RegisterHandlers(
programDataDirectory: dataDirectory,
sessionId: "(pre-runtime)",
tenantId: "(pre-runtime)",
agentVersion: GetAgentVersion());
// Startup self-update. Legacy parity: cleanup leftover .old files, load cached
// backend hash / downgrade policy, then attempt the update. Failures here never
// abort startup — we prefer to run the current version than delay.
@@ -2,6 +2,7 @@ using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Agent.V2.Core.Configuration;
using AutopilotMonitor.Agent.V2.Core.Diagnostics;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
@@ -558,6 +559,25 @@ namespace AutopilotMonitor.Agent.V2.Runtime
{
logger.Warning($"StartupEnvironmentProbes: outer exception: {ex.Message}");
}
// PR2 — Crash-dump capture: refresh the handler labels with the now-known
// session/tenant, then drain pending crash records from previous runs
// into the timeline as previous_crash_detected events.
try
{
var programData = Environment.ExpandEnvironmentVariables(@"%ProgramData%\AutopilotMonitor");
CrashDumpCapture.RegisterHandlers(
programDataDirectory: programData,
sessionId: agentConfig.SessionId ?? string.Empty,
tenantId: agentConfig.TenantId ?? string.Empty,
agentVersion: System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown");
PendingCrashReporter.ScanAndEmit(agentConfig, programData, logger, lifecyclePost);
}
catch (Exception ex)
{
logger.Warning($"PendingCrashReporter: outer exception: {ex.Message}");
}
});
logger.Info($"V2 agent runtime ready (session={agentConfig.SessionId}, tenant={agentConfig.TenantId}).");
@@ -1 +1 @@
894
898