diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs b/src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs
index a73a376d..786f52e5 100644
--- a/src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs
+++ b/src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs
@@ -152,6 +152,16 @@ namespace AutopilotMonitor.Agent.Core.Configuration
///
public int AgentMaxLifetimeMinutes { get; set; } = 360;
+ ///
+ /// Absolute maximum session age in hours across all agent restarts.
+ /// Emergency break: if the session has been alive longer than this, the agent
+ /// forces cleanup and self-destructs regardless of enrollment state.
+ /// Prevents zombie agents caused by unrecoverable logic errors.
+ /// Respects WhiteGlove scenarios (timer resets on Part 2 resume).
+ /// Default: 48 hours.
+ ///
+ public int AbsoluteMaxSessionHours { get; set; } = 48;
+
///
/// Bootstrap token for pre-MDM auth during OOBE.
/// Embedded by the OOBE bootstrapper script via --bootstrap-token CLI arg.
diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs
index 3fdccda8..94df7dfe 100644
--- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs
+++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs
@@ -717,6 +717,20 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
var eventTimestamp = (record.TimeCreated ?? DateTime.UtcNow).ToUniversalTime();
+ var eventData = new Dictionary
+ {
+ { "windowsEventId", eventId },
+ { "providerName", record.ProviderName ?? "" },
+ { "description", description },
+ { "eventLogChannel", ShellCoreEventLogChannel },
+ { "eventTime", eventTimestamp.ToString("o") }
+ };
+
+ if (eventType == "esp_failure" && _detectedEspFailureType != null)
+ {
+ eventData["failureType"] = _detectedEspFailureType;
+ }
+
_onEventCollected(new EnrollmentEvent
{
SessionId = _sessionId,
@@ -727,14 +741,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
Source = "EspAndHelloTracker",
Phase = EnrollmentPhase.Unknown, // Let EnrollmentTracker decide phase
Message = message,
- Data = new Dictionary
- {
- { "windowsEventId", eventId },
- { "providerName", record.ProviderName ?? "" },
- { "description", description },
- { "eventLogChannel", ShellCoreEventLogChannel },
- { "eventTime", eventTimestamp.ToString("o") }
- }
+ Data = eventData
});
_logger.Info($"Shell-Core event detected: {eventType} (EventID {eventId})");
@@ -746,7 +753,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
{
FinalizingSetupPhaseTriggered?.Invoke(this, finalizingSetupReason);
}
- catch { }
+ catch (Exception ex) { _logger.Error("FinalizingSetupPhaseTriggered handler failed", ex); }
}
// Fire WhiteGloveCompleted if this was a WhiteGlove success event.
@@ -758,7 +765,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
{
WhiteGloveCompleted?.Invoke(this, EventArgs.Empty);
}
- catch { }
+ catch (Exception ex) { _logger.Error("WhiteGloveCompleted handler failed", ex); }
}
// Fire EspFailureDetected if this was an ESP failure event.
@@ -772,7 +779,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
{
EspFailureDetected?.Invoke(this, failureType);
}
- catch { }
+ catch (Exception ex) { _logger.Error($"EspFailureDetected handler failed for '{failureType}'", ex); }
}
}
catch (Exception ex)
@@ -1046,7 +1053,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
{
_espExitDetected = true;
_logger.Info("Backfill: ESP exit event found in recent Shell-Core logs");
- try { FinalizingSetupPhaseTriggered?.Invoke(this, "esp_exiting"); } catch { }
+ try { FinalizingSetupPhaseTriggered?.Invoke(this, "esp_exiting"); } catch (Exception ex) { _logger.Error("Backfill: FinalizingSetupPhaseTriggered handler failed", ex); }
}
}
}
@@ -1061,7 +1068,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
{
var failureType = ExtractEspFailureType(description);
_logger.Info($"Backfill: ESP failure event found in recent Shell-Core logs: {failureType}");
- try { EspFailureDetected?.Invoke(this, failureType); } catch { }
+ try { EspFailureDetected?.Invoke(this, failureType); } catch (Exception ex) { _logger.Error($"Backfill: EspFailureDetected handler failed for '{failureType}'", ex); }
}
}
}
diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs
index 1d8ebefb..f12d39d4 100644
--- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs
+++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs
@@ -183,6 +183,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
Message = "WhiteGlove Part 2 — user enrollment started after pre-provisioning"
});
_sessionPersistence.ClearWhiteGloveComplete();
+ _sessionPersistence.ResetSessionCreatedAt();
}
// Collect and emit device geo-location asynchronously — HTTP call, not on critical path
diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs
index 9824ec3a..5f7acea6 100644
--- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs
+++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs
@@ -11,6 +11,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
private readonly string _sessionFilePath;
private readonly string _sequenceFilePath;
private readonly string _whiteGloveMarkerPath;
+ private readonly string _sessionCreatedPath;
private readonly object _lockObject = new object();
public SessionPersistence(string dataDirectory)
@@ -28,6 +29,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
_sessionFilePath = Path.Combine(dataDirectory, "session.id");
_sequenceFilePath = Path.Combine(dataDirectory, "session.seq");
_whiteGloveMarkerPath = Path.Combine(dataDirectory, "whiteglove.complete");
+ _sessionCreatedPath = Path.Combine(dataDirectory, "session.created");
}
///
@@ -55,6 +57,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
// Create new session ID if file doesn't exist or is invalid
var newSessionId = Guid.NewGuid().ToString();
File.WriteAllText(_sessionFilePath, newSessionId);
+ SaveSessionCreatedAt(DateTime.UtcNow);
return newSessionId;
}
catch (Exception ex)
@@ -82,6 +85,8 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
File.Delete(_sequenceFilePath);
if (File.Exists(_whiteGloveMarkerPath))
File.Delete(_whiteGloveMarkerPath);
+ if (File.Exists(_sessionCreatedPath))
+ File.Delete(_sessionCreatedPath);
}
catch (Exception)
{
@@ -176,6 +181,48 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
}
}
+ ///
+ /// Persists the session creation timestamp (UTC). Called once when a new session is created.
+ ///
+ public void SaveSessionCreatedAt(DateTime utcTimestamp)
+ {
+ lock (_lockObject)
+ {
+ try { File.WriteAllText(_sessionCreatedPath, utcTimestamp.ToString("O")); }
+ catch { }
+ }
+ }
+
+ ///
+ /// Loads the persisted session creation timestamp. Returns null if file is missing or invalid.
+ ///
+ public DateTime? LoadSessionCreatedAt()
+ {
+ lock (_lockObject)
+ {
+ try
+ {
+ if (File.Exists(_sessionCreatedPath))
+ {
+ var content = File.ReadAllText(_sessionCreatedPath).Trim();
+ if (DateTime.TryParse(content, null, System.Globalization.DateTimeStyles.RoundtripKind, out var dt))
+ return dt.ToUniversalTime();
+ }
+ }
+ catch { }
+ return null;
+ }
+ }
+
+ ///
+ /// Resets the session creation timestamp to now (UTC).
+ /// Used on WhiteGlove Part 2 resume to give the user enrollment phase a fresh timer.
+ ///
+ public void ResetSessionCreatedAt()
+ {
+ SaveSessionCreatedAt(DateTime.UtcNow);
+ }
+
///
/// Removes the WhiteGlove marker after the whiteglove_resumed event has been emitted.
///
diff --git a/src/Agent/AutopilotMonitor.Agent/Program.cs b/src/Agent/AutopilotMonitor.Agent/Program.cs
index c326941a..f1b87147 100644
--- a/src/Agent/AutopilotMonitor.Agent/Program.cs
+++ b/src/Agent/AutopilotMonitor.Agent/Program.cs
@@ -440,6 +440,13 @@ namespace AutopilotMonitor.Agent
return;
}
+ // Emergency break: absolute session age check across restarts.
+ // Prevents zombie agents that stay on the device forever due to logic errors.
+ if (CheckSessionAgeEmergencyBreak(config, logger, consoleMode))
+ {
+ return;
+ }
+
if (consoleMode)
{
Console.WriteLine($"Agent Version: {agentVersion}");
@@ -870,6 +877,82 @@ namespace AutopilotMonitor.Agent
return true;
}
+ ///
+ /// Emergency break: checks if the current session has been alive longer than the absolute maximum.
+ /// Prevents zombie agents that stay on the device forever due to logic errors preventing
+ /// normal enrollment completion. Respects WhiteGlove scenarios (skips check during resume).
+ /// Returns true if emergency break triggered and agent should exit.
+ ///
+ static bool CheckSessionAgeEmergencyBreak(AgentConfiguration config, AgentLogger logger, bool consoleMode)
+ {
+ try
+ {
+ var dataDir = Environment.ExpandEnvironmentVariables(Constants.AgentDataDirectory);
+ var persistence = new SessionPersistence(dataDir);
+
+ // Skip check if WhiteGlove-paused (device powered off between Part 1 and Part 2).
+ // The timer will be reset when MonitoringService.Start() processes the WhiteGlove resume.
+ if (persistence.IsWhiteGloveResume())
+ {
+ logger.Info("Emergency break: WhiteGlove resume detected — skipping session age check");
+ return false;
+ }
+
+ var sessionCreatedAt = persistence.LoadSessionCreatedAt();
+ if (sessionCreatedAt == null)
+ {
+ // Older agent without session.created — initialize the file now.
+ // The emergency break will only trigger on subsequent restarts.
+ if (persistence.SessionExists())
+ {
+ persistence.SaveSessionCreatedAt(DateTime.UtcNow);
+ logger.Info("Emergency break: Initialized session.created for existing session");
+ }
+ return false;
+ }
+
+ var sessionAgeHours = (DateTime.UtcNow - sessionCreatedAt.Value).TotalHours;
+ var maxAgeHours = config.AbsoluteMaxSessionHours;
+
+ if (sessionAgeHours <= maxAgeHours)
+ {
+ logger.Info($"Emergency break: Session age {sessionAgeHours:F1}h within limit ({maxAgeHours}h)");
+ return false;
+ }
+
+ // Session too old — emergency self-destruct
+ logger.Warning($"EMERGENCY BREAK: Session age {sessionAgeHours:F1}h exceeds maximum {maxAgeHours}h — forcing cleanup");
+
+ if (consoleMode)
+ Console.WriteLine($"EMERGENCY: Session is {sessionAgeHours:F1}h old (max: {maxAgeHours}h). Forcing cleanup.");
+
+ // Write enrollment-complete marker so next start exits cleanly
+ var stateDir = Environment.ExpandEnvironmentVariables(Constants.StateDirectory);
+ Directory.CreateDirectory(stateDir);
+ var markerPath = Path.Combine(stateDir, "enrollment-complete.marker");
+ File.WriteAllText(markerPath, $"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)");
+
+ if (config.SelfDestructOnComplete)
+ {
+ using (var service = new MonitoringService(config, logger, GetAgentVersion()))
+ {
+ service.TriggerCleanup();
+ }
+ }
+
+ // Clean up session files
+ persistence.DeleteSession();
+
+ logger.Info("Emergency break: Cleanup completed. Agent will exit.");
+ return true;
+ }
+ catch (Exception ex)
+ {
+ logger.Error($"Emergency break check failed: {ex.Message}", ex);
+ return false; // Don't block startup on check failure
+ }
+ }
+
static string GetAgentVersion()
{
try
diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt
index 1cf253f9..b4249c47 100644
--- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt
+++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt
@@ -1 +1 @@
-238
+239
diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs
index 550cca60..886ddaa8 100644
--- a/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs
@@ -303,6 +303,24 @@ namespace AutopilotMonitor.Functions.Functions
);
_logger.LogWarning("{SessionPrefix} Status: Failed - {FailureReason} (transitioned={Transitioned})", sessionPrefix, failureReason, statusTransitioned);
}
+ else if (espFailureEvent != null)
+ {
+ // Fallback: esp_failure arrived without enrollment_failed —
+ // agent-side handler likely threw an exception (catch {} swallowed it).
+ failureReason = espFailureEvent.Message ?? "ESP failure (backend fallback)";
+ statusTransitioned = await _storageService.UpdateSessionStatusAsync(
+ request.TenantId,
+ request.SessionId,
+ SessionStatus.Failed,
+ espFailureEvent.Phase,
+ failureReason,
+ completedAt: espFailureEvent.Timestamp,
+ earliestEventTimestamp: earliestEventTimestamp,
+ latestEventTimestamp: latestEventTimestamp
+ );
+ _logger.LogWarning("{SessionPrefix} Status: Failed via esp_failure fallback - {FailureReason} (transitioned={Transitioned})",
+ sessionPrefix, failureReason, statusTransitioned);
+ }
else if (gatherCompletionEvent != null)
{
// Gather rules collection completed — mark session as Succeeded.