fix esp_failure detection, added Agent - Emergency Break

src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs

@@ -152,6 +152,16 @@ namespace AutopilotMonitor.Agent.Core.Configuration
         /// </summary>
         public int AgentMaxLifetimeMinutes { get; set; } = 360;
 
+        /// <summary>
+        /// Absolute maximum session age in hours across all agent restarts.
+        /// Emergency break: if the session has been alive longer than this, the agent
+        /// forces cleanup and self-destructs regardless of enrollment state.
+        /// Prevents zombie agents caused by unrecoverable logic errors.
+        /// Respects WhiteGlove scenarios (timer resets on Part 2 resume).
+        /// Default: 48 hours.
+        /// </summary>
+        public int AbsoluteMaxSessionHours { get; set; } = 48;
+
         /// <summary>
         /// Bootstrap token for pre-MDM auth during OOBE.
         /// Embedded by the OOBE bootstrapper script via --bootstrap-token CLI arg.

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs

@@ -717,6 +717,20 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
 
                 var eventTimestamp = (record.TimeCreated ?? DateTime.UtcNow).ToUniversalTime();
 
+                var eventData = new Dictionary<string, object>
+                {
+                    { "windowsEventId", eventId },
+                    { "providerName", record.ProviderName ?? "" },
+                    { "description", description },
+                    { "eventLogChannel", ShellCoreEventLogChannel },
+                    { "eventTime", eventTimestamp.ToString("o") }
+                };
+
+                if (eventType == "esp_failure" && _detectedEspFailureType != null)
+                {
+                    eventData["failureType"] = _detectedEspFailureType;
+                }
+
                 _onEventCollected(new EnrollmentEvent
                 {
                     SessionId = _sessionId,
@@ -727,14 +741,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
                     Source = "EspAndHelloTracker",
                     Phase = EnrollmentPhase.Unknown, // Let EnrollmentTracker decide phase
                     Message = message,
-                    Data = new Dictionary<string, object>
-                    {
-                        { "windowsEventId", eventId },
-                        { "providerName", record.ProviderName ?? "" },
-                        { "description", description },
-                        { "eventLogChannel", ShellCoreEventLogChannel },
-                        { "eventTime", eventTimestamp.ToString("o") }
-                    }
+                    Data = eventData
                 });
 
                 _logger.Info($"Shell-Core event detected: {eventType} (EventID {eventId})");
@@ -746,7 +753,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
                     {
                         FinalizingSetupPhaseTriggered?.Invoke(this, finalizingSetupReason);
                     }
-                    catch { }
+                    catch (Exception ex) { _logger.Error("FinalizingSetupPhaseTriggered handler failed", ex); }
                 }
 
                 // Fire WhiteGloveCompleted if this was a WhiteGlove success event.
@@ -758,7 +765,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
                     {
                         WhiteGloveCompleted?.Invoke(this, EventArgs.Empty);
                     }
-                    catch { }
+                    catch (Exception ex) { _logger.Error("WhiteGloveCompleted handler failed", ex); }
                 }
 
                 // Fire EspFailureDetected if this was an ESP failure event.
@@ -772,7 +779,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
                     {
                         EspFailureDetected?.Invoke(this, failureType);
                     }
-                    catch { }
+                    catch (Exception ex) { _logger.Error($"EspFailureDetected handler failed for '{failureType}'", ex); }
                 }
             }
             catch (Exception ex)
@@ -1046,7 +1053,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
                                     {
                                         _espExitDetected = true;
                                         _logger.Info("Backfill: ESP exit event found in recent Shell-Core logs");
-                                        try { FinalizingSetupPhaseTriggered?.Invoke(this, "esp_exiting"); } catch { }
+                                        try { FinalizingSetupPhaseTriggered?.Invoke(this, "esp_exiting"); } catch (Exception ex) { _logger.Error("Backfill: FinalizingSetupPhaseTriggered handler failed", ex); }
                                     }
                                 }
                             }
@@ -1061,7 +1068,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
                             {
                                 var failureType = ExtractEspFailureType(description);
                                 _logger.Info($"Backfill: ESP failure event found in recent Shell-Core logs: {failureType}");
-                                try { EspFailureDetected?.Invoke(this, failureType); } catch { }
+                                try { EspFailureDetected?.Invoke(this, failureType); } catch (Exception ex) { _logger.Error($"Backfill: EspFailureDetected handler failed for '{failureType}'", ex); }
                             }
                         }
                     }

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs

@@ -183,6 +183,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
                     Message = "WhiteGlove Part 2 — user enrollment started after pre-provisioning"
                 });
                 _sessionPersistence.ClearWhiteGloveComplete();
+                _sessionPersistence.ResetSessionCreatedAt();
             }
 
             // Collect and emit device geo-location asynchronously — HTTP call, not on critical path

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs

@@ -11,6 +11,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
         private readonly string _sessionFilePath;
         private readonly string _sequenceFilePath;
         private readonly string _whiteGloveMarkerPath;
+        private readonly string _sessionCreatedPath;
         private readonly object _lockObject = new object();
 
         public SessionPersistence(string dataDirectory)
@@ -28,6 +29,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
             _sessionFilePath = Path.Combine(dataDirectory, "session.id");
             _sequenceFilePath = Path.Combine(dataDirectory, "session.seq");
             _whiteGloveMarkerPath = Path.Combine(dataDirectory, "whiteglove.complete");
+            _sessionCreatedPath = Path.Combine(dataDirectory, "session.created");
         }
 
         /// <summary>
@@ -55,6 +57,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
                     // Create new session ID if file doesn't exist or is invalid
                     var newSessionId = Guid.NewGuid().ToString();
                     File.WriteAllText(_sessionFilePath, newSessionId);
+                    SaveSessionCreatedAt(DateTime.UtcNow);
                     return newSessionId;
                 }
                 catch (Exception ex)
@@ -82,6 +85,8 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
                         File.Delete(_sequenceFilePath);
                     if (File.Exists(_whiteGloveMarkerPath))
                         File.Delete(_whiteGloveMarkerPath);
+                    if (File.Exists(_sessionCreatedPath))
+                        File.Delete(_sessionCreatedPath);
                 }
                 catch (Exception)
                 {
@@ -176,6 +181,48 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core
             }
         }
 
+        /// <summary>
+        /// Persists the session creation timestamp (UTC). Called once when a new session is created.
+        /// </summary>
+        public void SaveSessionCreatedAt(DateTime utcTimestamp)
+        {
+            lock (_lockObject)
+            {
+                try { File.WriteAllText(_sessionCreatedPath, utcTimestamp.ToString("O")); }
+                catch { }
+            }
+        }
+
+        /// <summary>
+        /// Loads the persisted session creation timestamp. Returns null if file is missing or invalid.
+        /// </summary>
+        public DateTime? LoadSessionCreatedAt()
+        {
+            lock (_lockObject)
+            {
+                try
+                {
+                    if (File.Exists(_sessionCreatedPath))
+                    {
+                        var content = File.ReadAllText(_sessionCreatedPath).Trim();
+                        if (DateTime.TryParse(content, null, System.Globalization.DateTimeStyles.RoundtripKind, out var dt))
+                            return dt.ToUniversalTime();
+                    }
+                }
+                catch { }
+                return null;
+            }
+        }
+
+        /// <summary>
+        /// Resets the session creation timestamp to now (UTC).
+        /// Used on WhiteGlove Part 2 resume to give the user enrollment phase a fresh timer.
+        /// </summary>
+        public void ResetSessionCreatedAt()
+        {
+            SaveSessionCreatedAt(DateTime.UtcNow);
+        }
+
         /// <summary>
         /// Removes the WhiteGlove marker after the whiteglove_resumed event has been emitted.
         /// </summary>

src/Agent/AutopilotMonitor.Agent/Program.cs

@@ -440,6 +440,13 @@ namespace AutopilotMonitor.Agent
                     return;
                 }
 
+                // Emergency break: absolute session age check across restarts.
+                // Prevents zombie agents that stay on the device forever due to logic errors.
+                if (CheckSessionAgeEmergencyBreak(config, logger, consoleMode))
+                {
+                    return;
+                }
+
                 if (consoleMode)
                 {
                     Console.WriteLine($"Agent Version: {agentVersion}");
@@ -870,6 +877,82 @@ namespace AutopilotMonitor.Agent
             return true;
         }
 
+        /// <summary>
+        /// Emergency break: checks if the current session has been alive longer than the absolute maximum.
+        /// Prevents zombie agents that stay on the device forever due to logic errors preventing
+        /// normal enrollment completion. Respects WhiteGlove scenarios (skips check during resume).
+        /// Returns true if emergency break triggered and agent should exit.
+        /// </summary>
+        static bool CheckSessionAgeEmergencyBreak(AgentConfiguration config, AgentLogger logger, bool consoleMode)
+        {
+            try
+            {
+                var dataDir = Environment.ExpandEnvironmentVariables(Constants.AgentDataDirectory);
+                var persistence = new SessionPersistence(dataDir);
+
+                // Skip check if WhiteGlove-paused (device powered off between Part 1 and Part 2).
+                // The timer will be reset when MonitoringService.Start() processes the WhiteGlove resume.
+                if (persistence.IsWhiteGloveResume())
+                {
+                    logger.Info("Emergency break: WhiteGlove resume detected — skipping session age check");
+                    return false;
+                }
+
+                var sessionCreatedAt = persistence.LoadSessionCreatedAt();
+                if (sessionCreatedAt == null)
+                {
+                    // Older agent without session.created — initialize the file now.
+                    // The emergency break will only trigger on subsequent restarts.
+                    if (persistence.SessionExists())
+                    {
+                        persistence.SaveSessionCreatedAt(DateTime.UtcNow);
+                        logger.Info("Emergency break: Initialized session.created for existing session");
+                    }
+                    return false;
+                }
+
+                var sessionAgeHours = (DateTime.UtcNow - sessionCreatedAt.Value).TotalHours;
+                var maxAgeHours = config.AbsoluteMaxSessionHours;
+
+                if (sessionAgeHours <= maxAgeHours)
+                {
+                    logger.Info($"Emergency break: Session age {sessionAgeHours:F1}h within limit ({maxAgeHours}h)");
+                    return false;
+                }
+
+                // Session too old — emergency self-destruct
+                logger.Warning($"EMERGENCY BREAK: Session age {sessionAgeHours:F1}h exceeds maximum {maxAgeHours}h — forcing cleanup");
+
+                if (consoleMode)
+                    Console.WriteLine($"EMERGENCY: Session is {sessionAgeHours:F1}h old (max: {maxAgeHours}h). Forcing cleanup.");
+
+                // Write enrollment-complete marker so next start exits cleanly
+                var stateDir = Environment.ExpandEnvironmentVariables(Constants.StateDirectory);
+                Directory.CreateDirectory(stateDir);
+                var markerPath = Path.Combine(stateDir, "enrollment-complete.marker");
+                File.WriteAllText(markerPath, $"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)");
+
+                if (config.SelfDestructOnComplete)
+                {
+                    using (var service = new MonitoringService(config, logger, GetAgentVersion()))
+                    {
+                        service.TriggerCleanup();
+                    }
+                }
+
+                // Clean up session files
+                persistence.DeleteSession();
+
+                logger.Info("Emergency break: Cleanup completed. Agent will exit.");
+                return true;
+            }
+            catch (Exception ex)
+            {
+                logger.Error($"Emergency break check failed: {ex.Message}", ex);
+                return false; // Don't block startup on check failure
+            }
+        }
+
         static string GetAgentVersion()
         {
             try

src/Agent/AutopilotMonitor.Agent/buildcounter.txt

@@ -1 +1 @@
-238
+239

src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs

@@ -303,6 +303,24 @@ namespace AutopilotMonitor.Functions.Functions
                     );
                     _logger.LogWarning("{SessionPrefix} Status: Failed - {FailureReason} (transitioned={Transitioned})", sessionPrefix, failureReason, statusTransitioned);
                 }
+                else if (espFailureEvent != null)
+                {
+                    // Fallback: esp_failure arrived without enrollment_failed —
+                    // agent-side handler likely threw an exception (catch {} swallowed it).
+                    failureReason = espFailureEvent.Message ?? "ESP failure (backend fallback)";
+                    statusTransitioned = await _storageService.UpdateSessionStatusAsync(
+                        request.TenantId,
+                        request.SessionId,
+                        SessionStatus.Failed,
+                        espFailureEvent.Phase,
+                        failureReason,
+                        completedAt: espFailureEvent.Timestamp,
+                        earliestEventTimestamp: earliestEventTimestamp,
+                        latestEventTimestamp: latestEventTimestamp
+                    );
+                    _logger.LogWarning("{SessionPrefix} Status: Failed via esp_failure fallback - {FailureReason} (transitioned={Transitioned})",
+                        sessionPrefix, failureReason, statusTransitioned);
+                }
                 else if (gatherCompletionEvent != null)
                 {
                     // Gather rules collection completed — mark session as Succeeded.