diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs b/src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs index a73a376d..786f52e5 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Configuration/AgentConfiguration.cs @@ -152,6 +152,16 @@ namespace AutopilotMonitor.Agent.Core.Configuration /// public int AgentMaxLifetimeMinutes { get; set; } = 360; + /// + /// Absolute maximum session age in hours across all agent restarts. + /// Emergency break: if the session has been alive longer than this, the agent + /// forces cleanup and self-destructs regardless of enrollment state. + /// Prevents zombie agents caused by unrecoverable logic errors. + /// Respects WhiteGlove scenarios (timer resets on Part 2 resume). + /// Default: 48 hours. + /// + public int AbsoluteMaxSessionHours { get; set; } = 48; + /// /// Bootstrap token for pre-MDM auth during OOBE. /// Embedded by the OOBE bootstrapper script via --bootstrap-token CLI arg. diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs index 3fdccda8..94df7dfe 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs @@ -717,6 +717,20 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors var eventTimestamp = (record.TimeCreated ?? DateTime.UtcNow).ToUniversalTime(); + var eventData = new Dictionary + { + { "windowsEventId", eventId }, + { "providerName", record.ProviderName ?? "" }, + { "description", description }, + { "eventLogChannel", ShellCoreEventLogChannel }, + { "eventTime", eventTimestamp.ToString("o") } + }; + + if (eventType == "esp_failure" && _detectedEspFailureType != null) + { + eventData["failureType"] = _detectedEspFailureType; + } + _onEventCollected(new EnrollmentEvent { SessionId = _sessionId, @@ -727,14 +741,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors Source = "EspAndHelloTracker", Phase = EnrollmentPhase.Unknown, // Let EnrollmentTracker decide phase Message = message, - Data = new Dictionary - { - { "windowsEventId", eventId }, - { "providerName", record.ProviderName ?? "" }, - { "description", description }, - { "eventLogChannel", ShellCoreEventLogChannel }, - { "eventTime", eventTimestamp.ToString("o") } - } + Data = eventData }); _logger.Info($"Shell-Core event detected: {eventType} (EventID {eventId})"); @@ -746,7 +753,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors { FinalizingSetupPhaseTriggered?.Invoke(this, finalizingSetupReason); } - catch { } + catch (Exception ex) { _logger.Error("FinalizingSetupPhaseTriggered handler failed", ex); } } // Fire WhiteGloveCompleted if this was a WhiteGlove success event. @@ -758,7 +765,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors { WhiteGloveCompleted?.Invoke(this, EventArgs.Empty); } - catch { } + catch (Exception ex) { _logger.Error("WhiteGloveCompleted handler failed", ex); } } // Fire EspFailureDetected if this was an ESP failure event. @@ -772,7 +779,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors { EspFailureDetected?.Invoke(this, failureType); } - catch { } + catch (Exception ex) { _logger.Error($"EspFailureDetected handler failed for '{failureType}'", ex); } } } catch (Exception ex) @@ -1046,7 +1053,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors { _espExitDetected = true; _logger.Info("Backfill: ESP exit event found in recent Shell-Core logs"); - try { FinalizingSetupPhaseTriggered?.Invoke(this, "esp_exiting"); } catch { } + try { FinalizingSetupPhaseTriggered?.Invoke(this, "esp_exiting"); } catch (Exception ex) { _logger.Error("Backfill: FinalizingSetupPhaseTriggered handler failed", ex); } } } } @@ -1061,7 +1068,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors { var failureType = ExtractEspFailureType(description); _logger.Info($"Backfill: ESP failure event found in recent Shell-Core logs: {failureType}"); - try { EspFailureDetected?.Invoke(this, failureType); } catch { } + try { EspFailureDetected?.Invoke(this, failureType); } catch (Exception ex) { _logger.Error($"Backfill: EspFailureDetected handler failed for '{failureType}'", ex); } } } } diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs index 1d8ebefb..f12d39d4 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs @@ -183,6 +183,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core Message = "WhiteGlove Part 2 — user enrollment started after pre-provisioning" }); _sessionPersistence.ClearWhiteGloveComplete(); + _sessionPersistence.ResetSessionCreatedAt(); } // Collect and emit device geo-location asynchronously — HTTP call, not on critical path diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs index 9824ec3a..5f7acea6 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/SessionPersistence.cs @@ -11,6 +11,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core private readonly string _sessionFilePath; private readonly string _sequenceFilePath; private readonly string _whiteGloveMarkerPath; + private readonly string _sessionCreatedPath; private readonly object _lockObject = new object(); public SessionPersistence(string dataDirectory) @@ -28,6 +29,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core _sessionFilePath = Path.Combine(dataDirectory, "session.id"); _sequenceFilePath = Path.Combine(dataDirectory, "session.seq"); _whiteGloveMarkerPath = Path.Combine(dataDirectory, "whiteglove.complete"); + _sessionCreatedPath = Path.Combine(dataDirectory, "session.created"); } /// @@ -55,6 +57,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core // Create new session ID if file doesn't exist or is invalid var newSessionId = Guid.NewGuid().ToString(); File.WriteAllText(_sessionFilePath, newSessionId); + SaveSessionCreatedAt(DateTime.UtcNow); return newSessionId; } catch (Exception ex) @@ -82,6 +85,8 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core File.Delete(_sequenceFilePath); if (File.Exists(_whiteGloveMarkerPath)) File.Delete(_whiteGloveMarkerPath); + if (File.Exists(_sessionCreatedPath)) + File.Delete(_sessionCreatedPath); } catch (Exception) { @@ -176,6 +181,48 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core } } + /// + /// Persists the session creation timestamp (UTC). Called once when a new session is created. + /// + public void SaveSessionCreatedAt(DateTime utcTimestamp) + { + lock (_lockObject) + { + try { File.WriteAllText(_sessionCreatedPath, utcTimestamp.ToString("O")); } + catch { } + } + } + + /// + /// Loads the persisted session creation timestamp. Returns null if file is missing or invalid. + /// + public DateTime? LoadSessionCreatedAt() + { + lock (_lockObject) + { + try + { + if (File.Exists(_sessionCreatedPath)) + { + var content = File.ReadAllText(_sessionCreatedPath).Trim(); + if (DateTime.TryParse(content, null, System.Globalization.DateTimeStyles.RoundtripKind, out var dt)) + return dt.ToUniversalTime(); + } + } + catch { } + return null; + } + } + + /// + /// Resets the session creation timestamp to now (UTC). + /// Used on WhiteGlove Part 2 resume to give the user enrollment phase a fresh timer. + /// + public void ResetSessionCreatedAt() + { + SaveSessionCreatedAt(DateTime.UtcNow); + } + /// /// Removes the WhiteGlove marker after the whiteglove_resumed event has been emitted. /// diff --git a/src/Agent/AutopilotMonitor.Agent/Program.cs b/src/Agent/AutopilotMonitor.Agent/Program.cs index c326941a..f1b87147 100644 --- a/src/Agent/AutopilotMonitor.Agent/Program.cs +++ b/src/Agent/AutopilotMonitor.Agent/Program.cs @@ -440,6 +440,13 @@ namespace AutopilotMonitor.Agent return; } + // Emergency break: absolute session age check across restarts. + // Prevents zombie agents that stay on the device forever due to logic errors. + if (CheckSessionAgeEmergencyBreak(config, logger, consoleMode)) + { + return; + } + if (consoleMode) { Console.WriteLine($"Agent Version: {agentVersion}"); @@ -870,6 +877,82 @@ namespace AutopilotMonitor.Agent return true; } + /// + /// Emergency break: checks if the current session has been alive longer than the absolute maximum. + /// Prevents zombie agents that stay on the device forever due to logic errors preventing + /// normal enrollment completion. Respects WhiteGlove scenarios (skips check during resume). + /// Returns true if emergency break triggered and agent should exit. + /// + static bool CheckSessionAgeEmergencyBreak(AgentConfiguration config, AgentLogger logger, bool consoleMode) + { + try + { + var dataDir = Environment.ExpandEnvironmentVariables(Constants.AgentDataDirectory); + var persistence = new SessionPersistence(dataDir); + + // Skip check if WhiteGlove-paused (device powered off between Part 1 and Part 2). + // The timer will be reset when MonitoringService.Start() processes the WhiteGlove resume. + if (persistence.IsWhiteGloveResume()) + { + logger.Info("Emergency break: WhiteGlove resume detected — skipping session age check"); + return false; + } + + var sessionCreatedAt = persistence.LoadSessionCreatedAt(); + if (sessionCreatedAt == null) + { + // Older agent without session.created — initialize the file now. + // The emergency break will only trigger on subsequent restarts. + if (persistence.SessionExists()) + { + persistence.SaveSessionCreatedAt(DateTime.UtcNow); + logger.Info("Emergency break: Initialized session.created for existing session"); + } + return false; + } + + var sessionAgeHours = (DateTime.UtcNow - sessionCreatedAt.Value).TotalHours; + var maxAgeHours = config.AbsoluteMaxSessionHours; + + if (sessionAgeHours <= maxAgeHours) + { + logger.Info($"Emergency break: Session age {sessionAgeHours:F1}h within limit ({maxAgeHours}h)"); + return false; + } + + // Session too old — emergency self-destruct + logger.Warning($"EMERGENCY BREAK: Session age {sessionAgeHours:F1}h exceeds maximum {maxAgeHours}h — forcing cleanup"); + + if (consoleMode) + Console.WriteLine($"EMERGENCY: Session is {sessionAgeHours:F1}h old (max: {maxAgeHours}h). Forcing cleanup."); + + // Write enrollment-complete marker so next start exits cleanly + var stateDir = Environment.ExpandEnvironmentVariables(Constants.StateDirectory); + Directory.CreateDirectory(stateDir); + var markerPath = Path.Combine(stateDir, "enrollment-complete.marker"); + File.WriteAllText(markerPath, $"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)"); + + if (config.SelfDestructOnComplete) + { + using (var service = new MonitoringService(config, logger, GetAgentVersion())) + { + service.TriggerCleanup(); + } + } + + // Clean up session files + persistence.DeleteSession(); + + logger.Info("Emergency break: Cleanup completed. Agent will exit."); + return true; + } + catch (Exception ex) + { + logger.Error($"Emergency break check failed: {ex.Message}", ex); + return false; // Don't block startup on check failure + } + } + static string GetAgentVersion() { try diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt index 1cf253f9..b4249c47 100644 --- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt @@ -1 +1 @@ -238 +239 diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs index 550cca60..886ddaa8 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs @@ -303,6 +303,24 @@ namespace AutopilotMonitor.Functions.Functions ); _logger.LogWarning("{SessionPrefix} Status: Failed - {FailureReason} (transitioned={Transitioned})", sessionPrefix, failureReason, statusTransitioned); } + else if (espFailureEvent != null) + { + // Fallback: esp_failure arrived without enrollment_failed — + // agent-side handler likely threw an exception (catch {} swallowed it). + failureReason = espFailureEvent.Message ?? "ESP failure (backend fallback)"; + statusTransitioned = await _storageService.UpdateSessionStatusAsync( + request.TenantId, + request.SessionId, + SessionStatus.Failed, + espFailureEvent.Phase, + failureReason, + completedAt: espFailureEvent.Timestamp, + earliestEventTimestamp: earliestEventTimestamp, + latestEventTimestamp: latestEventTimestamp + ); + _logger.LogWarning("{SessionPrefix} Status: Failed via esp_failure fallback - {FailureReason} (transitioned={Transitioned})", + sessionPrefix, failureReason, statusTransitioned); + } else if (gatherCompletionEvent != null) { // Gather rules collection completed — mark session as Succeeded.