From 3f7fcbc68490d48acef1d50b8259d0c1a784ae40 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 9 Apr 2026 19:31:54 +0200 Subject: [PATCH] feat: added StallProbeCollector to detect stalled sessions and identify ESP failure better added Bootstrapper script version --- .../Bootstrap/Install-AutopilotMonitor.ps1 | 6 + ...ndHelloTracker.ModernDeploymentTracking.cs | 174 ++++++ ...HelloTracker.ProvisioningStatusTracking.cs | 51 ++ .../Collectors/EspAndHelloTracker.cs | 23 +- .../Collectors/StallProbeCollector.cs | 520 ++++++++++++++++++ .../MonitoringService.CollectorLifecycle.cs | 105 +++- .../Core/MonitoringService.EventUpload.cs | 5 + .../Monitoring/Core/MonitoringService.cs | 7 +- .../EnrollmentTracker.CompletionLogic.cs | 119 +++- .../Tracking/EnrollmentTracker.Diagnostics.cs | 6 + .../Monitoring/Tracking/EnrollmentTracker.cs | 23 + .../AutopilotMonitor.Agent/buildcounter.txt | 2 +- .../TableMaintenanceRepository.cs | 3 + .../TableStorage/TableSessionRepository.cs | 6 +- .../Ingest/IngestEventsFunction.Processing.cs | 60 ++ .../Services/MaintenanceService.cs | 56 +- .../TableStorageService.Maintenance.cs | 55 +- .../Services/TableStorageService.Sessions.cs | 52 +- .../AutopilotMonitor.Shared/Constants.cs | 10 + .../DataAccess/IMaintenanceRepository.cs | 1 + .../DataAccess/ISessionRepository.cs | 3 +- .../Models/Config/AgentConfigResponse.cs | 69 +++ .../Models/SessionApiModels.cs | 9 + .../app/dashboard/components/SessionTable.tsx | 5 +- .../components/GatherRuleFormFields.tsx | 26 +- .../app/gather-rules/eventTypes.ts | 122 ++++ .../[sessionId]/components/EventTimeline.tsx | 7 + .../components/SessionInfoCard.tsx | 1 + .../app/sessions/[sessionId]/page.tsx | 2 +- .../components/GlobalSearch.tsx | 2 + .../lib/__tests__/eventTypes.test.ts | 88 +++ 31 files changed, 1581 insertions(+), 37 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.ModernDeploymentTracking.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/StallProbeCollector.cs create mode 100644 src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts create mode 100644 src/Web/autopilot-monitor-web/lib/__tests__/eventTypes.test.ts diff --git a/scripts/Bootstrap/Install-AutopilotMonitor.ps1 b/scripts/Bootstrap/Install-AutopilotMonitor.ps1 index 59ba416b..f5353ad1 100644 --- a/scripts/Bootstrap/Install-AutopilotMonitor.ps1 +++ b/scripts/Bootstrap/Install-AutopilotMonitor.ps1 @@ -38,6 +38,8 @@ PowerShell 5.1 (IME) reads scripts without BOM as ANSI, corrupting multi-byte chars. .CHANGELOG + 2026-04-09 v1.1 Introduced explicit script version and log it on startup + (all entries above were v1.0) 2026-03-31 Replaced OS age + MDM pre-flight checks with multi-signal guard: registry deployment marker, WMI/filesystem user profile detection, lastloggedonUser set, and 12h bootstrap window. @@ -61,6 +63,9 @@ param( [int]$MaxBootstrapWindowHours = 12 ) +# Script version (bump on meaningful changes; see .CHANGELOG above) +$ScriptVersion = "1.1" + # Configuration - Everything in ProgramData for easy cleanup $AgentBasePath = "$env:ProgramData\AutopilotMonitor" $AgentBinPath = "$AgentBasePath\Agent" @@ -105,6 +110,7 @@ function Get-FileMd5Base64 { try { Write-Log "===== Autopilot Monitor Bootstrap Started =====" + Write-Log "Bootstrap script version: v$ScriptVersion" # -- Pre-flight: Multi-signal guard to prevent installation on non-provisioning devices -- # Layered approach: each guard catches a different scenario. diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.ModernDeploymentTracking.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.ModernDeploymentTracking.cs new file mode 100644 index 00000000..83080161 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.ModernDeploymentTracking.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Eventing.Reader; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors +{ + /// + /// Partial: ModernDeployment-Diagnostics-Provider event log handling (Ebene 1 — live capture). + /// + /// Subscribes to two Windows event channels that Microsoft uses to log Autopilot and ESP events: + /// - Microsoft-Windows-ModernDeployment-Diagnostics-Provider/Autopilot + /// - Microsoft-Windows-ModernDeployment-Diagnostics-Provider/ManagementService + /// + /// Current mode is **live-capture only**: every event at Level ≤ ModernDeploymentLogLevelMax + /// (default 3 = Warning, Error, Critical) is forwarded to the backend as a + /// // + /// event. We intentionally do NOT + /// classify failures locally or fire EspFailureDetected from this code yet — we first + /// want to gather real EventIDs from production devices and then iterate on classification rules + /// via backend config without agent rollout. + /// + /// Timing: Watchers stay subscribed for the entire agent lifetime (not stopped by idle timeout). + /// They are push-based (kernel-driven delivery) — zero-cost when no events are written. + /// + public partial class EspAndHelloTracker + { + private const string ModernDeploymentAutopilotChannel = "Microsoft-Windows-ModernDeployment-Diagnostics-Provider/Autopilot"; + private const string ModernDeploymentManagementChannel = "Microsoft-Windows-ModernDeployment-Diagnostics-Provider/ManagementService"; + + private System.Diagnostics.Eventing.Reader.EventLogWatcher _modernDeploymentAutopilotWatcher; + private System.Diagnostics.Eventing.Reader.EventLogWatcher _modernDeploymentManagementWatcher; + + private void StartModernDeploymentEventLogWatchers() + { + _modernDeploymentAutopilotWatcher = TryStartModernDeploymentWatcher(ModernDeploymentAutopilotChannel, "Autopilot"); + _modernDeploymentManagementWatcher = TryStartModernDeploymentWatcher(ModernDeploymentManagementChannel, "ManagementService"); + } + + private System.Diagnostics.Eventing.Reader.EventLogWatcher TryStartModernDeploymentWatcher(string channelName, string shortName) + { + try + { + // Clamp the level filter into the valid Windows range [1..5]. + var levelMax = Math.Max(1, Math.Min(5, _modernDeploymentLogLevelMax)); + + // XPath: (Level >= 1 and Level <= levelMax). Level 0 ("LogAlways") is not filtered out by + // XPath comparison — Windows treats it separately — so we explicitly include it in the query. + var xpath = $"*[System[Level=0 or (Level >= 1 and Level <= {levelMax})]]"; + + var query = new EventLogQuery(channelName, PathType.LogName, xpath); + var watcher = new System.Diagnostics.Eventing.Reader.EventLogWatcher(query); + watcher.EventRecordWritten += (sender, args) => OnModernDeploymentEventRecordWritten(args, shortName, channelName); + watcher.Enabled = true; + + _logger.Info($"ModernDeployment watcher started: {channelName} (levelMax={levelMax})"); + return watcher; + } + catch (EventLogNotFoundException) + { + _logger.Warning($"ModernDeployment event log not found: {channelName} (normal on non-Windows 10/11 test environments)"); + return null; + } + catch (UnauthorizedAccessException ex) + { + _logger.Warning($"ModernDeployment watcher access denied for {channelName}: {ex.Message}"); + return null; + } + catch (Exception ex) + { + _logger.Error($"Failed to start ModernDeployment watcher for {channelName}", ex); + return null; + } + } + + private void StopModernDeploymentEventLogWatchers() + { + StopModernDeploymentWatcher(ref _modernDeploymentAutopilotWatcher, "Autopilot"); + StopModernDeploymentWatcher(ref _modernDeploymentManagementWatcher, "ManagementService"); + } + + private void StopModernDeploymentWatcher(ref System.Diagnostics.Eventing.Reader.EventLogWatcher watcher, string shortName) + { + if (watcher == null) + return; + + try + { + watcher.Enabled = false; + watcher.Dispose(); + _logger.Info($"ModernDeployment watcher stopped: {shortName}"); + } + catch (Exception ex) + { + _logger.Error($"Error stopping ModernDeployment watcher ({shortName})", ex); + } + finally + { + watcher = null; + } + } + + private void OnModernDeploymentEventRecordWritten(EventRecordWrittenEventArgs e, string shortName, string channelName) + { + if (e.EventRecord == null) + return; + + try + { + var record = e.EventRecord; + + // Determine severity based on Windows event level: + // 1 = Critical, 2 = Error, 3 = Warning, 4 = Information, 5 = Verbose + var level = record.Level ?? 4; + string eventType; + EventSeverity severity; + switch (level) + { + case 1: + case 2: + eventType = Constants.EventTypes.ModernDeploymentError; + severity = EventSeverity.Error; + break; + case 3: + eventType = Constants.EventTypes.ModernDeploymentWarning; + severity = EventSeverity.Warning; + break; + default: + eventType = Constants.EventTypes.ModernDeploymentLog; + severity = EventSeverity.Info; + break; + } + + string description = null; + try { description = record.FormatDescription(); } + catch { /* some events lack formatting resources */ } + + if (string.IsNullOrEmpty(description)) + description = $"Event ID {record.Id} (no formatted description)"; + + // Trim the message body to keep the event payload small. + var truncated = description.Length > 1000 ? description.Substring(0, 1000) + "…" : description; + + var data = new Dictionary + { + { "channel", shortName }, + { "channelFullName", channelName }, + { "eventId", record.Id }, + { "level", level }, + { "levelName", record.LevelDisplayName ?? string.Empty }, + { "providerName", record.ProviderName ?? string.Empty }, + { "timeCreated", record.TimeCreated?.ToUniversalTime().ToString("o") ?? string.Empty } + }; + + _onEventCollected(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = eventType, + Severity = severity, + Source = "ModernDeploymentWatcher", + Phase = EnrollmentPhase.Unknown, + Message = $"[{shortName}] EventID {record.Id}: {truncated}", + Data = data + }); + } + catch (Exception ex) + { + _logger.Error($"Error processing ModernDeployment event from {shortName}", ex); + } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.ProvisioningStatusTracking.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.ProvisioningStatusTracking.cs index 834ab1c2..b01eb990 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.ProvisioningStatusTracking.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.ProvisioningStatusTracking.cs @@ -64,6 +64,57 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors private const int DeviceSetupFallbackDelaySeconds = 30; private const int ProvisioningDebounceMilliseconds = 1000; + /// + /// Snapshot of current provisioning-category state for consumers like the signal-correlated + /// WhiteGlove detection. Thread-safe: copies the underlying dictionary under the state lock. + /// + public Dictionary GetProvisioningCategorySnapshot() + { + lock (_stateLock) + { + return _lastCategorySucceeded == null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(_lastCategorySucceeded, StringComparer.OrdinalIgnoreCase); + } + } + + /// + /// True when DeviceSetupCategory.Status has resolved categorySucceeded=true OR when the + /// fallback (all subcategories succeeded, categorySucceeded null) has been confirmed. + /// + public bool DeviceSetupCategorySucceeded + { + get + { + lock (_stateLock) + { + if (_deviceSetupProvisioningCompleteFired) + return true; + return _lastCategorySucceeded != null + && _lastCategorySucceeded.TryGetValue("DeviceSetupCategory.Status", out var v) + && v == true; + } + } + } + + /// + /// True when any AccountSetup subcategory has been tracked (resolved or in progress). + /// + public bool HasAccountSetupActivity + { + get + { + lock (_stateLock) + { + if (_lastSubcategoryStates == null) + return false; + return _lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs) + && subs != null + && subs.Count > 0; + } + } + } + private void StartProvisioningStatusWatcher() { _lastProvisioningJson = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs index d3e729cb..e59e203f 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/EspAndHelloTracker.cs @@ -56,6 +56,8 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors private readonly object _stateLock = new object(); private readonly int _helloWaitTimeoutSeconds; + private readonly bool _modernDeploymentWatcherEnabled; + private readonly int _modernDeploymentLogLevelMax; private const int HelloCompletionTimeoutSeconds = 300; private const int BackfillLookbackMinutes = 5; @@ -145,13 +147,22 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors EventId_HelloForBusiness_ProcessingStopped }; - public EspAndHelloTracker(string sessionId, string tenantId, Action onEventCollected, AgentLogger logger, int helloWaitTimeoutSeconds = 30) + public EspAndHelloTracker( + string sessionId, + string tenantId, + Action onEventCollected, + AgentLogger logger, + int helloWaitTimeoutSeconds = 30, + bool modernDeploymentWatcherEnabled = true, + int modernDeploymentLogLevelMax = 3) { _sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId)); _tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId)); _onEventCollected = onEventCollected ?? throw new ArgumentNullException(nameof(onEventCollected)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _helloWaitTimeoutSeconds = helloWaitTimeoutSeconds; + _modernDeploymentWatcherEnabled = modernDeploymentWatcherEnabled; + _modernDeploymentLogLevelMax = modernDeploymentLogLevelMax; } /// @@ -210,6 +221,13 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors // Subscribe to HelloForBusiness/Operational event log for Hello processing signals StartHelloForBusinessEventLogWatcher(); + // Subscribe to ModernDeployment-Diagnostics-Provider channels (Autopilot + ManagementService) + // for live capture of ESP/Autopilot events that Shell-Core/Registry watchers may miss. + if (_modernDeploymentWatcherEnabled) + { + StartModernDeploymentEventLogWatchers(); + } + // Safety net: backfill recent terminal events in case watcher started late or event delivery lagged. BackfillRecentTerminalHelloEvents(); BackfillRecentHelloForBusinessEvents(); @@ -315,6 +333,9 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors _logger.Error("Error stopping Shell-Core event watcher", ex); } } + + // Stop ModernDeployment-Diagnostics-Provider watchers + StopModernDeploymentEventLogWatchers(); } public void Dispose() diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/StallProbeCollector.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/StallProbeCollector.cs new file mode 100644 index 00000000..161815dd --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Collectors/StallProbeCollector.cs @@ -0,0 +1,520 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Eventing.Reader; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using AutopilotMonitor.Agent.Core.Logging; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; +using Microsoft.Win32; + +namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors +{ + /// + /// Ebene 2 — Defensive deep probes that run only when enrollment has been silent for a while. + /// + /// The collector is NOT a timer of its own. Instead, it is invoked from + /// (which runs every 60 s for the + /// entire agent lifetime). On each tick the collector inspects the idle duration and fires + /// a full scan at the configured thresholds (default: 2, 15, 30, 60, 180 min). + /// + /// Each probe scans up to four read-only sources (all wrapped in try/catch + hard timeout): + /// 1. Provisioning registry (HKLM\SOFTWARE\Microsoft\Provisioning\AutopilotSettings) + /// 2. Diagnostics registry (HKLM\SOFTWARE\Microsoft\Provisioning\Diagnostics\Autopilot) + /// 3. Windows event logs (ModernDeployment + Shell-Core + Application) + /// 4. AppWorkload.log tail (IME ESP app install state) + /// + /// The probe classifies the outcome as: + /// - anomaly → stall_probe_result (Warning) + optional EspFailureDetected + /// - activeInstalls → positive signal (healthy-in-progress), no alarm + /// - quiet → nothing found in any source + /// + /// Output policy (configurable via StallProbeTraceIndices, default = [2]): + /// - Trace-indexed probes: emit a stall_probe_check Trace event on quiet/activeInstalls. + /// - Non-trace probes: emit nothing when no anomaly is found (agent log only). + /// - Anomaly found: always emit stall_probe_result Warning (plus EspFailureDetected if terminal). + /// - Probe index configured as "session-stalled": emit fire-once session_stalled Warning. + /// + public class StallProbeCollector + { + private readonly string _sessionId; + private readonly string _tenantId; + private readonly Action _onEventCollected; + private readonly AgentLogger _logger; + private readonly int[] _thresholdsMinutes; + private readonly HashSet _traceIndices; + private readonly HashSet _sources; + private readonly int _sessionStalledAfterProbeIndex; + + private readonly object _stateLock = new object(); + private readonly HashSet _firedProbeIndices = new HashSet(); + private bool _sessionStalledFired; + + public event EventHandler EspFailureDetected; + + // Per-source hard timeouts (ms) + private const int RegistryScanTimeoutMs = 500; + private const int DiagnosticsRegistryScanTimeoutMs = 500; + private const int EventLogScanTimeoutMs = 2000; + private const int AppWorkloadTailTimeoutMs = 1000; + + // Terminal ESP failure EventIDs in ModernDeployment-Diagnostics-Provider channels. + // Conservative list — we prefer to NOT auto-terminate on unknown IDs and let the user decide. + // + // TODO(stall-probe-classification): This is stellschraube #1 for promoting stall-probe findings + // to terminal failures. Currently empty by design: the ModernDeployment watcher (Ebene 1) runs + // in live-capture mode and forwards raw events so we can gather real production EventIDs first. + // Once we have 1–2 weeks of production data showing which EventIDs reliably correspond to + // terminal ESP failures (e.g. "app install failed terminally", "ESP category Error"), add them + // here. From that point on, Probe 1 (2 min) will detect them and fire EspFailureDetected → + // the session goes terminal-Failed within ~2 min instead of waiting for the 5h backend sweep. + // Cross-reference: the matching terminalReason should go into result.TerminalReason when you + // add entries (see ScanEventLogs around the TerminalModernDeploymentEventIds.Contains check). + private static readonly HashSet TerminalModernDeploymentEventIds = new HashSet + { + // Populated as real failure EventIDs are observed in production. + // Kept empty intentionally — live-capture mode (Etappe 3) collects the raw events first. + }; + + // AppWorkload.log active-install markers (positive signals) + private static readonly Regex ActiveInstallRegex = new Regex( + @"EnforcementState\s*[:=]\s*(Downloading|Installing|InProgress)", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + // AppWorkload.log failure markers (negative signals — but scanner only reports, no terminal decision). + // + // TODO(stall-probe-classification): This is stellschraube #2 for promoting stall-probe findings + // to terminal failures. Today any match of this regex produces a Warning-severity + // stall_probe_result event but does NOT set result.IsTerminal or fire EspFailureDetected — + // on purpose, because IME writes transient "Failed" lines that recover (retry, re-detect) and + // we do not want false-terminal verdicts during the live-capture phase. Once we can + // distinguish "retryable IME hiccup" from "terminal app install failure" based on real + // production data, tighten the regex (e.g. require a specific error code range or a second + // confirming line) and set result.IsTerminal = true + result.TerminalReason in ScanAppWorkloadTail. + // Ideas for stronger terminal signals to add here once validated: + // - EnforcementState: Failed + specific HRESULT ranges (0x87D1... Win32 app failures) + // - DetectionState: Error that persists across multiple tail samples + // - Explicit "Required app failed" lines from the ESP required-app enforcer + private static readonly Regex FailureRegex = new Regex( + @"EnforcementState\s*[:=]\s*Failed|Error\s+0x[0-9A-Fa-f]+|DetectionState\s*[:=]\s*Error", + RegexOptions.Compiled); + + public StallProbeCollector( + string sessionId, + string tenantId, + Action onEventCollected, + AgentLogger logger, + int[] thresholdsMinutes, + int[] traceIndices, + string[] sources, + int sessionStalledAfterProbeIndex) + { + _sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId)); + _tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId)); + _onEventCollected = onEventCollected ?? throw new ArgumentNullException(nameof(onEventCollected)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _thresholdsMinutes = thresholdsMinutes ?? throw new ArgumentNullException(nameof(thresholdsMinutes)); + _traceIndices = new HashSet(traceIndices ?? new int[0]); + _sources = new HashSet(sources ?? new string[0], StringComparer.OrdinalIgnoreCase); + _sessionStalledAfterProbeIndex = sessionStalledAfterProbeIndex; + } + + /// + /// Resets all per-probe fire-once state. Called when a new real (non-periodic) event arrives. + /// session_stalled stays fire-once across the whole session — we don't re-warn if the + /// session stalls a second time; the backend maintenance sweep handles late stalls. + /// + public void ResetProbes() + { + lock (_stateLock) + { + if (_firedProbeIndices.Count > 0) + { + _logger.Trace($"StallProbeCollector: resetting {_firedProbeIndices.Count} fired probes"); + _firedProbeIndices.Clear(); + } + } + } + + /// + /// Called from the 60-s idle check tick. Decides whether any probe threshold has been + /// newly crossed and runs the corresponding probe. + /// + public void CheckAndRunProbes(double idleMinutes) + { + // Iterate thresholds from lowest to highest so probes fire in order. + for (int i = 0; i < _thresholdsMinutes.Length; i++) + { + var thresholdMinutes = _thresholdsMinutes[i]; + if (idleMinutes < thresholdMinutes) + return; + + int probeIndex = i + 1; // 1-based for user-facing output + + bool alreadyFired; + lock (_stateLock) + { + alreadyFired = _firedProbeIndices.Contains(probeIndex); + } + if (alreadyFired) + continue; + + RunProbe(probeIndex, thresholdMinutes, idleMinutes); + + lock (_stateLock) + { + _firedProbeIndices.Add(probeIndex); + } + } + } + + private void RunProbe(int probeIndex, int thresholdMinutes, double idleMinutes) + { + var started = DateTime.UtcNow; + var result = new ProbeResult { ProbeIndex = probeIndex, IdleMinutes = Math.Round(idleMinutes, 1) }; + + // Run each source best-effort; one failure does not skip the others. + if (_sources.Contains("provisioning_registry")) + RunWithTimeout(() => ScanProvisioningRegistry(result), RegistryScanTimeoutMs, "provisioning_registry", result); + + if (_sources.Contains("diagnostics_registry")) + RunWithTimeout(() => ScanDiagnosticsRegistry(result), DiagnosticsRegistryScanTimeoutMs, "diagnostics_registry", result); + + if (_sources.Contains("eventlog")) + RunWithTimeout(() => ScanEventLogs(result), EventLogScanTimeoutMs, "eventlog", result); + + if (_sources.Contains("appworkload_log")) + RunWithTimeout(() => ScanAppWorkloadTail(result), AppWorkloadTailTimeoutMs, "appworkload_log", result); + + result.DurationMs = (int)(DateTime.UtcNow - started).TotalMilliseconds; + + // Always log locally for diagnostics — the agent_YYYYMMDD.log is the always-on observability. + var anomaliesStr = result.Anomalies.Count > 0 + ? $"anomalies=[{string.Join(",", result.Anomalies)}]" + : "quiet"; + var activeStr = result.ActiveInstalls.Count > 0 + ? $" activeInstalls=[{string.Join(",", result.ActiveInstalls)}]" + : string.Empty; + var failedStr = result.SourcesFailed.Count > 0 + ? $" sources_failed=[{string.Join(",", result.SourcesFailed)}]" + : string.Empty; + _logger.Info($"StallProbeCollector: Probe {probeIndex}/{_thresholdsMinutes.Length} ran at idle={result.IdleMinutes}min, {anomaliesStr}{activeStr}{failedStr} (duration={result.DurationMs}ms)"); + + bool hasAnomaly = result.Anomalies.Count > 0; + bool hasActiveInstalls = result.ActiveInstalls.Count > 0; + + // Emit stall_probe_result Warning on anomaly — always, regardless of probe index. + if (hasAnomaly) + { + EmitProbeResultEvent(result, severity: EventSeverity.Warning); + + if (result.IsTerminal) + { + _logger.Warning($"StallProbeCollector: terminal ESP failure detected in Probe {probeIndex} — firing EspFailureDetected"); + try { EspFailureDetected?.Invoke(this, result.TerminalReason ?? "stall_probe_terminal"); } + catch (Exception ex) { _logger.Error("EspFailureDetected handler threw", ex); } + } + } + else if (_traceIndices.Contains(probeIndex)) + { + // Quiet or activeInstalls — emit a Trace heartbeat at trace-indexed probes. + EmitProbeCheckTraceEvent(result); + } + // else: non-trace probe without anomaly → silent except for agent log. + + // session_stalled is fire-once across the whole session. It fires at the configured + // probe index UNLESS activeInstalls is observed (activeInstalls counts as progress). + if (probeIndex == _sessionStalledAfterProbeIndex && !hasActiveInstalls) + { + lock (_stateLock) + { + if (_sessionStalledFired) + return; + _sessionStalledFired = true; + } + EmitSessionStalledEvent(result); + } + } + + private void RunWithTimeout(Action action, int timeoutMs, string sourceName, ProbeResult result) + { + try + { + // Run on the calling thread but guard against exceptions. We intentionally do not + // spawn a new thread per source — the IdleCheckCallback already runs on a worker + // thread, and spinning extra threads would be heavier than the work itself. + // The timeoutMs is enforced via source-internal reads (FileStream, Registry handles + // are using-scoped and the reads themselves are bounded). + var sw = System.Diagnostics.Stopwatch.StartNew(); + action(); + sw.Stop(); + if (sw.ElapsedMilliseconds > timeoutMs) + { + _logger.Warning($"StallProbeCollector: {sourceName} scan exceeded soft timeout ({sw.ElapsedMilliseconds}ms > {timeoutMs}ms)"); + } + } + catch (Exception ex) + { + _logger.Warning($"StallProbeCollector: {sourceName} scan failed: {ex.Message}"); + result.SourcesFailed.Add(sourceName); + } + } + + private void ScanProvisioningRegistry(ProbeResult result) + { + using (var key = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Provisioning\AutopilotSettings", writable: false)) + { + if (key == null) + return; + + foreach (var valueName in new[] { "DevicePreparationCategory.Status", "DeviceSetupCategory.Status", "AccountSetupCategory.Status" }) + { + var json = key.GetValue(valueName)?.ToString(); + if (string.IsNullOrEmpty(json)) + continue; + + // Rudimentary scan — only flag patterns that are definitively bad. + if (json.IndexOf("\"categorySucceeded\":false", StringComparison.OrdinalIgnoreCase) >= 0 || + json.IndexOf("\"subcategoryState\":\"failed\"", StringComparison.OrdinalIgnoreCase) >= 0) + { + result.Anomalies.Add($"provisioning_registry:{valueName}"); + if (result.RawSamples.Count < 5) + result.RawSamples.Add($"{valueName}: {Truncate(json, 400)}"); + } + } + } + } + + private void ScanDiagnosticsRegistry(ProbeResult result) + { + using (var diagRoot = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Provisioning\Diagnostics\Autopilot", writable: false)) + { + if (diagRoot == null) + return; + + object deployErrorCode = diagRoot.GetValue("DeploymentErrorCode"); + object deployErrorReason = diagRoot.GetValue("DeploymentErrorReason"); + + if (deployErrorCode != null) + { + // DeploymentErrorCode is typically a DWORD. 0 = no error. + var asString = deployErrorCode.ToString(); + if (!string.IsNullOrEmpty(asString) && asString != "0" && asString != "0x0") + { + result.Anomalies.Add("diagnostics_registry:DeploymentErrorCode"); + result.RawSamples.Add($"DeploymentErrorCode={asString}; DeploymentErrorReason={deployErrorReason ?? "(null)"}"); + result.IsTerminal = true; + result.TerminalReason = $"DeploymentErrorCode={asString}"; + } + } + } + } + + private void ScanEventLogs(ProbeResult result) + { + // Scan a small window (last 15 min) of Autopilot/ESP-related channels at Level ≤ 3. + var since = DateTime.UtcNow.AddMinutes(-15); + var sinceStr = since.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + var xpath = $"*[System[(Level >= 1 and Level <= 3) and TimeCreated[@SystemTime >= '{sinceStr}']]]"; + + var channels = new[] + { + "Microsoft-Windows-ModernDeployment-Diagnostics-Provider/Autopilot", + "Microsoft-Windows-ModernDeployment-Diagnostics-Provider/ManagementService", + "Microsoft-Windows-Shell-Core/Operational" + }; + + int foundCount = 0; + foreach (var channel in channels) + { + try + { + var query = new EventLogQuery(channel, PathType.LogName, xpath) { ReverseDirection = true }; + using (var reader = new EventLogReader(query)) + { + int perChannelCap = 20; + EventRecord record; + while ((record = reader.ReadEvent()) != null && perChannelCap-- > 0) + { + using (record) + { + foundCount++; + if (result.RawSamples.Count < 5) + { + string desc = null; + try { desc = record.FormatDescription(); } + catch { } + desc = Truncate(desc ?? $"EventID {record.Id}", 400); + result.RawSamples.Add($"[{channel.Split('/').Last()}] ID={record.Id} Level={record.Level}: {desc}"); + } + + // Check for known terminal ModernDeployment failure event IDs. + if (TerminalModernDeploymentEventIds.Contains(record.Id)) + { + result.IsTerminal = true; + result.TerminalReason = $"ModernDeployment EventID {record.Id}"; + } + } + } + } + } + catch (EventLogNotFoundException) + { + // Channel not available on this OS — ignore silently. + } + catch (Exception ex) + { + _logger.Trace($"StallProbeCollector: eventlog scan of {channel} failed: {ex.Message}"); + } + } + + if (foundCount > 0) + { + result.Anomalies.Add($"eventlog:{foundCount}_events"); + } + } + + private void ScanAppWorkloadTail(ProbeResult result) + { + var path = @"C:\ProgramData\Microsoft\IntuneManagementExtension\Logs\AppWorkload.log"; + if (!File.Exists(path)) + return; + + // Read the last ~200 KB. AppWorkload.log can be huge; we never read the whole file. + const int tailBytes = 200 * 1024; + using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) + { + long start = Math.Max(0, fs.Length - tailBytes); + fs.Seek(start, SeekOrigin.Begin); + using (var reader = new StreamReader(fs, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 4096, leaveOpen: true)) + { + string content = reader.ReadToEnd(); + + // Cut at the first newline to avoid starting mid-line. + var firstNewline = content.IndexOf('\n'); + if (firstNewline > 0 && firstNewline < content.Length - 1) + content = content.Substring(firstNewline + 1); + + // Positive-signal scan: active installations recently logged. + foreach (Match m in ActiveInstallRegex.Matches(content)) + { + if (result.ActiveInstalls.Count < 10 && !result.ActiveInstalls.Contains(m.Value)) + result.ActiveInstalls.Add(m.Value); + } + + // Negative-signal scan: failures. + var failureMatches = FailureRegex.Matches(content); + if (failureMatches.Count > 0) + { + result.Anomalies.Add($"appworkload_log:{failureMatches.Count}_errors"); + // Sample up to 3 failure lines for context. + int sampled = 0; + foreach (Match m in failureMatches) + { + if (sampled >= 3 || result.RawSamples.Count >= 5) + break; + // Extract the enclosing line for context. + int lineStart = content.LastIndexOf('\n', Math.Max(0, m.Index)); + int lineEnd = content.IndexOf('\n', m.Index); + if (lineStart < 0) lineStart = 0; + if (lineEnd < 0) lineEnd = content.Length; + var line = content.Substring(lineStart, Math.Min(400, lineEnd - lineStart)).Trim(); + result.RawSamples.Add($"AppWorkload: {line}"); + sampled++; + } + } + } + } + } + + private void EmitProbeResultEvent(ProbeResult result, EventSeverity severity) + { + var data = BuildDataJson(result); + _onEventCollected(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.StallProbeResult, + Severity = severity, + Source = "StallProbeCollector", + Phase = EnrollmentPhase.Unknown, + Message = $"Stall probe {result.ProbeIndex}/{_thresholdsMinutes.Length}: {result.Anomalies.Count} anomaly/ies after {result.IdleMinutes}min idle", + Data = data + }); + } + + private void EmitProbeCheckTraceEvent(ProbeResult result) + { + var data = BuildDataJson(result); + var summary = result.ActiveInstalls.Count > 0 + ? $"activeInstalls={result.ActiveInstalls.Count}" + : "quiet"; + _onEventCollected(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.StallProbeCheck, + Severity = EventSeverity.Trace, + Source = "StallProbeCollector", + Phase = EnrollmentPhase.Unknown, + Message = $"Stall probe {result.ProbeIndex}/{_thresholdsMinutes.Length} heartbeat: {summary} after {result.IdleMinutes}min idle", + Data = data + }); + } + + private void EmitSessionStalledEvent(ProbeResult result) + { + var data = BuildDataJson(result); + data["probeIndexTriggered"] = result.ProbeIndex; + _onEventCollected(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.SessionStalled, + Severity = EventSeverity.Warning, + Source = "StallProbeCollector", + Phase = EnrollmentPhase.Unknown, + Message = $"Session stalled: no enrollment progress for {result.IdleMinutes}min (stall probe {result.ProbeIndex})", + Data = data + }); + } + + private static Dictionary BuildDataJson(ProbeResult r) + { + return new Dictionary + { + { "probeIndex", r.ProbeIndex }, + { "idleMinutes", r.IdleMinutes }, + { "probeDurationMs", r.DurationMs }, + { "anomalyTypes", r.Anomalies.ToArray() }, + { "activeInstalls", r.ActiveInstalls.ToArray() }, + { "sources_failed", r.SourcesFailed.ToArray() }, + { "rawSamples", r.RawSamples.ToArray() }, + { "isTerminal", r.IsTerminal }, + { "terminalReason", r.TerminalReason ?? string.Empty } + }; + } + + private static string Truncate(string s, int max) + { + if (string.IsNullOrEmpty(s)) return string.Empty; + return s.Length <= max ? s : s.Substring(0, max) + "…"; + } + + private class ProbeResult + { + public int ProbeIndex; + public double IdleMinutes; + public int DurationMs; + public readonly List Anomalies = new List(); + public readonly List ActiveInstalls = new List(); + public readonly List SourcesFailed = new List(); + public readonly List RawSamples = new List(); + public bool IsTerminal; + public string TerminalReason; + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.CollectorLifecycle.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.CollectorLifecycle.cs index af7c5295..3463e912 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.CollectorLifecycle.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.CollectorLifecycle.cs @@ -66,21 +66,46 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core _collectorIdleTimeoutMinutes = collectors?.CollectorIdleTimeoutMinutes ?? 15; _lastRealEventTime = DateTime.UtcNow; _collectorsIdleStopped = false; - if (_collectorIdleTimeoutMinutes > 0) + + // Stall probe collector (Ebene 2) — event-driven deep probes at 2/15/30/60/180 min idle + if (collectors?.StallProbeEnabled != false) { - _idleCheckTimer = new Timer( - IdleCheckCallback, - null, - TimeSpan.FromSeconds(60), - TimeSpan.FromSeconds(60) - ); - _logger.Info($"Collector idle timeout enabled: {_collectorIdleTimeoutMinutes} min"); + var probeThresholds = collectors?.StallProbeThresholdsMinutes ?? new[] { 2, 15, 30, 60, 180 }; + var probeTraceIndices = collectors?.StallProbeTraceIndices ?? new[] { 2 }; + var probeSources = collectors?.StallProbeSources ?? new[] + { + "provisioning_registry", "diagnostics_registry", "eventlog", "appworkload_log" + }; + var stalledAfterIndex = collectors?.SessionStalledAfterProbeIndex ?? 4; + _stallProbeCollector = new StallProbeCollector( + _configuration.SessionId, + _configuration.TenantId, + EmitEvent, + _logger, + probeThresholds, + probeTraceIndices, + probeSources, + stalledAfterIndex); + _logger.Info($"StallProbeCollector enabled: thresholds=[{string.Join(",", probeThresholds)}]min, traces=[{string.Join(",", probeTraceIndices)}], sources={probeSources.Length}"); } else { - _logger.Info("Collector idle timeout disabled (0)"); + _logger.Info("StallProbeCollector disabled via config"); } + // The idle check timer drives BOTH the legacy collector idle timeout AND the stall probes. + // It runs every 60s for the entire agent lifetime (not stopped when collectors pause). + _idleCheckTimer = new Timer( + IdleCheckCallback, + null, + TimeSpan.FromSeconds(60), + TimeSpan.FromSeconds(60) + ); + if (_collectorIdleTimeoutMinutes > 0) + _logger.Info($"Collector idle timeout enabled: {_collectorIdleTimeoutMinutes} min"); + else + _logger.Info("Collector idle timeout disabled (0)"); + // EnrollmentTracker: smart enrollment tracking with IME log parsing // (replaces DownloadProgressCollector + EspUiStateCollector) // Skip when log replay is active - LogReplayService starts its own EnrollmentTracker @@ -105,6 +130,32 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core ); _enrollmentTracker.Start(); _logger.Info("EnrollmentTracker started — listening for IME patterns"); + + // Forward the previously-detected system reboot signal into the tracker so + // the signal-correlated WhiteGlove heuristic can see it. (The reboot event + // was already emitted from EmitStartupEvents before the tracker existed.) + if (_previousExitType == "reboot_kill" && _lastBootTimeUtc.HasValue) + { + _enrollmentTracker.NotifySystemRebootDetected(); + } + + // Wire up the stall probe's terminal-failure signal into the existing + // EnrollmentTracker failure pathway so downstream logic (grace period, + // enrollment_failed emission) stays unified. + if (_stallProbeCollector != null) + { + _stallProbeCollector.EspFailureDetected += (sender, failureType) => + { + try + { + _enrollmentTracker?.ReportEspFailureFromExternalSource(failureType); + } + catch (Exception ex) + { + _logger.Error("Failed to forward stall probe failure to EnrollmentTracker", ex); + } + }; + } } // Delivery Optimization collector — polls OS-level DO status for peer caching telemetry @@ -293,27 +344,48 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core } /// - /// Returns true for events generated by periodic timers (not real enrollment activity) + /// Returns true for events generated by periodic timers (not real enrollment activity). + /// These events do NOT reset _lastRealEventTime and do NOT restart periodic collectors — + /// the enrollment is considered "still idle" even if one of these events was emitted. + /// Stall-probe self-reports are classified periodic so the probe cannot unmark itself + /// as stalled (only a genuine enrollment signal can). /// private static bool IsPeriodicEvent(string eventType) { return eventType == "performance_snapshot" || eventType == "agent_metrics_snapshot" || eventType == "performance_collector_stopped" || - eventType == "agent_metrics_collector_stopped"; + eventType == "agent_metrics_collector_stopped" || + eventType == Constants.EventTypes.StallProbeCheck || + eventType == Constants.EventTypes.StallProbeResult || + eventType == Constants.EventTypes.SessionStalled; } /// /// Periodic check (every 60s) whether enrollment has gone idle. - /// When no real event has arrived within the configured timeout window, - /// periodic collectors are stopped to prevent session bloat. + /// Two responsibilities: + /// 1. Stop telemetry collectors after the configured idle window to prevent session bloat. + /// 2. Invoke the StallProbeCollector to check probe thresholds (2/15/30/60/180 min) and + /// scan for stuck ESP/IME state when silence persists. The timer keeps running even + /// after collectors are stopped so probe thresholds beyond the idle window are reachable. /// private void IdleCheckCallback(object state) { + var idleMinutes = (DateTime.UtcNow - _lastRealEventTime).TotalMinutes; + + // Delegate to stall probe first — it reads _lastRealEventTime and decides which probe to run. + try + { + _stallProbeCollector?.CheckAndRunProbes(idleMinutes); + } + catch (Exception ex) + { + _logger.Error("StallProbeCollector.CheckAndRunProbes threw", ex); + } + if (_collectorsIdleStopped) return; - var idleMinutes = (DateTime.UtcNow - _lastRealEventTime).TotalMinutes; if (idleMinutes < _collectorIdleTimeoutMinutes) return; @@ -373,8 +445,9 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core _collectorsIdleStopped = true; - // Stop the idle check timer — it will be restarted if collectors resume - _idleCheckTimer?.Change(Timeout.Infinite, Timeout.Infinite); + // NOTE: The idle check timer keeps running (previously stopped here) so the + // StallProbeCollector can still reach the higher probe thresholds (60/180 min). + // The early-return guard above prevents re-entry of the collector-stop logic. } /// diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.EventUpload.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.EventUpload.cs index d8ba9eab..8c6c4f74 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.EventUpload.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.EventUpload.cs @@ -38,6 +38,11 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core { _lastRealEventTime = DateTime.UtcNow; + // Reset stall probes — a real event proves the enrollment is still making progress, + // so the probe window starts fresh from 0. + try { _stallProbeCollector?.ResetProbes(); } + catch (Exception ex) { _logger.Verbose($"StallProbeCollector.ResetProbes failed: {ex.Message}"); } + // Restart periodic collectors if they were stopped due to idle if (_collectorsIdleStopped) { diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs index b67cb090..930b274e 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Core/MonitoringService.cs @@ -44,6 +44,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core private PerformanceCollector _performanceCollector; private AgentSelfMetricsCollector _agentSelfMetricsCollector; private DeliveryOptimizationCollector _deliveryOptimizationCollector; + private StallProbeCollector _stallProbeCollector; // Smart enrollment tracking (replaces DownloadProgressCollector + EspUiStateCollector) private EnrollmentTracker _enrollmentTracker; @@ -525,12 +526,16 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Core // Start ESP and Hello tracker (ESP exit, WhiteGlove, and WHfB provisioning tracking) var helloTimeout = _remoteConfigService?.CurrentConfig?.Collectors?.HelloWaitTimeoutSeconds ?? _configuration.HelloWaitTimeoutSeconds; + var modernDeploymentEnabled = _remoteConfigService?.CurrentConfig?.Collectors?.ModernDeploymentWatcherEnabled ?? true; + var modernDeploymentLevelMax = _remoteConfigService?.CurrentConfig?.Collectors?.ModernDeploymentLogLevelMax ?? 3; _espAndHelloTracker = new EspAndHelloTracker( _configuration.SessionId, _configuration.TenantId, EmitEvent, _logger, - helloTimeout + helloTimeout, + modernDeploymentWatcherEnabled: modernDeploymentEnabled, + modernDeploymentLogLevelMax: modernDeploymentLevelMax ); _espAndHelloTracker.Start(); diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.CompletionLogic.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.CompletionLogic.cs index 2914413e..1efa9f65 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.CompletionLogic.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.CompletionLogic.cs @@ -168,10 +168,22 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking /// Called when WhiteGlove (Pre-Provisioning) completes successfully. /// Emits the whiteglove_complete event. The MonitoringService handles /// the actual agent shutdown upon seeing this event type. + /// Idempotent: fire-once per session instance. A second invocation (e.g. Shell-Core + /// watcher fires after the signal-correlated detector already matched) is a no-op. /// private void OnWhiteGloveCompleted(object sender, EventArgs e) { - _logger.Info("EnrollmentTracker: WhiteGlove pre-provisioning completed"); + lock (_stateLock) + { + if (_signalCorrelatedWhiteGloveTriggered) + { + _logger.Debug("EnrollmentTracker: whiteglove_complete already emitted, skipping duplicate"); + return; + } + _signalCorrelatedWhiteGloveTriggered = true; + } + + _logger.Info($"EnrollmentTracker: WhiteGlove pre-provisioning completed (source: {sender?.GetType().Name ?? "unknown"})"); // Stop summary timer — no more app tracking needed _summaryTimer?.Change(System.Threading.Timeout.Infinite, System.Threading.Timeout.Infinite); @@ -195,11 +207,116 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking // Der State wird fuer Part 2 benoetigt falls der Tracker weiterlaufen muss. } + /// + /// Signal-correlated WhiteGlove detection (Ebene 2.6). Called periodically from the + /// summary timer (every 30 s). Evaluates the sealing pattern and, on a stable match + /// lasting ≥10 min, triggers the existing OnWhiteGloveCompleted pipeline — which emits + /// whiteglove_complete, persists the Part 1 marker, and starts graceful shutdown. + /// + /// Pattern (ALL required): + /// - All device-targeted apps installed, no errors (via PackageStates) + /// - DeviceSetupCategory.Status is succeeded + /// - A system reboot has been observed since session start + /// - AAD join status shows Not Joined + /// - AccountSetup category has no activity (no resolved subcategories) + /// - No desktop arrival event has been seen + /// - No Hello signals (Hello not completed, wizard not started) + /// + public void CheckSignalCorrelatedWhiteGlove() + { + // Fast-exit: guards that don't require I/O. + lock (_stateLock) + { + if (_signalCorrelatedWhiteGloveTriggered) + return; + if (_desktopArrived || _enrollmentCompleteEmitted) + return; + // Reboot observation is required — without it we are still in Part 1 pre-reboot. + if (!_systemRebootObserved) + return; + // AAD Not Joined is required — if the user already signed in, this is not sealing. + if (_aadJoinedWithUser) + return; + } + + // EspAndHelloTracker provides the authoritative provisioning signals. + if (_espAndHelloTracker == null) + return; + + bool deviceSetupOk = _espAndHelloTracker.DeviceSetupCategorySucceeded; + bool hasAccountSetupActivity = _espAndHelloTracker.HasAccountSetupActivity; + bool helloInFlight = _espAndHelloTracker.IsHelloCompleted; // true once Hello resolved (either success or skip) + + // ImeLogTracker provides the app-tracking state. + var packageStates = _imeLogTracker?.PackageStates; + bool appsDone = packageStates != null + && packageStates.CountAll > 0 + && packageStates.CountCompleted >= packageStates.CountAll + && packageStates.ErrorCount == 0; + + bool allConditionsMet = deviceSetupOk && appsDone && !hasAccountSetupActivity && !helloInFlight; + + DateTime? triggerAt = null; + bool shouldTrigger = false; + + lock (_stateLock) + { + if (!allConditionsMet) + { + // Stability window broken — reset. + if (_signalCorrelatedWhiteGloveStableSince != null) + { + _logger.Debug("EnrollmentTracker: signal-correlated WhiteGlove conditions no longer stable, resetting stability window"); + _signalCorrelatedWhiteGloveStableSince = null; + } + return; + } + + if (_signalCorrelatedWhiteGloveStableSince == null) + { + _signalCorrelatedWhiteGloveStableSince = DateTime.UtcNow; + _logger.Info($"EnrollmentTracker: signal-correlated WhiteGlove conditions met — starting {SignalCorrelatedWhiteGloveStabilityMinutes}min stability window"); + return; + } + + var stableDuration = DateTime.UtcNow - _signalCorrelatedWhiteGloveStableSince.Value; + if (stableDuration.TotalMinutes >= SignalCorrelatedWhiteGloveStabilityMinutes) + { + shouldTrigger = true; + triggerAt = _signalCorrelatedWhiteGloveStableSince.Value; + } + } + + if (shouldTrigger) + { + _logger.Info($"EnrollmentTracker: signal-correlated WhiteGlove stable since {triggerAt:HH:mm:ss} UTC — invoking OnWhiteGloveCompleted"); + try + { + OnWhiteGloveCompleted(this, EventArgs.Empty); + } + catch (Exception ex) + { + _logger.Error("Signal-correlated WhiteGlove invocation failed", ex); + } + } + } + /// /// Called when an ESP failure is detected (ESPProgress_Failure, _Timeout, _Abort, WhiteGlove_Failed, etc.). /// Terminal failures emit enrollment_failed immediately. /// Recoverable failures (e.g. ESPProgress_Timeout) get a grace period before failure. /// + /// + /// Public entry point for secondary ESP-failure detectors (StallProbeCollector) to report + /// a terminal failure finding. Reuses the same OnEspFailureDetected pathway as the + /// Shell-Core watcher so the downstream logic (grace period, enrollment_failed emission) + /// stays unified. + /// + public void ReportEspFailureFromExternalSource(string failureType) + { + OnEspFailureDetected(this, failureType); + } + private void OnEspFailureDetected(object sender, string failureType) { _logger.Info($"EnrollmentTracker: ESP failure detected: {failureType}"); diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.Diagnostics.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.Diagnostics.cs index 99b59cbf..bec0567f 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.Diagnostics.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.Diagnostics.cs @@ -69,6 +69,12 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking // Only emit if state-breakdown counters changed since last emission (backstop for missed events) EmitAppTrackingSummaryIfChanged(); + // Signal-correlated WhiteGlove detection (Ebene 2.6) — event-driven in the sense + // that it only triggers when the observed signal set matches; zero-cost when the + // session does not look like a sealing candidate. + try { CheckSignalCorrelatedWhiteGlove(); } + catch (Exception ex) { _logger.Error("CheckSignalCorrelatedWhiteGlove failed", ex); } + // Periodic state save (piggybacks on the existing 30s timer) bool dirty; EnrollmentStateData stateSnapshot; diff --git a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.cs b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.cs index b87a9bc3..09203668 100644 --- a/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Tracking/EnrollmentTracker.cs @@ -96,6 +96,29 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking // Detected via CollectAadJoinStatus() in DeviceInfoCollector. private bool _aadJoinedWithUser; + // Signal-correlated WhiteGlove detection (Ebene 2.6): + // Feeds the bestehenden OnWhiteGloveCompleted() callback when the Shell-Core detector + // does not match but the session shows the WhiteGlove sealing pattern (all Device-apps done, + // DeviceSetup category succeeded, system reboot observed, AAD Not Joined, no AccountSetup + // progress, no desktop arrival). Requires ≥10 min stability to avoid false-positives. + private DateTime? _signalCorrelatedWhiteGloveStableSince; + private bool _signalCorrelatedWhiteGloveTriggered; + private bool _systemRebootObserved; + private const int SignalCorrelatedWhiteGloveStabilityMinutes = 10; + + /// + /// Called by MonitoringService when a system_reboot_detected event is emitted + /// (agent restart after an unclean exit / machine reboot). Used as an input signal + /// for the signal-correlated WhiteGlove detection heuristic. + /// + public void NotifySystemRebootDetected() + { + lock (_stateLock) + { + _systemRebootObserved = true; + } + } + // True when no interactive user session is expected. // Self-Deploying (AutopilotMode=1) is always device-only. // SkipUserStatusPage=true alone does NOT mean device-only — admins commonly skip diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt index 7c890ad8..17ec7611 100644 --- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt @@ -1 +1 @@ -894 +901 diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs index 3bd54a8f..f097d522 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs @@ -41,6 +41,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime) => _storage.GetStalledSessionsAsync(tenantId, cutoffTime); + public Task> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) + => _storage.GetAgentSilentSessionsAsync(tenantId, silenceCutoff, hardCutoff); + public Task> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours) => _storage.GetExcessiveDataSendersAsync(tenantId, windowCutoff, maxSessionWindowHours); diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs index bf01e07b..51806ff6 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs @@ -47,10 +47,12 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, - bool? isUserDriven = null, DateTime? resumedAt = null) + bool? isUserDriven = null, DateTime? resumedAt = null, + DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false) => _storage.UpdateSessionStatusAsync(tenantId, sessionId, status, currentPhase, failureReason, completedAt, earliestEventTimestamp, - latestEventTimestamp, isPreProvisioned, isUserDriven, resumedAt); + latestEventTimestamp, isPreProvisioned, isUserDriven, resumedAt, + stalledAt, clearStalledAt, clearFailureReason); public Task IncrementSessionEventCountAsync( string tenantId, string sessionId, int increment, diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs index 2307243d..5a6f5049 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs @@ -567,14 +567,42 @@ namespace AutopilotMonitor.Functions.Functions.Ingest case "device_location": classification.DeviceLocationEvent = evt; break; + case "session_stalled": + classification.SessionStalledEvent = evt; + break; } + // Track whether this batch contains any non-periodic real event (used for + // Stalled→InProgress healing). Exclude telemetry snapshots and stall-probe + // self-reports which should not count as "agent is productively working". + if (!IsPeriodicOrStallEvent(evt.EventType)) + classification.HasNonPeriodicRealEvent = true; + AggregateAppInstallEvent(evt, storedEvents[0].TenantId!, storedEvents[0].SessionId!, classification.AppInstallUpdates); } return classification; } + /// + /// Classifies whether an event type should count as "real enrollment activity" for + /// Stalled→InProgress healing. Periodic telemetry and stall-probe self-reports are + /// excluded so that a silent agent that only emits heartbeats cannot unmark itself + /// as stalled without real progress. + /// + private static bool IsPeriodicOrStallEvent(string? eventType) => eventType switch + { + "performance_snapshot" => true, + "agent_metrics_snapshot" => true, + "performance_collector_stopped" => true, + "agent_metrics_collector_stopped" => true, + "stall_probe_check" => true, + "stall_probe_result" => true, + "session_stalled" => true, + "modern_deployment_log" => true, + _ => false + }; + /// /// Updates session status based on classified events. /// Returns (statusTransitioned, whiteGloveStatusTransitioned, failureReason). @@ -665,6 +693,36 @@ namespace AutopilotMonitor.Functions.Functions.Ingest } } + // Stalled-status transitions are independent of Succeeded/Failed/Pending/WhiteGlove paths. + // They can coexist with normal processing: if a batch contains session_stalled, mark Stalled; + // if a batch contains any real (non-periodic) event while the session is currently Stalled, + // heal it back to InProgress. + if (c.SessionStalledEvent != null) + { + var stalledReason = "Agent reported stall after 60min without progress (stall_probe)"; + var stalledTransitioned = await _sessionRepo.UpdateSessionStatusAsync( + request.TenantId, request.SessionId, SessionStatus.Stalled, + earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp, + stalledAt: c.SessionStalledEvent.Timestamp, failureReason: stalledReason); + if (stalledTransitioned) + _logger.LogWarning("{SessionPrefix} Status: Stalled (agent-reported via session_stalled event)", sessionPrefix); + } + else if (c.HasNonPeriodicRealEvent && !statusTransitioned && !whiteGloveStatusTransitioned) + { + // Only attempt healing if the current session is actually Stalled (one read). + // The repository guard will no-op this call when the session is in any other state. + var currentSession = await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId); + if (currentSession?.Status == SessionStatus.Stalled) + { + var healed = await _sessionRepo.UpdateSessionStatusAsync( + request.TenantId, request.SessionId, SessionStatus.InProgress, + earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp, + clearStalledAt: true, clearFailureReason: true); + if (healed) + _logger.LogInformation("{SessionPrefix} Status: InProgress (healed from Stalled by new real event)", sessionPrefix); + } + } + return (statusTransitioned, whiteGloveStatusTransitioned, failureReason); } @@ -930,6 +988,8 @@ namespace AutopilotMonitor.Functions.Functions.Ingest public EnrollmentEvent? WhiteGloveEvent { get; set; } public EnrollmentEvent? WhiteGloveResumedEvent { get; set; } public EnrollmentEvent? EspFailureEvent { get; set; } + public EnrollmentEvent? SessionStalledEvent { get; set; } + public bool HasNonPeriodicRealEvent { get; set; } public EnrollmentEvent? DeviceLocationEvent { get; set; } public DateTime? EarliestEventTimestamp { get; set; } public DateTime? LatestEventTimestamp { get; set; } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 530ff469..08dbde0d 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -189,7 +189,14 @@ namespace AutopilotMonitor.Functions.Services } /// - /// Marks sessions that have been in "InProgress" status for too long as "Failed - Timed Out" + /// Two-stage sweep for stuck sessions: + /// 1. **Agent-silent Stalled marker** (2h fixed): Sessions still InProgress but with no events + /// for more than 2h are marked as Stalled (non-terminal). Picks up agents that cannot emit + /// session_stalled themselves (bluescreen, network loss, power off). Excludes WhiteGlove. + /// 2. **Session timeout Failed** (5h default): Sessions that exceed the full SessionTimeoutHours + /// window (InProgress or Stalled) graduate to terminal Failed state. + /// Both stages run in the same 2h maintenance pass so no new timers are introduced + /// (preserving Container App scale-to-zero). /// private async Task MarkStalledSessionsAsTimedOutAsync() { @@ -200,6 +207,7 @@ namespace AutopilotMonitor.Functions.Services { var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync(); int totalSessionsTimedOut = 0; + int totalSessionsMarkedStalled = 0; foreach (var tenantId in tenantIds) { @@ -207,8 +215,50 @@ namespace AutopilotMonitor.Functions.Services { var config = await _tenantConfigService.GetConfigurationAsync(tenantId); var timeoutHours = config?.SessionTimeoutHours ?? 5; - var cutoffTime = DateTime.UtcNow.AddHours(-timeoutHours); + const int agentSilenceHours = 2; // fixed policy: 2h silence → Stalled intermediate + var now = DateTime.UtcNow; + var cutoffTime = now.AddHours(-timeoutHours); + var silenceCutoff = now.AddHours(-agentSilenceHours); + // -------- Stage 1: Agent-silent Stalled marker -------- + // InProgress sessions with no events in > 2h but not yet older than 5h + // (otherwise Stage 2 picks them up directly as Failed). + var silentSessions = await _maintenanceRepo.GetAgentSilentSessionsAsync( + tenantId, silenceCutoff, hardCutoff: cutoffTime); + + int silentMarked = 0; + foreach (var silent in silentSessions) + { + var lastEventAt = silent.LastEventAt ?? silent.StartedAt; + var silentMinutes = (int)(now - lastEventAt).TotalMinutes; + await _sessionRepo.UpdateSessionStatusAsync( + silent.TenantId, + silent.SessionId, + SessionStatus.Stalled, + stalledAt: now, + failureReason: $"Agent silent for {silentMinutes}min (detected by maintenance sweep)"); + silentMarked++; + } + + if (silentMarked > 0) + { + totalSessionsMarkedStalled += silentMarked; + _logger.LogInformation($"Tenant {tenantId}: Marked {silentMarked} agent-silent sessions as Stalled (silence threshold: {agentSilenceHours}h)"); + await _maintenanceRepo.LogAuditEntryAsync( + tenantId, + "SessionStalled", + "Session", + $"{silentMarked} sessions", + "System.Maintenance", + new Dictionary + { + { "SessionsMarkedStalled", silentMarked.ToString() }, + { "AgentSilenceHours", agentSilenceHours.ToString() }, + { "SilenceCutoff", silenceCutoff.ToString("yyyy-MM-ddTHH:mm:ss") } + }); + } + + // -------- Stage 2: Terminal timeout (5h default) -------- var stalledSessions = await _maintenanceRepo.GetStalledSessionsAsync(tenantId, cutoffTime); if (stalledSessions.Count == 0) @@ -255,7 +305,7 @@ namespace AutopilotMonitor.Functions.Services } stalledStart.Stop(); - _logger.LogInformation($"Stalled session check completed: {totalSessionsTimedOut} sessions timed out in {stalledStart.ElapsedMilliseconds}ms"); + _logger.LogInformation($"Stalled session check completed: {totalSessionsMarkedStalled} marked Stalled, {totalSessionsTimedOut} timed out in {stalledStart.ElapsedMilliseconds}ms"); } catch (Exception ex) { diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs index 6c577c57..7d799495 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs @@ -202,7 +202,15 @@ namespace AutopilotMonitor.Functions.Services { var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); - var filter = $"PartitionKey eq '{tenantId}' and Status eq 'InProgress' and StartedAt lt datetime'{cutoffTime:yyyy-MM-ddTHH:mm:ss}Z'"; + // Eligible for the 5h-timeout sweep: InProgress or Stalled sessions (Stalled is a + // non-terminal intermediate state set earlier by either the agent or the 2h sweep; + // when the 5h mark is reached without healing, the session graduates to Failed). + // IsPreProvisioned ne true → WhiteGlove sessions are intentionally long-lived and + // must never be timed out, even after months in storage. + var filter = $"PartitionKey eq '{tenantId}' " + + $"and (Status eq 'InProgress' or Status eq 'Stalled') " + + $"and StartedAt lt datetime'{cutoffTime:yyyy-MM-ddTHH:mm:ss}Z' " + + $"and IsPreProvisioned ne true"; var query = tableClient.QueryAsync(filter: filter); var sessions = new List(); @@ -220,6 +228,51 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// Gets sessions where the agent has gone completely silent for longer than the configured + /// agent-silence window (default 2h). These are candidates for the intermediate Stalled status. + /// Used by the 2h maintenance sweep as a backstop for agents that cannot send session_stalled + /// themselves (bluescreen, network loss, power off). + /// + /// Filter criteria: + /// - Status eq 'InProgress' — do not re-mark already-Stalled sessions + /// - LastEventAt < silenceCutoff — at least the configured window of agent silence + /// - StartedAt ge hardCutoff — do not catch sessions that will be picked up by the 5h timeout sweep + /// - IsPreProvisioned ne true — exclude sealed WhiteGlove sessions (intentionally long-lived) + /// + public async Task> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); + + var silenceCutoffStr = silenceCutoff.ToString("yyyy-MM-ddTHH:mm:ss"); + var hardCutoffStr = hardCutoff.ToString("yyyy-MM-ddTHH:mm:ss"); + var filter = $"PartitionKey eq '{tenantId}' " + + $"and Status eq 'InProgress' " + + $"and LastEventAt lt datetime'{silenceCutoffStr}Z' " + + $"and StartedAt ge datetime'{hardCutoffStr}Z' " + + $"and IsPreProvisioned ne true"; + + var query = tableClient.QueryAsync(filter: filter); + + var sessions = new List(); + await foreach (var entity in query) + { + sessions.Add(MapToSessionSummary(entity)); + } + + return sessions; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Failed to get agent-silent sessions for tenant {tenantId}"); + return new List(); + } + } + /// /// Gets sessions where the device has been actively sending data for longer than /// . diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index decffd11..071c320f 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -219,6 +219,7 @@ namespace AutopilotMonitor.Functions.Services IsPreProvisioned = entity.GetBoolean("IsPreProvisioned") ?? false, IsHybridJoin = entity.GetBoolean("IsHybridJoin") ?? false, ResumedAt = SafeGetDateTime(entity, "ResumedAt"), + StalledAt = SafeGetDateTime(entity, "StalledAt"), OsName = entity.GetString("OsName") ?? string.Empty, OsBuild = entity.GetString("OsBuild") ?? string.Empty, OsDisplayVersion = entity.GetString("OsDisplayVersion") ?? string.Empty, @@ -291,6 +292,7 @@ namespace AutopilotMonitor.Functions.Services int? durationSeconds = null; string? diagnosticsBlobName = null; DateTime? resumedAt = null; + DateTime? stalledAt = null; string geoCountry = string.Empty; string geoRegion = string.Empty; string geoCity = string.Empty; @@ -321,6 +323,7 @@ namespace AutopilotMonitor.Functions.Services durationSeconds = existingEntity.GetInt32("DurationSeconds"); diagnosticsBlobName = existingEntity.GetString("DiagnosticsBlobName"); resumedAt = existingEntity.GetDateTimeOffset("ResumedAt")?.UtcDateTime; + stalledAt = existingEntity.GetDateTimeOffset("StalledAt")?.UtcDateTime; geoCountry = existingEntity.GetString("GeoCountry") ?? string.Empty; geoRegion = existingEntity.GetString("GeoRegion") ?? string.Empty; geoCity = existingEntity.GetString("GeoCity") ?? string.Empty; @@ -401,6 +404,9 @@ namespace AutopilotMonitor.Functions.Services if (resumedAt.HasValue) entity["ResumedAt"] = EnsureUtc(resumedAt.Value); + if (stalledAt.HasValue) + entity["StalledAt"] = EnsureUtc(stalledAt.Value); + if (!string.IsNullOrEmpty(geoCountry)) entity["GeoCountry"] = geoCountry; if (!string.IsNullOrEmpty(geoRegion)) @@ -901,7 +907,7 @@ namespace AutopilotMonitor.Functions.Services /// Event count is maintained atomically by IncrementSessionEventCountAsync and is not /// recounted here — avoiding an expensive full-partition scan on every status change. /// - public async Task UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null) + public async Task UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false) { SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); @@ -934,11 +940,24 @@ namespace AutopilotMonitor.Functions.Services // Build a Merge update with only the fields that actually change var update = new TableEntity(tenantId, sessionId); - // InProgress is never passed here from IngestEventsFunction — status transitions - // are limited to Succeeded, Failed, Pending. The Pending→InProgress regression is - // structurally impossible because phase_changed events use IncrementSessionEventCountAsync - // (which never touches Status). Only StoreSessionAsync (re-registration) transitions - // Pending→InProgress for WhiteGlove Part 2. + // Status transitions from IngestEventsFunction: Succeeded, Failed, Pending, Stalled, + // and InProgress (the last one only for healing a Stalled session back to active). + // Guard: never regress a Pending (WhiteGlove) session to Stalled via the ingest + // path — WhiteGlove sessions are deliberately long-lived and handled via re-registration. + if (status == SessionStatus.Stalled && existingStatusStr == SessionStatus.Pending.ToString()) + { + _logger.LogInformation($"Session {sessionId} is in Pending (WhiteGlove) state, skipping Stalled transition"); + return false; + } + + // Guard: only allow InProgress transition if the current state is Stalled (healing). + // Other callers must not regress terminal or Pending state via this path. + if (status == SessionStatus.InProgress && existingStatusStr != SessionStatus.Stalled.ToString()) + { + _logger.LogInformation($"Session {sessionId} is in state '{existingStatusStr}', not Stalled — InProgress heal no-op"); + return false; + } + update["Status"] = status.ToString(); // Update current phase if provided @@ -1039,6 +1058,26 @@ namespace AutopilotMonitor.Functions.Services update["ResumedAt"] = EnsureUtc(resumedAt.Value); } + // Stalled-state bookkeeping: the agent emits session_stalled after 60 min idle, + // or the 2h maintenance sweep sets the state for completely silent agents. + if (stalledAt.HasValue) + { + update["StalledAt"] = EnsureUtc(stalledAt.Value); + if (!string.IsNullOrEmpty(failureReason)) + update["FailureReason"] = failureReason; + } + + // Heal: clear StalledAt + Stalled-phase FailureReason when the session goes + // back to InProgress (a new real event proved the agent is alive again). + if (clearStalledAt) + { + update["StalledAt"] = (DateTime?)null; + } + if (clearFailureReason) + { + update["FailureReason"] = string.Empty; + } + // Merge mode: only the fields set above are written; all other fields remain untouched. // This drastically reduces ETag conflicts when concurrent requests update different fields. await tableClient.UpdateEntityAsync(update, session.ETag, TableUpdateMode.Merge); @@ -1646,6 +1685,7 @@ namespace AutopilotMonitor.Functions.Services IsPreProvisioned = entity.GetBoolean("IsPreProvisioned") ?? false, IsHybridJoin = entity.GetBoolean("IsHybridJoin") ?? false, ResumedAt = SafeGetDateTime(entity, "ResumedAt"), + StalledAt = SafeGetDateTime(entity, "StalledAt"), OsName = entity.GetString("OsName") ?? string.Empty, OsBuild = entity.GetString("OsBuild") ?? string.Empty, OsDisplayVersion = entity.GetString("OsDisplayVersion") ?? string.Empty, diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 6040f5b6..b1ed4fbd 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -177,6 +177,16 @@ namespace AutopilotMonitor.Shared public const string VulnerabilityReport = "vulnerability_report"; public const string AgentSelfUpdated = "agent_self_updated"; public const string AgentSelfUpdateSkipped = "agent_self_update_skipped"; + + // Stall detection (Ebene 2 — StallProbeCollector) + public const string StallProbeCheck = "stall_probe_check"; // Trace heartbeat from Probe 2 (15 min) when no anomaly found + public const string StallProbeResult = "stall_probe_result"; // Warning when a probe found an anomaly + public const string SessionStalled = "session_stalled"; // Fire-once after Probe 4 (60 min) — triggers backend Stalled status + + // ModernDeployment EventLog Watcher (Ebene 1 — live capture) + public const string ModernDeploymentLog = "modern_deployment_log"; // Info-level live capture + public const string ModernDeploymentWarning = "modern_deployment_warning"; // Level 3 (Warning) + public const string ModernDeploymentError = "modern_deployment_error"; // Level 1-2 (Critical/Error) } // ----------------------------------------------------------------------- diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs index 91996621..7df05c2c 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs @@ -21,6 +21,7 @@ namespace AutopilotMonitor.Shared.DataAccess Task> GetSessionsOlderThanAsync(string tenantId, DateTime cutoffDate); Task> GetSessionsByDateRangeAsync(DateTime startDate, DateTime endDate, string? tenantId = null); Task> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime); + Task> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff); Task> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours); // --- Tenant Discovery --- diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs index e6cd9a5d..8ed31769 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs @@ -25,7 +25,8 @@ namespace AutopilotMonitor.Shared.DataAccess EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, - bool? isUserDriven = null, DateTime? resumedAt = null); + bool? isUserDriven = null, DateTime? resumedAt = null, + DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false); Task IncrementSessionEventCountAsync( string tenantId, string sessionId, int increment, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs b/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs index 2c1710ba..b66a2a2c 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs @@ -254,6 +254,75 @@ namespace AutopilotMonitor.Shared.Models /// public int DeliveryOptimizationIntervalSeconds { get; set; } = 3; + // ----------------------------------------------------------------------- + // Stall detection (Ebene 2 — StallProbeCollector) + // ----------------------------------------------------------------------- + + /// + /// Master switch for the stall probe mechanism. When enabled, probes run at + /// idle-time thresholds (see StallProbeThresholdsMinutes) to detect stuck + /// enrollments via registry, EventLog and AppWorkload log scans. + /// Default: true + /// + public bool StallProbeEnabled { get; set; } = true; + + /// + /// Idle-time thresholds in minutes at which probes run. Each probe is fire-once + /// per idle window; the counter resets on any real (non-periodic) event. + /// Default: [2, 15, 30, 60, 180] + /// Probe 1 (2 min): early quick-response, silent unless anomaly found. + /// Probe 2 (15 min): main gate with guaranteed trace heartbeat. + /// Probe 3 (30 min): silent unless found. + /// Probe 4 (60 min): fires session_stalled event → backend sets status to Stalled. + /// Probe 5 (180 min): last-chance scan before 5h backend timeout, silent unless found. + /// + public int[] StallProbeThresholdsMinutes { get; set; } = new[] { 2, 15, 30, 60, 180 }; + + /// + /// Probe indices (1-based) that emit a stall_probe_check trace event even when + /// no anomaly is found. Default: [2] — only Probe 2 (15 min) sends a heartbeat. + /// Set to [1, 2] for more observability at the cost of extra trace events. + /// + public int[] StallProbeTraceIndices { get; set; } = new[] { 2 }; + + /// + /// Which sources to scan per probe. Any source can be removed from the list + /// to disable it individually without affecting the others. + /// Default: all four sources enabled. + /// + public string[] StallProbeSources { get; set; } = new[] + { + "provisioning_registry", + "diagnostics_registry", + "eventlog", + "appworkload_log" + }; + + /// + /// After which probe index (1-based) the fire-once session_stalled event is emitted. + /// Default: 4 → after Probe 4 (60 min idle). + /// + public int SessionStalledAfterProbeIndex { get; set; } = 4; + + // ----------------------------------------------------------------------- + // ModernDeployment EventLog Watcher (Ebene 1 — live capture) + // ----------------------------------------------------------------------- + + /// + /// Master switch for the ModernDeployment EventLog watcher. When enabled, the agent + /// subscribes to two Windows event channels and forwards matching events as + /// modern_deployment_log events to the backend. + /// Default: true + /// + public bool ModernDeploymentWatcherEnabled { get; set; } = true; + + /// + /// Maximum Windows event level to capture from ModernDeployment channels. + /// Windows levels: 1=Critical, 2=Error, 3=Warning, 4=Information, 5=Verbose. + /// Default: 3 → captures Critical, Error and Warning. + /// + public int ModernDeploymentLogLevelMax { get; set; } = 3; + /// /// Creates default collector configuration /// diff --git a/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs index c408e2da..6be6307f 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs @@ -191,6 +191,14 @@ namespace AutopilotMonitor.Shared.Models /// public DateTime? ResumedAt { get; set; } + /// + /// Timestamp when the session was marked as Stalled. + /// Set when the agent sends a session_stalled event (after 60 min without progress) + /// or when the backend 2h maintenance sweep detects agent silence. + /// Cleared (null) when the session heals back to InProgress via a new real event. + /// + public DateTime? StalledAt { get; set; } + /// /// Whether the Autopilot profile indicates Hybrid Azure AD Join. /// Derived from CloudAssignedDomainJoinMethod == 1 in the Autopilot profile. @@ -246,6 +254,7 @@ namespace AutopilotMonitor.Shared.Models { InProgress, Pending, // WhiteGlove pre-provisioning complete, awaiting user enrollment + Stalled, // Agent reported no progress for >60 min, or backend sweep detected >2h silence. Non-terminal, can heal back to InProgress. Succeeded, Failed, Unknown diff --git a/src/Web/autopilot-monitor-web/app/dashboard/components/SessionTable.tsx b/src/Web/autopilot-monitor-web/app/dashboard/components/SessionTable.tsx index ed5aced6..0725c079 100644 --- a/src/Web/autopilot-monitor-web/app/dashboard/components/SessionTable.tsx +++ b/src/Web/autopilot-monitor-web/app/dashboard/components/SessionTable.tsx @@ -528,6 +528,7 @@ export function SessionTable({ Failed: "text-red-600", InProgress: "text-blue-600", Pending: "text-amber-600", + Stalled: "text-orange-600", }; return ( )} - {adminMode && (session?.status === 'InProgress' || session?.status === 'Pending') && ( + {adminMode && (session?.status === 'InProgress' || session?.status === 'Pending' || session?.status === 'Stalled') && ( <>