cahgned bootstrap guards and added more, new hybrid join completion logic

2026-10-24 04:32:32 +00:00 · 2026-03-31 22:00:08 +02:00
parent 1a89080f0f
commit 3b871b299d
9 changed files with 220 additions and 45 deletions
@@ -31,7 +31,7 @@
 .NOTES
    - Agent is temporary and auto-removes after enrollment
    - Everything in C:\ProgramData\AutopilotMonitor (easy cleanup)
-    - No Registry entries (clean machine after removal)
+    - One Registry key left after removal: HKLM\SOFTWARE\AutopilotMonitor\Deployed (prevents ghost re-installs)
    - Scheduled Task survives reboots during enrollment
    - SHA-256 integrity verification since v1.0.706+
    - IMPORTANT: This script MUST remain pure ASCII (no Unicode/UTF-8 special chars).
@@ -53,9 +53,9 @@
 param(
    [Parameter(Mandatory = $false)]
    [string]$AgentDownloadUrl = "https://autopilotmonitor.blob.core.windows.net/agent/AutopilotMonitor-Agent.zip",
-    
+
    [Parameter(Mandatory = $false)]
-    [int]$MaxOsAgeMinutes = 120
+    [int]$MaxBootstrapWindowHours = 12
 )

 # Configuration - Everything in ProgramData for easy cleanup
@@ -103,61 +103,75 @@ function Get-FileMd5Base64 {
 try {
    Write-Log "===== Autopilot Monitor Bootstrap Started ====="

-    # -- Pre-flight: Skip installation on already-enrolled devices --
+    # -- Pre-flight: Multi-signal guard to prevent installation on non-provisioning devices --
+    # Layered approach: each guard catches a different scenario.
+    # Guard 1: Ghost re-installs (registry marker from previous deployment)
+    # Guard 2: Productive devices not in OOBE
+    # Guard 3: Productive devices with stale OOBEInProgress flag (real user profile exists)
+    # Guard 4: Ultra edge case (no key, OOBE stuck, no user, but device running too long)
+    # Guard 5: Agent binary already present from a previous run

-    # Check 1: OS install date must be within threshold (fresh device)
-    # TODO: issue is the OS might got installed a long time ago but the OOBE just started now (e.g. old image, or re-flash).
-    # Ideally we want to detect the OOBE start time... need to investigate if there's a reliable way to get that (event log? registry? WMI?). 
-    # For now we just check OS install date as a proxy, which should work in most cases except for old images or re-flashed devices.
-    $osInstallDate = (Get-CimInstance Win32_OperatingSystem).InstallDate
-    $osAge = (Get-Date) - $osInstallDate
-    $osAgeMinutesRounded = [int][Math]::Round($osAge.TotalMinutes)
-    Write-Log "OS install date: $osInstallDate (age: ${osAgeMinutesRounded}m)"
-
-    if ($osAge.TotalMinutes -gt $MaxOsAgeMinutes) {
-        Write-Log "SKIP: OS was installed ${osAgeMinutesRounded}m ago (threshold: ${MaxOsAgeMinutes}m). Device is not freshly provisioned."
+    # Guard 1: Agent was already deployed on this device (registry marker survives self-destruct)
+    $deployed = (Get-ItemProperty -Path 'HKLM:\SOFTWARE\AutopilotMonitor' -Name 'Deployed' -ErrorAction SilentlyContinue).Deployed
+    if ($deployed) {
+        Write-Log "SKIP: Agent was previously deployed at $deployed."
        exit 0
    }

-    # Check 2: MDM enrollment must not be completed yet
-    $enrollmentPath = "HKLM:\SOFTWARE\Microsoft\Enrollments"
-    $mdmEnrolled = $false
-
-    if (Test-Path $enrollmentPath) {
-        $enrollmentEntries = Get-ChildItem -Path $enrollmentPath -ErrorAction SilentlyContinue |
-            Where-Object { $_.PSChildName -match '^[0-9A-Fa-f\-]{36}$' }
-
-        foreach ($entry in $enrollmentEntries) {
-            $providerID = (Get-ItemProperty -Path $entry.PSPath -Name "ProviderID" -ErrorAction SilentlyContinue).ProviderID
-            if ($providerID) {
-                $mdmEnrolled = $true
-                Write-Log "Found MDM enrollment: ProviderID=$providerID (EnrollmentID=$($entry.PSChildName))"
-                break
-            }
-        }
+    # Guard 2: Device must be in OOBE (= actively provisioning)
+    $oobeInProgress = (Get-ItemProperty -Path 'HKLM:\SYSTEM\Setup' -Name 'OOBEInProgress' -ErrorAction SilentlyContinue).OOBEInProgress
+    Write-Log "OOBEInProgress=$oobeInProgress"
+    if ($oobeInProgress -ne 1) {
+        Write-Log "SKIP: Device is not in OOBE (OOBEInProgress=$oobeInProgress). Not a fresh enrollment."
+        exit 0
    }

-    if ($mdmEnrolled) {
-        # Double-check: Is enrollment status tracking showing completed?
-        $espPath = "HKLM:\SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking\Device\Setup"
-        $hasCompleted = (Get-ItemProperty -Path $espPath -Name "HasProvisioningCompleted" -ErrorAction SilentlyContinue).HasProvisioningCompleted
+    # Guard 3: No real user profile should exist yet (catches stale OOBEInProgress on productive devices)
+    # Combines WMI (Win32_UserProfile.Special flag) and filesystem for maximum reliability.
+    $excludePattern = '^(defaultuser\d*|Public|Default( User)?|All Users)$'

-        if ($hasCompleted -eq 1) {
-            Write-Log "SKIP: Autopilot enrollment already completed (HasProvisioningCompleted=1). Device is already enrolled."
-            exit 0
-        }
+    $profileNames = @(
+        # WMI/CIM view -- Special flag reliably excludes SYSTEM/LocalService/NetworkService
+        try {
+            Get-CimInstance Win32_UserProfile -ErrorAction Stop |
+                Where-Object { -not $_.Special -and $_.LocalPath -like 'C:\Users\*' } |
+                ForEach-Object { Split-Path $_.LocalPath -Leaf }
+        } catch { Write-Log "INFO: WMI profile query failed, continuing with filesystem check." }
+
+        # Filesystem view -- catches profiles WMI might miss
+        (Get-ChildItem 'C:\Users' -Directory -ErrorAction SilentlyContinue).Name
+    ) | Where-Object { $_ -and $_ -notmatch $excludePattern } |
+        Select-Object -Unique
+
+    if ($profileNames) {
+        $names = ($profileNames | Select-Object -First 3) -join ', '
+        Write-Log "SKIP: Real user profile(s) found ($names). Device appears productive."
+        exit 0
    }

-    # Check 3: Is the agent already installed? (leftover from previous run)
+    # Guard 4: Bootstrap window check (ultra edge case: no key, OOBE stuck, no user profiles)
+    # NOT "how long may enrollment take" -- agent handles that internally (6h emergency break).
+    # This is "how old can the OOBE state be before I no longer trust it for initial install".
+    # 12h bootstrap window vs 6h agent emergency break = consistent layered approach.
+    # Sleep/standby does NOT reset uptime -- only real boot/restart does.
+    $lastBoot = (Get-CimInstance Win32_OperatingSystem).LastBootUpTime
+    $uptimeHours = ((Get-Date) - $lastBoot).TotalHours
+    Write-Log "Device uptime: $([int]$uptimeHours)h (last boot: $lastBoot)"
+    if ($uptimeHours -gt $MaxBootstrapWindowHours) {
+        Write-Log "SKIP: Device uptime is $([int]$uptimeHours)h. OOBE state is older than accepted bootstrap window of ${MaxBootstrapWindowHours}h."
+        exit 0
+    }
+
+    # Guard 5: Is the agent already installed? (leftover from previous run)
    if (Test-Path $AgentBinPath) {
        $existingAgent = Get-ChildItem -Path $AgentBinPath -Filter "AutopilotMonitor.Agent.exe" -ErrorAction SilentlyContinue
        if ($existingAgent) {
-            Write-Log "SKIP: Agent already installed at $($existingAgent.FullName). Previous enrollment may have been in progress. Please check if enrollment completed or clean up C:\ProgramData\AutopilotMonitor before re-running."
+            Write-Log "SKIP: Agent already installed at $($existingAgent.FullName)."
            exit 0
        }
    }

-    Write-Log "Pre-flight checks passed - device is freshly provisioned and enrollment in progress"
+    Write-Log "Pre-flight checks passed -- device is in OOBE, no prior deployment, no real user profiles"

    # Download and extract agent binaries
    $agentExePath = Join-Path $AgentBinPath "AutopilotMonitor.Agent.exe"
@@ -390,6 +390,28 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Collectors
            }
        }

+        /// <summary>
+        /// Resets Hello tracking state when ESP resumes after a mid-enrollment reboot (hybrid join).
+        /// Stops all running timers and clears Hello state so the timer chain restarts fresh when
+        /// ESP exits again for real.
+        /// </summary>
+        public void ResetForEspResumption()
+        {
+            lock (_stateLock)
+            {
+                _helloWaitTimer?.Dispose();
+                _helloWaitTimer = null;
+                _helloCompletionTimer?.Dispose();
+                _helloCompletionTimer = null;
+
+                _isHelloCompleted = false;
+                _helloWizardStarted = false;
+                _espExitDetected = false;
+                HelloOutcome = null;
+            }
+            _logger.Info("EspAndHelloTracker: Reset for ESP resumption — Hello tracking restarted");
+        }
+
        /// <summary>
        /// Starts the Hello wait timer. Should be called by EnrollmentTracker when AccountSetup phase exits.
        /// The timer waits for Hello wizard to start (event 62404) within the configured timeout.
@@ -117,6 +117,9 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
        // True when AAD join status shows a joined device with a real user email
        public bool AadJoinedWithUser { get; set; }

+        // True when Autopilot profile indicates Hybrid Azure AD Join (CloudAssignedDomainJoinMethod == 1)
+        public bool IsHybridJoin { get; set; }
+
        // Signal timestamps for audit trail
        public DateTime? EspFirstSeenUtc { get; set; }
        public DateTime? EspFinalExitUtc { get; set; }
@@ -550,6 +550,8 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
                _skipDeviceStatusPage = loaded.SkipDeviceStatusPage;
                _autopilotMode = loaded.AutopilotMode;
                _aadJoinedWithUser = loaded.AadJoinedWithUser;
+                // Restore hybrid join flag from persisted state; re-detect as fallback
+                _isHybridJoin = loaded.IsHybridJoin || DetectHybridJoinStatic();
                _stateData = loaded;
            }

@@ -632,6 +634,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
            // and OnWaitingForHelloSafetyTimeout) both read _enrollmentCompleteEmitted=false simultaneously.
            bool helloResolved;
            bool espGateBlocked;
+            bool hybridRebootGateBlocked;
            bool alreadyEmitted;
            string enrollmentType;
            bool espEverSeen, espFinalExitSeen, desktopArrived;
@@ -646,6 +649,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
                    alreadyEmitted = true;
                    helloResolved = false;
                    espGateBlocked = false;
+                    hybridRebootGateBlocked = false;
                    enrollmentType = _enrollmentType;
                    espEverSeen = _espEverSeen;
                    espFinalExitSeen = _espFinalExitSeen;
@@ -674,8 +678,23 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
                    espGateBlocked = (source == "desktop_arrival" || source == "desktop_hello")
                        && enrollmentType != "v2" && espEverSeen && !espFinalExitSeen;

+                    // GUARD 3: Hybrid Join reboot gate
+                    // In hybrid join, ESP exit may be for a mid-enrollment reboot (domain user
+                    // login required). Block esp_hello_composite unless we have stronger confirmation:
+                    // - IME user session completed (strongest signal: IME says enrollment is done)
+                    // - Agent restarted after ESP exit (reboot happened, we survived it)
+                    hybridRebootGateBlocked = false;
+                    if (_isHybridJoin && source == "esp_hello_composite")
+                    {
+                        bool hasImeCompletion = _stateData.ImePatternSeenUtc.HasValue;
+                        bool agentRestartedAfterEspExit = _stateData.EspFinalExitUtc.HasValue
+                            && _agentStartTimeUtc > _stateData.EspFinalExitUtc.Value;
+
+                        hybridRebootGateBlocked = !hasImeCompletion && !agentRestartedAfterEspExit;
+                    }
+
                    // If all guards pass, claim the flag atomically
-                    if (helloResolved && !espGateBlocked)
+                    if (helloResolved && !espGateBlocked && !hybridRebootGateBlocked)
                    {
                        _enrollmentCompleteEmitted = true;
                        _stateData.EnrollmentCompleteEmitted = true;
@@ -698,7 +717,8 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
            _logger.Debug($"EnrollmentTracker: TryEmitEnrollmentComplete('{source}') — evaluating guards " +
                          $"[espEverSeen={espEverSeen}, espFinalExitSeen={espFinalExitSeen}, desktopArrived={desktopArrived}, " +
                          $"helloCompleted={_espAndHelloTracker?.IsHelloCompleted ?? false}, helloPolicyConfigured={_espAndHelloTracker?.IsPolicyConfigured ?? false}, " +
-                          $"enrollmentType={enrollmentType}, autopilotMode={autopilotMode}, isSelfDeploying={isSelfDeploying}, isDeviceOnlyDeployment={isDeviceOnly}]");
+                          $"enrollmentType={enrollmentType}, autopilotMode={autopilotMode}, isSelfDeploying={isSelfDeploying}, isDeviceOnlyDeployment={isDeviceOnly}, " +
+                          $"isHybridJoin={_isHybridJoin}]");

            if (!helloResolved)
            {
@@ -714,6 +734,14 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
                return;
            }

+            if (hybridRebootGateBlocked)
+            {
+                _logger.Info($"EnrollmentTracker: Completion source '{source}' blocked — hybrid join reboot gate " +
+                             $"(waiting for IME completion or agent restart after ESP exit)");
+                EmitCompletionCheck(source, "blocked", "hybrid_reboot_gate");
+                return;
+            }
+
            _logger.Info($"EnrollmentTracker: all guards passed for source '{source}' — emitting enrollment_complete");

            // Stop timers
@@ -50,6 +50,45 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
                return;
            }

+            // ESP resumed after final exit (e.g., hybrid join reboot recovery).
+            // If ESP was marked as "final exited" but we see the same phase again from IME,
+            // ESP has resumed after a reboot. Reset completion state so the agent waits for the
+            // real final exit instead of self-destructing prematurely.
+            bool espFinalExitSeen;
+            lock (_stateLock) { espFinalExitSeen = _espFinalExitSeen; }
+
+            if (espFinalExitSeen && string.Equals(phase, lastPhase, StringComparison.OrdinalIgnoreCase))
+            {
+                _logger.Info($"EnrollmentTracker: ESP phase '{phase}' re-detected after final exit — " +
+                             "ESP has resumed (hybrid join reboot recovery), resetting completion state");
+                lock (_stateLock)
+                {
+                    _espFinalExitSeen = false;
+                    _stateData.EspFinalExitSeen = false;
+                    _stateData.EspFinalExitUtc = null;
+                }
+                RecordSignal("esp_resumed");
+                _espAndHelloTracker?.ResetForEspResumption();
+
+                _emitEvent(new EnrollmentEvent
+                {
+                    SessionId = _sessionId,
+                    TenantId = _tenantId,
+                    EventType = "esp_resumed",
+                    Severity = EventSeverity.Info,
+                    Source = "EnrollmentTracker",
+                    Phase = EnrollmentPhase.AccountSetup,
+                    Message = $"ESP phase '{phase}' re-detected after final exit — enrollment resumed after reboot",
+                    Data = new Dictionary<string, object>
+                    {
+                        { "espPhase", phase },
+                        { "isHybridJoin", _isHybridJoin }
+                    },
+                    ImmediateUpload = true
+                });
+                return;
+            }
+
            // Only emit event if the phase has actually changed
            if (string.Equals(phase, lastPhase, StringComparison.OrdinalIgnoreCase))
            {
@@ -72,6 +72,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
        private bool _espFinalExitSeen;
        private bool _enrollmentCompleteEmitted;
        private bool _desktopArrived;
+        private bool _isHybridJoin;
        private readonly DateTime _agentStartTimeUtc = DateTime.UtcNow;

        // Device-Only ESP detection (Phase 3C)
@@ -189,6 +190,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
        {
            _isBootstrapMode = isBootstrapMode;
            _sendTraceEvents = sendTraceEvents;
+            _isHybridJoin = DetectHybridJoinStatic();
            _sessionId = sessionId;
            _tenantId = tenantId;
            _emitEvent = emitEvent;
@@ -359,6 +361,7 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Tracking
                _stateData.SkipDeviceStatusPage = _skipDeviceStatusPage;
                _stateData.AutopilotMode = _autopilotMode;
                _stateData.AadJoinedWithUser = _aadJoinedWithUser;
+                _stateData.IsHybridJoin = _isHybridJoin;
                _stateDirty = true;
            }

@@ -222,6 +222,56 @@ namespace AutopilotMonitor.Agent
        /// </summary>
        static bool CheckEnrollmentCompleteMarker(AgentConfiguration config, AgentLogger logger, bool consoleMode)
        {
+            // Registry guard: if "Deployed" key exists but no active session file,
+            // this is a ghost restart after previous self-destruct. Exit immediately.
+            try
+            {
+                using (var regKey = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\AutopilotMonitor"))
+                {
+                    var deployed = regKey?.GetValue("Deployed") as string;
+                    if (!string.IsNullOrEmpty(deployed))
+                    {
+                        var dataDir = Environment.ExpandEnvironmentVariables(Constants.AgentDataDirectory);
+                        var sessionPersistence = new SessionPersistence(dataDir);
+
+                        if (!sessionPersistence.SessionExists())
+                        {
+                            logger.Info($"Agent was previously deployed ({deployed}) but no active session — ghost restart detected");
+
+                            if (config.SelfDestructOnComplete)
+                            {
+                                logger.Info("Attempting cleanup retry (scheduled task may still be registered)...");
+                                try
+                                {
+                                    using (var service = new MonitoringService(config, logger, GetAgentVersion()))
+                                    {
+                                        service.TriggerCleanup();
+                                    }
+                                    logger.Info("Cleanup retry completed.");
+                                }
+                                catch (Exception cleanupEx)
+                                {
+                                    logger.Warning($"Cleanup retry failed: {cleanupEx.Message}");
+                                }
+                            }
+
+                            if (consoleMode)
+                                Console.WriteLine("Agent was previously deployed but no active session. Exiting.");
+
+                            return true;
+                        }
+
+                        // session.id exists → normal restart (Part 2, crash recovery) → continue
+                        logger.Info($"Deployment marker found ({deployed}) with active session — continuing normally");
+                    }
+                }
+            }
+            catch (Exception regEx)
+            {
+                logger.Warning($"Registry deployment marker check failed: {regEx.Message}");
+            }
+
+            // File-based enrollment-complete marker (original guard)
            var stateDirectory = Environment.ExpandEnvironmentVariables(Constants.StateDirectory);
            var markerPath = Path.Combine(stateDirectory, "enrollment-complete.marker");

@@ -6,6 +6,7 @@ using System.Reflection;
 using AutopilotMonitor.Agent.Core.Configuration;
 using AutopilotMonitor.Agent.Core.Logging;
 using AutopilotMonitor.Shared;
+using Microsoft.Win32;

 namespace AutopilotMonitor.Agent
 {
@@ -103,6 +104,21 @@ namespace AutopilotMonitor.Agent

                logger.Info($"Scheduled Task '{taskName}' started successfully");

+                // Write deployment marker — survives self-destruct, prevents ghost re-installs.
+                // Written here (not in bootstrap script) so manual --install also sets it.
+                try
+                {
+                    using (var regKey = Registry.LocalMachine.CreateSubKey(@"SOFTWARE\AutopilotMonitor"))
+                    {
+                        regKey.SetValue("Deployed", DateTime.UtcNow.ToString("O"));
+                    }
+                    logger.Info("Deployment registry marker written (HKLM\\SOFTWARE\\AutopilotMonitor\\Deployed)");
+                }
+                catch (Exception regEx)
+                {
+                    logger.Warning($"Failed to write deployment registry marker: {regEx.Message}");
+                }
+
                if (consoleMode)
                {
                    Console.WriteLine("Installation completed successfully.");
@@ -1 +1 @@
-734
+735