agent StateMachine fixes

src/Agent/AutopilotMonitor.Agent.Core.Tests/Collectors/ProvisioningStatusTrackerTests.cs

@@ -440,5 +440,90 @@ namespace AutopilotMonitor.Agent.Core.Tests.Collectors
             t.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", json);
             Assert.False(t.HasSaveWhiteGloveSuccessResult);
         }
+
+        [Fact]
+        public void ProcessCategoryStatus_DeviceSetup_WhiteGloveStateTransition_EmitsRawDumpForEachChange()
+        {
+            var t = CreateTracker();
+
+            // Poll 1: SaveWhiteGloveSuccessResult=notStarted
+            var jsonNotStarted = @"{
+                ""categoryState"":""inProgress"",
+                ""DeviceSetup.SecurityPoliciesSubcategory"":{""subcategoryState"":""succeeded"",""subcategoryStatusText"":""ok""},
+                ""DeviceSetup.SaveWhiteGloveSuccessResult"":{""subcategoryState"":""notStarted""},
+                ""categoryStatusText"":""working""
+            }";
+
+            t.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", jsonNotStarted);
+
+            // Should emit raw dump for first WG state observation
+            var raws = OfType("esp_provisioning_raw");
+            var wgRaws = raws.Where(e => e.Message.Contains("whiteglove_signal_")).ToList();
+            Assert.Single(wgRaws);
+            Assert.Contains("whiteglove_signal_notStarted", wgRaws[0].Message);
+            Assert.False(t.HasSaveWhiteGloveSuccessResult);
+
+            // Poll 2: SaveWhiteGloveSuccessResult=succeeded
+            var jsonSucceeded = @"{
+                ""categoryState"":""succeeded"",
+                ""DeviceSetup.SecurityPoliciesSubcategory"":{""subcategoryState"":""succeeded"",""subcategoryStatusText"":""ok""},
+                ""DeviceSetup.SaveWhiteGloveSuccessResult"":{""subcategoryState"":""succeeded"",""subcategoryStatusText"":""succeeded""},
+                ""categoryStatusText"":""Complete""
+            }";
+
+            t.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", jsonSucceeded);
+
+            // Should now have 2 WG raw dumps (ConcurrentBag has no guaranteed order)
+            raws = OfType("esp_provisioning_raw");
+            wgRaws = raws.Where(e => e.Message.Contains("whiteglove_signal_")).ToList();
+            Assert.Equal(2, wgRaws.Count);
+            Assert.Single(wgRaws, e => e.Message.Contains("whiteglove_signal_notStarted"));
+            Assert.Single(wgRaws, e => e.Message.Contains("whiteglove_signal_succeeded"));
+            Assert.True(t.HasSaveWhiteGloveSuccessResult);
+        }
+
+        [Fact]
+        public void ProcessCategoryStatus_DeviceSetup_WhiteGloveStateSame_NoDuplicateRawDump()
+        {
+            var t = CreateTracker();
+
+            var json = @"{
+                ""categoryState"":""inProgress"",
+                ""DeviceSetup.SecurityPoliciesSubcategory"":{""subcategoryState"":""succeeded"",""subcategoryStatusText"":""ok""},
+                ""DeviceSetup.SaveWhiteGloveSuccessResult"":{""subcategoryState"":""notStarted""},
+                ""categoryStatusText"":""working""
+            }";
+
+            // Same JSON twice — should only emit WG raw dump once
+            t.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", json);
+            t.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", json);
+
+            var raws = OfType("esp_provisioning_raw");
+            var wgRaws = raws.Where(e => e.Message.Contains("whiteglove_signal_")).ToList();
+            Assert.Single(wgRaws);
+            Assert.False(t.HasSaveWhiteGloveSuccessResult);
+        }
+
+        [Fact]
+        public void ProcessCategoryStatus_DeviceSetup_WhiteGloveStaysNotStarted_FlagNeverSet()
+        {
+            var t = CreateTracker();
+
+            var json = @"{
+                ""categoryState"":""succeeded"",
+                ""DeviceSetup.SecurityPoliciesSubcategory"":{""subcategoryState"":""succeeded"",""subcategoryStatusText"":""ok""},
+                ""DeviceSetup.SaveWhiteGloveSuccessResult"":{""subcategoryState"":""notStarted""},
+                ""categoryStatusText"":""Complete""
+            }";
+
+            t.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", json);
+
+            // Raw dump emitted (for observability), but flag stays false
+            var wgRaws = OfType("esp_provisioning_raw")
+                .Where(e => e.Message.Contains("whiteglove_signal_")).ToList();
+            Assert.Single(wgRaws);
+            Assert.Contains("whiteglove_signal_notStarted", wgRaws[0].Message);
+            Assert.False(t.HasSaveWhiteGloveSuccessResult);
+        }
     }
 }

src/Agent/AutopilotMonitor.Agent.Core.Tests/Tracking/CompletionStateMachineTests.cs

@@ -1143,18 +1143,19 @@ namespace AutopilotMonitor.Agent.Core.Tests.Tracking
         // ===== WhiteGlove Guard Tests =====
 
         [Fact]
-        public void DeviceOnly_ProvisioningComplete_WithWhiteGloveStart_TransitionsToWhiteGloveCompleted()
+        public void DeviceOnly_ProvisioningComplete_WithWhiteGloveStartOnly_CompletesAsSelfDeploying()
         {
+            // EventID 509 alone (soft signal) must NOT trigger WhiteGlove — fires on hybrid-join too
             var sm = new CompletionStateMachine();
             var ctx = DeviceOnlyContext();
             ctx.WhiteGloveStartDetected = true;
+            ctx.HasSaveWhiteGloveSuccessResult = false;
 
             var result = sm.ProcessTrigger("device_setup_provisioning_complete", ctx);
 
-            Assert.Equal(EnrollmentCompletionState.WhiteGloveCompleted, sm.CurrentState);
-            Assert.True(result.ShouldEmitWhiteGloveComplete);
-            Assert.False(result.ShouldEmitEnrollmentComplete);
-            Assert.Contains("whiteglove_guard_activated", result.SignalsToRecord);
+            Assert.Equal(EnrollmentCompletionState.Completed, sm.CurrentState);
+            Assert.True(result.ShouldEmitEnrollmentComplete);
+            Assert.False(result.ShouldEmitWhiteGloveComplete);
         }
 
         [Fact]
@@ -1220,7 +1221,7 @@ namespace AutopilotMonitor.Agent.Core.Tests.Tracking
         {
             var sm = new CompletionStateMachine();
             var ctx = DeviceOnlyContext();
-            ctx.WhiteGloveStartDetected = true;
+            ctx.HasSaveWhiteGloveSuccessResult = true;
             sm.ProcessTrigger("device_setup_provisioning_complete", ctx);
 
             Assert.Equal(EnrollmentCompletionState.WhiteGloveCompleted, sm.CurrentState);
@@ -1232,23 +1233,21 @@ namespace AutopilotMonitor.Agent.Core.Tests.Tracking
         }
 
         [Fact]
-        public void EspExiting_SkipUser_WithWhiteGloveStart_TransitionsToWhiteGloveCompleted()
+        public void EspExiting_SkipUser_WithWhiteGloveStartOnly_CompletesAsDeviceOnly()
         {
+            // EventID 509 alone (soft signal) must NOT trigger WhiteGlove on ESP exit
             var sm = new CompletionStateMachine();
 
-            // ESP phase detected first
             var ctx = DefaultContext(autopilotMode: 0, skipUserStatusPage: true, aadJoinedWithUser: false,
                 isHelloPolicyConfigured: false, whiteGloveStartDetected: true);
+            ctx.HasSaveWhiteGloveSuccessResult = false;
             sm.ProcessTrigger("esp_phase_changed", ctx);
 
-            // ESP exits with SkipUserStatusPage=true + WhiteGlove signal
             ctx.LastEspPhase = "DeviceSetup";
             var result = sm.ProcessTrigger("esp_exiting", ctx);
 
-            Assert.Equal(EnrollmentCompletionState.WhiteGloveCompleted, sm.CurrentState);
-            Assert.True(result.ShouldEmitWhiteGloveComplete);
-            Assert.False(result.ShouldEmitEnrollmentComplete);
-            Assert.Contains("whiteglove_guard_esp_exiting", result.SignalsToRecord);
+            Assert.NotEqual(EnrollmentCompletionState.WhiteGloveCompleted, sm.CurrentState);
+            Assert.False(result.ShouldEmitWhiteGloveComplete);
         }
 
         [Fact]

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Enrollment/Completion/CompletionStateMachine.cs

@@ -38,7 +38,9 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Enrollment.Completion
         // WhiteGlove detection signals
         public bool WhiteGloveStartDetected { get; set; }
         public bool HasSaveWhiteGloveSuccessResult { get; set; }
-        public bool IsWhiteGloveActive => WhiteGloveStartDetected || HasSaveWhiteGloveSuccessResult;
+        // Only SaveWhiteGloveSuccessResult=succeeded from ESP registry is a reliable WG confirmation.
+        // WhiteGloveStartDetected (EventID 509) is a soft signal — fires on hybrid-join too.
+        public bool IsWhiteGloveActive => HasSaveWhiteGloveSuccessResult;
 
         // Timing
         public DateTime AgentStartTimeUtc { get; set; }

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Enrollment/EnrollmentTracker.CompletionLogic.cs

@@ -112,22 +112,22 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Enrollment
                 return;
             }
 
-            // WhiteGlove guard: When EventID 509 was detected OR the DeviceSetup registry
-            // contains SaveWhiteGloveSuccessResult=succeeded, this is a WhiteGlove session.
-            // Route to OnWhiteGloveCompleted instead of self_deploying_provisioning_complete.
+            // WhiteGlove guard: Only SaveWhiteGloveSuccessResult=succeeded from the ESP registry
+            // is a reliable WhiteGlove confirmation. EventID 509 is a soft signal only — it fires
+            // on hybrid-join devices too and must NOT trigger WG completion on its own.
             bool whiteGloveStarted = _espAndHelloTracker?.IsWhiteGloveStartDetected ?? false;
             bool whiteGloveSuccessResult = _espAndHelloTracker?.HasSaveWhiteGloveSuccessResult ?? false;
 
-            if (whiteGloveStarted || whiteGloveSuccessResult)
+            if (whiteGloveSuccessResult)
             {
-                _logger.Info($"EnrollmentTracker: Device-only deployment with WhiteGlove signals detected " +
+                _logger.Info($"EnrollmentTracker: Device-only deployment with WhiteGlove confirmed " +
                              $"(whiteGloveStarted={whiteGloveStarted}, saveWhiteGloveSuccessResult={whiteGloveSuccessResult}) — " +
                              "routing to WhiteGlove completion instead of self_deploying_provisioning_complete");
 
                 RecordSignal("whiteglove_guard_activated");
 
                 EmitTraceEvent("whiteglove_guard_activated",
-                    "DeviceSetup provisioning complete but WhiteGlove signals present — routing to WhiteGlove path",
+                    "DeviceSetup provisioning complete with SaveWhiteGloveSuccessResult confirmed — routing to WhiteGlove path",
                     new Dictionary<string, object>
                     {
                         { "whiteGloveStartDetected", whiteGloveStarted },
@@ -662,21 +662,20 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Enrollment
                 }
                 else if (skipUser == true)
                 {
-                    // WhiteGlove guard: same check as in OnDeviceSetupProvisioningComplete.
-                    // If the ESP exit happens in a WhiteGlove session (SkipUserStatusPage=true, but
-                    // WhiteGlove signals present), route to WhiteGlove completion instead of device-only.
+                    // WhiteGlove guard: Only SaveWhiteGloveSuccessResult=succeeded is reliable.
+                    // EventID 509 is a soft signal — fires on hybrid-join too, must not trigger WG.
                     bool wgStarted = _espAndHelloTracker?.IsWhiteGloveStartDetected ?? false;
                     bool wgSuccessResult = _espAndHelloTracker?.HasSaveWhiteGloveSuccessResult ?? false;
 
-                    if (wgStarted || wgSuccessResult)
+                    if (wgSuccessResult)
                     {
-                        _logger.Info($"EnrollmentTracker: ESP exiting with SkipUserStatusPage=true but WhiteGlove signals present " +
+                        _logger.Info($"EnrollmentTracker: ESP exiting with SkipUserStatusPage=true and WhiteGlove confirmed " +
                                      $"(whiteGloveStarted={wgStarted}, saveWhiteGloveSuccessResult={wgSuccessResult}) — " +
                                      "routing to WhiteGlove completion instead of device_only_esp_registry");
 
                         RecordSignal("whiteglove_guard_activated");
                         EmitTraceEvent("whiteglove_guard_esp_exiting",
-                            "ESP exiting with SkipUserStatusPage=true but WhiteGlove signals present — routing to WhiteGlove path",
+                            "ESP exiting with SaveWhiteGloveSuccessResult confirmed — routing to WhiteGlove path",
                             new Dictionary<string, object>
                             {
                                 { "whiteGloveStartDetected", wgStarted },
@@ -1207,7 +1206,9 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Enrollment
                     { "skipDeviceStatusPage", _skipDeviceStatusPage?.ToString() ?? "unknown" },
                     { "autopilotMode", _autopilotMode?.ToString() ?? "unknown" },
                     { "isSelfDeploying", IsSelfDeploying },
-                    { "isDeviceOnlyDeployment", IsDeviceOnlyDeployment }
+                    { "isDeviceOnlyDeployment", IsDeviceOnlyDeployment },
+                    { "whiteGloveStartDetected", _espAndHelloTracker?.IsWhiteGloveStartDetected ?? false },
+                    { "saveWhiteGloveSuccessResult", _espAndHelloTracker?.HasSaveWhiteGloveSuccessResult ?? false }
                 };
             }
 

src/Agent/AutopilotMonitor.Agent.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs

@@ -76,6 +76,8 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Enrollment.SystemSignals
         private bool _deviceSetupProvisioningCompleteFired;
         // WhiteGlove confirmation: DeviceSetup registry contains SaveWhiteGloveSuccessResult=succeeded
         private bool _saveWhiteGloveSuccessResultSeen;
+        // Track SaveWhiteGloveSuccessResult state transitions for observability (null → notStarted → succeeded)
+        private string _lastSaveWhiteGloveState;
         private readonly object _stateLock = new object();
 
         public event EventHandler<string> EspFailureDetected;
@@ -556,25 +558,38 @@ namespace AutopilotMonitor.Agent.Core.Monitoring.Enrollment.SystemSignals
                     string categoryStatusMessage = SafeGetString(root, "categoryStatusMessage");
                     var subcategories = ParseSubcategories(root);
 
-                    // Detect WhiteGlove confirmation signal in DeviceSetup category.
+                    // Detect WhiteGlove signal in DeviceSetup category.
                     // SaveWhiteGloveSuccessResult is NOT a *Subcategory-suffixed property, so
                     // ParseSubcategories skips it. We scan the raw JSON explicitly.
-                    if (string.Equals(categoryLabel, "DeviceSetup", StringComparison.OrdinalIgnoreCase)
-                        && !_saveWhiteGloveSuccessResultSeen)
+                    // Track state transitions for full observability (notStarted → succeeded).
+                    if (string.Equals(categoryLabel, "DeviceSetup", StringComparison.OrdinalIgnoreCase))
                     {
                         foreach (var prop in root.EnumerateObject())
                         {
                             if (prop.Name.IndexOf("SaveWhiteGloveSuccessResult", StringComparison.OrdinalIgnoreCase) >= 0
                                 && prop.Value.ValueKind == JsonValueKind.Object)
                             {
-                                var state = SafeGetString(prop.Value, "subcategoryState");
-                                if (string.Equals(state, "succeeded", StringComparison.OrdinalIgnoreCase))
+                                var state = SafeGetString(prop.Value, "subcategoryState") ?? "unknown";
+
+                                if (!string.Equals(state, _lastSaveWhiteGloveState, StringComparison.OrdinalIgnoreCase))
                                 {
-                                    _saveWhiteGloveSuccessResultSeen = true;
-                                    _logger.Info("ProvisioningStatusTracker: SaveWhiteGloveSuccessResult=succeeded detected " +
-                                                 "— WhiteGlove confirmation signal");
-                                    break;
+                                    var previousState = _lastSaveWhiteGloveState ?? "not_seen";
+                                    _lastSaveWhiteGloveState = state;
+
+                                    _logger.Info($"ProvisioningStatusTracker: SaveWhiteGloveSuccessResult state " +
+                                                 $"transition: {previousState} -> {state}");
+
+                                    EmitRawRegistryDump(categoryLabel, jsonValue,
+                                        $"whiteglove_signal_{state}");
+
+                                    if (string.Equals(state, "succeeded", StringComparison.OrdinalIgnoreCase))
+                                    {
+                                        _saveWhiteGloveSuccessResultSeen = true;
+                                        _logger.Info("ProvisioningStatusTracker: SaveWhiteGloveSuccessResult=succeeded " +
+                                                     "— WhiteGlove confirmation signal");
+                                    }
                                 }
+                                break;
                             }
                         }
                     }

src/Agent/AutopilotMonitor.Agent/buildcounter.txt

@@ -1 +1 @@
-1008
+1013

src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestEventsFunction.Processing.cs

@@ -748,21 +748,12 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
                 }
             }
 
-            // Early WhiteGlove classification: if whiteglove_started event is in this batch,
-            // mark the session as pre-provisioned immediately (safety net for old agents that may
-            // still send self_deploying_provisioning_complete instead of whiteglove_complete).
+            // whiteglove_started (EventID 509) is a soft signal only — it fires on hybrid-join
+            // devices too. Do NOT set IsPreProvisioned here. Only whiteglove_complete (confirmed by
+            // SaveWhiteGloveSuccessResult=succeeded in ESP registry) should mark IsPreProvisioned.
             if (c.WhiteGloveStartedEvent != null)
             {
-                try
-                {
-                    await _sessionRepo.SetSessionPreProvisionedAsync(
-                        request.TenantId, request.SessionId, true, isUserDriven: false);
-                    _logger.LogInformation("{SessionPrefix} whiteglove_started detected — marked IsPreProvisioned=true early", sessionPrefix);
-                }
-                catch (Exception ex)
-                {
-                    _logger.LogWarning(ex, "{SessionPrefix} Failed to set early IsPreProvisioned for whiteglove_started", sessionPrefix);
-                }
+                _logger.LogInformation("{SessionPrefix} whiteglove_started detected (soft signal — not setting IsPreProvisioned, awaiting whiteglove_complete)", sessionPrefix);
             }
 
             // Stalled-status transitions are independent of Succeeded/Failed/Pending/WhiteGlove paths.