fix WhiteGlove spool drain 500 loop that blocks event upload and causes data loss

The IngestEventsFunction verification check returned HTTP 500 when the
WhiteGlove IsPreProvisioned session update failed, causing the agent to
retry the same batch in a tight loop (same whiteglove_complete event,
same failing update, same 500). After 20 retries the agent gave up with
events stuck in the local spool and the session metadata broken.

- Add unconditional fallback (ETag.All merge) via SetSessionPreProvisionedAsync
  when UpdateSessionStatusAsync fails despite internal ETag retries
- Replace the 500 verification response with a warning log + continue
  with 200 so the agent can drain its spool
- Simplify SetSessionPreProvisionedAsync to use ETag.All unconditional
  merge, removing the unnecessary read-then-write pattern

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs

@@ -281,6 +281,24 @@ namespace AutopilotMonitor.Functions.Functions
                         isPreProvisioned: true
                     );
 
+                    // Fallback: if the optimistic-concurrency merge failed despite internal retries
+                    // (5 ETag retries + force-write), use an unconditional single-field write to at
+                    // least persist the critical IsPreProvisioned flag.
+                    if (!whiteGloveStatusTransitioned)
+                    {
+                        _logger.LogWarning("{SessionPrefix} WhiteGlove UpdateSessionStatusAsync failed, attempting unconditional fallback for IsPreProvisioned", sessionPrefix);
+                        try
+                        {
+                            await _storageService.SetSessionPreProvisionedAsync(request.TenantId, request.SessionId, true);
+                            whiteGloveStatusTransitioned = true;
+                            _logger.LogInformation("{SessionPrefix} WhiteGlove fallback succeeded: IsPreProvisioned set via unconditional merge", sessionPrefix);
+                        }
+                        catch (Exception fallbackEx)
+                        {
+                            _logger.LogError(fallbackEx, "{SessionPrefix} WhiteGlove fallback SetSessionPreProvisionedAsync also failed", sessionPrefix);
+                        }
+                    }
+
                     _logger.LogInformation("{SessionPrefix} Status: Pending (WhiteGlove pre-provisioning complete, transitioned={Transitioned})", sessionPrefix, whiteGloveStatusTransitioned);
                 }
                 else if (processedCount > 0)
@@ -351,16 +369,19 @@ namespace AutopilotMonitor.Functions.Functions
                 // Retrieve updated session data to include in SignalR messages
                 var updatedSession = await _storageService.GetSessionAsync(request.TenantId, request.SessionId);
 
-                // Verify whiteglove status update was persisted — return error so agent retries if not.
+                // Log warning if WhiteGlove status update was not persisted despite retries and fallback.
-                // Events are already stored (idempotent UpsertReplace), only the session update needs retry.
+                // Events are already stored (idempotent UpsertReplace). Do NOT return 500 here:
+                // the agent retries the same batch containing whiteglove_complete, hitting the same
+                // failing update path, creating a self-reinforcing 500 loop that blocks spool drain
+                // and causes data loss on the monitoring session. Returning 200 lets the agent
+                // drain its spool; the session may need manual correction in the worst case.
                 if (whiteGloveEvent != null && updatedSession?.IsPreProvisioned != true)
                 {
                     _logger.LogError(
-                        "{SessionPrefix} CRITICAL: WhiteGlove status update not persisted. " +
+                        "{SessionPrefix} WhiteGlove status update not persisted after retries and fallback. " +
-                        "IsPreProvisioned={IsPreProvisioned}, Status={Status}. Returning 500 for agent retry.",
+                        "IsPreProvisioned={IsPreProvisioned}, Status={Status}. " +
+                        "Proceeding with 200 to allow agent spool drain.",
                         sessionPrefix, updatedSession?.IsPreProvisioned, updatedSession?.Status);
-                    return await CreateErrorResponse(req, HttpStatusCode.InternalServerError,
-                        "WhiteGlove session status update failed");
                 }
 
                 // Send Teams notification on enrollment completion (fire-and-forget, non-fatal)

src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs

@@ -694,30 +694,24 @@ namespace AutopilotMonitor.Functions.Services
         }
 
         /// <summary>
-        /// Sets the IsPreProvisioned flag on an existing session (Merge-mode, single field update).
+        /// Sets the IsPreProvisioned flag on an existing session (unconditional Merge-mode write).
+        /// Uses ETag.All to bypass optimistic-concurrency conflicts, making this suitable as a
+        /// last-resort fallback when ETag-based updates have been exhausted.
         /// </summary>
         public async Task SetSessionPreProvisionedAsync(string tenantId, string sessionId, bool isPreProvisioned)
         {
             SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
             SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
 
-            try
+            var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
-            {
-                var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
-                var entity = await tableClient.GetEntityAsync<TableEntity>(tenantId, sessionId);
 
-                var update = new TableEntity(tenantId, sessionId)
+            var update = new TableEntity(tenantId, sessionId)
-                {
-                    ["IsPreProvisioned"] = isPreProvisioned
-                };
-
-                await tableClient.UpdateEntityAsync(update, entity.Value.ETag, Azure.Data.Tables.TableUpdateMode.Merge);
-                _logger.LogInformation($"Set IsPreProvisioned={isPreProvisioned} for session {sessionId}");
-            }
-            catch (Exception ex)
             {
-                _logger.LogError(ex, $"Failed to set IsPreProvisioned for session {sessionId}");
+                ["IsPreProvisioned"] = isPreProvisioned
-            }
+            };
+
+            await tableClient.UpdateEntityAsync(update, ETag.All, TableUpdateMode.Merge);
+            _logger.LogInformation($"Set IsPreProvisioned={isPreProvisioned} for session {sessionId} (unconditional merge)");
         }
 
         /// <summary>