diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs index b34c3364..a3d2225b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs @@ -281,6 +281,24 @@ namespace AutopilotMonitor.Functions.Functions isPreProvisioned: true ); + // Fallback: if the optimistic-concurrency merge failed despite internal retries + // (5 ETag retries + force-write), use an unconditional single-field write to at + // least persist the critical IsPreProvisioned flag. + if (!whiteGloveStatusTransitioned) + { + _logger.LogWarning("{SessionPrefix} WhiteGlove UpdateSessionStatusAsync failed, attempting unconditional fallback for IsPreProvisioned", sessionPrefix); + try + { + await _storageService.SetSessionPreProvisionedAsync(request.TenantId, request.SessionId, true); + whiteGloveStatusTransitioned = true; + _logger.LogInformation("{SessionPrefix} WhiteGlove fallback succeeded: IsPreProvisioned set via unconditional merge", sessionPrefix); + } + catch (Exception fallbackEx) + { + _logger.LogError(fallbackEx, "{SessionPrefix} WhiteGlove fallback SetSessionPreProvisionedAsync also failed", sessionPrefix); + } + } + _logger.LogInformation("{SessionPrefix} Status: Pending (WhiteGlove pre-provisioning complete, transitioned={Transitioned})", sessionPrefix, whiteGloveStatusTransitioned); } else if (processedCount > 0) @@ -351,16 +369,19 @@ namespace AutopilotMonitor.Functions.Functions // Retrieve updated session data to include in SignalR messages var updatedSession = await _storageService.GetSessionAsync(request.TenantId, request.SessionId); - // Verify whiteglove status update was persisted — return error so agent retries if not. - // Events are already stored (idempotent UpsertReplace), only the session update needs retry. + // Log warning if WhiteGlove status update was not persisted despite retries and fallback. + // Events are already stored (idempotent UpsertReplace). Do NOT return 500 here: + // the agent retries the same batch containing whiteglove_complete, hitting the same + // failing update path, creating a self-reinforcing 500 loop that blocks spool drain + // and causes data loss on the monitoring session. Returning 200 lets the agent + // drain its spool; the session may need manual correction in the worst case. if (whiteGloveEvent != null && updatedSession?.IsPreProvisioned != true) { _logger.LogError( - "{SessionPrefix} CRITICAL: WhiteGlove status update not persisted. " + - "IsPreProvisioned={IsPreProvisioned}, Status={Status}. Returning 500 for agent retry.", + "{SessionPrefix} WhiteGlove status update not persisted after retries and fallback. " + + "IsPreProvisioned={IsPreProvisioned}, Status={Status}. " + + "Proceeding with 200 to allow agent spool drain.", sessionPrefix, updatedSession?.IsPreProvisioned, updatedSession?.Status); - return await CreateErrorResponse(req, HttpStatusCode.InternalServerError, - "WhiteGlove session status update failed"); } // Send Teams notification on enrollment completion (fire-and-forget, non-fatal) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index 533ab2cc..bf912ebe 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -694,30 +694,24 @@ namespace AutopilotMonitor.Functions.Services } /// - /// Sets the IsPreProvisioned flag on an existing session (Merge-mode, single field update). + /// Sets the IsPreProvisioned flag on an existing session (unconditional Merge-mode write). + /// Uses ETag.All to bypass optimistic-concurrency conflicts, making this suitable as a + /// last-resort fallback when ETag-based updates have been exhausted. /// public async Task SetSessionPreProvisionedAsync(string tenantId, string sessionId, bool isPreProvisioned) { SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); - try - { - var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); - var entity = await tableClient.GetEntityAsync(tenantId, sessionId); + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); - var update = new TableEntity(tenantId, sessionId) - { - ["IsPreProvisioned"] = isPreProvisioned - }; - - await tableClient.UpdateEntityAsync(update, entity.Value.ETag, Azure.Data.Tables.TableUpdateMode.Merge); - _logger.LogInformation($"Set IsPreProvisioned={isPreProvisioned} for session {sessionId}"); - } - catch (Exception ex) + var update = new TableEntity(tenantId, sessionId) { - _logger.LogError(ex, $"Failed to set IsPreProvisioned for session {sessionId}"); - } + ["IsPreProvisioned"] = isPreProvisioned + }; + + await tableClient.UpdateEntityAsync(update, ETag.All, TableUpdateMode.Merge); + _logger.LogInformation($"Set IsPreProvisioned={isPreProvisioned} for session {sessionId} (unconditional merge)"); } ///