diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs
index b34c3364..a3d2225b 100644
--- a/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Functions/IngestEventsFunction.cs
@@ -281,6 +281,24 @@ namespace AutopilotMonitor.Functions.Functions
isPreProvisioned: true
);
+ // Fallback: if the optimistic-concurrency merge failed despite internal retries
+ // (5 ETag retries + force-write), use an unconditional single-field write to at
+ // least persist the critical IsPreProvisioned flag.
+ if (!whiteGloveStatusTransitioned)
+ {
+ _logger.LogWarning("{SessionPrefix} WhiteGlove UpdateSessionStatusAsync failed, attempting unconditional fallback for IsPreProvisioned", sessionPrefix);
+ try
+ {
+ await _storageService.SetSessionPreProvisionedAsync(request.TenantId, request.SessionId, true);
+ whiteGloveStatusTransitioned = true;
+ _logger.LogInformation("{SessionPrefix} WhiteGlove fallback succeeded: IsPreProvisioned set via unconditional merge", sessionPrefix);
+ }
+ catch (Exception fallbackEx)
+ {
+ _logger.LogError(fallbackEx, "{SessionPrefix} WhiteGlove fallback SetSessionPreProvisionedAsync also failed", sessionPrefix);
+ }
+ }
+
_logger.LogInformation("{SessionPrefix} Status: Pending (WhiteGlove pre-provisioning complete, transitioned={Transitioned})", sessionPrefix, whiteGloveStatusTransitioned);
}
else if (processedCount > 0)
@@ -351,16 +369,19 @@ namespace AutopilotMonitor.Functions.Functions
// Retrieve updated session data to include in SignalR messages
var updatedSession = await _storageService.GetSessionAsync(request.TenantId, request.SessionId);
- // Verify whiteglove status update was persisted — return error so agent retries if not.
- // Events are already stored (idempotent UpsertReplace), only the session update needs retry.
+ // Log warning if WhiteGlove status update was not persisted despite retries and fallback.
+ // Events are already stored (idempotent UpsertReplace). Do NOT return 500 here:
+ // the agent retries the same batch containing whiteglove_complete, hitting the same
+ // failing update path, creating a self-reinforcing 500 loop that blocks spool drain
+ // and causes data loss on the monitoring session. Returning 200 lets the agent
+ // drain its spool; the session may need manual correction in the worst case.
if (whiteGloveEvent != null && updatedSession?.IsPreProvisioned != true)
{
_logger.LogError(
- "{SessionPrefix} CRITICAL: WhiteGlove status update not persisted. " +
- "IsPreProvisioned={IsPreProvisioned}, Status={Status}. Returning 500 for agent retry.",
+ "{SessionPrefix} WhiteGlove status update not persisted after retries and fallback. " +
+ "IsPreProvisioned={IsPreProvisioned}, Status={Status}. " +
+ "Proceeding with 200 to allow agent spool drain.",
sessionPrefix, updatedSession?.IsPreProvisioned, updatedSession?.Status);
- return await CreateErrorResponse(req, HttpStatusCode.InternalServerError,
- "WhiteGlove session status update failed");
}
// Send Teams notification on enrollment completion (fire-and-forget, non-fatal)
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs
index 533ab2cc..bf912ebe 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs
@@ -694,30 +694,24 @@ namespace AutopilotMonitor.Functions.Services
}
///
- /// Sets the IsPreProvisioned flag on an existing session (Merge-mode, single field update).
+ /// Sets the IsPreProvisioned flag on an existing session (unconditional Merge-mode write).
+ /// Uses ETag.All to bypass optimistic-concurrency conflicts, making this suitable as a
+ /// last-resort fallback when ETag-based updates have been exhausted.
///
public async Task SetSessionPreProvisionedAsync(string tenantId, string sessionId, bool isPreProvisioned)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
- try
- {
- var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
- var entity = await tableClient.GetEntityAsync(tenantId, sessionId);
+ var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
- var update = new TableEntity(tenantId, sessionId)
- {
- ["IsPreProvisioned"] = isPreProvisioned
- };
-
- await tableClient.UpdateEntityAsync(update, entity.Value.ETag, Azure.Data.Tables.TableUpdateMode.Merge);
- _logger.LogInformation($"Set IsPreProvisioned={isPreProvisioned} for session {sessionId}");
- }
- catch (Exception ex)
+ var update = new TableEntity(tenantId, sessionId)
{
- _logger.LogError(ex, $"Failed to set IsPreProvisioned for session {sessionId}");
- }
+ ["IsPreProvisioned"] = isPreProvisioned
+ };
+
+ await tableClient.UpdateEntityAsync(update, ETag.All, TableUpdateMode.Merge);
+ _logger.LogInformation($"Set IsPreProvisioned={isPreProvisioned} for session {sessionId} (unconditional merge)");
}
///