From bb2eaa1a56a0a2f88fdbccce8202175e49d8e3cd Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 19 Mar 2026 13:33:40 +0100 Subject: [PATCH] fix ghost session entries --- .../AutopilotMonitor.Agent/buildcounter.txt | 2 +- .../Services/MaintenanceService.cs | 5 ++ .../TableStorageService.Maintenance.cs | 77 +++++++++++++++++++ .../Services/TableStorageService.Sessions.cs | 36 +++++++++ 4 files changed, 119 insertions(+), 1 deletion(-) diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt index 1b79f38e..cc5027ee 100644 --- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt @@ -1 +1 @@ -500 +502 diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 931df1c8..a56080f6 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -89,6 +89,11 @@ namespace AutopilotMonitor.Functions.Services // Safety net: backfill any sessions missing from SessionsIndex await _storageService.BackfillSessionIndexAsync(); + // One-time cleanup: remove ghost SessionsIndex entries caused by the + // StoreSessionAsync Replace-mode IndexRowKey bug (now fixed). + // TODO: Remove after 2026-06-01 + await _storageService.CleanupGhostSessionIndexEntriesAsync(); + // Backfill OnboardedAt for tenants that don't have it yet await BackfillTenantOnboardedAtAsync(); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs index 5104af96..59f0b9af 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs @@ -466,6 +466,83 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// One-time cleanup: removes ghost entries from SessionsIndex where a SessionId has + /// multiple index rows (caused by a bug where StoreSessionAsync's Replace mode deleted + /// IndexRowKey, preventing old index entries from being cleaned up when StartedAt shifted). + /// Fixed in the same release — this cleanup can be removed once all environments have run it. + /// TODO: Remove after 2026-06-01 (3 months grace period). + /// + public async Task CleanupGhostSessionIndexEntriesAsync() + { + try + { + var indexTable = _tableServiceClient.GetTableClient(Constants.TableNames.SessionsIndex); + var sessionsTable = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); + + // Build a map of SessionId → list of index RowKeys (across all tenants) + var sessionIndexEntries = new Dictionary>(); + + var query = indexTable.QueryAsync( + select: new[] { "PartitionKey", "RowKey", "SessionId", "EventCount" }); + + await foreach (var entity in query) + { + var sessionId = entity.GetString("SessionId") ?? ExtractSessionIdFromIndexRowKey(entity.RowKey); + var key = $"{entity.PartitionKey}_{sessionId}"; + var eventCount = entity.GetInt32("EventCount") ?? 0; + + if (!sessionIndexEntries.ContainsKey(key)) + sessionIndexEntries[key] = new List<(string, string, int)>(); + + sessionIndexEntries[key].Add((entity.PartitionKey, entity.RowKey, eventCount)); + } + + int deletedCount = 0; + + foreach (var (key, entries) in sessionIndexEntries) + { + if (entries.Count <= 1) + continue; // No duplicates + + // Keep the entry with the highest EventCount (most up-to-date); + // delete all others as ghosts. + var sorted = entries.OrderByDescending(e => e.EventCount).ToList(); + var keep = sorted[0]; + + for (int i = 1; i < sorted.Count; i++) + { + var ghost = sorted[i]; + try + { + await indexTable.DeleteEntityAsync(ghost.PartitionKey, ghost.RowKey); + deletedCount++; + _logger.LogInformation( + "Deleted ghost SessionsIndex entry: {PartitionKey}/{RowKey} (EventCount={GhostCount}, kept entry has EventCount={KeptCount})", + ghost.PartitionKey, ghost.RowKey, ghost.EventCount, keep.EventCount); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to delete ghost index entry {PartitionKey}/{RowKey}", + ghost.PartitionKey, ghost.RowKey); + } + } + } + + if (deletedCount > 0) + { + _logger.LogInformation("Ghost SessionsIndex cleanup completed: {Count} ghost entries removed", deletedCount); + } + + return deletedCount; + } + catch (Exception ex) + { + _logger.LogError(ex, "Ghost SessionsIndex cleanup failed"); + return 0; + } + } + /// /// Checks if the SessionsIndex table is empty. /// Used by startup backfill to determine if a full migration is needed. diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index cf306bf9..b5713607 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -294,6 +294,7 @@ namespace AutopilotMonitor.Functions.Services string geoRegion = string.Empty; string geoCity = string.Empty; string geoLoc = string.Empty; + string? existingIndexRowKey = null; try { @@ -323,6 +324,7 @@ namespace AutopilotMonitor.Functions.Services geoRegion = existingEntity.GetString("GeoRegion") ?? string.Empty; geoCity = existingEntity.GetString("GeoCity") ?? string.Empty; geoLoc = existingEntity.GetString("GeoLoc") ?? string.Empty; + existingIndexRowKey = existingEntity.GetString("IndexRowKey"); // Guard: never regress a terminal status (Succeeded/Failed) back to InProgress. // StoreSessionAsync uses UpsertEntity (Replace mode) which overwrites all fields. @@ -407,6 +409,12 @@ namespace AutopilotMonitor.Functions.Services if (!string.IsNullOrEmpty(geoLoc)) entity["GeoLoc"] = geoLoc; + // Preserve IndexRowKey through the Replace so that: + // 1. Concurrent UpdateSessionStatusAsync calls can find it (prevents index stale-status) + // 2. UpsertSessionIndexAsync can detect and delete old index entries when startedAt shifts + if (!string.IsNullOrEmpty(existingIndexRowKey)) + entity["IndexRowKey"] = existingIndexRowKey; + await tableClient.UpsertEntityAsync(entity); // Dual-write: upsert into SessionsIndex for time-sorted listing @@ -933,6 +941,19 @@ namespace AutopilotMonitor.Functions.Services // Dual-write: merge the same fields into SessionsIndex var indexRowKey = session.GetString("IndexRowKey"); + + // Fallback: if IndexRowKey was lost (e.g. StoreSessionAsync Replace race), + // compute it from StartedAt + SessionId so the index still gets updated. + if (string.IsNullOrEmpty(indexRowKey)) + { + var sessionStartedAt = session.GetDateTimeOffset("StartedAt")?.UtcDateTime; + if (sessionStartedAt.HasValue) + { + indexRowKey = ComputeIndexRowKey(sessionStartedAt.Value, sessionId); + _logger.LogWarning("Session {SessionId}: IndexRowKey was null, computed fallback from StartedAt", sessionId); + } + } + await MergeSessionIndexAsync(tenantId, indexRowKey, update); _logger.LogInformation($"Updated session {sessionId} status to {status}"); @@ -1041,6 +1062,15 @@ namespace AutopilotMonitor.Functions.Services // Dual-write: merge the same fields into SessionsIndex var forceIndexRowKey = freshSession.GetString("IndexRowKey"); + if (string.IsNullOrEmpty(forceIndexRowKey)) + { + var forceStartedAt = freshSession.GetDateTimeOffset("StartedAt")?.UtcDateTime; + if (forceStartedAt.HasValue) + { + forceIndexRowKey = ComputeIndexRowKey(forceStartedAt.Value, sessionId); + _logger.LogWarning("Session {SessionId}: IndexRowKey was null in force-write path, computed fallback from StartedAt", sessionId); + } + } await MergeSessionIndexAsync(tenantId, forceIndexRowKey, forceUpdate); _logger.LogInformation($"Force-updated session {sessionId} status to {status} (unconditional merge after ETag exhaustion)"); @@ -1137,6 +1167,12 @@ namespace AutopilotMonitor.Functions.Services // Dual-write: merge the same fields into SessionsIndex var indexRowKey = entity.Value.GetString("IndexRowKey"); + if (string.IsNullOrEmpty(indexRowKey)) + { + var incStartedAt = entity.Value.GetDateTimeOffset("StartedAt")?.UtcDateTime; + if (incStartedAt.HasValue) + indexRowKey = ComputeIndexRowKey(incStartedAt.Value, sessionId); + } await MergeSessionIndexAsync(tenantId, indexRowKey, update); return;