diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt
index 1b79f38e..cc5027ee 100644
--- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt
+++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt
@@ -1 +1 @@
-500
+502
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
index 931df1c8..a56080f6 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
@@ -89,6 +89,11 @@ namespace AutopilotMonitor.Functions.Services
// Safety net: backfill any sessions missing from SessionsIndex
await _storageService.BackfillSessionIndexAsync();
+ // One-time cleanup: remove ghost SessionsIndex entries caused by the
+ // StoreSessionAsync Replace-mode IndexRowKey bug (now fixed).
+ // TODO: Remove after 2026-06-01
+ await _storageService.CleanupGhostSessionIndexEntriesAsync();
+
// Backfill OnboardedAt for tenants that don't have it yet
await BackfillTenantOnboardedAtAsync();
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs
index 5104af96..59f0b9af 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs
@@ -466,6 +466,83 @@ namespace AutopilotMonitor.Functions.Services
}
}
+ ///
+ /// One-time cleanup: removes ghost entries from SessionsIndex where a SessionId has
+ /// multiple index rows (caused by a bug where StoreSessionAsync's Replace mode deleted
+ /// IndexRowKey, preventing old index entries from being cleaned up when StartedAt shifted).
+ /// Fixed in the same release — this cleanup can be removed once all environments have run it.
+ /// TODO: Remove after 2026-06-01 (3 months grace period).
+ ///
+ public async Task CleanupGhostSessionIndexEntriesAsync()
+ {
+ try
+ {
+ var indexTable = _tableServiceClient.GetTableClient(Constants.TableNames.SessionsIndex);
+ var sessionsTable = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
+
+ // Build a map of SessionId → list of index RowKeys (across all tenants)
+ var sessionIndexEntries = new Dictionary>();
+
+ var query = indexTable.QueryAsync(
+ select: new[] { "PartitionKey", "RowKey", "SessionId", "EventCount" });
+
+ await foreach (var entity in query)
+ {
+ var sessionId = entity.GetString("SessionId") ?? ExtractSessionIdFromIndexRowKey(entity.RowKey);
+ var key = $"{entity.PartitionKey}_{sessionId}";
+ var eventCount = entity.GetInt32("EventCount") ?? 0;
+
+ if (!sessionIndexEntries.ContainsKey(key))
+ sessionIndexEntries[key] = new List<(string, string, int)>();
+
+ sessionIndexEntries[key].Add((entity.PartitionKey, entity.RowKey, eventCount));
+ }
+
+ int deletedCount = 0;
+
+ foreach (var (key, entries) in sessionIndexEntries)
+ {
+ if (entries.Count <= 1)
+ continue; // No duplicates
+
+ // Keep the entry with the highest EventCount (most up-to-date);
+ // delete all others as ghosts.
+ var sorted = entries.OrderByDescending(e => e.EventCount).ToList();
+ var keep = sorted[0];
+
+ for (int i = 1; i < sorted.Count; i++)
+ {
+ var ghost = sorted[i];
+ try
+ {
+ await indexTable.DeleteEntityAsync(ghost.PartitionKey, ghost.RowKey);
+ deletedCount++;
+ _logger.LogInformation(
+ "Deleted ghost SessionsIndex entry: {PartitionKey}/{RowKey} (EventCount={GhostCount}, kept entry has EventCount={KeptCount})",
+ ghost.PartitionKey, ghost.RowKey, ghost.EventCount, keep.EventCount);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Failed to delete ghost index entry {PartitionKey}/{RowKey}",
+ ghost.PartitionKey, ghost.RowKey);
+ }
+ }
+ }
+
+ if (deletedCount > 0)
+ {
+ _logger.LogInformation("Ghost SessionsIndex cleanup completed: {Count} ghost entries removed", deletedCount);
+ }
+
+ return deletedCount;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Ghost SessionsIndex cleanup failed");
+ return 0;
+ }
+ }
+
///
/// Checks if the SessionsIndex table is empty.
/// Used by startup backfill to determine if a full migration is needed.
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs
index cf306bf9..b5713607 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs
@@ -294,6 +294,7 @@ namespace AutopilotMonitor.Functions.Services
string geoRegion = string.Empty;
string geoCity = string.Empty;
string geoLoc = string.Empty;
+ string? existingIndexRowKey = null;
try
{
@@ -323,6 +324,7 @@ namespace AutopilotMonitor.Functions.Services
geoRegion = existingEntity.GetString("GeoRegion") ?? string.Empty;
geoCity = existingEntity.GetString("GeoCity") ?? string.Empty;
geoLoc = existingEntity.GetString("GeoLoc") ?? string.Empty;
+ existingIndexRowKey = existingEntity.GetString("IndexRowKey");
// Guard: never regress a terminal status (Succeeded/Failed) back to InProgress.
// StoreSessionAsync uses UpsertEntity (Replace mode) which overwrites all fields.
@@ -407,6 +409,12 @@ namespace AutopilotMonitor.Functions.Services
if (!string.IsNullOrEmpty(geoLoc))
entity["GeoLoc"] = geoLoc;
+ // Preserve IndexRowKey through the Replace so that:
+ // 1. Concurrent UpdateSessionStatusAsync calls can find it (prevents index stale-status)
+ // 2. UpsertSessionIndexAsync can detect and delete old index entries when startedAt shifts
+ if (!string.IsNullOrEmpty(existingIndexRowKey))
+ entity["IndexRowKey"] = existingIndexRowKey;
+
await tableClient.UpsertEntityAsync(entity);
// Dual-write: upsert into SessionsIndex for time-sorted listing
@@ -933,6 +941,19 @@ namespace AutopilotMonitor.Functions.Services
// Dual-write: merge the same fields into SessionsIndex
var indexRowKey = session.GetString("IndexRowKey");
+
+ // Fallback: if IndexRowKey was lost (e.g. StoreSessionAsync Replace race),
+ // compute it from StartedAt + SessionId so the index still gets updated.
+ if (string.IsNullOrEmpty(indexRowKey))
+ {
+ var sessionStartedAt = session.GetDateTimeOffset("StartedAt")?.UtcDateTime;
+ if (sessionStartedAt.HasValue)
+ {
+ indexRowKey = ComputeIndexRowKey(sessionStartedAt.Value, sessionId);
+ _logger.LogWarning("Session {SessionId}: IndexRowKey was null, computed fallback from StartedAt", sessionId);
+ }
+ }
+
await MergeSessionIndexAsync(tenantId, indexRowKey, update);
_logger.LogInformation($"Updated session {sessionId} status to {status}");
@@ -1041,6 +1062,15 @@ namespace AutopilotMonitor.Functions.Services
// Dual-write: merge the same fields into SessionsIndex
var forceIndexRowKey = freshSession.GetString("IndexRowKey");
+ if (string.IsNullOrEmpty(forceIndexRowKey))
+ {
+ var forceStartedAt = freshSession.GetDateTimeOffset("StartedAt")?.UtcDateTime;
+ if (forceStartedAt.HasValue)
+ {
+ forceIndexRowKey = ComputeIndexRowKey(forceStartedAt.Value, sessionId);
+ _logger.LogWarning("Session {SessionId}: IndexRowKey was null in force-write path, computed fallback from StartedAt", sessionId);
+ }
+ }
await MergeSessionIndexAsync(tenantId, forceIndexRowKey, forceUpdate);
_logger.LogInformation($"Force-updated session {sessionId} status to {status} (unconditional merge after ETag exhaustion)");
@@ -1137,6 +1167,12 @@ namespace AutopilotMonitor.Functions.Services
// Dual-write: merge the same fields into SessionsIndex
var indexRowKey = entity.Value.GetString("IndexRowKey");
+ if (string.IsNullOrEmpty(indexRowKey))
+ {
+ var incStartedAt = entity.Value.GetDateTimeOffset("StartedAt")?.UtcDateTime;
+ if (incStartedAt.HasValue)
+ indexRowKey = ComputeIndexRowKey(incStartedAt.Value, sessionId);
+ }
await MergeSessionIndexAsync(tenantId, indexRowKey, update);
return;