bulk delete speedup, session runaway alarm, Autopilot validation permissions sentance docs

src/Agent/AutopilotMonitor.Agent/buildcounter.txt

@@ -1 +1 @@
-989
+995

src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs

@@ -84,6 +84,12 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
         public Task UpdateSessionImeAgentVersionAsync(string tenantId, string sessionId, string version)
             => _storage.UpdateSessionImeAgentVersionAsync(tenantId, sessionId, version);
 
+        public Task<List<SessionSummary>> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold)
+            => _storage.GetSessionsWithEventCountAboveAsync(tenantId, threshold);
+
+        public Task MarkExcessiveEventsAlertedAsync(string tenantId, string sessionId)
+            => _storage.MarkExcessiveEventsAlertedAsync(tenantId, sessionId);
+
         public Task<bool> RecordImeVersionAsync(string version, string tenantId, string sessionId)
             => _storage.RecordImeVersionAsync(version, tenantId, sessionId);
 

src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs

@@ -103,6 +103,7 @@ namespace AutopilotMonitor.Functions.Services
             try
             {
                 await MarkStalledSessionsAsTimedOutAsync();
+                await DetectExcessiveEventSessionsAsync();
                 await BlockExcessiveDataSendersAsync();
                 await AggregateMetricsWithCatchUpAsync();
                 await CleanupOldDataAsync();
@@ -142,6 +143,7 @@ namespace AutopilotMonitor.Functions.Services
                 {
                     await MarkStalledSessionsAsTimedOutAsync();
                     result.StalledSessionsChecked = true;
+                    await DetectExcessiveEventSessionsAsync();
                 }
 
                 var dateToAggregate = targetDate ?? DateTime.UtcNow.AddDays(-1).Date;
@@ -313,6 +315,54 @@ namespace AutopilotMonitor.Functions.Services
             }
         }
 
+        /// <summary>
+        /// Runaway-session threshold: sessions with more events than this are very likely produced
+        /// by an agent loop bug (largest real sessions observed to date are &lt;~500 events).
+        /// </summary>
+        private const int ExcessiveEventCountThreshold = 2000;
+
+        /// <summary>
+        /// Scans every tenant for sessions whose EventCount exceeds <see cref="ExcessiveEventCountThreshold"/>.
+        /// Emits one ExcessiveSessionEvents ops event per session (dispatched to Telegram/Teams via OpsAlertDispatchService)
+        /// and marks the session so subsequent maintenance runs do not re-alert.
+        /// </summary>
+        private async Task DetectExcessiveEventSessionsAsync()
+        {
+            try
+            {
+                var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
+                int totalAlerted = 0;
+
+                foreach (var tenantId in tenantIds)
+                {
+                    try
+                    {
+                        var runaways = await _sessionRepo.GetSessionsWithEventCountAboveAsync(tenantId, ExcessiveEventCountThreshold);
+                        foreach (var session in runaways)
+                        {
+                            if (session.ExcessiveEventsAlerted) continue;
+
+                            await _opsEventService.RecordExcessiveSessionEventsAsync(
+                                tenantId, session.SessionId, session.EventCount, ExcessiveEventCountThreshold);
+                            await _sessionRepo.MarkExcessiveEventsAlertedAsync(tenantId, session.SessionId);
+                            totalAlerted++;
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(ex, "Failed excessive-event scan for tenant {TenantId}", tenantId);
+                    }
+                }
+
+                if (totalAlerted > 0)
+                    _logger.LogWarning("Excessive-event scan: {Count} new runaway session(s) alerted (threshold {Threshold})", totalAlerted, ExcessiveEventCountThreshold);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to scan for excessive-event sessions");
+            }
+        }
+
         /// <summary>
         /// Detects devices that are still actively sending data beyond the configured maximum session
         /// window and blocks them automatically. Status-independent: uses LastEventAt (written on every

src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs

@@ -91,6 +91,11 @@ namespace AutopilotMonitor.Functions.Services
                 $"{sessionCount} session(s) timed out after {timeoutHours}h",
                 tenantId, "System.Maintenance", new { sessionCount, timeoutHours });
 
+        public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold)
+            => WriteAsync(OpsEventCategory.Agent, "ExcessiveSessionEvents", OpsEventSeverity.Warning,
+                $"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug",
+                tenantId, "System.Maintenance", new { sessionId, eventCount, threshold });
+
         public Task RecordNewImeVersionDetectedAsync(string version, string tenantId, string sessionId)
             => WriteAsync(OpsEventCategory.Agent, "NewImeVersionDetected", OpsEventSeverity.Warning,
                 $"New IME agent version detected: {version}",

src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs

@@ -1,3 +1,4 @@
+using Azure;
 using Azure.Data.Tables;
 using AutopilotMonitor.Functions.Security;
 using AutopilotMonitor.Shared;
@@ -376,40 +377,13 @@ namespace AutopilotMonitor.Functions.Services
             SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
             SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
 
-            try
-            {
-                var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Events);
-
-                // Query all events for this session using the same PartitionKey format as StoreEventAsync
-                var partitionKey = $"{tenantId}_{sessionId}";
-                var filter = $"PartitionKey eq '{partitionKey}'";
-                var events = tableClient.QueryAsync<TableEntity>(filter);
-
-                int deletedCount = 0;
-                var batch = new List<TableTransactionAction>();
-                await foreach (var eventEntity in events)
-                {
-                    batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, eventEntity));
-                    deletedCount++;
-                    if (batch.Count >= 100)
-                    {
-                        await tableClient.SubmitTransactionAsync(batch);
-                        batch.Clear();
-                    }
-                }
-                if (batch.Count > 0)
-                {
-                    await tableClient.SubmitTransactionAsync(batch);
-                }
-
-                _logger.LogInformation($"Deleted {deletedCount} events for session {sessionId}");
-                return deletedCount;
-            }
-            catch (Exception ex)
-            {
-                _logger.LogError(ex, $"Failed to delete events for session {sessionId}");
-                return 0;
-            }
+            var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Events);
+            var partitionKey = $"{tenantId}_{sessionId}";
+            var filter = $"PartitionKey eq '{partitionKey}'";
+            var deleted = await DeleteByFilterInBatchesAsync(tableClient, filter, $"events for session {sessionId}");
+            if (deleted > 0)
+                _logger.LogInformation($"Deleted {deleted} events for session {sessionId}");
+            return deleted;
         }
 
         /// <summary>
@@ -420,36 +394,10 @@ namespace AutopilotMonitor.Functions.Services
             SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
             SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
 
-            try
-            {
-                var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleResults);
-                var partitionKey = $"{tenantId}_{sessionId}";
-                var query = tableClient.QueryAsync<TableEntity>(filter: $"PartitionKey eq '{partitionKey}'");
-
-                int deletedCount = 0;
-                var batch = new List<TableTransactionAction>();
-                await foreach (var entity in query)
-                {
-                    batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity));
-                    deletedCount++;
-                    if (batch.Count >= 100)
-                    {
-                        await tableClient.SubmitTransactionAsync(batch);
-                        batch.Clear();
-                    }
-                }
-                if (batch.Count > 0)
-                {
-                    await tableClient.SubmitTransactionAsync(batch);
-                }
-
-                return deletedCount;
-            }
-            catch (Exception ex)
-            {
-                _logger.LogError(ex, $"Failed to delete rule results for session {sessionId}");
-                return 0;
-            }
+            var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleResults);
+            var partitionKey = $"{tenantId}_{sessionId}";
+            var filter = $"PartitionKey eq '{partitionKey}'";
+            return await DeleteByFilterInBatchesAsync(tableClient, filter, $"rule results for session {sessionId}");
         }
 
         /// <summary>
@@ -460,34 +408,66 @@ namespace AutopilotMonitor.Functions.Services
             SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
             SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
 
+            var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.AppInstallSummaries);
+            // AppInstallSummaries PK=tenantId, so all filtered rows still share the same PK → batch-tx-safe.
+            var filter = $"PartitionKey eq '{tenantId}' and SessionId eq '{sessionId}'";
+            return await DeleteByFilterInBatchesAsync(tableClient, filter, $"app install summaries for session {sessionId}");
+        }
+
+        /// <summary>
+        /// Deletes all entities matching the filter from the given table.
+        /// Uses projected query (PK/RK only) to minimize payload, and submits 100-entity
+        /// batch transactions in parallel (up to 4 in flight) for faster bulk delete.
+        /// REQUIRES all matched rows to share the same PartitionKey (Table Storage batch constraint).
+        /// </summary>
+        private async Task<int> DeleteByFilterInBatchesAsync(TableClient tableClient, string filter, string contextForLogs)
+        {
+            const int maxParallelBatches = 4;
+            const int batchSize = 100;
+
             try
             {
-                var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.AppInstallSummaries);
-                var query = tableClient.QueryAsync<TableEntity>(
-                    filter: $"PartitionKey eq '{tenantId}' and SessionId eq '{sessionId}'");
+                // Project to PK/RK only — drastically reduces query bytes for large sessions.
+                var query = tableClient.QueryAsync<TableEntity>(filter: filter, select: new[] { "PartitionKey", "RowKey" });
 
                 int deletedCount = 0;
-                var batch = new List<TableTransactionAction>();
+                var batch = new List<TableTransactionAction>(batchSize);
+                var gate = new SemaphoreSlim(maxParallelBatches);
+                var inFlight = new List<Task>();
+
+                async Task SubmitAsync(List<TableTransactionAction> snapshot)
+                {
+                    await gate.WaitAsync().ConfigureAwait(false);
+                    try { await tableClient.SubmitTransactionAsync(snapshot).ConfigureAwait(false); }
+                    finally { gate.Release(); }
+                }
+
                 await foreach (var entity in query)
                 {
-                    batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity));
+                    // ETag.All → unconditional delete (safe for maintenance cleanup; no optimistic concurrency needed).
+                    var stub = new TableEntity(entity.PartitionKey, entity.RowKey) { ETag = ETag.All };
+                    batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, stub));
                     deletedCount++;
-                    if (batch.Count >= 100)
+                    if (batch.Count >= batchSize)
                     {
-                        await tableClient.SubmitTransactionAsync(batch);
-                        batch.Clear();
+                        var snapshot = batch;
+                        batch = new List<TableTransactionAction>(batchSize);
+                        inFlight.Add(SubmitAsync(snapshot));
                     }
                 }
                 if (batch.Count > 0)
                 {
-                    await tableClient.SubmitTransactionAsync(batch);
+                    inFlight.Add(SubmitAsync(batch));
                 }
 
+                if (inFlight.Count > 0)
+                    await Task.WhenAll(inFlight).ConfigureAwait(false);
+
                 return deletedCount;
             }
             catch (Exception ex)
             {
-                _logger.LogError(ex, $"Failed to delete app install summaries for session {sessionId}");
+                _logger.LogError(ex, $"Failed to delete {contextForLogs}");
                 return 0;
             }
         }

src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs

@@ -236,7 +236,8 @@ namespace AutopilotMonitor.Functions.Services
                 GeoCity = entity.GetString("GeoCity") ?? string.Empty,
                 GeoLoc = entity.GetString("GeoLoc") ?? string.Empty,
                 PlatformScriptCount = SafeGetInt32(entity, "PlatformScriptCount") ?? 0,
-                RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0
+                RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0,
+                ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false
             };
         }
 
@@ -1732,7 +1733,8 @@ namespace AutopilotMonitor.Functions.Services
                 GeoCity = entity.GetString("GeoCity") ?? string.Empty,
                 GeoLoc = entity.GetString("GeoLoc") ?? string.Empty,
                 PlatformScriptCount = SafeGetInt32(entity, "PlatformScriptCount") ?? 0,
-                RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0
+                RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0,
+                ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false
             };
         }
 
@@ -1886,5 +1888,81 @@ namespace AutopilotMonitor.Functions.Services
         }
 
         #endregion
+
+        #region Excessive-Event Detection
+
+        /// <summary>
+        /// Returns sessions in the given tenant whose EventCount exceeds the threshold.
+        /// Projects only the fields needed for the idempotency check — avoids full entity read.
+        /// Never throws: on failure returns an empty list (maintenance scan is best-effort).
+        /// </summary>
+        public async Task<List<SessionSummary>> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold)
+        {
+            SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
+
+            var matches = new List<SessionSummary>();
+            try
+            {
+                var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
+                var query = tableClient.QueryAsync<TableEntity>(
+                    filter: $"PartitionKey eq '{tenantId}' and EventCount gt {threshold}",
+                    select: new[] { "PartitionKey", "RowKey", "EventCount", "ExcessiveEventsAlerted" });
+
+                await foreach (var entity in query)
+                {
+                    matches.Add(new SessionSummary
+                    {
+                        TenantId = entity.PartitionKey,
+                        SessionId = entity.RowKey,
+                        EventCount = SafeGetInt32(entity, "EventCount") ?? 0,
+                        ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false
+                    });
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Failed to query sessions with EventCount > {Threshold} for tenant {TenantId}", threshold, tenantId);
+            }
+            return matches;
+        }
+
+        /// <summary>
+        /// Sets ExcessiveEventsAlerted=true on the session entity to make the ops-alert emission
+        /// idempotent across maintenance runs. Best-effort Merge write; failures are logged only.
+        /// </summary>
+        public async Task MarkExcessiveEventsAlertedAsync(string tenantId, string sessionId)
+        {
+            SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
+            SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
+
+            try
+            {
+                var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
+                var update = new TableEntity(tenantId, sessionId)
+                {
+                    ["ExcessiveEventsAlerted"] = true
+                };
+                await tableClient.UpdateEntityAsync(update, ETag.All, TableUpdateMode.Merge);
+
+                // Dual-write into SessionsIndex so dashboard queries see the flag without a separate read.
+                try
+                {
+                    var entity = await tableClient.GetEntityAsync<TableEntity>(tenantId, sessionId,
+                        select: new[] { "IndexRowKey" });
+                    var indexRowKey = entity.Value.GetString("IndexRowKey");
+                    await MergeSessionIndexAsync(tenantId, indexRowKey, update);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogDebug(ex, "SessionsIndex merge skipped for ExcessiveEventsAlerted on {SessionId}", sessionId);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Failed to mark session {SessionId} as ExcessiveEventsAlerted", sessionId);
+            }
+        }
+
+        #endregion
     }
 }

src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs

@@ -54,6 +54,18 @@ namespace AutopilotMonitor.Shared.DataAccess
         /// </summary>
         Task<List<ServerAction>> FetchAndClearPendingActionsAsync(string tenantId, string sessionId);
 
+        // --- Excessive-Event Detection ---
+        /// <summary>
+        /// Returns sessions in <paramref name="tenantId"/> whose EventCount exceeds <paramref name="threshold"/>.
+        /// Used by maintenance to surface runaway sessions (likely agent loop bugs).
+        /// </summary>
+        Task<List<SessionSummary>> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold);
+
+        /// <summary>
+        /// Marks the session as already-alerted so maintenance only emits one ops event per runaway session.
+        /// </summary>
+        Task MarkExcessiveEventsAlertedAsync(string tenantId, string sessionId);
+
         // --- IME Version History ---
         Task<bool> RecordImeVersionAsync(string version, string tenantId, string sessionId);
         Task<List<ImeVersionHistoryEntry>> GetImeVersionHistoryAsync();

src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs

@@ -263,6 +263,12 @@ namespace AutopilotMonitor.Shared.Models
         public int PlatformScriptCount { get; set; }
         public int RemediationScriptCount { get; set; }
 
+        /// <summary>
+        /// True once maintenance has emitted an ExcessiveSessionEvents ops alert for this session.
+        /// Prevents duplicate alerts on subsequent maintenance runs for the same runaway session.
+        /// </summary>
+        public bool ExcessiveEventsAlerted { get; set; }
+
         /// <summary>
         /// JSON-serialized <see cref="System.Collections.Generic.List{T}"/> of <see cref="ServerAction"/>
         /// pending delivery to the agent. Empty string when no actions are queued.

src/Web/autopilot-monitor-web/app/settings/components/AutopilotValidationSection.tsx

@@ -91,7 +91,7 @@ export default function AutopilotValidationSection({
       <div className="p-6 space-y-5">
         <div className="bg-gray-50 border border-gray-200 rounded-lg p-3">
           <p className="text-sm text-gray-700">
-            Both validations require the <strong>DeviceManagementServiceConfig.Read.All</strong> permission. Enabling either option starts Microsoft Entra admin consent if not already granted. After consent, the setting is saved automatically.
+            Both validations require the <strong>DeviceManagementServiceConfig.Read.All</strong> permission. Enabling either option starts Microsoft Entra admin consent if not already granted. After consent, the setting is saved automatically. Granting consent requires at least the <strong>Application Administrator</strong> or <strong>Global Administrator</strong> Entra role.
           </p>
         </div>