diff --git a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt index 6dbdf1f4..6e1fc58d 100644 --- a/src/Agent/AutopilotMonitor.Agent/buildcounter.txt +++ b/src/Agent/AutopilotMonitor.Agent/buildcounter.txt @@ -1 +1 @@ -989 +995 diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs index 7e0fae7c..5b9af591 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs @@ -84,6 +84,12 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task UpdateSessionImeAgentVersionAsync(string tenantId, string sessionId, string version) => _storage.UpdateSessionImeAgentVersionAsync(tenantId, sessionId, version); + public Task> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold) + => _storage.GetSessionsWithEventCountAboveAsync(tenantId, threshold); + + public Task MarkExcessiveEventsAlertedAsync(string tenantId, string sessionId) + => _storage.MarkExcessiveEventsAlertedAsync(tenantId, sessionId); + public Task RecordImeVersionAsync(string version, string tenantId, string sessionId) => _storage.RecordImeVersionAsync(version, tenantId, sessionId); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 08dbde0d..d350bf8d 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -103,6 +103,7 @@ namespace AutopilotMonitor.Functions.Services try { await MarkStalledSessionsAsTimedOutAsync(); + await DetectExcessiveEventSessionsAsync(); await BlockExcessiveDataSendersAsync(); await AggregateMetricsWithCatchUpAsync(); await CleanupOldDataAsync(); @@ -142,6 +143,7 @@ namespace AutopilotMonitor.Functions.Services { await MarkStalledSessionsAsTimedOutAsync(); result.StalledSessionsChecked = true; + await DetectExcessiveEventSessionsAsync(); } var dateToAggregate = targetDate ?? DateTime.UtcNow.AddDays(-1).Date; @@ -313,6 +315,54 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// Runaway-session threshold: sessions with more events than this are very likely produced + /// by an agent loop bug (largest real sessions observed to date are <~500 events). + /// + private const int ExcessiveEventCountThreshold = 2000; + + /// + /// Scans every tenant for sessions whose EventCount exceeds . + /// Emits one ExcessiveSessionEvents ops event per session (dispatched to Telegram/Teams via OpsAlertDispatchService) + /// and marks the session so subsequent maintenance runs do not re-alert. + /// + private async Task DetectExcessiveEventSessionsAsync() + { + try + { + var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync(); + int totalAlerted = 0; + + foreach (var tenantId in tenantIds) + { + try + { + var runaways = await _sessionRepo.GetSessionsWithEventCountAboveAsync(tenantId, ExcessiveEventCountThreshold); + foreach (var session in runaways) + { + if (session.ExcessiveEventsAlerted) continue; + + await _opsEventService.RecordExcessiveSessionEventsAsync( + tenantId, session.SessionId, session.EventCount, ExcessiveEventCountThreshold); + await _sessionRepo.MarkExcessiveEventsAlertedAsync(tenantId, session.SessionId); + totalAlerted++; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed excessive-event scan for tenant {TenantId}", tenantId); + } + } + + if (totalAlerted > 0) + _logger.LogWarning("Excessive-event scan: {Count} new runaway session(s) alerted (threshold {Threshold})", totalAlerted, ExcessiveEventCountThreshold); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to scan for excessive-event sessions"); + } + } + /// /// Detects devices that are still actively sending data beyond the configured maximum session /// window and blocks them automatically. Status-independent: uses LastEventAt (written on every diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index ac43d15c..817d9431 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -91,6 +91,11 @@ namespace AutopilotMonitor.Functions.Services $"{sessionCount} session(s) timed out after {timeoutHours}h", tenantId, "System.Maintenance", new { sessionCount, timeoutHours }); + public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold) + => WriteAsync(OpsEventCategory.Agent, "ExcessiveSessionEvents", OpsEventSeverity.Warning, + $"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug", + tenantId, "System.Maintenance", new { sessionId, eventCount, threshold }); + public Task RecordNewImeVersionDetectedAsync(string version, string tenantId, string sessionId) => WriteAsync(OpsEventCategory.Agent, "NewImeVersionDetected", OpsEventSeverity.Warning, $"New IME agent version detected: {version}", diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs index 7d799495..0e7d4b73 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs @@ -1,3 +1,4 @@ +using Azure; using Azure.Data.Tables; using AutopilotMonitor.Functions.Security; using AutopilotMonitor.Shared; @@ -376,40 +377,13 @@ namespace AutopilotMonitor.Functions.Services SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); - try - { - var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Events); - - // Query all events for this session using the same PartitionKey format as StoreEventAsync - var partitionKey = $"{tenantId}_{sessionId}"; - var filter = $"PartitionKey eq '{partitionKey}'"; - var events = tableClient.QueryAsync(filter); - - int deletedCount = 0; - var batch = new List(); - await foreach (var eventEntity in events) - { - batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, eventEntity)); - deletedCount++; - if (batch.Count >= 100) - { - await tableClient.SubmitTransactionAsync(batch); - batch.Clear(); - } - } - if (batch.Count > 0) - { - await tableClient.SubmitTransactionAsync(batch); - } - - _logger.LogInformation($"Deleted {deletedCount} events for session {sessionId}"); - return deletedCount; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Failed to delete events for session {sessionId}"); - return 0; - } + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Events); + var partitionKey = $"{tenantId}_{sessionId}"; + var filter = $"PartitionKey eq '{partitionKey}'"; + var deleted = await DeleteByFilterInBatchesAsync(tableClient, filter, $"events for session {sessionId}"); + if (deleted > 0) + _logger.LogInformation($"Deleted {deleted} events for session {sessionId}"); + return deleted; } /// @@ -420,36 +394,10 @@ namespace AutopilotMonitor.Functions.Services SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); - try - { - var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleResults); - var partitionKey = $"{tenantId}_{sessionId}"; - var query = tableClient.QueryAsync(filter: $"PartitionKey eq '{partitionKey}'"); - - int deletedCount = 0; - var batch = new List(); - await foreach (var entity in query) - { - batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity)); - deletedCount++; - if (batch.Count >= 100) - { - await tableClient.SubmitTransactionAsync(batch); - batch.Clear(); - } - } - if (batch.Count > 0) - { - await tableClient.SubmitTransactionAsync(batch); - } - - return deletedCount; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Failed to delete rule results for session {sessionId}"); - return 0; - } + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.RuleResults); + var partitionKey = $"{tenantId}_{sessionId}"; + var filter = $"PartitionKey eq '{partitionKey}'"; + return await DeleteByFilterInBatchesAsync(tableClient, filter, $"rule results for session {sessionId}"); } /// @@ -460,34 +408,66 @@ namespace AutopilotMonitor.Functions.Services SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.AppInstallSummaries); + // AppInstallSummaries PK=tenantId, so all filtered rows still share the same PK → batch-tx-safe. + var filter = $"PartitionKey eq '{tenantId}' and SessionId eq '{sessionId}'"; + return await DeleteByFilterInBatchesAsync(tableClient, filter, $"app install summaries for session {sessionId}"); + } + + /// + /// Deletes all entities matching the filter from the given table. + /// Uses projected query (PK/RK only) to minimize payload, and submits 100-entity + /// batch transactions in parallel (up to 4 in flight) for faster bulk delete. + /// REQUIRES all matched rows to share the same PartitionKey (Table Storage batch constraint). + /// + private async Task DeleteByFilterInBatchesAsync(TableClient tableClient, string filter, string contextForLogs) + { + const int maxParallelBatches = 4; + const int batchSize = 100; + try { - var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.AppInstallSummaries); - var query = tableClient.QueryAsync( - filter: $"PartitionKey eq '{tenantId}' and SessionId eq '{sessionId}'"); + // Project to PK/RK only — drastically reduces query bytes for large sessions. + var query = tableClient.QueryAsync(filter: filter, select: new[] { "PartitionKey", "RowKey" }); int deletedCount = 0; - var batch = new List(); + var batch = new List(batchSize); + var gate = new SemaphoreSlim(maxParallelBatches); + var inFlight = new List(); + + async Task SubmitAsync(List snapshot) + { + await gate.WaitAsync().ConfigureAwait(false); + try { await tableClient.SubmitTransactionAsync(snapshot).ConfigureAwait(false); } + finally { gate.Release(); } + } + await foreach (var entity in query) { - batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, entity)); + // ETag.All → unconditional delete (safe for maintenance cleanup; no optimistic concurrency needed). + var stub = new TableEntity(entity.PartitionKey, entity.RowKey) { ETag = ETag.All }; + batch.Add(new TableTransactionAction(TableTransactionActionType.Delete, stub)); deletedCount++; - if (batch.Count >= 100) + if (batch.Count >= batchSize) { - await tableClient.SubmitTransactionAsync(batch); - batch.Clear(); + var snapshot = batch; + batch = new List(batchSize); + inFlight.Add(SubmitAsync(snapshot)); } } if (batch.Count > 0) { - await tableClient.SubmitTransactionAsync(batch); + inFlight.Add(SubmitAsync(batch)); } + if (inFlight.Count > 0) + await Task.WhenAll(inFlight).ConfigureAwait(false); + return deletedCount; } catch (Exception ex) { - _logger.LogError(ex, $"Failed to delete app install summaries for session {sessionId}"); + _logger.LogError(ex, $"Failed to delete {contextForLogs}"); return 0; } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index 8f75ad65..db9dd76a 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -236,7 +236,8 @@ namespace AutopilotMonitor.Functions.Services GeoCity = entity.GetString("GeoCity") ?? string.Empty, GeoLoc = entity.GetString("GeoLoc") ?? string.Empty, PlatformScriptCount = SafeGetInt32(entity, "PlatformScriptCount") ?? 0, - RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0 + RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0, + ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false }; } @@ -1732,7 +1733,8 @@ namespace AutopilotMonitor.Functions.Services GeoCity = entity.GetString("GeoCity") ?? string.Empty, GeoLoc = entity.GetString("GeoLoc") ?? string.Empty, PlatformScriptCount = SafeGetInt32(entity, "PlatformScriptCount") ?? 0, - RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0 + RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0, + ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false }; } @@ -1886,5 +1888,81 @@ namespace AutopilotMonitor.Functions.Services } #endregion + + #region Excessive-Event Detection + + /// + /// Returns sessions in the given tenant whose EventCount exceeds the threshold. + /// Projects only the fields needed for the idempotency check — avoids full entity read. + /// Never throws: on failure returns an empty list (maintenance scan is best-effort). + /// + public async Task> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + + var matches = new List(); + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); + var query = tableClient.QueryAsync( + filter: $"PartitionKey eq '{tenantId}' and EventCount gt {threshold}", + select: new[] { "PartitionKey", "RowKey", "EventCount", "ExcessiveEventsAlerted" }); + + await foreach (var entity in query) + { + matches.Add(new SessionSummary + { + TenantId = entity.PartitionKey, + SessionId = entity.RowKey, + EventCount = SafeGetInt32(entity, "EventCount") ?? 0, + ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false + }); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to query sessions with EventCount > {Threshold} for tenant {TenantId}", threshold, tenantId); + } + return matches; + } + + /// + /// Sets ExcessiveEventsAlerted=true on the session entity to make the ops-alert emission + /// idempotent across maintenance runs. Best-effort Merge write; failures are logged only. + /// + public async Task MarkExcessiveEventsAlertedAsync(string tenantId, string sessionId) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); + + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); + var update = new TableEntity(tenantId, sessionId) + { + ["ExcessiveEventsAlerted"] = true + }; + await tableClient.UpdateEntityAsync(update, ETag.All, TableUpdateMode.Merge); + + // Dual-write into SessionsIndex so dashboard queries see the flag without a separate read. + try + { + var entity = await tableClient.GetEntityAsync(tenantId, sessionId, + select: new[] { "IndexRowKey" }); + var indexRowKey = entity.Value.GetString("IndexRowKey"); + await MergeSessionIndexAsync(tenantId, indexRowKey, update); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "SessionsIndex merge skipped for ExcessiveEventsAlerted on {SessionId}", sessionId); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to mark session {SessionId} as ExcessiveEventsAlerted", sessionId); + } + } + + #endregion } } diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs index 372d7852..52655e84 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs @@ -54,6 +54,18 @@ namespace AutopilotMonitor.Shared.DataAccess /// Task> FetchAndClearPendingActionsAsync(string tenantId, string sessionId); + // --- Excessive-Event Detection --- + /// + /// Returns sessions in whose EventCount exceeds . + /// Used by maintenance to surface runaway sessions (likely agent loop bugs). + /// + Task> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold); + + /// + /// Marks the session as already-alerted so maintenance only emits one ops event per runaway session. + /// + Task MarkExcessiveEventsAlertedAsync(string tenantId, string sessionId); + // --- IME Version History --- Task RecordImeVersionAsync(string version, string tenantId, string sessionId); Task> GetImeVersionHistoryAsync(); diff --git a/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs index ab2a1fa1..a8925d4a 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/SessionApiModels.cs @@ -263,6 +263,12 @@ namespace AutopilotMonitor.Shared.Models public int PlatformScriptCount { get; set; } public int RemediationScriptCount { get; set; } + /// + /// True once maintenance has emitted an ExcessiveSessionEvents ops alert for this session. + /// Prevents duplicate alerts on subsequent maintenance runs for the same runaway session. + /// + public bool ExcessiveEventsAlerted { get; set; } + /// /// JSON-serialized of /// pending delivery to the agent. Empty string when no actions are queued. diff --git a/src/Web/autopilot-monitor-web/app/settings/components/AutopilotValidationSection.tsx b/src/Web/autopilot-monitor-web/app/settings/components/AutopilotValidationSection.tsx index 8c04bd36..0d462531 100644 --- a/src/Web/autopilot-monitor-web/app/settings/components/AutopilotValidationSection.tsx +++ b/src/Web/autopilot-monitor-web/app/settings/components/AutopilotValidationSection.tsx @@ -91,7 +91,7 @@ export default function AutopilotValidationSection({

- Both validations require the DeviceManagementServiceConfig.Read.All permission. Enabling either option starts Microsoft Entra admin consent if not already granted. After consent, the setting is saved automatically. + Both validations require the DeviceManagementServiceConfig.Read.All permission. Enabling either option starts Microsoft Entra admin consent if not already granted. After consent, the setting is saved automatically. Granting consent requires at least the Application Administrator or Global Administrator Entra role.