diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceExcessiveEventAutoActionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceExcessiveEventAutoActionTests.cs index 09df09a6..daf70de8 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceExcessiveEventAutoActionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceExcessiveEventAutoActionTests.cs @@ -116,6 +116,53 @@ public class MaintenanceServiceExcessiveEventAutoActionTests Assert.False(MaintenanceService.IsAutoActionEligible(status)); } + // ── Recount gate ────────────────────────────────────────────────────────── + // The stored counter doubles on replayed batches, so every warn and every block is + // decided on a recount. The gate keeps that Events-partition scan to candidates that + // still have something to fire. + + [Fact] + public void HasPendingAction_WarnNotYetFired_IsPending() + { + Assert.True(MaintenanceService.HasPendingExcessiveEventAction( + storedEventCount: 2001, SessionStatus.Succeeded, alreadyAlerted: false, alreadyAutoActioned: false, + warnThreshold: 2000, autoMode: "Off", autoThreshold: 2500)); + } + + [Fact] + public void HasPendingAction_BlockNotYetFiredOnRunningSession_IsPending() + { + Assert.True(MaintenanceService.HasPendingExcessiveEventAction( + storedEventCount: 2501, SessionStatus.InProgress, alreadyAlerted: true, alreadyAutoActioned: false, + warnThreshold: 2000, autoMode: "Block", autoThreshold: 2500)); + } + + [Fact] + public void HasPendingAction_FinishedSessionAboveBlockThreshold_IsNotRescannedForever() + { + // Warned long ago, can never be blocked any more: without the eligibility term this + // row would cost a full partition scan on every sweep until retention removes it. + Assert.False(MaintenanceService.HasPendingExcessiveEventAction( + storedEventCount: 9000, SessionStatus.Incomplete, alreadyAlerted: true, alreadyAutoActioned: false, + warnThreshold: 2000, autoMode: "Block", autoThreshold: 2500)); + } + + [Fact] + public void HasPendingAction_BothAlreadyFired_IsNotPending() + { + Assert.False(MaintenanceService.HasPendingExcessiveEventAction( + storedEventCount: 9000, SessionStatus.InProgress, alreadyAlerted: true, alreadyAutoActioned: true, + warnThreshold: 2000, autoMode: "Block", autoThreshold: 2500)); + } + + [Fact] + public void HasPendingAction_WarnDisabledAndModeOff_IsNotPending() + { + Assert.False(MaintenanceService.HasPendingExcessiveEventAction( + storedEventCount: 9000, SessionStatus.InProgress, alreadyAlerted: false, alreadyAutoActioned: false, + warnThreshold: 0, autoMode: "Off", autoThreshold: 2500)); + } + // ── Status parsing feeding that gate ────────────────────────────────────── // The runaway query projects a subset of columns. SessionStatus.InProgress is ordinal 0, // so an unparsed Status would leave every row looking in-progress and the gate above diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 729bbfb8..27252cae 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -815,6 +815,23 @@ namespace AutopilotMonitor.Functions.Services /// internal static bool IsAutoActionEligible(SessionStatus status) => status == SessionStatus.InProgress; + /// + /// Pure gate: would the STORED counter trigger a warn or an auto-action that has not + /// fired yet? Only then is the authoritative recount worth its Events-partition scan. + /// The auto tier counts only while the session is still eligible — otherwise a finished + /// session above the auto threshold would be rescanned on every sweep until retention. + /// + internal static bool HasPendingExcessiveEventAction( + int storedEventCount, SessionStatus status, bool alreadyAlerted, bool alreadyAutoActioned, + int warnThreshold, string? autoMode, int autoThreshold) + { + var warnPending = warnThreshold > 0 && storedEventCount > warnThreshold && !alreadyAlerted; + var autoPending = !alreadyAutoActioned + && IsAutoActionEligible(status) + && DecideAutoAction(storedEventCount, autoMode, autoThreshold) != null; + return warnPending || autoPending; + } + /// /// Scans every tenant for sessions whose EventCount exceeds the configured warn /// threshold () or the @@ -880,8 +897,20 @@ namespace AutopilotMonitor.Functions.Services try { var runaways = await _sessionRepo.GetSessionsWithEventCountAboveAsync(tenantId, queryThreshold); - foreach (var session in runaways) + foreach (var candidate in runaways) { + if (!HasPendingExcessiveEventAction( + candidate.EventCount, candidate.Status, + candidate.ExcessiveEventsAlerted, candidate.ExcessiveEventsAutoActioned, + warnEnabled ? warnThreshold : 0, autoEnabled ? autoMode : "Off", autoThreshold)) + continue; + + // The stored EventCount is an at-least-once increment: replayed batches + // dedupe as rows but double the counter, and nothing corrects it while + // the session is still running. Warn and block on the recounted rows only. + var session = await RecountRunawayCandidateAsync(tenantId, candidate); + if (session == null) continue; + // Warn-tier: emit once per session, regardless of auto-action state. if (warnEnabled && session.EventCount > warnThreshold && !session.ExcessiveEventsAlerted) { @@ -961,6 +990,32 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// Replaces the stored counters of a runaway candidate with the authoritative Events row + /// count and returns the re-read session. Returns null when the count could not be + /// verified — the candidate is then left alone until the next sweep rather than warned + /// about or blocked on an unverified number. + /// + private async Task RecountRunawayCandidateAsync(string tenantId, SessionSummary candidate) + { + var scan = await _sessionRepo.ReconcileSessionCountersAsync(tenantId, candidate.SessionId); + var recounted = scan == null ? null : await _sessionRepo.GetSessionAsync(tenantId, candidate.SessionId); + if (recounted == null) + { + _logger.LogWarning( + "Excessive-event scan: could not verify the event count of session {SessionId} (tenant {TenantId}, stored {StoredCount}); skipped until the next sweep", + candidate.SessionId, tenantId, candidate.EventCount); + return null; + } + + if (recounted.EventCount != candidate.EventCount) + _logger.LogWarning( + "Excessive-event scan: session {SessionId} (tenant {TenantId}) stored EventCount {StoredCount} corrected to {ActualCount} rows", + candidate.SessionId, tenantId, candidate.EventCount, recounted.EventCount); + + return recounted; + } + /// /// Aggregates metrics for any missed days in the last 7 days, plus yesterday. /// Checks the UsageMetrics table for existing snapshots to avoid re-aggregation. diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 5cb68acf..a39f34be 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -984,7 +984,7 @@ namespace AutopilotMonitor.Functions.Services public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.ExcessiveSessionEvents, OpsEventSeverity.Warning, - $"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug", + $"Session {sessionId} has {eventCount} events (threshold {threshold}) — unusually high event volume", tenantId, "System.Maintenance", new { sessionId, eventCount, threshold }); /// diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs index a61029ad..28d4f15c 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs @@ -178,7 +178,8 @@ namespace AutopilotMonitor.Shared.DataAccess // --- Excessive-Event Detection --- /// /// Returns sessions in whose EventCount exceeds . - /// Used by maintenance to surface runaway sessions (likely agent loop bugs). + /// Used by maintenance to find runaway candidates; the stored counter can overcount, so + /// the caller recounts before acting on it. /// Task> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold);