diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceExcessiveEventAutoActionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceExcessiveEventAutoActionTests.cs
index 09df09a6..daf70de8 100644
--- a/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceExcessiveEventAutoActionTests.cs
+++ b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceExcessiveEventAutoActionTests.cs
@@ -116,6 +116,53 @@ public class MaintenanceServiceExcessiveEventAutoActionTests
Assert.False(MaintenanceService.IsAutoActionEligible(status));
}
+ // ── Recount gate ──────────────────────────────────────────────────────────
+ // The stored counter doubles on replayed batches, so every warn and every block is
+ // decided on a recount. The gate keeps that Events-partition scan to candidates that
+ // still have something to fire.
+
+ [Fact]
+ public void HasPendingAction_WarnNotYetFired_IsPending()
+ {
+ Assert.True(MaintenanceService.HasPendingExcessiveEventAction(
+ storedEventCount: 2001, SessionStatus.Succeeded, alreadyAlerted: false, alreadyAutoActioned: false,
+ warnThreshold: 2000, autoMode: "Off", autoThreshold: 2500));
+ }
+
+ [Fact]
+ public void HasPendingAction_BlockNotYetFiredOnRunningSession_IsPending()
+ {
+ Assert.True(MaintenanceService.HasPendingExcessiveEventAction(
+ storedEventCount: 2501, SessionStatus.InProgress, alreadyAlerted: true, alreadyAutoActioned: false,
+ warnThreshold: 2000, autoMode: "Block", autoThreshold: 2500));
+ }
+
+ [Fact]
+ public void HasPendingAction_FinishedSessionAboveBlockThreshold_IsNotRescannedForever()
+ {
+ // Warned long ago, can never be blocked any more: without the eligibility term this
+ // row would cost a full partition scan on every sweep until retention removes it.
+ Assert.False(MaintenanceService.HasPendingExcessiveEventAction(
+ storedEventCount: 9000, SessionStatus.Incomplete, alreadyAlerted: true, alreadyAutoActioned: false,
+ warnThreshold: 2000, autoMode: "Block", autoThreshold: 2500));
+ }
+
+ [Fact]
+ public void HasPendingAction_BothAlreadyFired_IsNotPending()
+ {
+ Assert.False(MaintenanceService.HasPendingExcessiveEventAction(
+ storedEventCount: 9000, SessionStatus.InProgress, alreadyAlerted: true, alreadyAutoActioned: true,
+ warnThreshold: 2000, autoMode: "Block", autoThreshold: 2500));
+ }
+
+ [Fact]
+ public void HasPendingAction_WarnDisabledAndModeOff_IsNotPending()
+ {
+ Assert.False(MaintenanceService.HasPendingExcessiveEventAction(
+ storedEventCount: 9000, SessionStatus.InProgress, alreadyAlerted: false, alreadyAutoActioned: false,
+ warnThreshold: 0, autoMode: "Off", autoThreshold: 2500));
+ }
+
// ── Status parsing feeding that gate ──────────────────────────────────────
// The runaway query projects a subset of columns. SessionStatus.InProgress is ordinal 0,
// so an unparsed Status would leave every row looking in-progress and the gate above
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
index 729bbfb8..27252cae 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs
@@ -815,6 +815,23 @@ namespace AutopilotMonitor.Functions.Services
///
internal static bool IsAutoActionEligible(SessionStatus status) => status == SessionStatus.InProgress;
+ ///
+ /// Pure gate: would the STORED counter trigger a warn or an auto-action that has not
+ /// fired yet? Only then is the authoritative recount worth its Events-partition scan.
+ /// The auto tier counts only while the session is still eligible — otherwise a finished
+ /// session above the auto threshold would be rescanned on every sweep until retention.
+ ///
+ internal static bool HasPendingExcessiveEventAction(
+ int storedEventCount, SessionStatus status, bool alreadyAlerted, bool alreadyAutoActioned,
+ int warnThreshold, string? autoMode, int autoThreshold)
+ {
+ var warnPending = warnThreshold > 0 && storedEventCount > warnThreshold && !alreadyAlerted;
+ var autoPending = !alreadyAutoActioned
+ && IsAutoActionEligible(status)
+ && DecideAutoAction(storedEventCount, autoMode, autoThreshold) != null;
+ return warnPending || autoPending;
+ }
+
///
/// Scans every tenant for sessions whose EventCount exceeds the configured warn
/// threshold () or the
@@ -880,8 +897,20 @@ namespace AutopilotMonitor.Functions.Services
try
{
var runaways = await _sessionRepo.GetSessionsWithEventCountAboveAsync(tenantId, queryThreshold);
- foreach (var session in runaways)
+ foreach (var candidate in runaways)
{
+ if (!HasPendingExcessiveEventAction(
+ candidate.EventCount, candidate.Status,
+ candidate.ExcessiveEventsAlerted, candidate.ExcessiveEventsAutoActioned,
+ warnEnabled ? warnThreshold : 0, autoEnabled ? autoMode : "Off", autoThreshold))
+ continue;
+
+ // The stored EventCount is an at-least-once increment: replayed batches
+ // dedupe as rows but double the counter, and nothing corrects it while
+ // the session is still running. Warn and block on the recounted rows only.
+ var session = await RecountRunawayCandidateAsync(tenantId, candidate);
+ if (session == null) continue;
+
// Warn-tier: emit once per session, regardless of auto-action state.
if (warnEnabled && session.EventCount > warnThreshold && !session.ExcessiveEventsAlerted)
{
@@ -961,6 +990,32 @@ namespace AutopilotMonitor.Functions.Services
}
}
+ ///
+ /// Replaces the stored counters of a runaway candidate with the authoritative Events row
+ /// count and returns the re-read session. Returns null when the count could not be
+ /// verified — the candidate is then left alone until the next sweep rather than warned
+ /// about or blocked on an unverified number.
+ ///
+ private async Task RecountRunawayCandidateAsync(string tenantId, SessionSummary candidate)
+ {
+ var scan = await _sessionRepo.ReconcileSessionCountersAsync(tenantId, candidate.SessionId);
+ var recounted = scan == null ? null : await _sessionRepo.GetSessionAsync(tenantId, candidate.SessionId);
+ if (recounted == null)
+ {
+ _logger.LogWarning(
+ "Excessive-event scan: could not verify the event count of session {SessionId} (tenant {TenantId}, stored {StoredCount}); skipped until the next sweep",
+ candidate.SessionId, tenantId, candidate.EventCount);
+ return null;
+ }
+
+ if (recounted.EventCount != candidate.EventCount)
+ _logger.LogWarning(
+ "Excessive-event scan: session {SessionId} (tenant {TenantId}) stored EventCount {StoredCount} corrected to {ActualCount} rows",
+ candidate.SessionId, tenantId, candidate.EventCount, recounted.EventCount);
+
+ return recounted;
+ }
+
///
/// Aggregates metrics for any missed days in the last 7 days, plus yesterday.
/// Checks the UsageMetrics table for existing snapshots to avoid re-aggregation.
diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs
index 5cb68acf..a39f34be 100644
--- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs
+++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs
@@ -984,7 +984,7 @@ namespace AutopilotMonitor.Functions.Services
public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.ExcessiveSessionEvents, OpsEventSeverity.Warning,
- $"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug",
+ $"Session {sessionId} has {eventCount} events (threshold {threshold}) — unusually high event volume",
tenantId, "System.Maintenance", new { sessionId, eventCount, threshold });
///
diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs
index a61029ad..28d4f15c 100644
--- a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs
+++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs
@@ -178,7 +178,8 @@ namespace AutopilotMonitor.Shared.DataAccess
// --- Excessive-Event Detection ---
///
/// Returns sessions in whose EventCount exceeds .
- /// Used by maintenance to surface runaway sessions (likely agent loop bugs).
+ /// Used by maintenance to find runaway candidates; the stored counter can overcount, so
+ /// the caller recounts before acting on it.
///
Task> GetSessionsWithEventCountAboveAsync(string tenantId, int threshold);