diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ModernDeploymentTrackerRollupTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ModernDeploymentTrackerRollupTests.cs new file mode 100644 index 00000000..95626719 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ModernDeploymentTrackerRollupTests.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals +{ + /// + /// Session 8bc1180f (2026-06-12) — harmless-EventId burst rollup. Windows dumped 689 + /// EventID-100 "Autopilot policy not found" records in one minute; every one was + /// forwarded as an individual Debug event, saturating the signal-ingress queue + /// (256/256 back-pressure). The tracker now forwards the first + /// occurrences per + /// EventId individually and afterwards only every + /// th, carrying the + /// cumulative count. + /// + public sealed class ModernDeploymentTrackerRollupTests : IDisposable + { + private static readonly DateTime At = new DateTime(2026, 6, 12, 9, 37, 0, DateTimeKind.Utc); + + private readonly TempDirectory _tmp = new TempDirectory(); + private readonly FakeSignalIngressSink _sink; + private readonly ModernDeploymentTracker _tracker; + + public ModernDeploymentTrackerRollupTests() + { + _sink = new FakeSignalIngressSink(); + var post = new InformationalEventPost(_sink, new VirtualClock(At)); + var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info); + _tracker = new ModernDeploymentTracker( + sessionId: "sess-rollup", + tenantId: "tenant-rollup", + post: post, + logger: logger, + backfillEnabled: false); + } + + public void Dispose() => _tmp.Dispose(); + + private void ProcessHarmless(int eventId, int times, int level = 2) + { + for (var i = 0; i < times; i++) + { + _tracker.ProcessEvent( + eventId: eventId, + level: level, + levelDisplayName: "Error", + providerName: "ModernDeployment-Diagnostics-Provider", + timeCreatedUtc: At.AddSeconds(i), + formattedDescription: $"Autopilot policy [SomePolicy] not found.", + shortName: "Autopilot", + channelName: ModernDeploymentTracker.AutopilotChannel, + isBackfill: false); + } + } + + private IReadOnlyList PostedLogs() => + _sink.Posted.Where(p => + p.Kind == DecisionSignalKind.InformationalEvent + && p.Payload != null + && p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et) + && et == "modern_deployment_log").ToList(); + + [Fact] + public void FirstThreeOccurrences_AreForwardedIndividually() + { + ProcessHarmless(eventId: 100, times: ModernDeploymentTracker.HarmlessRollupIndividualLimit); + + Assert.Equal(ModernDeploymentTracker.HarmlessRollupIndividualLimit, PostedLogs().Count); + } + + [Fact] + public void BurstOf689_IsRolledUpToIndividualLimitPlusEveryHundredth() + { + // The real session-8bc1180f burst shape: 689 EventID-100 records in one minute. + ProcessHarmless(eventId: 100, times: 689); + + // 3 individual + rollups at occurrence 100, 200, ..., 600 = 9 total (was 689). + var posted = PostedLogs(); + Assert.Equal( + ModernDeploymentTracker.HarmlessRollupIndividualLimit + 689 / ModernDeploymentTracker.HarmlessRollupEmitEvery, + posted.Count); + + // The last rollup emission carries the cumulative count of 600 in the message. + Assert.Contains("600 occurrences so far", posted[posted.Count - 1].Payload![SignalPayloadKeys.Message]); + } + + [Fact] + public void RollupCounters_AreIndependentPerEventId() + { + ProcessHarmless(eventId: 100, times: 10); + ProcessHarmless(eventId: 1005, times: 2); + + // EventId 100: 3 individual (4..10 suppressed). EventId 1005: both individual — + // its counter is untouched by the 100-burst. + Assert.Equal(ModernDeploymentTracker.HarmlessRollupIndividualLimit + 2, PostedLogs().Count); + } + + [Fact] + public void NonHarmlessEventIds_AreNeverSuppressed() + { + // EventId 999 is not in the default harmless set {100, 1005, 1010} — every + // occurrence keeps flowing (as modern_deployment_error for level 2). + for (var i = 0; i < 10; i++) + { + _tracker.ProcessEvent( + eventId: 999, + level: 2, + levelDisplayName: "Error", + providerName: "ModernDeployment-Diagnostics-Provider", + timeCreatedUtc: At.AddSeconds(i), + formattedDescription: "Genuine deployment error.", + shortName: "Autopilot", + channelName: ModernDeploymentTracker.AutopilotChannel, + isBackfill: false); + } + + var errors = _sink.Posted.Where(p => + p.Payload != null + && p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et) + && et == "modern_deployment_error").ToList(); + Assert.Equal(10, errors.Count); + } + + [Fact] + public void InfoLevelEvents_AreNotRollupGated() + { + // Level 4 (Informational) events are not part of the harmless Level-2/3 downgrade + // and therefore not rollup-gated. (The watcher XPath only subscribes to Level<=3 + // in production; this guards the ProcessEvent contract itself.) + for (var i = 0; i < 10; i++) + { + _tracker.ProcessEvent( + eventId: 100, + level: 4, + levelDisplayName: "Information", + providerName: "ModernDeployment-Diagnostics-Provider", + timeCreatedUtc: At.AddSeconds(i), + formattedDescription: "Informational record.", + shortName: "Autopilot", + channelName: ModernDeploymentTracker.AutopilotChannel, + isBackfill: false); + } + + Assert.Equal(10, PostedLogs().Count); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/SignalActivityClassifierTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/SignalActivityClassifierTests.cs index c75a8571..c5de6b2c 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/SignalActivityClassifierTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/SignalActivityClassifierTests.cs @@ -32,6 +32,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration [InlineData("stall_probe_check")] [InlineData("stall_probe_result")] [InlineData("session_stalled")] + // OS-eventlog forwarding — observability, not enrollment progress (session 8bc1180f: + // an EventID-100 burst must not keep the periodic collectors alive). + [InlineData("modern_deployment_log")] // Agent health / control / transport — not device/enrollment progress (P2). [InlineData("collector_degraded")] [InlineData("telemetry_upload_poisoned")] @@ -62,6 +65,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration Assert.True(SignalActivityClassifier.IsRealActivity(DecisionSignalKind.InformationalEvent, payload)); } + [Theory] + // Genuine ModernDeployment warnings/errors stay real activity — only the Info/Debug + // `modern_deployment_log` forwarding is excluded. + [InlineData("modern_deployment_warning")] + [InlineData("modern_deployment_error")] + public void Modern_deployment_warning_and_error_remain_activity(string eventType) + { + var payload = new Dictionary { [SignalPayloadKeys.EventType] = eventType }; + Assert.True(SignalActivityClassifier.IsRealActivity(DecisionSignalKind.InformationalEvent, payload)); + } + [Fact] public void Informational_event_without_payload_or_eventType_is_activity() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ModernDeploymentTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ModernDeploymentTracker.cs index 8e8b883c..0f0a8763 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ModernDeploymentTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ModernDeploymentTracker.cs @@ -38,6 +38,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals internal const string WhiteGloveBackfillStateFileName = "whiteglove-backfill.json"; + /// + /// Per-EventId emission caps for harmless-downgraded events (session 8bc1180f, + /// 2026-06-12). Windows can dump hundreds of identical EventID-100 "Autopilot policy + /// not found" records in a single minute (observed: 689/min → signal-ingress queue + /// saturated at 256/256). Forwarding each one as a Debug event has zero diagnostic + /// value beyond a count, so per EventId the first + /// occurrences are forwarded individually + /// and afterwards only every th occurrence is + /// emitted, carrying the cumulative occurrenceCount. Counter-based (no timers) + /// so the suppression is deterministic and replay-safe; the tail below the next + /// multiple is intentionally absorbed — the last emitted rollup carries the running + /// total, which bounds the loss to < one bucket. + /// + internal const int HarmlessRollupIndividualLimit = 3; + + /// See . + internal const int HarmlessRollupEmitEvery = 100; + private readonly AgentLogger _logger; private readonly string _sessionId; private readonly string _tenantId; @@ -53,6 +71,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals private bool _whiteGloveStartDetected; private readonly object _stateLock = new object(); + // Cumulative per-EventId occurrence counters for harmless-downgraded events. + // EventRecordWritten callbacks can run concurrently → guarded by its own lock. + private readonly Dictionary _harmlessOccurrenceCounts = new Dictionary(); + private readonly object _harmlessCountLock = new object(); + public ModernDeploymentTracker( string sessionId, string tenantId, @@ -268,10 +291,26 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals // 1005, 1010 "Autopilot.dll WIL hardwareinfo.cpp HRESULT 0x80070002" — no real // enrollment impact). Events stay visible in the timeline (Debug severity) for // troubleshooting. - if ((effectiveLevel == 2 || effectiveLevel == 3) && _harmlessEventIds.Contains(eventId)) + var harmlessDowngraded = (effectiveLevel == 2 || effectiveLevel == 3) && _harmlessEventIds.Contains(eventId); + var occurrenceCount = 0; + if (harmlessDowngraded) { eventType = Constants.EventTypes.ModernDeploymentLog; severity = EventSeverity.Debug; + + // Burst rollup (session 8bc1180f): forward the first N occurrences per + // EventId individually, then only every Kth with the cumulative count. + lock (_harmlessCountLock) + { + _harmlessOccurrenceCounts.TryGetValue(eventId, out occurrenceCount); + occurrenceCount++; + _harmlessOccurrenceCounts[eventId] = occurrenceCount; + } + if (occurrenceCount > HarmlessRollupIndividualLimit + && occurrenceCount % HarmlessRollupEmitEvery != 0) + { + return; // suppressed — counted toward the next rollup emission + } } var description = string.IsNullOrEmpty(formattedDescription) @@ -292,6 +331,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals { "backfilled", isBackfill } }; + var message = $"[{shortName}] EventID {eventId}: {truncated}"; + if (harmlessDowngraded) + { + data["occurrenceCount"] = occurrenceCount; + if (occurrenceCount > HarmlessRollupIndividualLimit) + { + data["rollup"] = true; + message = $"[{shortName}] EventID {eventId}: {occurrenceCount} occurrences so far " + + $"(harmless-ID rollup, intermediate occurrences suppressed). Last: {truncated}"; + } + } + _post.Emit(new EnrollmentEvent { SessionId = _sessionId, @@ -300,7 +351,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals Severity = severity, Source = "ModernDeploymentWatcher", Phase = EnrollmentPhase.Unknown, - Message = $"[{shortName}] EventID {eventId}: {truncated}", + Message = message, Data = data }); } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/SignalActivityClassifier.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/SignalActivityClassifier.cs index b674bfc3..31c6d2eb 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/SignalActivityClassifier.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/SignalActivityClassifier.cs @@ -47,6 +47,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration SharedConstants.EventTypes.StallProbeResult, SharedConstants.EventTypes.SessionStalled, + // OS-eventlog forwarding — observability, not device/enrollment progress. Windows + // re-reads its Autopilot policy cache at arbitrary times (observed: 689 EventID-100 + // records in one minute, session 8bc1180f) entirely decoupled from enrollment + // activity; letting those bursts reset the idle clocks would keep the periodic + // collectors alive indefinitely. Only the Info/Debug-severity `modern_deployment_log` + // type is excluded — `modern_deployment_warning` / `modern_deployment_error` stay + // real activity (rare, and worth keeping diagnostics running for). + SharedConstants.EventTypes.ModernDeploymentLog, + // Agent health / control / transport — not device/enrollment progress. SharedConstants.EventTypes.CollectorDegraded, SharedConstants.EventTypes.TelemetryUploadPoisoned,