fix(agent-v2): roll up harmless ModernDeployment event bursts, exclude them from idle clocks

Session 8bc1180f: Windows dumped 689 EventID-100 "Autopilot policy not found"
records in one minute. Each was downgraded to Debug but still forwarded as an
individual modern_deployment_log event - 701 of 1116 session events (63%),
signal-ingress queue saturated (ingress_backpressure 256/256), and the burst
deferred the periodic-collector idle stop.

Two changes:

- ModernDeploymentTracker: harmless-downgraded EventIds (default 100/1005/1010,
  Level 2/3 -> Debug) are now rollup-gated per EventId - first 3 occurrences
  forwarded individually, afterwards only every 100th, carrying the cumulative
  occurrenceCount (plus a rollup marker in Data). Counter-based, no timers;
  warnings/errors and non-harmless IDs are untouched. The 689-burst now
  produces 9 events instead of 689.
- SignalActivityClassifier: modern_deployment_log added to the non-activity
  denylist - OS-eventlog forwarding is observability, not enrollment progress,
  and a policy-cache re-read burst must not keep the periodic collectors
  alive. modern_deployment_warning/error remain real activity.

V2.Core rollup + classifier tests 33/33 green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-06-12 13:00:33 +02:00
parent a77ad21e76
commit 30f71e1b54
4 changed files with 232 additions and 2 deletions
@@ -0,0 +1,156 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
using AutopilotMonitor.DecisionCore.Engine;
using AutopilotMonitor.DecisionCore.Signals;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
{
/// <summary>
/// Session 8bc1180f (2026-06-12) — harmless-EventId burst rollup. Windows dumped 689
/// EventID-100 "Autopilot policy not found" records in one minute; every one was
/// forwarded as an individual Debug event, saturating the signal-ingress queue
/// (256/256 back-pressure). The tracker now forwards the first
/// <see cref="ModernDeploymentTracker.HarmlessRollupIndividualLimit"/> occurrences per
/// EventId individually and afterwards only every
/// <see cref="ModernDeploymentTracker.HarmlessRollupEmitEvery"/>th, carrying the
/// cumulative count.
/// </summary>
public sealed class ModernDeploymentTrackerRollupTests : IDisposable
{
private static readonly DateTime At = new DateTime(2026, 6, 12, 9, 37, 0, DateTimeKind.Utc);
private readonly TempDirectory _tmp = new TempDirectory();
private readonly FakeSignalIngressSink _sink;
private readonly ModernDeploymentTracker _tracker;
public ModernDeploymentTrackerRollupTests()
{
_sink = new FakeSignalIngressSink();
var post = new InformationalEventPost(_sink, new VirtualClock(At));
var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
_tracker = new ModernDeploymentTracker(
sessionId: "sess-rollup",
tenantId: "tenant-rollup",
post: post,
logger: logger,
backfillEnabled: false);
}
public void Dispose() => _tmp.Dispose();
private void ProcessHarmless(int eventId, int times, int level = 2)
{
for (var i = 0; i < times; i++)
{
_tracker.ProcessEvent(
eventId: eventId,
level: level,
levelDisplayName: "Error",
providerName: "ModernDeployment-Diagnostics-Provider",
timeCreatedUtc: At.AddSeconds(i),
formattedDescription: $"Autopilot policy [SomePolicy] not found.",
shortName: "Autopilot",
channelName: ModernDeploymentTracker.AutopilotChannel,
isBackfill: false);
}
}
private IReadOnlyList<FakeSignalIngressSink.PostedSignal> PostedLogs() =>
_sink.Posted.Where(p =>
p.Kind == DecisionSignalKind.InformationalEvent
&& p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == "modern_deployment_log").ToList();
[Fact]
public void FirstThreeOccurrences_AreForwardedIndividually()
{
ProcessHarmless(eventId: 100, times: ModernDeploymentTracker.HarmlessRollupIndividualLimit);
Assert.Equal(ModernDeploymentTracker.HarmlessRollupIndividualLimit, PostedLogs().Count);
}
[Fact]
public void BurstOf689_IsRolledUpToIndividualLimitPlusEveryHundredth()
{
// The real session-8bc1180f burst shape: 689 EventID-100 records in one minute.
ProcessHarmless(eventId: 100, times: 689);
// 3 individual + rollups at occurrence 100, 200, ..., 600 = 9 total (was 689).
var posted = PostedLogs();
Assert.Equal(
ModernDeploymentTracker.HarmlessRollupIndividualLimit + 689 / ModernDeploymentTracker.HarmlessRollupEmitEvery,
posted.Count);
// The last rollup emission carries the cumulative count of 600 in the message.
Assert.Contains("600 occurrences so far", posted[posted.Count - 1].Payload![SignalPayloadKeys.Message]);
}
[Fact]
public void RollupCounters_AreIndependentPerEventId()
{
ProcessHarmless(eventId: 100, times: 10);
ProcessHarmless(eventId: 1005, times: 2);
// EventId 100: 3 individual (4..10 suppressed). EventId 1005: both individual —
// its counter is untouched by the 100-burst.
Assert.Equal(ModernDeploymentTracker.HarmlessRollupIndividualLimit + 2, PostedLogs().Count);
}
[Fact]
public void NonHarmlessEventIds_AreNeverSuppressed()
{
// EventId 999 is not in the default harmless set {100, 1005, 1010} — every
// occurrence keeps flowing (as modern_deployment_error for level 2).
for (var i = 0; i < 10; i++)
{
_tracker.ProcessEvent(
eventId: 999,
level: 2,
levelDisplayName: "Error",
providerName: "ModernDeployment-Diagnostics-Provider",
timeCreatedUtc: At.AddSeconds(i),
formattedDescription: "Genuine deployment error.",
shortName: "Autopilot",
channelName: ModernDeploymentTracker.AutopilotChannel,
isBackfill: false);
}
var errors = _sink.Posted.Where(p =>
p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == "modern_deployment_error").ToList();
Assert.Equal(10, errors.Count);
}
[Fact]
public void InfoLevelEvents_AreNotRollupGated()
{
// Level 4 (Informational) events are not part of the harmless Level-2/3 downgrade
// and therefore not rollup-gated. (The watcher XPath only subscribes to Level<=3
// in production; this guards the ProcessEvent contract itself.)
for (var i = 0; i < 10; i++)
{
_tracker.ProcessEvent(
eventId: 100,
level: 4,
levelDisplayName: "Information",
providerName: "ModernDeployment-Diagnostics-Provider",
timeCreatedUtc: At.AddSeconds(i),
formattedDescription: "Informational record.",
shortName: "Autopilot",
channelName: ModernDeploymentTracker.AutopilotChannel,
isBackfill: false);
}
Assert.Equal(10, PostedLogs().Count);
}
}
}
@@ -32,6 +32,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
[InlineData("stall_probe_check")]
[InlineData("stall_probe_result")]
[InlineData("session_stalled")]
// OS-eventlog forwarding — observability, not enrollment progress (session 8bc1180f:
// an EventID-100 burst must not keep the periodic collectors alive).
[InlineData("modern_deployment_log")]
// Agent health / control / transport — not device/enrollment progress (P2).
[InlineData("collector_degraded")]
[InlineData("telemetry_upload_poisoned")]
@@ -62,6 +65,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
Assert.True(SignalActivityClassifier.IsRealActivity(DecisionSignalKind.InformationalEvent, payload));
}
[Theory]
// Genuine ModernDeployment warnings/errors stay real activity — only the Info/Debug
// `modern_deployment_log` forwarding is excluded.
[InlineData("modern_deployment_warning")]
[InlineData("modern_deployment_error")]
public void Modern_deployment_warning_and_error_remain_activity(string eventType)
{
var payload = new Dictionary<string, string> { [SignalPayloadKeys.EventType] = eventType };
Assert.True(SignalActivityClassifier.IsRealActivity(DecisionSignalKind.InformationalEvent, payload));
}
[Fact]
public void Informational_event_without_payload_or_eventType_is_activity()
{
@@ -38,6 +38,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
internal const string WhiteGloveBackfillStateFileName = "whiteglove-backfill.json";
/// <summary>
/// Per-EventId emission caps for harmless-downgraded events (session 8bc1180f,
/// 2026-06-12). Windows can dump hundreds of identical EventID-100 "Autopilot policy
/// not found" records in a single minute (observed: 689/min → signal-ingress queue
/// saturated at 256/256). Forwarding each one as a Debug event has zero diagnostic
/// value beyond a count, so per EventId the first
/// <see cref="HarmlessRollupIndividualLimit"/> occurrences are forwarded individually
/// and afterwards only every <see cref="HarmlessRollupEmitEvery"/>th occurrence is
/// emitted, carrying the cumulative <c>occurrenceCount</c>. Counter-based (no timers)
/// so the suppression is deterministic and replay-safe; the tail below the next
/// multiple is intentionally absorbed — the last emitted rollup carries the running
/// total, which bounds the loss to &lt; one bucket.
/// </summary>
internal const int HarmlessRollupIndividualLimit = 3;
/// <summary>See <see cref="HarmlessRollupIndividualLimit"/>.</summary>
internal const int HarmlessRollupEmitEvery = 100;
private readonly AgentLogger _logger;
private readonly string _sessionId;
private readonly string _tenantId;
@@ -53,6 +71,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
private bool _whiteGloveStartDetected;
private readonly object _stateLock = new object();
// Cumulative per-EventId occurrence counters for harmless-downgraded events.
// EventRecordWritten callbacks can run concurrently → guarded by its own lock.
private readonly Dictionary<int, int> _harmlessOccurrenceCounts = new Dictionary<int, int>();
private readonly object _harmlessCountLock = new object();
public ModernDeploymentTracker(
string sessionId,
string tenantId,
@@ -268,10 +291,26 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
// 1005, 1010 "Autopilot.dll WIL hardwareinfo.cpp HRESULT 0x80070002" — no real
// enrollment impact). Events stay visible in the timeline (Debug severity) for
// troubleshooting.
if ((effectiveLevel == 2 || effectiveLevel == 3) && _harmlessEventIds.Contains(eventId))
var harmlessDowngraded = (effectiveLevel == 2 || effectiveLevel == 3) && _harmlessEventIds.Contains(eventId);
var occurrenceCount = 0;
if (harmlessDowngraded)
{
eventType = Constants.EventTypes.ModernDeploymentLog;
severity = EventSeverity.Debug;
// Burst rollup (session 8bc1180f): forward the first N occurrences per
// EventId individually, then only every Kth with the cumulative count.
lock (_harmlessCountLock)
{
_harmlessOccurrenceCounts.TryGetValue(eventId, out occurrenceCount);
occurrenceCount++;
_harmlessOccurrenceCounts[eventId] = occurrenceCount;
}
if (occurrenceCount > HarmlessRollupIndividualLimit
&& occurrenceCount % HarmlessRollupEmitEvery != 0)
{
return; // suppressed — counted toward the next rollup emission
}
}
var description = string.IsNullOrEmpty(formattedDescription)
@@ -292,6 +331,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
{ "backfilled", isBackfill }
};
var message = $"[{shortName}] EventID {eventId}: {truncated}";
if (harmlessDowngraded)
{
data["occurrenceCount"] = occurrenceCount;
if (occurrenceCount > HarmlessRollupIndividualLimit)
{
data["rollup"] = true;
message = $"[{shortName}] EventID {eventId}: {occurrenceCount} occurrences so far " +
$"(harmless-ID rollup, intermediate occurrences suppressed). Last: {truncated}";
}
}
_post.Emit(new EnrollmentEvent
{
SessionId = _sessionId,
@@ -300,7 +351,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
Severity = severity,
Source = "ModernDeploymentWatcher",
Phase = EnrollmentPhase.Unknown,
Message = $"[{shortName}] EventID {eventId}: {truncated}",
Message = message,
Data = data
});
}
@@ -47,6 +47,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
SharedConstants.EventTypes.StallProbeResult,
SharedConstants.EventTypes.SessionStalled,
// OS-eventlog forwarding — observability, not device/enrollment progress. Windows
// re-reads its Autopilot policy cache at arbitrary times (observed: 689 EventID-100
// records in one minute, session 8bc1180f) entirely decoupled from enrollment
// activity; letting those bursts reset the idle clocks would keep the periodic
// collectors alive indefinitely. Only the Info/Debug-severity `modern_deployment_log`
// type is excluded — `modern_deployment_warning` / `modern_deployment_error` stay
// real activity (rare, and worth keeping diagnostics running for).
SharedConstants.EventTypes.ModernDeploymentLog,
// Agent health / control / transport — not device/enrollment progress.
SharedConstants.EventTypes.CollectorDegraded,
SharedConstants.EventTypes.TelemetryUploadPoisoned,