mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
fix(agent-v2): roll up harmless ModernDeployment event bursts, exclude them from idle clocks
Session 8bc1180f: Windows dumped 689 EventID-100 "Autopilot policy not found" records in one minute. Each was downgraded to Debug but still forwarded as an individual modern_deployment_log event - 701 of 1116 session events (63%), signal-ingress queue saturated (ingress_backpressure 256/256), and the burst deferred the periodic-collector idle stop. Two changes: - ModernDeploymentTracker: harmless-downgraded EventIds (default 100/1005/1010, Level 2/3 -> Debug) are now rollup-gated per EventId - first 3 occurrences forwarded individually, afterwards only every 100th, carrying the cumulative occurrenceCount (plus a rollup marker in Data). Counter-based, no timers; warnings/errors and non-harmless IDs are untouched. The 689-burst now produces 9 events instead of 689. - SignalActivityClassifier: modern_deployment_log added to the non-activity denylist - OS-eventlog forwarding is observability, not enrollment progress, and a policy-cache re-read burst must not keep the periodic collectors alive. modern_deployment_warning/error remain real activity. V2.Core rollup + classifier tests 33/33 green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+156
@@ -0,0 +1,156 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Session 8bc1180f (2026-06-12) — harmless-EventId burst rollup. Windows dumped 689
|
||||
/// EventID-100 "Autopilot policy not found" records in one minute; every one was
|
||||
/// forwarded as an individual Debug event, saturating the signal-ingress queue
|
||||
/// (256/256 back-pressure). The tracker now forwards the first
|
||||
/// <see cref="ModernDeploymentTracker.HarmlessRollupIndividualLimit"/> occurrences per
|
||||
/// EventId individually and afterwards only every
|
||||
/// <see cref="ModernDeploymentTracker.HarmlessRollupEmitEvery"/>th, carrying the
|
||||
/// cumulative count.
|
||||
/// </summary>
|
||||
public sealed class ModernDeploymentTrackerRollupTests : IDisposable
|
||||
{
|
||||
private static readonly DateTime At = new DateTime(2026, 6, 12, 9, 37, 0, DateTimeKind.Utc);
|
||||
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
private readonly FakeSignalIngressSink _sink;
|
||||
private readonly ModernDeploymentTracker _tracker;
|
||||
|
||||
public ModernDeploymentTrackerRollupTests()
|
||||
{
|
||||
_sink = new FakeSignalIngressSink();
|
||||
var post = new InformationalEventPost(_sink, new VirtualClock(At));
|
||||
var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
|
||||
_tracker = new ModernDeploymentTracker(
|
||||
sessionId: "sess-rollup",
|
||||
tenantId: "tenant-rollup",
|
||||
post: post,
|
||||
logger: logger,
|
||||
backfillEnabled: false);
|
||||
}
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
private void ProcessHarmless(int eventId, int times, int level = 2)
|
||||
{
|
||||
for (var i = 0; i < times; i++)
|
||||
{
|
||||
_tracker.ProcessEvent(
|
||||
eventId: eventId,
|
||||
level: level,
|
||||
levelDisplayName: "Error",
|
||||
providerName: "ModernDeployment-Diagnostics-Provider",
|
||||
timeCreatedUtc: At.AddSeconds(i),
|
||||
formattedDescription: $"Autopilot policy [SomePolicy] not found.",
|
||||
shortName: "Autopilot",
|
||||
channelName: ModernDeploymentTracker.AutopilotChannel,
|
||||
isBackfill: false);
|
||||
}
|
||||
}
|
||||
|
||||
private IReadOnlyList<FakeSignalIngressSink.PostedSignal> PostedLogs() =>
|
||||
_sink.Posted.Where(p =>
|
||||
p.Kind == DecisionSignalKind.InformationalEvent
|
||||
&& p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == "modern_deployment_log").ToList();
|
||||
|
||||
[Fact]
|
||||
public void FirstThreeOccurrences_AreForwardedIndividually()
|
||||
{
|
||||
ProcessHarmless(eventId: 100, times: ModernDeploymentTracker.HarmlessRollupIndividualLimit);
|
||||
|
||||
Assert.Equal(ModernDeploymentTracker.HarmlessRollupIndividualLimit, PostedLogs().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BurstOf689_IsRolledUpToIndividualLimitPlusEveryHundredth()
|
||||
{
|
||||
// The real session-8bc1180f burst shape: 689 EventID-100 records in one minute.
|
||||
ProcessHarmless(eventId: 100, times: 689);
|
||||
|
||||
// 3 individual + rollups at occurrence 100, 200, ..., 600 = 9 total (was 689).
|
||||
var posted = PostedLogs();
|
||||
Assert.Equal(
|
||||
ModernDeploymentTracker.HarmlessRollupIndividualLimit + 689 / ModernDeploymentTracker.HarmlessRollupEmitEvery,
|
||||
posted.Count);
|
||||
|
||||
// The last rollup emission carries the cumulative count of 600 in the message.
|
||||
Assert.Contains("600 occurrences so far", posted[posted.Count - 1].Payload![SignalPayloadKeys.Message]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RollupCounters_AreIndependentPerEventId()
|
||||
{
|
||||
ProcessHarmless(eventId: 100, times: 10);
|
||||
ProcessHarmless(eventId: 1005, times: 2);
|
||||
|
||||
// EventId 100: 3 individual (4..10 suppressed). EventId 1005: both individual —
|
||||
// its counter is untouched by the 100-burst.
|
||||
Assert.Equal(ModernDeploymentTracker.HarmlessRollupIndividualLimit + 2, PostedLogs().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonHarmlessEventIds_AreNeverSuppressed()
|
||||
{
|
||||
// EventId 999 is not in the default harmless set {100, 1005, 1010} — every
|
||||
// occurrence keeps flowing (as modern_deployment_error for level 2).
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
_tracker.ProcessEvent(
|
||||
eventId: 999,
|
||||
level: 2,
|
||||
levelDisplayName: "Error",
|
||||
providerName: "ModernDeployment-Diagnostics-Provider",
|
||||
timeCreatedUtc: At.AddSeconds(i),
|
||||
formattedDescription: "Genuine deployment error.",
|
||||
shortName: "Autopilot",
|
||||
channelName: ModernDeploymentTracker.AutopilotChannel,
|
||||
isBackfill: false);
|
||||
}
|
||||
|
||||
var errors = _sink.Posted.Where(p =>
|
||||
p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == "modern_deployment_error").ToList();
|
||||
Assert.Equal(10, errors.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InfoLevelEvents_AreNotRollupGated()
|
||||
{
|
||||
// Level 4 (Informational) events are not part of the harmless Level-2/3 downgrade
|
||||
// and therefore not rollup-gated. (The watcher XPath only subscribes to Level<=3
|
||||
// in production; this guards the ProcessEvent contract itself.)
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
_tracker.ProcessEvent(
|
||||
eventId: 100,
|
||||
level: 4,
|
||||
levelDisplayName: "Information",
|
||||
providerName: "ModernDeployment-Diagnostics-Provider",
|
||||
timeCreatedUtc: At.AddSeconds(i),
|
||||
formattedDescription: "Informational record.",
|
||||
shortName: "Autopilot",
|
||||
channelName: ModernDeploymentTracker.AutopilotChannel,
|
||||
isBackfill: false);
|
||||
}
|
||||
|
||||
Assert.Equal(10, PostedLogs().Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
+14
@@ -32,6 +32,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
[InlineData("stall_probe_check")]
|
||||
[InlineData("stall_probe_result")]
|
||||
[InlineData("session_stalled")]
|
||||
// OS-eventlog forwarding — observability, not enrollment progress (session 8bc1180f:
|
||||
// an EventID-100 burst must not keep the periodic collectors alive).
|
||||
[InlineData("modern_deployment_log")]
|
||||
// Agent health / control / transport — not device/enrollment progress (P2).
|
||||
[InlineData("collector_degraded")]
|
||||
[InlineData("telemetry_upload_poisoned")]
|
||||
@@ -62,6 +65,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
Assert.True(SignalActivityClassifier.IsRealActivity(DecisionSignalKind.InformationalEvent, payload));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// Genuine ModernDeployment warnings/errors stay real activity — only the Info/Debug
|
||||
// `modern_deployment_log` forwarding is excluded.
|
||||
[InlineData("modern_deployment_warning")]
|
||||
[InlineData("modern_deployment_error")]
|
||||
public void Modern_deployment_warning_and_error_remain_activity(string eventType)
|
||||
{
|
||||
var payload = new Dictionary<string, string> { [SignalPayloadKeys.EventType] = eventType };
|
||||
Assert.True(SignalActivityClassifier.IsRealActivity(DecisionSignalKind.InformationalEvent, payload));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Informational_event_without_payload_or_eventType_is_activity()
|
||||
{
|
||||
|
||||
+53
-2
@@ -38,6 +38,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
|
||||
internal const string WhiteGloveBackfillStateFileName = "whiteglove-backfill.json";
|
||||
|
||||
/// <summary>
|
||||
/// Per-EventId emission caps for harmless-downgraded events (session 8bc1180f,
|
||||
/// 2026-06-12). Windows can dump hundreds of identical EventID-100 "Autopilot policy
|
||||
/// not found" records in a single minute (observed: 689/min → signal-ingress queue
|
||||
/// saturated at 256/256). Forwarding each one as a Debug event has zero diagnostic
|
||||
/// value beyond a count, so per EventId the first
|
||||
/// <see cref="HarmlessRollupIndividualLimit"/> occurrences are forwarded individually
|
||||
/// and afterwards only every <see cref="HarmlessRollupEmitEvery"/>th occurrence is
|
||||
/// emitted, carrying the cumulative <c>occurrenceCount</c>. Counter-based (no timers)
|
||||
/// so the suppression is deterministic and replay-safe; the tail below the next
|
||||
/// multiple is intentionally absorbed — the last emitted rollup carries the running
|
||||
/// total, which bounds the loss to < one bucket.
|
||||
/// </summary>
|
||||
internal const int HarmlessRollupIndividualLimit = 3;
|
||||
|
||||
/// <summary>See <see cref="HarmlessRollupIndividualLimit"/>.</summary>
|
||||
internal const int HarmlessRollupEmitEvery = 100;
|
||||
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly string _sessionId;
|
||||
private readonly string _tenantId;
|
||||
@@ -53,6 +71,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
private bool _whiteGloveStartDetected;
|
||||
private readonly object _stateLock = new object();
|
||||
|
||||
// Cumulative per-EventId occurrence counters for harmless-downgraded events.
|
||||
// EventRecordWritten callbacks can run concurrently → guarded by its own lock.
|
||||
private readonly Dictionary<int, int> _harmlessOccurrenceCounts = new Dictionary<int, int>();
|
||||
private readonly object _harmlessCountLock = new object();
|
||||
|
||||
public ModernDeploymentTracker(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
@@ -268,10 +291,26 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// 1005, 1010 "Autopilot.dll WIL hardwareinfo.cpp HRESULT 0x80070002" — no real
|
||||
// enrollment impact). Events stay visible in the timeline (Debug severity) for
|
||||
// troubleshooting.
|
||||
if ((effectiveLevel == 2 || effectiveLevel == 3) && _harmlessEventIds.Contains(eventId))
|
||||
var harmlessDowngraded = (effectiveLevel == 2 || effectiveLevel == 3) && _harmlessEventIds.Contains(eventId);
|
||||
var occurrenceCount = 0;
|
||||
if (harmlessDowngraded)
|
||||
{
|
||||
eventType = Constants.EventTypes.ModernDeploymentLog;
|
||||
severity = EventSeverity.Debug;
|
||||
|
||||
// Burst rollup (session 8bc1180f): forward the first N occurrences per
|
||||
// EventId individually, then only every Kth with the cumulative count.
|
||||
lock (_harmlessCountLock)
|
||||
{
|
||||
_harmlessOccurrenceCounts.TryGetValue(eventId, out occurrenceCount);
|
||||
occurrenceCount++;
|
||||
_harmlessOccurrenceCounts[eventId] = occurrenceCount;
|
||||
}
|
||||
if (occurrenceCount > HarmlessRollupIndividualLimit
|
||||
&& occurrenceCount % HarmlessRollupEmitEvery != 0)
|
||||
{
|
||||
return; // suppressed — counted toward the next rollup emission
|
||||
}
|
||||
}
|
||||
|
||||
var description = string.IsNullOrEmpty(formattedDescription)
|
||||
@@ -292,6 +331,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{ "backfilled", isBackfill }
|
||||
};
|
||||
|
||||
var message = $"[{shortName}] EventID {eventId}: {truncated}";
|
||||
if (harmlessDowngraded)
|
||||
{
|
||||
data["occurrenceCount"] = occurrenceCount;
|
||||
if (occurrenceCount > HarmlessRollupIndividualLimit)
|
||||
{
|
||||
data["rollup"] = true;
|
||||
message = $"[{shortName}] EventID {eventId}: {occurrenceCount} occurrences so far " +
|
||||
$"(harmless-ID rollup, intermediate occurrences suppressed). Last: {truncated}";
|
||||
}
|
||||
}
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
@@ -300,7 +351,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
Severity = severity,
|
||||
Source = "ModernDeploymentWatcher",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"[{shortName}] EventID {eventId}: {truncated}",
|
||||
Message = message,
|
||||
Data = data
|
||||
});
|
||||
}
|
||||
|
||||
@@ -47,6 +47,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
SharedConstants.EventTypes.StallProbeResult,
|
||||
SharedConstants.EventTypes.SessionStalled,
|
||||
|
||||
// OS-eventlog forwarding — observability, not device/enrollment progress. Windows
|
||||
// re-reads its Autopilot policy cache at arbitrary times (observed: 689 EventID-100
|
||||
// records in one minute, session 8bc1180f) entirely decoupled from enrollment
|
||||
// activity; letting those bursts reset the idle clocks would keep the periodic
|
||||
// collectors alive indefinitely. Only the Info/Debug-severity `modern_deployment_log`
|
||||
// type is excluded — `modern_deployment_warning` / `modern_deployment_error` stay
|
||||
// real activity (rare, and worth keeping diagnostics running for).
|
||||
SharedConstants.EventTypes.ModernDeploymentLog,
|
||||
|
||||
// Agent health / control / transport — not device/enrollment progress.
|
||||
SharedConstants.EventTypes.CollectorDegraded,
|
||||
SharedConstants.EventTypes.TelemetryUploadPoisoned,
|
||||
|
||||
Reference in New Issue
Block a user