From f0a3f04398247036cf65dbf9e63155c9943d8da6 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Fri, 4 Sep 2026 13:11:28 +0200 Subject: [PATCH] fix(agent,rules): AccountSetup activity means observed progress; same-event filter for event_data conditions (q8n) Agent: ProvisioningStatusTracker.HasAccountSetupActivity now requires at least one AccountSetup subcategory past notStarted. Windows writes AccountSetupCategory.Status with every subcategory notStarted BEFORE the Device-ESP page exits, so the presence check confirmed the Device->Account handoff (Shell-Core 62407) as the final ESP exit; the user-apps-settled synthesis then completed AccountSetup and released the User-ESP keep-awake minutes before the real exit (fleet: 77 % of keep-awake sessions, median 3 min; on battery that is enough for Modern Standby). D-183. Rule engine: filterField/filterOperator/filterValue now apply to event_data and event_data_array as a same-event filter (shared ApplyValueFilter with event_count). ANALYZE-DEV-012 v1.1.0 excludes backfilled pre-session episodes (backfilled equals false); a second independent condition could not, since each condition scans all events on its own. D-184. Schema, Shared model, MCP authoring guide and generated types follow. Tests: ProvisioningStatusTrackerAccountSetupActivityTests (the verbatim all-notStarted registry JSON), SleepEpisodeRuleTests (+3 backfill cases), RuleEngineEventDataFilterTests. Solution build 0 warnings; agent suite 2820 and backend suite 5791 green. Co-Authored-By: Claude Fable 5.1 --- internal | 2 +- rules/analyze/ANALYZE-DEV-012.json | 7 +- rules/dist/analyze-rules.json | 7 +- rules/schema/analyze-rule.schema.json | 6 +- ...gStatusTrackerAccountSetupActivityTests.cs | 131 +++++++++++ .../SystemSignals/EspAndHelloTracker.cs | 16 +- .../ProvisioningStatusTracker.cs | 20 +- .../RuleEngineEventDataFilterTests.cs | 215 ++++++++++++++++++ .../SleepEpisodeRuleTests.cs | 57 ++++- .../RuleEngine.ConditionEvaluators.cs | 35 ++- .../src/generated/wire-types.generated.ts | 6 +- .../src/rule-authoring-guide.ts | 5 +- .../src/rule-authoring.generated.ts | 6 +- .../Models/Rules/AnalyzeRule.cs | 20 +- .../utils/shared-manifests.json | 6 +- .../utils/wire-types.generated.ts | 6 +- 16 files changed, 494 insertions(+), 51 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningStatusTrackerAccountSetupActivityTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEventDataFilterTests.cs diff --git a/internal b/internal index d97bc98d..0b5da6ce 160000 --- a/internal +++ b/internal @@ -1 +1 @@ -Subproject commit d97bc98dc6580a470c8c41ef285bc7a1d497f616 +Subproject commit 0b5da6ce1e0a11b7d3a3c9b2584697ddc250abb6 diff --git a/rules/analyze/ANALYZE-DEV-012.json b/rules/analyze/ANALYZE-DEV-012.json index e3ec29bb..5a87ffcc 100644 --- a/rules/analyze/ANALYZE-DEV-012.json +++ b/rules/analyze/ANALYZE-DEV-012.json @@ -2,10 +2,10 @@ "$schema": "../schema/analyze-rule.schema.json", "ruleId": "ANALYZE-DEV-012", "title": "Device Entered Standby During Enrollment", - "description": "Detects that the device completed a sleep, hibernate or Modern Standby episode of at least 5 minutes while the enrollment was running (system_sleep_episode event from the agent's system timeline watcher, recorded retroactively at wake from the Windows event log — pre-agent episodes are backfilled). Uses a single required condition on the episode's durationSeconds so the match is pinned to one event instance and the interpolated duration is the matched value. Short Modern Standby screen-off scenarios (under 60 s of real sleep) never emit the event, and this rule adds a 5-minute floor so brief lid-close moments do not fire it.", + "description": "Detects that the device completed a sleep, hibernate or Modern Standby episode of at least 5 minutes while the enrollment was running (system_sleep_episode event from the agent's system timeline watcher, recorded retroactively at wake from the Windows event log). The watcher also backfills pre-agent episodes from the last 24 h for timeline context (backfilled=true); those ended before the enrollment started and are excluded via the same-event filter backfilled equals false — a separate condition could not do that, because each condition scans all events on its own and a pre-session hibernate would still satisfy the duration test. A single required condition keeps the match pinned to one event instance so the interpolated duration is the matched value. Short Modern Standby screen-off scenarios (under 60 s of real sleep) never emit the event, and this rule adds a 5-minute floor so brief lid-close moments do not fire it.", "severity": "info", "category": "device", - "version": "1.0.0", + "version": "1.1.0", "enabled": true, "evaluateOn": [ "enrollment_end", @@ -23,6 +23,9 @@ "dataField": "durationSeconds", "operator": "gte", "value": "300", + "filterField": "backfilled", + "filterOperator": "equals", + "filterValue": "false", "required": true } ], diff --git a/rules/dist/analyze-rules.json b/rules/dist/analyze-rules.json index 3278ce58..34ff5ae7 100644 --- a/rules/dist/analyze-rules.json +++ b/rules/dist/analyze-rules.json @@ -1772,10 +1772,10 @@ { "ruleId": "ANALYZE-DEV-012", "title": "Device Entered Standby During Enrollment", - "description": "Detects that the device completed a sleep, hibernate or Modern Standby episode of at least 5 minutes while the enrollment was running (system_sleep_episode event from the agent's system timeline watcher, recorded retroactively at wake from the Windows event log — pre-agent episodes are backfilled). Uses a single required condition on the episode's durationSeconds so the match is pinned to one event instance and the interpolated duration is the matched value. Short Modern Standby screen-off scenarios (under 60 s of real sleep) never emit the event, and this rule adds a 5-minute floor so brief lid-close moments do not fire it.", + "description": "Detects that the device completed a sleep, hibernate or Modern Standby episode of at least 5 minutes while the enrollment was running (system_sleep_episode event from the agent's system timeline watcher, recorded retroactively at wake from the Windows event log). The watcher also backfills pre-agent episodes from the last 24 h for timeline context (backfilled=true); those ended before the enrollment started and are excluded via the same-event filter backfilled equals false — a separate condition could not do that, because each condition scans all events on its own and a pre-session hibernate would still satisfy the duration test. A single required condition keeps the match pinned to one event instance so the interpolated duration is the matched value. Short Modern Standby screen-off scenarios (under 60 s of real sleep) never emit the event, and this rule adds a 5-minute floor so brief lid-close moments do not fire it.", "severity": "info", "category": "device", - "version": "1.0.0", + "version": "1.1.0", "enabled": true, "evaluateOn": [ "enrollment_end", @@ -1793,6 +1793,9 @@ "dataField": "durationSeconds", "operator": "gte", "value": "300", + "filterField": "backfilled", + "filterOperator": "equals", + "filterValue": "false", "required": true } ], diff --git a/rules/schema/analyze-rule.schema.json b/rules/schema/analyze-rule.schema.json index 3974b8de..f0337db2 100644 --- a/rules/schema/analyze-rule.schema.json +++ b/rules/schema/analyze-rule.schema.json @@ -204,16 +204,16 @@ }, "filterField": { "type": "string", - "description": "For event_count only: optional value filter — only events whose filterField satisfies filterOperator/filterValue are counted (e.g. count only performance_snapshot events with memory_used_percent > 90). Applies before counting for both count_gte and count_per_group_gte." + "description": "Optional same-event value filter for event_count, event_data and event_data_array: only events whose filterField satisfies filterOperator/filterValue are considered. For event_count it applies before counting (both count_gte and count_per_group_gte, e.g. count only performance_snapshot events with memory_used_percent > 90). For event_data/event_data_array the filter and the main dataField/operator test are evaluated on the SAME event instance — the only way to pin two fields of one event together, since separate conditions each scan all events independently (e.g. system_sleep_episode with durationSeconds gte 300 AND backfilled equals false)." }, "filterOperator": { "type": "string", - "description": "Operator for the event_count value filter.", + "description": "Operator for the same-event value filter. Same vocabulary as operator.", "enum": ["equals", "not_equals", "contains", "not_contains", "regex", "not_regex", "gt", "lt", "gte", "lte", "exists", "not_exists", "in", "not_in"] }, "filterValue": { "type": "string", - "description": "Value for the event_count value filter." + "description": "Value for the same-event value filter. Boolean values are stringified (\"true\"/\"false\")." }, "correlateEventType": { "type": "string", diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningStatusTrackerAccountSetupActivityTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningStatusTrackerAccountSetupActivityTests.cs new file mode 100644 index 00000000..14f61bf4 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningStatusTrackerAccountSetupActivityTests.cs @@ -0,0 +1,131 @@ +using System; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals +{ + /// + /// Backlog q8n — must mean + /// observed progress, not registry presence. Windows writes + /// AccountSetupCategory.Status with every subcategory notStarted BEFORE the + /// Device-ESP page exits, so a presence check labelled the intermediate Device→Account + /// Shell-Core 62407 as the final ESP exit; the user-apps-settled synthesis then completed + /// AccountSetup minutes early and released the User-ESP keep-awake while the ESP page was + /// still up (session a2256107, standby on battery at 09:43 with the real exit at 09:47). + /// Both EspAndHelloTracker guards (IsIntermediateDeviceEspExit, + /// IsConfirmedPostAccountSetupExit) hang off this property. + /// + public sealed class ProvisioningStatusTrackerAccountSetupActivityTests + { + private static readonly DateTime Fixed = new DateTime(2026, 9, 4, 9, 37, 42, DateTimeKind.Utc); + + // Verbatim shape of the pre-written registry JSON seen at first_seen on 25H2/26200. + private const string AllNotStartedJson = @"{ + ""categoryState"":""notStarted"", + ""AccountSetup.WaitingForAadRegistrationSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.PrepareMultifactorAuth"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.SecurityPoliciesSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.CertificatesSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.NetworkConnectionsSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.AppsSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.SendResultsToMdmServer"":{""subcategoryState"":""notStarted""} + }"; + + private const string FirstProgressJson = @"{ + ""categoryState"":""inProgress"", + ""AccountSetup.WaitingForAadRegistrationSubcategory"":{""subcategoryState"":""succeeded"",""subcategoryStatusText"":""succeeded""}, + ""AccountSetup.PrepareMultifactorAuth"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.SecurityPoliciesSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.CertificatesSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.NetworkConnectionsSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.AppsSubcategory"":{""subcategoryState"":""notStarted""}, + ""AccountSetup.SendResultsToMdmServer"":{""subcategoryState"":""notStarted""} + }"; + + private sealed class Fixture : IDisposable + { + public TempDirectory Tmp { get; } = new TempDirectory(); + public FakeSignalIngressSink Sink { get; } = new FakeSignalIngressSink(); + public ProvisioningStatusTracker Tracker { get; } + + public Fixture() + { + var clock = new VirtualClock(Fixed); + var post = new InformationalEventPost(Sink, clock); + var logger = new AgentLogger(Tmp.Path, AgentLogLevel.Info); + Tracker = new ProvisioningStatusTracker( + sessionId: "S1", + tenantId: "T1", + post: post, + logger: logger); + } + + public void Dispose() + { + Tracker.Dispose(); + Tmp.Dispose(); + } + } + + [Fact] + public void NothingTracked_IsNotActivity() + { + using var f = new Fixture(); + Assert.False(f.Tracker.HasAccountSetupActivity); + } + + [Fact] + public void PreWrittenAllNotStartedJson_IsNotActivity() + { + // The registry JSON exists (7 subcategories) but nothing has started — this is the + // state at the intermediate Device-ESP exit and must NOT confirm it as final. + using var f = new Fixture(); + f.Tracker.ProcessCategoryStatusForTest("AccountSetupCategory.Status", AllNotStartedJson); + + Assert.False(f.Tracker.HasAccountSetupActivity); + } + + [Fact] + public void FirstSubcategoryLeavingNotStarted_IsActivity() + { + using var f = new Fixture(); + f.Tracker.ProcessCategoryStatusForTest("AccountSetupCategory.Status", AllNotStartedJson); + f.Tracker.ProcessCategoryStatusForTest("AccountSetupCategory.Status", FirstProgressJson); + + Assert.True(f.Tracker.HasAccountSetupActivity); + } + + [Theory] + [InlineData("inProgress")] + [InlineData("in_progress")] + [InlineData("failed")] + [InlineData("notRequired")] + public void AnyNonNotStartedState_IsActivity(string state) + { + using var f = new Fixture(); + f.Tracker.ProcessCategoryStatusForTest("AccountSetupCategory.Status", + "{\"categoryState\":\"inProgress\"," + + "\"AccountSetup.AppsSubcategory\":{\"subcategoryState\":\"" + state + "\"}," + + "\"AccountSetup.CertificatesSubcategory\":{\"subcategoryState\":\"notStarted\"}}"); + + Assert.True(f.Tracker.HasAccountSetupActivity); + } + + [Fact] + public void DeviceSetupProgress_DoesNotCountAsAccountSetupActivity() + { + // Adversarial: a fully progressed DeviceSetup category must not leak into the + // AccountSetup answer. + using var f = new Fixture(); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", + "{\"categoryState\":\"succeeded\",\"DeviceSetup.AppsSubcategory\":{\"subcategoryState\":\"succeeded\"}}"); + f.Tracker.ProcessCategoryStatusForTest("AccountSetupCategory.Status", AllNotStartedJson); + + Assert.False(f.Tracker.HasAccountSetupActivity); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs index f9f9a665..6a624e73 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs @@ -201,7 +201,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals public bool DeviceSetupCategorySucceeded => _provisioningTracker?.DeviceSetupCategorySucceeded ?? false; /// - /// True when any AccountSetup subcategory has been tracked (resolved or in progress). + /// True once an AccountSetup subcategory has left notStarted (observed progress, not + /// registry presence — see ). /// public bool HasAccountSetupActivity => _provisioningTracker?.HasAccountSetupActivity ?? false; @@ -558,9 +559,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals /// /// SkipUser is unknown or explicitly true (device-only / SkipUser flow — /// AwaitingHello is legitimately reachable directly after Device-ESP). - /// The provisioning tracker has observed AccountSetup activity (registry JSON - /// under AccountSetupCategory.Status has surfaced at least one subcategory), - /// i.e. the current exit is the final post-Account-ESP one. + /// The provisioning tracker has observed AccountSetup progress (at least one + /// subcategory under AccountSetupCategory.Status has left notStarted), + /// i.e. the current exit is the final post-Account-ESP one. Presence of the JSON + /// alone is not evidence — Windows writes it all-notStarted before the + /// Device-ESP exit (q8n). /// /// private bool IsIntermediateDeviceEspExit() @@ -593,8 +596,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals /// it back. So this one demands positive evidence and treats "unknown" as "not confirmed": /// /// - /// the provisioning tracker has seen AccountSetup activity — the page that just - /// tore down is demonstrably the AccountSetup one; or + /// the provisioning tracker has seen AccountSetup progress (a subcategory past + /// notStarted) — the page that just tore down is demonstrably the + /// AccountSetup one; or /// SkipUser is explicitly true — the profile has no user ESP at all, so the /// Device-ESP exit IS the final one and no second exit is coming. /// diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs index 857f8f5b..0ea68d15 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs @@ -208,7 +208,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals } /// - /// True when any AccountSetup subcategory has been tracked (resolved or in progress). + /// True when at least one AccountSetup subcategory has left notStarted — i.e. the + /// user ESP page has demonstrably started working (in progress, succeeded, failed or + /// notRequired). Mere presence of AccountSetupCategory.Status does NOT count: + /// Windows writes that JSON with every subcategory notStarted BEFORE the Device-ESP + /// page exits (observed on 25H2/26200 across tenants, session a2256107: JSON at 09:37:42, + /// intermediate Shell-Core 62407 at 09:37:53, first real progress at 09:38:25). A + /// presence check therefore mislabelled the Device→Account handoff exit as the final + /// exit, which released the User-ESP keep-awake minutes early (backlog q8n). /// public bool HasAccountSetupActivity { @@ -218,9 +225,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals { if (_lastSubcategoryStates == null) return false; - return _lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs) - && subs != null - && subs.Count > 0; + if (!_lastSubcategoryStates.TryGetValue("AccountSetupCategory.Status", out var subs) || subs == null) + return false; + foreach (var state in subs.Values) + { + if (!string.Equals(state, "notStarted", StringComparison.OrdinalIgnoreCase)) + return true; + } + return false; } } } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEventDataFilterTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEventDataFilterTests.cs new file mode 100644 index 00000000..6e35ee33 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEventDataFilterTests.cs @@ -0,0 +1,215 @@ +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Pins the same-event value filter (FilterField/FilterOperator/FilterValue) on the +/// event_data and event_data_array condition sources (backlog q8n). Every +/// condition scans all events of its type independently, so two conditions cannot express +/// "field A AND field B on the SAME event" — the filter can. Mirrors the event_count filter +/// semantics (): no filter = unchanged behaviour, +/// a missing filter field stringifies to empty. +/// +public class RuleEngineEventDataFilterTests +{ + private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + private const string SessionId = "c3d4e5f6-a7b8-9012-cdef-123456789012"; + + [Fact] + public async Task EventData_FilterAndMainTest_MustHoldOnTheSameEvent() + { + // Event 1 passes the main test but fails the filter; event 2 passes the filter but + // fails the main test. Independent evaluation would fire — the same-event filter must not. + var events = new List + { + Episode(sequence: 1, durationSeconds: 5000, backfilled: true), + Episode(sequence: 2, durationSeconds: 90, backfilled: false), + }; + + var outcome = await RunAsync(MakeEventDataRule(), events); + + Assert.Empty(outcome.Results); + } + + [Fact] + public async Task EventData_MatchesTheEventThatSatisfiesBoth_AndReportsItAsEvidence() + { + var events = new List + { + Episode(sequence: 1, durationSeconds: 5000, backfilled: true), + Episode(sequence: 2, durationSeconds: 900, backfilled: false), + }; + + var outcome = await RunAsync(MakeEventDataRule(), events); + + var result = Assert.Single(outcome.Results); + var evidence = Assert.IsAssignableFrom>(result.MatchedConditions["long_live_episode"]); + Assert.Equal(2, Convert.ToInt32(evidence["sequence"])); + Assert.Equal("900", evidence["value"]?.ToString()); + } + + [Fact] + public async Task EventData_NoFilterConfigured_KeepsIndependentBehaviour() + { + var rule = MakeEventDataRule(); + rule.Conditions[0].FilterField = null!; + rule.Conditions[0].FilterOperator = null!; + rule.Conditions[0].FilterValue = null!; + + var events = new List + { + Episode(sequence: 1, durationSeconds: 5000, backfilled: true), + }; + + var outcome = await RunAsync(rule, events); + + Assert.Single(outcome.Results); + } + + [Fact] + public async Task EventData_MissingFilterField_IsExcludedByEquals_AndKeptByNotEquals() + { + var evt = Episode(sequence: 1, durationSeconds: 5000, backfilled: false); + evt.Data!.Remove("backfilled"); + var events = new List { evt }; + + // equals "false" on a missing field → empty string ≠ "false" → excluded. + Assert.Empty((await RunAsync(MakeEventDataRule(), events)).Results); + + // not_equals "true" on a missing field → kept. + var lenient = MakeEventDataRule(); + lenient.Conditions[0].FilterOperator = "not_equals"; + lenient.Conditions[0].FilterValue = "true"; + Assert.Single((await RunAsync(lenient, events)).Results); + } + + [Fact] + public async Task EventDataArray_FilterSelectsWhichEventsArraysAreIterated() + { + // The array element would match in both events; only the event passing the filter counts. + var rule = new AnalyzeRule + { + RuleId = "ANALYZE-TST-011", + Title = "Array with same-event filter", + Severity = "warning", + Category = "security", + Enabled = true, + IsBuiltIn = false, + BaseConfidence = 80, + ConfidenceThreshold = 40, + Conditions = new List + { + new() + { + Signal = "unexpected_artifact", + Source = "event_data_array", + EventType = "provisioning_package_scan", + DataField = "artifacts", + ItemField = "identity", + Operator = "regex", + Value = "^Evil", + FilterField = "source", + FilterOperator = "equals", + FilterValue = "registry", + Required = true + } + }, + Explanation = "test" + }; + + var fileScan = ScanEvent(sequence: 1, source: "file", "Evil.ppkg"); + Assert.Empty((await RunAsync(rule, new List { fileScan })).Results); + + var registryScan = ScanEvent(sequence: 2, source: "registry", "Evil.ppkg"); + var outcome = await RunAsync(rule, new List { fileScan, registryScan }); + var result = Assert.Single(outcome.Results); + var evidence = Assert.IsAssignableFrom>(result.MatchedConditions["unexpected_artifact"]); + Assert.Equal(2, Convert.ToInt32(evidence["sequence"])); + } + + // ===== Builders ===== + + private static AnalyzeRule MakeEventDataRule() => new() + { + RuleId = "ANALYZE-TST-010", + Title = "Long live sleep episode", + Severity = "info", + Category = "device", + Enabled = true, + // Synthetic rule ID outside the embedded catalog — see RuleEngineEventCountFilterTests. + IsBuiltIn = false, + BaseConfidence = 90, + ConfidenceThreshold = 40, + Conditions = new List + { + new() + { + Signal = "long_live_episode", + Source = "event_data", + EventType = "system_sleep_episode", + DataField = "durationSeconds", + Operator = "gte", + Value = "300", + FilterField = "backfilled", + FilterOperator = "equals", + FilterValue = "false", + Required = true + } + }, + Explanation = "test" + }; + + private static EnrollmentEvent Episode(int sequence, long durationSeconds, bool backfilled) => new() + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "system_sleep_episode", + Timestamp = DateTime.UtcNow.AddMinutes(sequence), + Sequence = sequence, + Data = new Dictionary + { + ["kind"] = "modern_standby", + ["durationSeconds"] = durationSeconds, + ["backfilled"] = backfilled, + } + }; + + private static EnrollmentEvent ScanEvent(int sequence, string source, params string[] identities) => new() + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "provisioning_package_scan", + Timestamp = DateTime.UtcNow.AddMinutes(sequence), + Sequence = sequence, + Data = new Dictionary + { + ["source"] = source, + ["artifacts"] = identities + .Select(id => (object)new Dictionary { ["identity"] = id }) + .ToList(), + } + }; + + private static async Task RunAsync(AnalyzeRule rule, List events) + { + var ruleRepo = new Mock(); + ruleRepo.Setup(r => r.GetAnalyzeRulesAsync("global")).ReturnsAsync(new List { rule }); + ruleRepo.Setup(r => r.GetAnalyzeRulesAsync(TenantId)).ReturnsAsync(new List()); + ruleRepo.Setup(r => r.GetRuleStatesAsync(It.IsAny())).ReturnsAsync(new Dictionary()); + ruleRepo.Setup(r => r.GetRuleResultsAsync(It.IsAny(), It.IsAny())).ReturnsAsync(new List()); + + var sessionRepo = new Mock(); + sessionRepo.Setup(s => s.GetSessionEventsStrictAsync(TenantId, SessionId, It.IsAny())).ReturnsAsync(events); + + var ruleService = new AnalyzeRuleService(ruleRepo.Object, NullLogger.Instance); + var engine = new RuleEngine(ruleService, ruleRepo.Object, sessionRepo.Object, NullLogger.Instance); + + return await engine.AnalyzeSessionAsync(TenantId, SessionId); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SleepEpisodeRuleTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SleepEpisodeRuleTests.cs index ec9aaaee..98f3f51f 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/SleepEpisodeRuleTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SleepEpisodeRuleTests.cs @@ -90,9 +90,62 @@ public class SleepEpisodeRuleTests Assert.Empty(outcome.Results); } + [Fact] + public async Task ANALYZE_DEV_012_does_not_fire_on_a_backfilled_pre_session_episode() + { + // Backlog q8n: the watcher backfills pre-agent episodes for timeline context. A 91 h + // hibernate that ended before the enrollment started (session a2256107) satisfied the + // duration test and fired the rule although the device never slept while enrolling. + var rule = BuiltInAnalyzeRules.GetAll().First(r => r.RuleId == "ANALYZE-DEV-012"); + + var events = new List + { + SleepEpisode(durationSeconds: 329_140, kind: "hibernate", backfilled: true), + }; + + var outcome = await RunAsync(rule, events); + Assert.Empty(outcome.Results); + } + + [Fact] + public async Task ANALYZE_DEV_012_pins_duration_and_backfilled_to_the_same_episode() + { + // The adversarial mix from a2256107: a long backfilled episode plus a short live one. + // Two independent conditions (duration >= 300, backfilled = false) would each find a + // match on a different event and fire; the same-event filter must not. + var rule = BuiltInAnalyzeRules.GetAll().First(r => r.RuleId == "ANALYZE-DEV-012"); + + var events = new List + { + SleepEpisode(durationSeconds: 329_140, kind: "hibernate", backfilled: true), + SleepEpisode(durationSeconds: 89, kind: "modern_standby", backfilled: false), + }; + + var outcome = await RunAsync(rule, events); + Assert.Empty(outcome.Results); + } + + [Fact] + public async Task ANALYZE_DEV_012_fires_on_a_live_episode_next_to_a_backfilled_one() + { + var rule = BuiltInAnalyzeRules.GetAll().First(r => r.RuleId == "ANALYZE-DEV-012"); + + var events = new List + { + SleepEpisode(durationSeconds: 329_140, kind: "hibernate", backfilled: true), + SleepEpisode(durationSeconds: 600, kind: "modern_standby", backfilled: false), + }; + + var outcome = await RunAsync(rule, events); + + var result = Assert.Single(outcome.Results); + var matched = AsDict(result.MatchedConditions["slept_during_enrollment"]); + Assert.Equal("600", AsString(matched["value"])); // the live episode, not the backfilled one + } + // ===== Event builder — mirrors the SystemTimelineTracker emit shape ===== - private static EnrollmentEvent SleepEpisode(long durationSeconds, string kind) => new() + private static EnrollmentEvent SleepEpisode(long durationSeconds, string kind, bool backfilled = false) => new() { EventId = Guid.NewGuid().ToString(), TenantId = TenantId, @@ -106,7 +159,7 @@ public class SleepEpisodeRuleTests ["enteredAt"] = DateTime.UtcNow.AddSeconds(-durationSeconds).ToString("o"), ["exitedAt"] = DateTime.UtcNow.ToString("o"), ["durationSeconds"] = durationSeconds, - ["backfilled"] = false, + ["backfilled"] = backfilled, } }; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs index f373f7db..e6037ae6 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs @@ -163,9 +163,31 @@ namespace AutopilotMonitor.Functions.Services return (true, existsEvidence); } + /// + /// Optional same-event value filter (FilterField/FilterOperator/FilterValue): keeps only + /// events whose filter field satisfies the operator. Shared by event_count (which + /// events are counted) and event_data/event_data_array (which event + /// instances the main dataField/operator may match). The filter and the main test are + /// evaluated on the SAME event, which independent conditions cannot express — each + /// condition scans all events on its own, so "durationSeconds ≥ 300" and + /// "backfilled = false" as two conditions would match two different episodes + /// (ANALYZE-DEV-012 firing on a pre-session backfilled hibernate, backlog q8n). + /// A missing filter field stringifies to empty, so equals excludes the event + /// and not_equals keeps it. No filter configured = unchanged behaviour. + /// + private List ApplyValueFilter(RuleCondition condition, List matchingEvents) + { + if (string.IsNullOrEmpty(condition.FilterField) || string.IsNullOrEmpty(condition.FilterOperator)) + return matchingEvents; + + return matchingEvents + .Where(e => MatchesOperator(GetDataFieldValue(e, condition.FilterField) ?? string.Empty, condition.FilterOperator, condition.FilterValue)) + .ToList(); + } + private (bool matched, object evidence) EvaluateEventDataCondition(RuleCondition condition, List events) { - var matchingEvents = events.Where(e => MatchesEventType(e, condition.EventType)).ToList(); + var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList()); foreach (var evt in matchingEvents) { @@ -214,7 +236,7 @@ namespace AutopilotMonitor.Functions.Services /// private (bool matched, object evidence) EvaluateEventDataArrayCondition(RuleCondition condition, List events) { - var matchingEvents = events.Where(e => MatchesEventType(e, condition.EventType)).ToList(); + var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList()); foreach (var evt in matchingEvents) { @@ -362,16 +384,9 @@ namespace AutopilotMonitor.Functions.Services private (bool matched, object evidence) EvaluateEventCountCondition(RuleCondition condition, List events) { - var matchingEvents = events.Where(e => MatchesEventType(e, condition.EventType)).ToList(); - // Optional value filter: count only events whose FilterField satisfies FilterOperator/FilterValue // (e.g. only performance_snapshot events with memory_used_percent > 90). - if (!string.IsNullOrEmpty(condition.FilterField) && !string.IsNullOrEmpty(condition.FilterOperator)) - { - matchingEvents = matchingEvents - .Where(e => MatchesOperator(GetDataFieldValue(e, condition.FilterField) ?? string.Empty, condition.FilterOperator, condition.FilterValue)) - .ToList(); - } + var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList()); // count_per_group_gte: group by DataField value (e.g. appId), fire if any group >= threshold if (condition.Operator == "count_per_group_gte" && !string.IsNullOrEmpty(condition.DataField) && int.TryParse(condition.Value, out var groupThreshold)) diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index fe4f3b00..0ee2bb37 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -2842,11 +2842,11 @@ export interface RuleCondition { value: string; /** Whether this condition must match for the rule to fire If false, it only contributes to confidence scoring */ required: boolean; - /** Optional value filter for "event_count": only events whose FilterField satisfies FilterOperator/FilterValue are counted (e.g. count only performance_snapshot events with memory_used_percent > 90). Applies before counting for both count_gte and count_per_group_gte. */ + /** Optional same-event value filter: only events whose FilterField satisfies FilterOperator/FilterValue are considered by the condition. For "event_count": applies before counting (both count_gte and count_per_group_gte), e.g. count only performance_snapshot events with memory_used_percent > 90. For "event_data"/"event_data_array": the filter and the main DataField/Operator test are evaluated on the SAME event instance — the only way to pin two fields of one event together, since separate conditions each scan all events independently (e.g. system_sleep_episode with durationSeconds >= 300 AND backfilled = false). */ filterField: string; - /** Operator for the count filter. Uses same operators as the main Operator field. */ + /** Operator for the same-event filter. Uses the same operators as the main Operator field. */ filterOperator: string; - /** Value for the count filter. */ + /** Value for the same-event filter. */ filterValue: string; /** The second event type to correlate with (Event B). Example: "app_install_failed" */ correlateEventType: string; diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts index 8b52bd4d..e4874bea 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts @@ -145,7 +145,10 @@ export const RULE_AUTHORING_GUIDE = { 'on a field inside those events. Most common source.', event_data: 'Compare a data field inside events of eventType (dataField + operator + value). Supports ' + - 'dot-paths into nested objects; "message" resolves to the event message.', + 'dot-paths into nested objects; "message" resolves to the event message. Optional ' + + 'filterField/filterOperator/filterValue pin a second field of the SAME event (e.g. ' + + 'durationSeconds gte 300 with filter backfilled equals false) — two separate conditions ' + + 'would match two different events.', event_data_array: 'Iterate an ARRAY field (dataField) inside ONE event and test a sub-field of each element ' + '(itemField; empty = element itself). Matches if ANY element passes. Allow-list pattern: ' + diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts index a76f84b5..09b555c2 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts @@ -496,11 +496,11 @@ export const ANALYZE_RULE_SCHEMA: Record = { }, "filterField": { "type": "string", - "description": "For event_count only: optional value filter — only events whose filterField satisfies filterOperator/filterValue are counted (e.g. count only performance_snapshot events with memory_used_percent > 90). Applies before counting for both count_gte and count_per_group_gte." + "description": "Optional same-event value filter for event_count, event_data and event_data_array: only events whose filterField satisfies filterOperator/filterValue are considered. For event_count it applies before counting (both count_gte and count_per_group_gte, e.g. count only performance_snapshot events with memory_used_percent > 90). For event_data/event_data_array the filter and the main dataField/operator test are evaluated on the SAME event instance — the only way to pin two fields of one event together, since separate conditions each scan all events independently (e.g. system_sleep_episode with durationSeconds gte 300 AND backfilled equals false)." }, "filterOperator": { "type": "string", - "description": "Operator for the event_count value filter.", + "description": "Operator for the same-event value filter. Same vocabulary as operator.", "enum": [ "equals", "not_equals", @@ -520,7 +520,7 @@ export const ANALYZE_RULE_SCHEMA: Record = { }, "filterValue": { "type": "string", - "description": "Value for the event_count value filter." + "description": "Value for the same-event value filter. Boolean values are stringified (\"true\"/\"false\")." }, "correlateEventType": { "type": "string", diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Rules/AnalyzeRule.cs b/src/Shared/AutopilotMonitor.Shared/Models/Rules/AnalyzeRule.cs index 48cfa5f2..0dc25c2b 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Rules/AnalyzeRule.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Rules/AnalyzeRule.cs @@ -279,24 +279,28 @@ namespace AutopilotMonitor.Shared.Models /// public bool Required { get; set; } = false; - // ===== Event Count Filter Properties ===== - // Used only when Source = "event_count" + // ===== Same-Event Value Filter Properties ===== + // Used when Source = "event_count", "event_data" or "event_data_array" /// - /// Optional value filter for "event_count": only events whose FilterField - /// satisfies FilterOperator/FilterValue are counted (e.g. count only - /// performance_snapshot events with memory_used_percent > 90). - /// Applies before counting for both count_gte and count_per_group_gte. + /// Optional same-event value filter: only events whose FilterField satisfies + /// FilterOperator/FilterValue are considered by the condition. + /// For "event_count": applies before counting (both count_gte and count_per_group_gte), + /// e.g. count only performance_snapshot events with memory_used_percent > 90. + /// For "event_data"/"event_data_array": the filter and the main DataField/Operator test + /// are evaluated on the SAME event instance — the only way to pin two fields of one + /// event together, since separate conditions each scan all events independently + /// (e.g. system_sleep_episode with durationSeconds >= 300 AND backfilled = false). /// public string FilterField { get; set; } = default!; /// - /// Operator for the count filter. Uses same operators as the main Operator field. + /// Operator for the same-event filter. Uses the same operators as the main Operator field. /// public string FilterOperator { get; set; } = default!; /// - /// Value for the count filter. + /// Value for the same-event filter. /// public string FilterValue { get; set; } = default!; diff --git a/src/Web/autopilot-monitor-web/utils/shared-manifests.json b/src/Web/autopilot-monitor-web/utils/shared-manifests.json index b5e2e93a..00fa10bc 100644 --- a/src/Web/autopilot-monitor-web/utils/shared-manifests.json +++ b/src/Web/autopilot-monitor-web/utils/shared-manifests.json @@ -13257,7 +13257,7 @@ "kind": "primitive", "name": "string" }, - "doc": "Optional value filter for \"event_count\": only events whose FilterField satisfies FilterOperator/FilterValue are counted (e.g. count only performance_snapshot events with memory_used_percent > 90). Applies before counting for both count_gte and count_per_group_gte." + "doc": "Optional same-event value filter: only events whose FilterField satisfies FilterOperator/FilterValue are considered by the condition. For \"event_count\": applies before counting (both count_gte and count_per_group_gte), e.g. count only performance_snapshot events with memory_used_percent > 90. For \"event_data\"/\"event_data_array\": the filter and the main DataField/Operator test are evaluated on the SAME event instance — the only way to pin two fields of one event together, since separate conditions each scan all events independently (e.g. system_sleep_episode with durationSeconds >= 300 AND backfilled = false)." }, { "name": "filterOperator", @@ -13265,7 +13265,7 @@ "kind": "primitive", "name": "string" }, - "doc": "Operator for the count filter. Uses same operators as the main Operator field." + "doc": "Operator for the same-event filter. Uses the same operators as the main Operator field." }, { "name": "filterValue", @@ -13273,7 +13273,7 @@ "kind": "primitive", "name": "string" }, - "doc": "Value for the count filter." + "doc": "Value for the same-event filter." }, { "name": "correlateEventType", diff --git a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts index 6c4c97b6..3fb3794b 100644 --- a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts @@ -2842,11 +2842,11 @@ export interface RuleCondition { value: string; /** Whether this condition must match for the rule to fire If false, it only contributes to confidence scoring */ required: boolean; - /** Optional value filter for "event_count": only events whose FilterField satisfies FilterOperator/FilterValue are counted (e.g. count only performance_snapshot events with memory_used_percent > 90). Applies before counting for both count_gte and count_per_group_gte. */ + /** Optional same-event value filter: only events whose FilterField satisfies FilterOperator/FilterValue are considered by the condition. For "event_count": applies before counting (both count_gte and count_per_group_gte), e.g. count only performance_snapshot events with memory_used_percent > 90. For "event_data"/"event_data_array": the filter and the main DataField/Operator test are evaluated on the SAME event instance — the only way to pin two fields of one event together, since separate conditions each scan all events independently (e.g. system_sleep_episode with durationSeconds >= 300 AND backfilled = false). */ filterField: string; - /** Operator for the count filter. Uses same operators as the main Operator field. */ + /** Operator for the same-event filter. Uses the same operators as the main Operator field. */ filterOperator: string; - /** Value for the count filter. */ + /** Value for the same-event filter. */ filterValue: string; /** The second event type to correlate with (Event B). Example: "app_install_failed" */ correlateEventType: string;