diff --git a/rules/schema/analyze-rule.schema.json b/rules/schema/analyze-rule.schema.json index 62130c59..404c71de 100644 --- a/rules/schema/analyze-rule.schema.json +++ b/rules/schema/analyze-rule.schema.json @@ -89,12 +89,14 @@ "type": "array", "description": "Conditions evaluated against the event stream. All required conditions must match for the rule to fire.", "items": { "$ref": "#/$defs/ruleCondition" }, - "minItems": 1 + "minItems": 1, + "maxItems": 20 }, "preconditions": { "type": "array", "description": "Optional device-fact gates evaluated BEFORE conditions. ALL preconditions must pass; if any fails the rule is silently skipped — no result, no UI card. Useful for filtering out hardware/OS profiles where the rule does not apply (e.g. virtual machines).", "items": { "$ref": "#/$defs/rulePrecondition" }, + "maxItems": 10, "default": [] }, "baseConfidence": { diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AnalyzeRuleShapeLimitsTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AnalyzeRuleShapeLimitsTests.cs new file mode 100644 index 00000000..d85a3567 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/AnalyzeRuleShapeLimitsTests.cs @@ -0,0 +1,184 @@ +using AutopilotMonitor.Functions.Functions.Rules; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// The size limits of a tenant-authored analyze rule (D-410): at most 20 conditions and 10 +/// preconditions, and every value a regex or not_regex operator compares — condition value, +/// filter value, event A filter value, precondition value — at most 2,000 characters and +/// parseable. One implementation feeds the dry-run validation and every write path (create, +/// update, template copy); the boundary itself is always allowed. +/// +public class AnalyzeRuleShapeLimitsTests +{ + private const string TenantId = "tenant-limits"; + + [Fact] + public void A_rule_at_every_limit_passes() + { + var rule = Rule(AnalyzeRuleLimits.MaxConditions, AnalyzeRuleLimits.MaxPreconditions); + var atLimit = new string('a', AnalyzeRuleLimits.MaxRegexLength); + rule.Conditions[0].Operator = "regex"; + rule.Conditions[0].Value = atLimit; + rule.Conditions[1].FilterField = "f"; + rule.Conditions[1].FilterOperator = "not_regex"; + rule.Conditions[1].FilterValue = atLimit; + rule.Conditions[2].EventAFilterOperator = "REGEX"; + rule.Conditions[2].EventAFilterValue = atLimit; + rule.Preconditions[0].Operator = "regex"; + rule.Preconditions[0].Value = atLimit; + + Assert.Empty(AnalyzeRuleService.ShapeLimitErrors(rule)); + } + + [Fact] + public void One_condition_or_precondition_over_the_limit_is_named() + { + var errors = AnalyzeRuleService.ShapeLimitErrors(Rule(AnalyzeRuleLimits.MaxConditions + 1, AnalyzeRuleLimits.MaxPreconditions + 1)); + + Assert.Contains(errors, e => e.Contains("at most 20 conditions (this one has 21)")); + Assert.Contains(errors, e => e.Contains("at most 10 preconditions (this one has 11)")); + } + + [Theory] + [InlineData("conditions[0].value")] + [InlineData("conditions[0].filterValue")] + [InlineData("conditions[0].eventAFilterValue")] + [InlineData("preconditions[0].value")] + public void A_regex_over_the_length_limit_is_rejected_wherever_it_sits(string field) + { + var rule = Rule(1, 1); + Place(rule, field, "regex", new string('a', AnalyzeRuleLimits.MaxRegexLength + 1)); + + var error = Assert.Single(AnalyzeRuleService.ShapeLimitErrors(rule)); + Assert.StartsWith($"{field}: a regex may be at most 2000 characters (this one has 2001)", error); + } + + [Theory] + [InlineData("conditions[0].filterValue")] + [InlineData("conditions[0].eventAFilterValue")] + [InlineData("preconditions[0].value")] + public void An_unparseable_regex_is_rejected_beyond_the_condition_value(string field) + { + var rule = Rule(1, 1); + Place(rule, field, "not_regex", "([unclosed"); + + var error = Assert.Single(AnalyzeRuleService.ShapeLimitErrors(rule)); + Assert.StartsWith($"{field}: value is not a valid regex", error); + } + + [Fact] + public void A_long_value_of_any_other_operator_is_not_limited() + { + var rule = Rule(1, 0); + rule.Conditions[0].Operator = "not_in"; + rule.Conditions[0].Value = string.Join(",", Enumerable.Range(0, 1000).Select(i => $"app-{i:D5}")); + + Assert.Empty(AnalyzeRuleService.ShapeLimitErrors(rule)); + } + + [Fact] + public void The_dry_run_validation_carries_the_limits() + { + var rule = Rule(AnalyzeRuleLimits.MaxConditions + 1, 0); + + Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("at most 20 conditions")); + } + + // ===== write paths ===== + + [Fact] + public async Task Create_rejects_an_oversized_rule_before_anything_is_stored() + { + var repo = new Mock(); + var service = new AnalyzeRuleService(repo.Object, NullLogger.Instance); + + await Assert.ThrowsAsync(() => service.CreateRuleAsync(TenantId, Rule(AnalyzeRuleLimits.MaxConditions + 1, 0))); + repo.Verify(r => r.StoreAnalyzeRuleAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task Update_of_a_custom_rule_rejects_an_oversized_regex_before_anything_is_stored() + { + var repo = new Mock(); + var service = new AnalyzeRuleService(repo.Object, NullLogger.Instance); + var rule = Rule(1, 0); + Place(rule, "conditions[0].value", "not_regex", new string('a', AnalyzeRuleLimits.MaxRegexLength + 1)); + + await Assert.ThrowsAsync(() => service.UpdateRuleAsync(TenantId, rule)); + repo.Verify(r => r.StoreAnalyzeRuleAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task Template_copy_rejects_an_oversized_regex_from_its_variable() + { + // ANALYZE-SEC-006 shape: the template variable IS the tenant's not_regex allow-list. + var template = Rule(1, 0); + template.RuleId = "ANALYZE-SEC-006"; + template.IsBuiltIn = true; + template.Conditions[0].Operator = "not_regex"; + template.TemplateVariables = new List { new() { Name = "allowList", ConditionIndex = 0, Field = "value" } }; + + var repo = new Mock(); + repo.Setup(r => r.GetAnalyzeRulesAsync("global")).ReturnsAsync(new List { template }); + repo.Setup(r => r.GetAnalyzeRulesAsync(TenantId)).ReturnsAsync(new List()); + repo.Setup(r => r.GetRuleStatesAsync(TenantId)).ReturnsAsync(new Dictionary()); + var service = new AnalyzeRuleService(repo.Object, NullLogger.Instance); + + await Assert.ThrowsAsync(() => service.CreateFromTemplateAsync(TenantId, "ANALYZE-SEC-006", + new Dictionary { ["allowList"] = new string('a', AnalyzeRuleLimits.MaxRegexLength + 1) })); + repo.Verify(r => r.StoreAnalyzeRuleAsync(It.IsAny(), TenantId), Times.Never); + } + + // ===== fixtures ===== + + private static AnalyzeRule Rule(int conditions, int preconditions) => new() + { + RuleId = "CUSTOM-LIMITS", + IsBuiltIn = false, // the model defaults to a built-in rule + Title = "limits", + Severity = "warning", + Category = "test", + BaseConfidence = 80, + ConfidenceThreshold = 40, + Conditions = Enumerable.Range(0, conditions).Select(i => new RuleCondition + { + Signal = $"s{i}", Source = "event_data", EventType = "app_install_failed", DataField = "errorCode", + Operator = "equals", Value = "1", Required = true, + }).ToList(), + Preconditions = Enumerable.Range(0, preconditions).Select(_ => new RulePrecondition + { + Source = "event_data", EventType = "os_info", DataField = "edition", Operator = "equals", Value = "Enterprise", + }).ToList(), + }; + + private static void Place(AnalyzeRule rule, string field, string op, string value) + { + switch (field) + { + case "conditions[0].value": + rule.Conditions[0].Operator = op; + rule.Conditions[0].Value = value; + break; + case "conditions[0].filterValue": + rule.Conditions[0].FilterField = "f"; + rule.Conditions[0].FilterOperator = op; + rule.Conditions[0].FilterValue = value; + break; + case "conditions[0].eventAFilterValue": + rule.Conditions[0].EventAFilterField = "f"; + rule.Conditions[0].EventAFilterOperator = op; + rule.Conditions[0].EventAFilterValue = value; + break; + default: + rule.Preconditions[0].Operator = op; + rule.Preconditions[0].Value = value; + break; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/DerivedWriterLivenessTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/DerivedWriterLivenessTests.cs index 1360ef53..df790752 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/DerivedWriterLivenessTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/DerivedWriterLivenessTests.cs @@ -95,6 +95,7 @@ public class DerivedWriterLivenessTests configService: null!, channelDispatcher: null!, GuardFor(row), + stopReporter: null!, NullLogger.Instance); await handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope @@ -116,6 +117,7 @@ public class DerivedWriterLivenessTests ruleService: null!, Mock.Of(), Mock.Of(), Mock.Of(), signalRNotification: null!, configService: null!, channelDispatcher: null!, new SessionDeletionGuard(reader.Object, NullLogger.Instance), + stopReporter: null!, NullLogger.Instance); await Assert.ThrowsAsync(() => handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/HardwareWhitelistValidatorTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/HardwareWhitelistValidatorTests.cs new file mode 100644 index 00000000..687ee54f --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/HardwareWhitelistValidatorTests.cs @@ -0,0 +1,105 @@ +using System.Diagnostics; +using System.Text.RegularExpressions; +using AutopilotMonitor.Functions.Security; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// The hardware whitelist matches tenant wildcards against device headers on every agent request +/// without building a Regex (D-411): same results as the former regex construction for every shape +/// an entry can take, linear work for the worst entry a tenant can write, and a device value longer +/// than any real hardware string only meets "*". +/// +public class HardwareWhitelistValidatorTests +{ + /// The construction this replaced, kept as the oracle: '*' → '.*', '?' → '.', the rest escaped. + private static bool FormerRegexMatch(string text, string pattern) + { + if (pattern == "*") + return true; + var regex = "^" + Regex.Escape(pattern).Replace("\\*", ".*").Replace("\\?", ".") + "$"; + return Regex.IsMatch(text, regex, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(5)); + } + + [Theory] + [InlineData("Dell Inc.", "Dell*")] + [InlineData("dell inc.", "DELL*")] + [InlineData("Hewlett-Packard", "HP*")] + [InlineData("HP", "HP*")] + [InlineData("LENOVO", "Lenovo*")] + [InlineData("Microsoft Corporation", "Microsoft Corporation")] + [InlineData("Microsoft Corporation ", "Microsoft Corporation")] + [InlineData("Microsoft Corp", "Microsoft Corporation")] + [InlineData("ThinkPad T14 Gen 6", "*T14*")] + [InlineData("ThinkPad T14 Gen 6", "*Gen ?")] + [InlineData("ThinkPad T14 Gen 16", "*Gen ?")] + [InlineData("ThinkPad T14 Gen 6", "ThinkPad*Gen*")] + [InlineData("VMware20,1", "VMware??,?")] + [InlineData("VMware20,1", "VMware?,?")] + [InlineData("a.b", "a.b")] + [InlineData("axb", "a.b")] + [InlineData("f(x)", "f(x)")] + [InlineData("f(x)", "f(*)")] + [InlineData("a\\b", "a\\b")] + [InlineData("a\\xyzb", "a\\*b")] + [InlineData("a*b", "a\\*b")] + [InlineData("[abc]", "[abc]")] + [InlineData("a", "[abc]")] + [InlineData("x+y", "x+y")] + [InlineData("xxy", "x+y")] + [InlineData("$1.00", "$1.00")] + [InlineData("^start", "^start")] + [InlineData("abc", "**")] + [InlineData("abc", "*?*")] + [InlineData("ab", "???")] + [InlineData("abc", "???")] + [InlineData("abcc", "*c")] + [InlineData("ÄÖÜ GmbH", "äöü*")] + [InlineData("straße", "STRASSE")] + public void Matches_like_the_former_regex_construction(string value, string pattern) + { + Assert.Equal(FormerRegexMatch(value, pattern), HardwareWhitelistValidator.MatchesWildcard(value, pattern)); + } + + [Fact] + public void Case_folding_is_ordinal_the_one_deliberate_difference() + { + // The regex engine folded Turkish dotted 'İ' to 'i'; ordinal folding does not. SMBIOS strings are ASCII. + Assert.True(FormerRegexMatch("İstanbul", "istanbul*")); + Assert.False(HardwareWhitelistValidator.MatchesWildcard("İstanbul", "istanbul*")); + } + + [Fact] + public void The_worst_whitelist_a_tenant_can_write_costs_milliseconds_per_request() + { + // 200 entries of 128 characters, each a chain of wildcards that never matches, against the + // longest value still compared (the cap) and a 32 KB header (past the cap). + var entry = string.Concat(Enumerable.Repeat("*a", 63)) + "*b"; + var whitelist = Enumerable.Repeat(entry, 200).ToArray(); + var atCap = new string('a', HardwareWhitelistValidator.MaxMatchedValueLength); + var huge = new string('a', 32 * 1024); + + var clock = Stopwatch.StartNew(); + var atCapResult = HardwareWhitelistValidator.ValidateHardware(atCap, "Model", whitelist, new[] { "*" }); + var hugeResult = HardwareWhitelistValidator.ValidateHardware(huge, "Model", whitelist, new[] { "*" }); + clock.Stop(); + + Assert.False(atCapResult.IsValid); + Assert.False(hugeResult.IsValid); + // The former construction spent up to 200 × 100 ms per list on this. + Assert.True(clock.ElapsedMilliseconds < 500, $"took {clock.ElapsedMilliseconds} ms"); + } + + [Fact] + public void A_value_longer_than_any_hardware_string_only_meets_the_star() + { + var longValue = "Dell" + new string('x', HardwareWhitelistValidator.MaxMatchedValueLength); + + Assert.False(HardwareWhitelistValidator.MatchesWildcard(longValue, "Dell*")); + Assert.True(HardwareWhitelistValidator.MatchesWildcard(longValue, "*")); + + var rejected = HardwareWhitelistValidator.ValidateHardware(longValue, "Model", new[] { "Dell*" }, new[] { "*" }); + Assert.False(rejected.IsValid); + Assert.True(rejected.ErrorMessage!.Length < HardwareWhitelistValidator.MaxMatchedValueLength + 64, "the rejection message carries a shortened value"); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ReportUploadGateTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ReportUploadGateTests.cs index 99cdfc11..daa169d3 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/ReportUploadGateTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ReportUploadGateTests.cs @@ -56,10 +56,11 @@ public class ReportUploadGateTests var gate = CreateGate(limit: 1); var start = new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc); - Assert.True(gate.TakeAlert(start)); - Assert.False(gate.TakeAlert(start.AddMinutes(59))); - Assert.True(gate.TakeAlert(start + ReportUploadGate.AlertWindow)); - Assert.False(gate.TakeAlert(start + ReportUploadGate.AlertWindow)); + var window = ReportUploadGate.AlertWindow; + Assert.True(gate.TakeAlert(start, window)); + Assert.False(gate.TakeAlert(start.AddMinutes(59), window)); + Assert.True(gate.TakeAlert(start + window, window)); + Assert.False(gate.TakeAlert(start + window, window)); } [Fact] diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEvaluationBudgetTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEvaluationBudgetTests.cs new file mode 100644 index 00000000..de4f2741 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEvaluationBudgetTests.cs @@ -0,0 +1,316 @@ +using System.Diagnostics; +using System.Net; +using System.Text.Json; +using AutopilotMonitor.Functions.Functions.Rules; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// The evaluation budget of tenant-authored rules (D-410): the first regex match timeout stops the +/// rule instead of costing one second per event, a spent run budget leaves the later rules +/// unevaluated, every scan without an operator call is checked too, a rule that did not finish keeps +/// its stored row, and the dry-run reports the stop as verdict "stopped". +/// +public class RuleEvaluationBudgetTests +{ + private const string TenantId = "tenant-budget"; + private const string SessionId = "session-budget"; + + /// Nested quantifier over input it cannot match: catastrophic backtracking, ends at the 1 s match timeout. + private const string CatastrophicPattern = @"^(\w+\s?)+$"; + private static readonly string NonMatchingInput = new string('a', 60) + "!"; + + // ===== analysis run ===== + + [Fact] + public async Task Regex_timeout_stops_the_rule_after_one_match_and_the_next_rule_still_runs() + { + var events = Enumerable.Range(1, 200).Select(i => Event("app_install_failed", i, ("errorDetail", NonMatchingInput))).ToList(); + var redos = CustomRule("TEST-REDOS", new RuleCondition + { + Signal = "slow", Source = "event_data", EventType = "app_install_failed", + DataField = "errorDetail", Operator = "regex", Value = CatastrophicPattern, Required = true, + }); + var next = CustomRule("TEST-NEXT", ExistsCondition("app_install_failed")); + + var clock = Stopwatch.StartNew(); + var outcome = await Harness(events, redos, next).Engine().AnalyzeSessionAsync(TenantId, SessionId); + clock.Stop(); + + // One timed-out match, not 200 of them. + Assert.True(clock.ElapsedMilliseconds < 4000, $"took {clock.ElapsedMilliseconds} ms"); + var stop = Assert.Single(outcome.Stops); + Assert.Equal(new RuleEvaluationStop("TEST-REDOS", EvaluationStopReasons.RegexTimeout, IsCatalogRule: false), stop); + Assert.Empty(outcome.SkippedRuleIds); + Assert.Equal(new[] { "TEST-NEXT" }, outcome.Results.Select(r => r.RuleId)); + } + + [Fact] + public async Task Spent_budget_stops_the_run_keeps_every_unfinished_row_and_resolves_nothing() + { + var events = new List { Event("app_install_failed", 1, ("appId", "app-1")) }; + var first = CustomRule("TEST-A", new RuleCondition + { + Signal = "a", Source = "event_data", EventType = "app_install_failed", + DataField = "appId", Operator = "equals", Value = "app-1", Required = true, + }); + var second = CustomRule("TEST-B", ExistsCondition("app_install_failed")); + var third = CustomRule("TEST-C", ExistsCondition("app_install_failed")); + var interimA = StoredRow("TEST-A", isInterim: true); + var interimB = StoredRow("TEST-B", isInterim: true); + + var harness = Harness(events, first, second, third); + harness.ExistingResults.AddRange(new[] { interimA, interimB }); + var engine = harness.Engine(TimeSpan.Zero); + + var outcome = await engine.AnalyzeSessionAsync(TenantId, SessionId, AnalyzeRunContext.Terminal()); + + var stop = Assert.Single(outcome.Stops); + Assert.Equal("TEST-A", stop.RuleId); + Assert.Equal(EvaluationStopReasons.TimeBudget, stop.Reason); + Assert.Equal(new[] { "TEST-B", "TEST-C" }, outcome.SkippedRuleIds); + Assert.Empty(outcome.Results); + Assert.Empty(outcome.ResolvedResults); // a stop is no evidence that a finding went away + Assert.Equal(new[] { interimA, interimB }, outcome.KeptResults); + Assert.Null(interimA.ResolvedAt); + Assert.True(interimA.IsInterim); + } + + /// + /// Each of these evaluators scans the session per matching event without calling an operator + /// (quadratic in the session size); without its own check a spent budget would not stop it, and + /// the rule would simply be evaluated. + /// + [Theory] + [InlineData("event_correlation pairing")] + [InlineData("suppressByEvent scan")] + [InlineData("app_install_duration start search")] + public async Task Scans_without_an_operator_call_check_the_budget_too(string scan) + { + var (condition, events) = scan switch + { + "event_correlation pairing" => ( + new RuleCondition + { + Signal = "pair", Source = "event_correlation", EventType = "app_install_started", + CorrelateEventType = "app_install_failed", JoinField = "appId", Required = true, + }, + new[] { Event("app_install_started", 1, ("appId", "x")), Event("app_install_failed", 2, ("appId", "x")) }), + "suppressByEvent scan" => ( + new RuleCondition + { + Signal = "failed", Source = "event_type", EventType = "app_install_failed", Operator = "exists", Required = true, + SuppressByEvent = new SuppressByEventConfig { EventType = "app_install_completed", JoinField = "appId" }, + }, + new[] { Event("app_install_failed", 1, ("appId", "x")) }), + _ => ( + new RuleCondition { Signal = "slow", Source = "app_install_duration", Operator = "gt", Value = "0", Required = true }, + new[] { Event("app_install_completed", 1, ("appId", "x")) }), + }; + + var outcome = await Harness(events.ToList(), CustomRule("TEST-SCAN", condition)).Engine(TimeSpan.Zero) + .AnalyzeSessionAsync(TenantId, SessionId); + + var stop = Assert.Single(outcome.Stops); + Assert.Equal(EvaluationStopReasons.TimeBudget, stop.Reason); + Assert.Empty(outcome.Results); + } + + // ===== dry-run ===== + + [Fact] + public void Dry_run_regex_timeout_ends_with_verdict_stopped_and_a_note_naming_the_condition() + { + var rule = CustomRule("TEST-DRY", + ExistsCondition("app_install_failed"), + new RuleCondition + { + Signal = "slow", Source = "event_data", EventType = "app_install_failed", + DataField = "errorDetail", Operator = "regex", Value = CatastrophicPattern, Required = true, + }); + var events = Enumerable.Range(1, 50).Select(i => Event("app_install_failed", i, ("errorDetail", NonMatchingInput))).ToList(); + + var dry = Harness(events).Engine().DryRunEvaluate(rule, events, new RuleEvaluationBudget(RuleEvaluationBudget.DryRun)); + + Assert.Equal(RuleDryRunVerdict.Stopped, dry.Verdict); + Assert.Equal(EvaluationStopReasons.RegexTimeout, dry.StopReason); + Assert.StartsWith("conditions[1] ('slow'): a regex hit the 1 s match timeout", dry.StopNote); + // Only what finished before the stop; no half-evaluated evidence map. + Assert.Equal(new[] { "exists_app_install_failed" }, dry.Conditions.Select(c => c.Signal)); + Assert.Null(dry.MatchedConditions); + Assert.Null(dry.FinalConfidence); + } + + [Fact] + public void Dry_run_spent_budget_stops_at_the_first_operator() + { + var rule = CustomRule("TEST-DRY", ExistsCondition("app_install_failed")); + rule.Preconditions = new List + { + new() { Source = "event_data", EventType = "os_info", DataField = "edition", Operator = "equals", Value = "Enterprise" }, + }; + var events = new List { Event("os_info", 1, ("edition", "Enterprise")) }; + + var dry = Harness(events).Engine().DryRunEvaluate(rule, events, new RuleEvaluationBudget(TimeSpan.Zero)); + + Assert.Equal(RuleDryRunVerdict.Stopped, dry.Verdict); + Assert.Equal(EvaluationStopReasons.TimeBudget, dry.StopReason); + Assert.StartsWith("preconditions[0] (os_info): the dry-run used up its 0 s time budget on this session's 1 events", dry.StopNote); + Assert.Empty(dry.Preconditions); + Assert.Empty(dry.Conditions); + } + + [Fact] + public async Task Dry_run_of_a_caller_who_went_away_stops_as_cancelled() + { + var events = new List { Event("app_install_failed", 1, ("appId", "x")) }; + var rule = CustomRule("TEST-DRY", new RuleCondition + { + Signal = "a", Source = "event_data", EventType = "app_install_failed", DataField = "appId", Operator = "equals", Value = "x", Required = true, + }); + using var gone = new CancellationTokenSource(); + gone.Cancel(); + + var dry = await Harness(events).Engine().DryRunRuleAsync(TenantId, SessionId, rule, gone.Token); + + Assert.Equal(RuleDryRunVerdict.Stopped, dry.Verdict); + Assert.Equal(EvaluationStopReasons.Cancelled, dry.StopReason); + } + + // ===== "Analyze now" ===== + + [Fact] + public async Task Reanalyze_with_a_full_gate_answers_429_and_raises_one_ops_event() + { + var opsEvents = new List(); + var harness = Harness(new List()); + var gate = new ReanalyzeGate(1, TestNotifications.CapturingOpsEvents(opsEvents), NullLogger.Instance); + using var busy = gate.TryEnter(); + var function = harness.Function(gate, opsEvents); + + var (req, _) = EndpointHarness.Request(TenantId, queryString: "?reanalyze=true"); + var response = await function.Run(req, SessionId); + + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + Assert.Equal(Constants.ApiErrorCodes.RateLimited, EndpointHarness.ErrorCode(response)); + Assert.True(response.Headers.TryGetValues("Retry-After", out var retryAfter)); + Assert.Equal("5", retryAfter!.Single()); + var alert = Assert.Single(opsEvents); + Assert.Equal(OpsEventTypes.RuleEvaluationLimitReached, alert.EventType); + Assert.Contains("re-analysis", alert.Details); + harness.Maintenance.Verify(m => m.DeleteSessionRuleResultsAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task Reanalyze_writes_back_the_row_of_a_stopped_rule_and_names_it() + { + var events = Enumerable.Range(1, 20).Select(i => Event("app_install_failed", i, ("errorDetail", NonMatchingInput))).ToList(); + var redos = CustomRule("TEST-REDOS", new RuleCondition + { + Signal = "slow", Source = "event_data", EventType = "app_install_failed", + DataField = "errorDetail", Operator = "regex", Value = CatastrophicPattern, Required = true, + }); + var earlierFinding = StoredRow("TEST-REDOS", isInterim: false); + var harness = Harness(events, redos); + harness.ExistingResults.Add(earlierFinding); + var opsEvents = new List(); + var function = harness.Function( + new ReanalyzeGate(TestNotifications.CapturingOpsEvents(opsEvents), NullLogger.Instance), opsEvents); + + var (req, _) = EndpointHarness.Request(TenantId, queryString: "?reanalyze=true"); + var response = await function.Run(req, SessionId); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + harness.Maintenance.Verify(m => m.DeleteSessionRuleResultsAsync(TenantId, SessionId), Times.Once); + harness.Rules.Verify(r => r.StoreRuleResultAsync(earlierFinding), Times.Once); + + response.Body.Position = 0; + using var body = JsonDocument.Parse(response.Body); + Assert.Equal(new[] { "TEST-REDOS" }, body.RootElement.GetProperty("stoppedRuleIds").EnumerateArray().Select(e => e.GetString())); + + var stopped = Assert.Single(opsEvents); + Assert.Equal(OpsEventTypes.RuleEvaluationStopped, stopped.EventType); + Assert.Equal(OpsEventSeverity.Warning, stopped.Severity); + } + + // ===== fixtures ===== + + private static RuleCondition ExistsCondition(string eventType) => new() + { + Signal = "exists_" + eventType, Source = "event_type", EventType = eventType, Operator = "exists", Required = true, + }; + + private static AnalyzeRule CustomRule(string ruleId, params RuleCondition[] conditions) => new() + { + RuleId = ruleId, + IsBuiltIn = false, // the model defaults to a built-in rule + Title = ruleId, + Severity = "warning", + Category = "test", + Enabled = true, + BaseConfidence = 80, + ConfidenceThreshold = 40, + Conditions = conditions.ToList(), + Explanation = "test", + }; + + private static RuleResult StoredRow(string ruleId, bool isInterim) => new() + { + RuleId = ruleId, + ResultId = "result-" + ruleId, + TenantId = TenantId, + SessionId = SessionId, + IsInterim = isInterim, + DetectedAt = DateTime.UtcNow.AddMinutes(-30), + }; + + private static EnrollmentEvent Event(string eventType, int sequence, params (string Key, object Value)[] data) => new() + { + EventId = $"evt-{sequence}", + TenantId = TenantId, + SessionId = SessionId, + EventType = eventType, + Sequence = sequence, + Timestamp = DateTime.UtcNow.AddMinutes(sequence), + Data = data.ToDictionary(d => d.Key, d => d.Value), + }; + + private static RuleHarness Harness(List events, params AnalyzeRule[] customRules) => new(events, customRules); + + /// The tenant's custom rules behind a real AnalyzeRuleService, mocked storage. + private sealed class RuleHarness + { + public Mock Rules { get; } = new(); + public Mock Sessions { get; } = new(); + public Mock Maintenance { get; } = new(); + public List ExistingResults { get; } = new(); + + public RuleHarness(List events, AnalyzeRule[] customRules) + { + Rules.Setup(r => r.GetAnalyzeRulesAsync("global")).ReturnsAsync(new List()); + Rules.Setup(r => r.GetAnalyzeRulesAsync(TenantId)).ReturnsAsync(customRules.ToList()); + Rules.Setup(r => r.GetRuleStatesAsync(It.IsAny())).ReturnsAsync(new Dictionary()); + Rules.Setup(r => r.GetRuleResultsAsync(TenantId, SessionId)).ReturnsAsync(() => ExistingResults.ToList()); + Rules.Setup(r => r.StoreRuleResultAsync(It.IsAny())).ReturnsAsync(true); + Sessions.Setup(s => s.GetSessionEventsStrictAsync(TenantId, SessionId, It.IsAny())).ReturnsAsync(events); + } + + private AnalyzeRuleService Service() => new(Rules.Object, NullLogger.Instance); + + public RuleEngine Engine(TimeSpan? runBudget = null) => + new(Service(), Rules.Object, Sessions.Object, NullLogger.Instance) + { + RunBudget = runBudget ?? RuleEvaluationBudget.AnalysisRun, + }; + + public GetRuleResultsFunction Function(ReanalyzeGate gate, List opsEvents) => + new(NullLogger.Instance, Rules.Object, Maintenance.Object, Sessions.Object, Service(), + gate, new RuleEvaluationStopReporter(TestNotifications.CapturingOpsEvents(opsEvents))); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEvidenceKeysTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEvidenceKeysTests.cs index a7b58f1f..bdc7f41d 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEvidenceKeysTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEvidenceKeysTests.cs @@ -69,7 +69,7 @@ public class RuleEvidenceKeysTests Required = true, }; - var dry = Engine().DryRunEvaluate(Rule(condition), Events()); + var dry = Engine().DryRunEvaluate(Rule(condition), Events(), new RuleEvaluationBudget(RuleEvaluationBudget.DryRun)); var matched = Assert.IsAssignableFrom>(dry.MatchedConditions); Assert.True(matched.ContainsKey("sig"), $"{source}/{op} did not match the fixture"); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RulePresentationTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RulePresentationTests.cs index 62172dda..8cce8113 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/RulePresentationTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RulePresentationTests.cs @@ -344,6 +344,11 @@ public class RulePresentationTests Assert.Equal(RulePresentationLimits.TemplateMaxLength, fact.GetProperty("value").GetProperty("maxLength").GetInt32()); Assert.Equal(RulePresentationLimits.TemplateMaxLength, subject.GetProperty("key").GetProperty("maxLength").GetInt32()); Assert.Equal(RulePresentationLimits.TemplateMaxLength, subject.GetProperty("name").GetProperty("maxLength").GetInt32()); + + // The rule-shape counts (D-410): the schema lets validate_rule refuse what the backend refuses. + var analyzeRule = defs.GetProperty("analyzeRule").GetProperty("properties"); + Assert.Equal(AnalyzeRuleLimits.MaxConditions, analyzeRule.GetProperty("conditions").GetProperty("maxItems").GetInt32()); + Assert.Equal(AnalyzeRuleLimits.MaxPreconditions, analyzeRule.GetProperty("preconditions").GetProperty("maxItems").GetInt32()); } private static string[] Strings(JsonElement array) => array.EnumerateArray().Select(e => e.GetString()!).ToArray(); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RuleWorkGateTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RuleWorkGateTests.cs new file mode 100644 index 00000000..1042575f --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RuleWorkGateTests.cs @@ -0,0 +1,127 @@ +using System.Net; +using System.Text.Json; +using AutopilotMonitor.Functions.Functions.Rules; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging.Abstractions; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// The per-instance caps on synchronous rule work (D-379, D-410) and the ops events behind them: a +/// caller past a gate's limit gets 429 with a retry hint, the first refusal of a window raises one +/// RuleEvaluationLimitReached naming the gate, and stopped rules reach the operators through a +/// reporter that a tenant's rule count cannot turn into an alarm storm. +/// +public class RuleWorkGateTests +{ + private const string Upn = "admin@example.com"; + + private readonly List _opsEvents = new(); + + [Theory] + [InlineData("dry-run", 5)] + [InlineData("re-analysis", 5)] + [InlineData("pattern-test", 3)] + public async Task Caller_past_the_limit_gets_429_with_a_retry_hint_and_one_ops_event_per_window(string gateName, int retryAfter) + { + var gate = Gate(gateName, limit: 1); + using var busy = gate.TryEnter(); + Assert.NotNull(busy); + Assert.Null(gate.TryEnter()); + + var (req, _) = EndpointHarness.Request("tenant-a", Upn); + var refused = await gate.RefuseAsync(req, "tenant-a", Upn); + await gate.RefuseAsync(req, "tenant-a", Upn); + + Assert.Equal(HttpStatusCode.TooManyRequests, refused.StatusCode); + Assert.Equal(Constants.ApiErrorCodes.RateLimited, EndpointHarness.ErrorCode(refused)); + Assert.True(refused.Headers.TryGetValues("Retry-After", out var header)); + Assert.Equal(retryAfter.ToString(), header!.Single()); + // The situation only: the portal and the MCP phrase the retry from Retry-After. + Assert.EndsWith("running right now.", ErrorText(refused)); + + var alert = Assert.Single(_opsEvents); + Assert.Equal(OpsEventTypes.RuleEvaluationLimitReached, alert.EventType); + Assert.Equal(OpsEventSeverity.Warning, alert.Severity); + Assert.Equal("tenant-a", alert.TenantId); + using var details = JsonDocument.Parse(alert.Details!); + Assert.Equal(gateName, details.RootElement.GetProperty("gate").GetString()); + } + + [Fact] + public void Default_limits_never_refuse_a_person() + { + var ops = TestNotifications.CapturingOpsEvents(_opsEvents); + Assert.Equal(2, new RuleDryRunGate(ops, NullLogger.Instance).Limit); + Assert.Equal(4, new ReanalyzeGate(ops, NullLogger.Instance).Limit); + } + + // ===== RuleEvaluationStopReporter ===== + + [Fact] + public async Task A_stopped_custom_rule_is_a_warning_and_a_stopped_catalog_rule_an_error() + { + var reporter = new RuleEvaluationStopReporter(TestNotifications.CapturingOpsEvents(_opsEvents)); + var outcome = new AnalysisOutcome(); + outcome.Stops.Add(new RuleEvaluationStop("CUSTOM-1", EvaluationStopReasons.RegexTimeout, IsCatalogRule: false)); + outcome.Stops.Add(new RuleEvaluationStop("ANALYZE-APP-002", EvaluationStopReasons.TimeBudget, IsCatalogRule: true)); + outcome.SkippedRuleIds.AddRange(new[] { "CUSTOM-2", "CUSTOM-3" }); + + await reporter.ReportAsync("tenant-a", "session-1", AnalyzeRunKind.Terminal, outcome); + + Assert.Equal(2, _opsEvents.Count); + Assert.All(_opsEvents, e => Assert.Equal(OpsEventTypes.RuleEvaluationStopped, e.EventType)); + Assert.Equal(OpsEventSeverity.Warning, _opsEvents[0].Severity); + Assert.Equal(OpsEventSeverity.Error, _opsEvents[1].Severity); + using var budgetStop = JsonDocument.Parse(_opsEvents[1].Details!); + Assert.Equal(2, budgetStop.RootElement.GetProperty("skippedRules").GetInt32()); + using var regexStop = JsonDocument.Parse(_opsEvents[0].Details!); + Assert.Equal(0, regexStop.RootElement.GetProperty("skippedRules").GetInt32()); + } + + [Fact] + public void One_event_per_tenant_and_rule_per_window() + { + var reporter = new RuleEvaluationStopReporter(TestNotifications.CapturingOpsEvents(_opsEvents)); + var start = new DateTime(2026, 11, 5, 12, 0, 0, DateTimeKind.Utc); + + Assert.True(reporter.Take("tenant-a", "CUSTOM-1", start)); + Assert.False(reporter.Take("tenant-a", "CUSTOM-1", start.AddMinutes(59))); + Assert.True(reporter.Take("tenant-b", "CUSTOM-1", start.AddMinutes(1))); + Assert.True(reporter.Take("tenant-a", "CUSTOM-1", start + RuleEvaluationStopReporter.Window)); + } + + [Fact] + public void A_tenant_with_many_rules_cannot_flood_the_alarm() + { + var reporter = new RuleEvaluationStopReporter(TestNotifications.CapturingOpsEvents(_opsEvents)); + var start = new DateTime(2026, 11, 5, 12, 0, 0, DateTimeKind.Utc); + + var taken = Enumerable.Range(0, 1000).Count(i => reporter.Take("tenant-a", $"CUSTOM-{i}", start.AddSeconds(i))); + + Assert.Equal(RuleEvaluationStopReporter.MaxEventsPerWindow, taken); + Assert.True(reporter.Take("tenant-a", "CUSTOM-new", start + RuleEvaluationStopReporter.Window)); + } + + private RuleWorkGate Gate(string name, int limit) + { + var ops = TestNotifications.CapturingOpsEvents(_opsEvents); + return name switch + { + "dry-run" => new RuleDryRunGate(limit, ops, NullLogger.Instance), + "re-analysis" => new ReanalyzeGate(limit, ops, NullLogger.Instance), + _ => new TestLogPatternGate(limit, ops, NullLogger.Instance), + }; + } + + private static string ErrorText(Microsoft.Azure.Functions.Worker.Http.HttpResponseData response) + { + response.Body.Position = 0; + using var doc = JsonDocument.Parse(response.Body); + return doc.RootElement.GetProperty("error").GetString()!; + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RulesWireParityTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RulesWireParityTests.cs index a1b57bbb..e625e181 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/RulesWireParityTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RulesWireParityTests.cs @@ -140,6 +140,26 @@ public class RulesWireParityTests new DryRunAnalyzeRuleResponse { Success = true, SessionId = sessionId, Result = result }); } + [Fact] + public void DryRunAnalyzeRuleResponse_carries_a_stop_with_its_reason_and_note() + { + var sessionId = "7c1e2a90-1111-4b7c-a1d2-52f3bbbb0006"; + var result = new RuleDryRun + { + Verdict = RuleDryRunVerdict.Stopped, + EventCount = 4200, + BaseConfidence = 50, + ConfidenceThreshold = 60, + // D-410: verdict "stopped" — the trace keeps only the entries evaluated before the stop. + StopReason = EvaluationStopReasons.RegexTimeout, + StopNote = "conditions[0] ('slow'): a regex hit the 1 s match timeout on this session's data", + }; + + AssertParity( + new { success = true, sessionId, result }, + new DryRunAnalyzeRuleResponse { Success = true, SessionId = sessionId, Result = result }); + } + // ---- GetRuleResults ------------------------------------------------------------------ [Fact] @@ -178,6 +198,8 @@ public class RulesWireParityTests }; var openResults = results.Where(r => r.ResolvedAt == null).ToList(); var persistFailureRuleIds = new List { "ANALYZE-NET-001" }; + // D-410: rules the reanalyze stopped at its evaluation budget, omitted when there are none. + var stoppedRuleIds = new List { "CUSTOM-SLOW-001" }; AssertParity( new @@ -190,7 +212,8 @@ public class RulesWireParityTests highCount = openResults.Count(r => r.Severity == "high"), warningCount = openResults.Count(r => r.Severity == "warning"), persistFailureCount = persistFailureRuleIds.Count, - persistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null + persistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null, + stoppedRuleIds = stoppedRuleIds.Count > 0 ? stoppedRuleIds : null }, new GetRuleResultsResponse { @@ -202,7 +225,8 @@ public class RulesWireParityTests HighCount = openResults.Count(r => r.Severity == "high"), WarningCount = openResults.Count(r => r.Severity == "warning"), PersistFailureCount = persistFailureRuleIds.Count, - PersistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null + PersistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null, + StoppedRuleIds = stoppedRuleIds.Count > 0 ? stoppedRuleIds : null }); } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TestLogPatternFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TestLogPatternFunctionTests.cs index 6ff2e333..43eade4d 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/TestLogPatternFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TestLogPatternFunctionTests.cs @@ -227,7 +227,7 @@ public class TestLogPatternFunctionTests Assert.Equal(1, result.TimeoutCount); Assert.Single(result.Lines); Assert.Equal("regex_timeout", result.Lines[0].Outcome); - Assert.Equal(TestLogPatternFunction.StopReasonRegexTimeout, result.StopReason); + Assert.Equal(EvaluationStopReasons.RegexTimeout, result.StopReason); Assert.Contains(result.Notes, n => n.Contains("line 1 hit") && n.Contains("lines 2-200 were not evaluated")); Assert.DoesNotContain(result.Notes, n => n.Contains("case-SENSITIVE")); Assert.True(sw.ElapsedMilliseconds < 4000, $"catastrophic request took {sw.ElapsedMilliseconds} ms"); @@ -259,7 +259,7 @@ public class TestLogPatternFunctionTests Assert.Single(result.Lines); Assert.Equal("matched", result.Lines[0].Outcome); - Assert.Equal(TestLogPatternFunction.StopReasonTimeBudget, result.StopReason); + Assert.Equal(EvaluationStopReasons.TimeBudget, result.StopReason); Assert.Contains(result.Notes, n => n.Contains("after line 1") && n.Contains("lines 2-3 were not evaluated")); var twoLines = TestLogPatternFunction.EvaluatePattern( @@ -282,16 +282,19 @@ public class TestLogPatternFunctionTests TestLogPatternFunction.EvaluationBudget, cts.Token); Assert.Empty(result.Lines); - Assert.Equal(TestLogPatternFunction.StopReasonCancelled, result.StopReason); + Assert.Equal(EvaluationStopReasons.Cancelled, result.StopReason); Assert.Equal(0, result.MatchCount); } // ===== the per-instance gate ===== + private static TestLogPatternGate Gate(int limit) + => new(limit, TestNotifications.CapturingOpsEvents(new()), Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + [Fact] public void Gate_HandsOutLimitSlots_AndTakesThemBackOnDispose() { - var gate = new TestLogPatternGate(1); + var gate = Gate(1); Assert.Equal(1, gate.Limit); var first = gate.TryEnter(); @@ -309,7 +312,7 @@ public class TestLogPatternFunctionTests [Fact] public void Gate_DefaultLimit_IsTwoPerInstance() { - var gate = new TestLogPatternGate(); + var gate = new TestLogPatternGate(TestNotifications.CapturingOpsEvents(new()), Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); Assert.Equal(2, gate.Limit); Assert.Equal(TestLogPatternGate.DefaultLimit, gate.Limit); } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TestNotifications.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TestNotifications.cs index 915f929d..e560f3b6 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/TestNotifications.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TestNotifications.cs @@ -1,6 +1,8 @@ using System.Net.Http; using AutopilotMonitor.Functions.Services; using AutopilotMonitor.Functions.Services.Notifications; +using AutopilotMonitor.Shared.DataAccess; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -24,4 +26,15 @@ internal static class TestNotifications internal static OpsAlertDispatchService InertOpsAlertDispatch(AdminConfigurationService adminConfigService) => new(adminConfigService, InertDispatcher(), NullLogger.Instance); + + /// An that records every saved event in and sends nothing. + internal static OpsEventService CapturingOpsEvents(List sink) + { + var repo = new Mock(); + repo.Setup(r => r.SaveOpsEventAsync(It.IsAny())) + .Returns(e => { lock (sink) sink.Add(e); return Task.CompletedTask; }); + var adminConfig = new Mock( + Mock.Of(), NullLogger.Instance, new MemoryCache(new MemoryCacheOptions())); + return new OpsEventService(repo.Object, NullLogger.Instance, InertOpsAlertDispatch(adminConfig.Object)); + } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Reports/ReportUploadGate.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Reports/ReportUploadGate.cs index 82896332..0585e1d6 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Reports/ReportUploadGate.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Reports/ReportUploadGate.cs @@ -24,7 +24,6 @@ namespace AutopilotMonitor.Functions.Functions.Reports private readonly OpsEventService _opsEvents; private readonly ILogger _logger; - private long _nextAlertTicks; public ReportUploadGate(OpsEventService opsEvents, ILogger logger) : this(DefaultLimit, opsEvents, logger) @@ -44,20 +43,12 @@ namespace AutopilotMonitor.Functions.Functions.Reports "Report upload refused on {Route}: {Limit} uploads already in progress on this instance (tenant {TenantId})", route, Limit, tenantId); - if (TakeAlert(DateTime.UtcNow)) + if (TakeAlert(DateTime.UtcNow, AlertWindow)) await _opsEvents.RecordReportUploadLimitReachedAsync(route, Limit, AlertWindow, tenantId, upn); return await req.ErrorAsync(HttpStatusCode.TooManyRequests, Constants.ApiErrorCodes.RateLimited, - "Too many reports are being uploaded right now. Try again in a few seconds.", + "Too many reports are being uploaded right now.", retryAfterSeconds: RetryAfterSeconds); } - - /// True for the first refusal of an alert window, the one that raises the ops event. - internal bool TakeAlert(DateTime utcNow) - { - var next = Interlocked.Read(ref _nextAlertTicks); - return utcNow.Ticks >= next - && Interlocked.CompareExchange(ref _nextAlertTicks, utcNow.Add(AlertWindow).Ticks, next) == next; - } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs index f67cb8de..66495560 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs @@ -19,6 +19,11 @@ namespace AutopilotMonitor.Functions.Functions.Rules /// modified (the engine's dry-run path has no MarkSessionAsFailed side effect by construction). /// Global-scope callers may dry-run against any tenant's session — the tenant is resolved from /// the session, same as GetRuleResultsFunction. + /// + /// The draft is caller-authored work on the shared app, so it is bounded (D-410): the draft's + /// size is capped (), the evaluation stops at + /// the first regex match timeout, at the 2 s budget or when the caller goes away (verdict + /// "stopped"), and caps how many dry-runs run per instance. /// public class DryRunAnalyzeRuleFunction { @@ -26,21 +31,25 @@ namespace AutopilotMonitor.Functions.Functions.Rules private readonly IRuleRepository _ruleRepo; private readonly ISessionRepository _sessionRepo; private readonly AnalyzeRuleService _analyzeRuleService; + private readonly RuleDryRunGate _gate; public DryRunAnalyzeRuleFunction( ILogger logger, IRuleRepository ruleRepo, ISessionRepository sessionRepo, - AnalyzeRuleService analyzeRuleService) + AnalyzeRuleService analyzeRuleService, + RuleDryRunGate gate) { _logger = logger; _ruleRepo = ruleRepo; _sessionRepo = sessionRepo; _analyzeRuleService = analyzeRuleService; + _gate = gate; } [Function("DryRunAnalyzeRule")] public async Task Run( - [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "rules/analyze/dryrun")] HttpRequestData req) + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "rules/analyze/dryrun")] HttpRequestData req, + CancellationToken cancellationToken) { // Authentication + TenantAdminOrGlobalReader authorization enforced by PolicyEnforcementMiddleware. // Read-only diagnostic, so the read-only Global Reader is admitted alongside Tenant Admins and GAs. @@ -66,6 +75,11 @@ namespace AutopilotMonitor.Functions.Functions.Rules var sessionId = request.SessionId!.Trim(); var rule = request.Rule!; + // A slot of this instance before any tenant data is read; held to the response. + using var slot = _gate.TryEnter(); + if (slot == null) + return await _gate.RefuseAsync(req, requestCtx.TenantId, requestCtx.UserPrincipalName); + // Global-scope cross-tenant fallback: resolve the session's actual tenant so a GA / // Global Reader can dry-run against any tenant's session (mirrors GetRuleResultsFunction). effectiveTenantId = await requestCtx.ResolveSessionScopeAsync(_sessionRepo, sessionId); @@ -77,11 +91,23 @@ namespace AutopilotMonitor.Functions.Functions.Rules } var ruleEngine = new RuleEngine(_analyzeRuleService, _ruleRepo, _sessionRepo, _logger); - var result = await ruleEngine.DryRunRuleAsync(effectiveTenantId, sessionId, rule); + var result = await ruleEngine.DryRunRuleAsync(effectiveTenantId, sessionId, rule, cancellationToken); - _logger.LogInformation( - "Dry-run of draft rule {RuleId} against session {SessionId}: verdict={Verdict}, confidence={Confidence}", - string.IsNullOrEmpty(rule.RuleId) ? "(no id)" : rule.RuleId, sessionId, result.Verdict, result.FinalConfidence); + var ruleLabel = string.IsNullOrEmpty(rule.RuleId) ? "(no id)" : rule.RuleId; + if (result.Verdict == RuleDryRunVerdict.Stopped) + { + // Warning so it reaches App Insights: one hit is an author with a broken rule, a + // cluster from one caller is load aimed at this endpoint (the gate reports that). + _logger.LogWarning( + "Analyze dry-run stopped: {Reason} for draft rule {RuleId} against session {SessionId} ({EventCount} events): {Note}", + result.StopReason, ruleLabel, sessionId, result.EventCount, result.StopNote); + } + else + { + _logger.LogInformation( + "Dry-run of draft rule {RuleId} against session {SessionId}: verdict={Verdict}, confidence={Confidence}", + ruleLabel, sessionId, result.Verdict, result.FinalConfidence); + } var response = req.CreateResponse(HttpStatusCode.OK); await response.WriteAsJsonAsync(new DryRunAnalyzeRuleResponse { Success = true, SessionId = sessionId, Result = result }); @@ -124,8 +150,9 @@ namespace AutopilotMonitor.Functions.Functions.Rules /// /// Structural validation of a draft rule before evaluation. Returns human-readable errors /// (empty list = valid). Deliberately limited to what would make the dry-run silently - /// meaningless or throw — full authoring lint (schema, guardrails, event-type catalog) - /// lives client-side in the MCP `validate_rule` tool. + /// meaningless or throw, plus the size limits every write path enforces too — full + /// authoring lint (schema, guardrails, event-type catalog) lives client-side in the MCP + /// `validate_rule` tool. Also the gate of community rule submissions. /// internal static List ValidateDraftRule(AnalyzeRule? rule) { @@ -185,22 +212,11 @@ namespace AutopilotMonitor.Functions.Functions.Rules System.Globalization.CultureInfo.InvariantCulture, out var skewThreshold) || skewThreshold <= 0) errors.Add($"{label}: clock_skew requires a positive numeric value (threshold in seconds)"); } - - if ((string.Equals(c.Operator, "regex", StringComparison.OrdinalIgnoreCase) - || string.Equals(c.Operator, "not_regex", StringComparison.OrdinalIgnoreCase)) - && !string.IsNullOrEmpty(c.Value)) - { - try - { - _ = new Regex(c.Value, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); - } - catch (ArgumentException ex) - { - errors.Add($"{label}: value is not a valid regex: {ex.Message}"); - } - } } + // Counts and every regex-bearing value (length first, then it must parse). + errors.AddRange(AnalyzeRuleService.ShapeLimitErrors(rule)); + for (var i = 0; i < (rule.ConfidenceFactors?.Count ?? 0); i++) { var f = rule.ConfidenceFactors![i]; @@ -214,4 +230,27 @@ namespace AutopilotMonitor.Functions.Functions.Rules return errors; } } + + /// + /// Per-instance cap on concurrent analyze dry-runs (singleton). A slot is held from before the + /// session is read to the response, so this endpoint never occupies more than + /// threads of an instance, whatever the request rate. + /// Retry-After 5 s: an event read plus the 2 s budget plus one running match. + /// + public sealed class RuleDryRunGate : RuleWorkGate + { + internal const int DefaultLimit = 2; + + public RuleDryRunGate(OpsEventService opsEvents, ILogger logger) + : this(DefaultLimit, opsEvents, logger) + { + } + + internal RuleDryRunGate(int limit, OpsEventService opsEvents, ILogger logger) + : base(limit, "dry-run", + "Too many rule tests are running right now.", + retryAfterSeconds: 5, opsEvents, logger) + { + } + } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Rules/GetRuleResultsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/GetRuleResultsFunction.cs index f3f40b1d..b3f84514 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Rules/GetRuleResultsFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/GetRuleResultsFunction.cs @@ -13,6 +13,8 @@ namespace AutopilotMonitor.Functions.Functions.Rules /// Returns analysis results (rule evaluations) for a session. /// Supports on-demand re-analysis via ?reanalyze=true query parameter. /// Global admins can request analysis for any tenant by passing ?tenantId=... + /// The re-analysis runs the tenant's rules synchronously, so it takes a slot of + /// and its run is bounded by the evaluation budget (D-410). /// public class GetRuleResultsFunction { @@ -21,19 +23,25 @@ namespace AutopilotMonitor.Functions.Functions.Rules private readonly IMaintenanceRepository _maintenanceRepo; private readonly ISessionRepository _sessionRepo; private readonly AnalyzeRuleService _analyzeRuleService; + private readonly ReanalyzeGate _gate; + private readonly RuleEvaluationStopReporter _stopReporter; public GetRuleResultsFunction( ILogger logger, IRuleRepository ruleRepo, IMaintenanceRepository maintenanceRepo, ISessionRepository sessionRepo, - AnalyzeRuleService analyzeRuleService) + AnalyzeRuleService analyzeRuleService, + ReanalyzeGate gate, + RuleEvaluationStopReporter stopReporter) { _logger = logger; _ruleRepo = ruleRepo; _maintenanceRepo = maintenanceRepo; _sessionRepo = sessionRepo; _analyzeRuleService = analyzeRuleService; + _gate = gate; + _stopReporter = stopReporter; } [Function("GetRuleResults")] @@ -65,6 +73,14 @@ namespace AutopilotMonitor.Functions.Functions.Rules // and a 500 would blank the page even when most rules persisted fine. var persistFailureRuleIds = new List(); + // Rules the re-analysis could not evaluate within its budget; they keep their rows. + var stoppedRuleIds = new List(); + + // A slot of this instance for the whole rebuild (analysis, delete, rewrite). + using var slot = reanalyze ? _gate.TryEnter() : null; + if (reanalyze && slot == null) + return await _gate.RefuseAsync(req, requestCtx.TenantId, requestCtx.UserPrincipalName); + if (reanalyze) { try @@ -76,12 +92,16 @@ namespace AutopilotMonitor.Functions.Functions.Rules // channel notification. Rules that no longer fire come back as ResolvedResults // (kept for audit) instead of silently vanishing. var outcome = await ruleEngine.AnalyzeSessionAsync(effectiveTenantId, sessionId, reanalyze: true); + stoppedRuleIds.AddRange(outcome.Stops.Select(s => s.RuleId).Concat(outcome.SkippedRuleIds)); + await _stopReporter.ReportAsync(effectiveTenantId, sessionId, AnalyzeRunKind.Reanalyze, outcome); // Delete existing results so stale entries (e.g. rows from since-deleted or - // disabled rules) don't persist after re-analysis + // disabled rules) don't persist after re-analysis. Rows of rules the run could + // not evaluate (KeptResults) go back unchanged — a stop is no evidence that a + // finding went away. await _maintenanceRepo.DeleteSessionRuleResultsAsync(effectiveTenantId, sessionId); - foreach (var result in outcome.Results.Concat(outcome.ResolvedResults)) + foreach (var result in outcome.Results.Concat(outcome.ResolvedResults).Concat(outcome.KeptResults)) { var stored = await _ruleRepo.StoreRuleResultAsync(result); if (!stored) @@ -133,9 +153,33 @@ namespace AutopilotMonitor.Functions.Functions.Rules HighCount = openResults.Count(r => r.Severity == "high"), WarningCount = openResults.Count(r => r.Severity == "warning"), PersistFailureCount = persistFailureRuleIds.Count, - PersistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null + PersistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null, + StoppedRuleIds = stoppedRuleIds.Count > 0 ? stoppedRuleIds : null }); return response; } } + + /// + /// Per-instance cap on concurrent "Analyze now" rebuilds (singleton). A slot is held from the + /// analysis to the last rewritten row. The button is the only caller and a rebuild takes a + /// fraction of a second, so four slots never refuse a person; they stop a request flood from + /// holding every thread of an instance. Retry-After 5 s: one run's budget. + /// + public sealed class ReanalyzeGate : RuleWorkGate + { + internal const int DefaultLimit = 4; + + public ReanalyzeGate(OpsEventService opsEvents, ILogger logger) + : this(DefaultLimit, opsEvents, logger) + { + } + + internal ReanalyzeGate(int limit, OpsEventService opsEvents, ILogger logger) + : base(limit, "re-analysis", + "Too many analyses are running right now.", + retryAfterSeconds: 5, opsEvents, logger) + { + } + } } diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs index e8142028..0002030b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs @@ -5,6 +5,7 @@ using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.Logging; using AutopilotMonitor.Shared.Models; using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Services; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.Extensions.Logging; @@ -61,16 +62,10 @@ namespace AutopilotMonitor.Functions.Functions.Rules /// /// Wall-clock budget for one whole test. A legitimate pattern finishes 200 lines in /// milliseconds; the worst case per request is this budget plus one running match - /// (≤ ), which is what promises. + /// (≤ ), which is what the gate's Retry-After promises. /// internal static readonly TimeSpan EvaluationBudget = TimeSpan.FromSeconds(2); - private const int RetryAfterSeconds = 3; - - internal const string StopReasonRegexTimeout = "regex_timeout"; - internal const string StopReasonTimeBudget = "time_budget"; - internal const string StopReasonCancelled = "cancelled"; - [Function("TestLogPattern")] public async Task Run( [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "rules/gather/test-pattern")] HttpRequestData req, @@ -94,12 +89,8 @@ namespace AutopilotMonitor.Functions.Functions.Rules var slot = _gate.TryEnter(); if (slot == null) { - _logger.LogWarning( - "Log-pattern test refused: {Limit} evaluations already running on this instance", - _gate.Limit); - return await req.ErrorAsync(HttpStatusCode.TooManyRequests, Constants.ApiErrorCodes.RateLimited, - "too many pattern tests are running right now — retry in a few seconds", - retryAfterSeconds: RetryAfterSeconds); + var requestCtx = req.GetRequestContext(); + return await _gate.RefuseAsync(req, requestCtx.TenantId, requestCtx.UserPrincipalName); } using (slot) @@ -190,7 +181,7 @@ namespace AutopilotMonitor.Functions.Functions.Rules { if (cancellationToken.IsCancellationRequested) { - Stop(result, StopReasonCancelled, + Stop(result, EvaluationStopReasons.Cancelled, i == 0 ? "evaluation cancelled before the first line" : $"evaluation cancelled after line {i}", i, lines.Count); break; @@ -198,7 +189,7 @@ namespace AutopilotMonitor.Functions.Functions.Rules // Checked before every line but the first: the previous line spent the budget. if (i > 0 && clock.Elapsed >= budget) { - Stop(result, StopReasonTimeBudget, + Stop(result, EvaluationStopReasons.TimeBudget, $"evaluation stopped after line {i}: the test exceeded its {budget.TotalSeconds:0.#} s budget — a pattern this slow would strain the device too; anchor it (^) and avoid unbounded .* scans", i, lines.Count); break; @@ -236,7 +227,7 @@ namespace AutopilotMonitor.Functions.Functions.Rules { row.Outcome = "regex_timeout"; result.TimeoutCount++; - Stop(result, StopReasonRegexTimeout, + Stop(result, EvaluationStopReasons.RegexTimeout, $"line {i + 1} hit the agent's {MatchTimeout.TotalSeconds:0.#} s match timeout (catastrophic backtracking) — on the device such lines are skipped and only counted as regex timeouts; remove nested or overlapping quantifiers before deploying", i + 1, lines.Count); break; @@ -308,17 +299,21 @@ namespace AutopilotMonitor.Functions.Functions.Rules /// construction to the end of the evaluation; a caller finding no free slot is answered 429 /// at once, so this endpoint can never occupy more than /// worker threads of an instance, whatever the request rate. Nothing but this test waits on - /// this gate. + /// this gate. Retry-After 3 s: the 2 s budget plus one running match. /// - public sealed class TestLogPatternGate : ConcurrencyGate + public sealed class TestLogPatternGate : RuleWorkGate { internal const int DefaultLimit = 2; - public TestLogPatternGate() : this(DefaultLimit) + public TestLogPatternGate(OpsEventService opsEvents, ILogger logger) + : this(DefaultLimit, opsEvents, logger) { } - internal TestLogPatternGate(int limit) : base(limit) + internal TestLogPatternGate(int limit, OpsEventService opsEvents, ILogger logger) + : base(limit, "pattern-test", + "Too many pattern tests are running right now.", + retryAfterSeconds: 3, opsEvents, logger) { } } diff --git a/src/Backend/AutopilotMonitor.Functions/Helpers/ConcurrencyGate.cs b/src/Backend/AutopilotMonitor.Functions/Helpers/ConcurrencyGate.cs index 57caccb1..958da5db 100644 --- a/src/Backend/AutopilotMonitor.Functions/Helpers/ConcurrencyGate.cs +++ b/src/Backend/AutopilotMonitor.Functions/Helpers/ConcurrencyGate.cs @@ -10,6 +10,7 @@ namespace AutopilotMonitor.Functions.Helpers; public abstract class ConcurrencyGate { private readonly SemaphoreSlim _slots; + private long _nextAlertTicks; protected ConcurrencyGate(int limit) { @@ -22,6 +23,17 @@ public abstract class ConcurrencyGate /// A slot to dispose when the work ends, or null when every slot is taken. public IDisposable? TryEnter() => _slots.Wait(0) ? new Slot(_slots) : null; + /// + /// True for the first refusal of an alert window, the one that raises the gate's ops event; + /// later refusals in that window get false, so a full gate can never flood the alarm. + /// + internal bool TakeAlert(DateTime utcNow, TimeSpan window) + { + var next = Interlocked.Read(ref _nextAlertTicks); + return utcNow.Ticks >= next + && Interlocked.CompareExchange(ref _nextAlertTicks, utcNow.Add(window).Ticks, next) == next; + } + private sealed class Slot : IDisposable { private SemaphoreSlim? _slots; diff --git a/src/Backend/AutopilotMonitor.Functions/Helpers/RuleWorkGate.cs b/src/Backend/AutopilotMonitor.Functions/Helpers/RuleWorkGate.cs new file mode 100644 index 00000000..6b5a86a4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Helpers/RuleWorkGate.cs @@ -0,0 +1,54 @@ +using System.Net; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Helpers; + +/// +/// Per-instance cap on rule work a caller starts synchronously: the analyze dry-run, the manual +/// re-analysis and the logparser pattern test (D-379, D-410), each its own singleton subclass. The +/// work runs caller-authored patterns on the shared app, so the request rate must not decide how +/// many threads it holds. A caller who finds every slot taken is answered 429 at once with a retry +/// hint; the first refusal per instance, gate and hour raises RuleEvaluationLimitReached for +/// the operators, later refusals in that window stay in the Warning trace. Legitimate use never +/// fills a gate, so a refusal is load aimed at it. +/// +public abstract class RuleWorkGate : ConcurrencyGate +{ + internal static readonly TimeSpan AlertWindow = TimeSpan.FromHours(1); + + private readonly string _gate; + private readonly string _busyMessage; + private readonly int _retryAfterSeconds; + private readonly OpsEventService _opsEvents; + private readonly ILogger _logger; + + protected RuleWorkGate(int limit, string gate, string busyMessage, int retryAfterSeconds, OpsEventService opsEvents, ILogger logger) + : base(limit) + { + _gate = gate; + _busyMessage = busyMessage; + _retryAfterSeconds = retryAfterSeconds; + _opsEvents = opsEvents; + _logger = logger; + } + + /// + /// The 429 for a caller who found every slot taken. The message names the situation only; + /// clients phrase the retry from Retry-After (the portal appends "Try again in N s."). + /// + public async Task RefuseAsync(HttpRequestData req, string? tenantId, string? upn) + { + _logger.LogWarning( + "Rule work refused: the {Gate} gate has {Limit} evaluations running on this instance (tenant {TenantId})", + _gate, Limit, tenantId); + + if (TakeAlert(DateTime.UtcNow, AlertWindow)) + await _opsEvents.RecordRuleEvaluationLimitReachedAsync(_gate, Limit, AlertWindow, tenantId, upn); + + return await req.ErrorAsync(HttpStatusCode.TooManyRequests, Constants.ApiErrorCodes.RateLimited, + _busyMessage, retryAfterSeconds: _retryAfterSeconds); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index facee9c0..0fdbe460 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -158,8 +158,13 @@ builder.Services.AddMemoryCache(); builder.Services.AddHttpClient(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); -// Per-instance cap on concurrent logparser pattern tests (caller-controlled regex work, D-379). +// Per-instance caps on rule work a caller starts synchronously, each with an ops event when it is +// hit: logparser pattern tests (D-379), analyze dry-runs and "Analyze now" rebuilds (D-410). builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +// Throttled ops events for rules the analysis budget stopped (D-410). +builder.Services.AddSingleton(); // Per-instance cap on concurrent report uploads (20 MiB bodies), with an ops event when it is hit. builder.Services.AddSingleton(); // Shared QueueClient bootstrap (code-quality audit D2): single Managed-Identity/connection-string diff --git a/src/Backend/AutopilotMonitor.Functions/Security/HardwareWhitelistValidator.cs b/src/Backend/AutopilotMonitor.Functions/Security/HardwareWhitelistValidator.cs index 0ef18ad4..c0d80700 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/HardwareWhitelistValidator.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/HardwareWhitelistValidator.cs @@ -1,6 +1,6 @@ using System; +using System.IO.Enumeration; using System.Linq; -using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; namespace AutopilotMonitor.Functions.Security @@ -11,6 +11,13 @@ namespace AutopilotMonitor.Functions.Security /// public static class HardwareWhitelistValidator { + /// + /// Longest device value an entry other than "*" is compared with. Real SMBIOS manufacturer + /// and model strings are a few dozen characters; a longer header is not one, and the + /// comparison costs value × entry length on every agent request (D-411). + /// + internal const int MaxMatchedValueLength = 256; + /// /// Validates hardware against whitelist /// @@ -50,11 +57,11 @@ namespace AutopilotMonitor.Functions.Security var manufacturerMatch = manufacturerWhitelist.Any(pattern => MatchesWildcard(manufacturer, pattern)); if (!manufacturerMatch) { - logger?.LogWarning($"Hardware rejected: Manufacturer '{manufacturer}' not in whitelist"); + logger?.LogWarning($"Hardware rejected: Manufacturer '{Shorten(manufacturer)}' not in whitelist"); return new HardwareValidationResult { IsValid = false, - ErrorMessage = $"Manufacturer '{manufacturer}' is not authorized", + ErrorMessage = $"Manufacturer '{Shorten(manufacturer)}' is not authorized", Manufacturer = manufacturer, Model = model }; @@ -64,11 +71,11 @@ namespace AutopilotMonitor.Functions.Security var modelMatch = modelWhitelist.Any(pattern => MatchesWildcard(model, pattern)); if (!modelMatch) { - logger?.LogWarning($"Hardware rejected: Model '{model}' not in whitelist"); + logger?.LogWarning($"Hardware rejected: Model '{Shorten(model)}' not in whitelist"); return new HardwareValidationResult { IsValid = false, - ErrorMessage = $"Model '{model}' is not authorized", + ErrorMessage = $"Model '{Shorten(model)}' is not authorized", Manufacturer = manufacturer, Model = model }; @@ -84,30 +91,26 @@ namespace AutopilotMonitor.Functions.Security } /// - /// Matches a string against a wildcard pattern (supports * and ?) + /// Whole-value wildcard match, case-insensitive: '*' is any run of characters (also none), + /// '?' exactly one, everything else literal. The tenant writes the entries and the device + /// the value, so no Regex is built from them: the BCL's simple-expression matcher needs at + /// most value × entry steps and never backtracks (D-411). Its escape character is '\', + /// which is literal in an entry, so it is doubled first. Case folding is ordinal — SMBIOS + /// strings are ASCII. /// - private static bool MatchesWildcard(string text, string pattern) + internal static bool MatchesWildcard(string text, string pattern) { if (pattern == "*") return true; - - // Convert wildcard pattern to regex - // Escape regex special chars, then replace * and ? - var regexPattern = "^" + Regex.Escape(pattern) - .Replace("\\*", ".*") - .Replace("\\?", ".") + "$"; - - try - { - // Use timeout to prevent ReDoS attacks with pathological input - return Regex.IsMatch(text, regexPattern, RegexOptions.IgnoreCase, TimeSpan.FromMilliseconds(100)); - } - catch (RegexMatchTimeoutException) - { - // Timeout indicates potential ReDoS attempt - treat as non-match + if (text.Length > MaxMatchedValueLength) return false; - } + + return FileSystemName.MatchesSimpleExpression(pattern.Replace("\\", "\\\\"), text, ignoreCase: true); } + + /// The value as a rejection message shows it: never longer than a real hardware string. + private static string Shorten(string value) + => value.Length <= MaxMatchedValueLength ? value : value.Substring(0, MaxMatchedValueLength) + "…"; } /// diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AnalyzeOnEnrollmentEndHandler.cs b/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AnalyzeOnEnrollmentEndHandler.cs index 97fae50c..5e9211e5 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AnalyzeOnEnrollmentEndHandler.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Analyze/AnalyzeOnEnrollmentEndHandler.cs @@ -51,6 +51,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze private readonly TenantConfigurationService _configService; private readonly Notifications.NotificationChannelDispatcher _channelDispatcher; private readonly Deletion.SessionDeletionGuard _deletionGuard; + private readonly RuleEvaluationStopReporter _stopReporter; private readonly ILogger _logger; public const string ReasonEnrollmentComplete = "enrollment_complete"; @@ -68,6 +69,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze TenantConfigurationService configService, Notifications.NotificationChannelDispatcher channelDispatcher, Deletion.SessionDeletionGuard deletionGuard, + RuleEvaluationStopReporter stopReporter, ILogger logger) { _ruleService = ruleService; @@ -78,6 +80,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze _configService = configService; _channelDispatcher = channelDispatcher; _deletionGuard = deletionGuard; + _stopReporter = stopReporter; _logger = logger; } @@ -131,6 +134,10 @@ namespace AutopilotMonitor.Functions.Services.Analyze var ruleEngine = new RuleEngine(_ruleService, _ruleRepo, _sessionRepo, _logger); var outcome = await ruleEngine.AnalyzeSessionAsync(envelope.TenantId, envelope.SessionId, context).ConfigureAwait(false); + // Rules the evaluation budget stopped (D-410) become throttled ops events. Their stored + // rows were left alone by the engine; this path only upserts, so KeptResults is moot. + await _stopReporter.ReportAsync(envelope.TenantId, envelope.SessionId, context.Kind, outcome).ConfigureAwait(false); + foreach (var result in outcome.Results.Concat(outcome.ResolvedResults)) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/AnalyzeRuleService.cs b/src/Backend/AutopilotMonitor.Functions/Services/AnalyzeRuleService.cs index 4e79963b..bbc28655 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/AnalyzeRuleService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/AnalyzeRuleService.cs @@ -1,3 +1,4 @@ +using System.Text.RegularExpressions; using AutopilotMonitor.Functions.Services.Caching; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; @@ -334,6 +335,7 @@ namespace AutopilotMonitor.Functions.Services ValidateEvaluateOn(rule); ValidateRelatedDocs(rule); ValidatePresentation(rule); + ValidateShapeLimits(rule); rule.IsBuiltIn = false; rule.IsCommunity = false; @@ -373,6 +375,7 @@ namespace AutopilotMonitor.Functions.Services ValidateEvaluateOn(rule); ValidateRelatedDocs(rule); ValidatePresentation(rule); + ValidateShapeLimits(rule); // Custom rules are fully tenant-owned: fold a notify override into the row default // (there is no separate RuleState for them) — mirrors how the UI edits Enabled. @@ -473,6 +476,9 @@ namespace AutopilotMonitor.Functions.Services } } + // A template variable can carry the tenant's own regex (ANALYZE-SEC-006's allow-list). + ValidateShapeLimits(customRule); + // Store the custom rule in the tenant partition var success = await _ruleRepo.StoreAnalyzeRuleAsync(customRule, tenantId); if (!success) @@ -825,6 +831,77 @@ namespace AutopilotMonitor.Functions.Services private static bool IsTemplateWithin(string? text, int maxLength) => !string.IsNullOrWhiteSpace(text) && text.Length <= maxLength; + /// + /// Rejects a rule over the with + /// (mapped to 400) naming every offending field. Runs on create, custom-rule update (also the + /// portal's enable/notify toggles, which send the whole rule) and template copy. + /// + internal static void ValidateShapeLimits(AnalyzeRule rule) + { + var errors = ShapeLimitErrors(rule); + if (errors.Count > 0) + throw new ArgumentException("Invalid rule: " + string.Join("; ", errors)); + } + + /// + /// The size limits of a tenant-authored rule (D-410): the number of conditions and + /// preconditions, and every value compared with regex or not_regex (condition value, + /// filter value, event A filter value, precondition value). The length is checked before a + /// Regex is built; a value within it must then parse. Returns user-facing errors, empty when + /// the rule fits. One implementation for the dry-run validation and every write path. + /// + internal static List ShapeLimitErrors(AnalyzeRule rule) + { + var errors = new List(); + var conditions = rule.Conditions ?? new List(); + var preconditions = rule.Preconditions ?? new List(); + + if (conditions.Count > AnalyzeRuleLimits.MaxConditions) + errors.Add($"a rule holds at most {AnalyzeRuleLimits.MaxConditions} conditions (this one has {conditions.Count})"); + if (preconditions.Count > AnalyzeRuleLimits.MaxPreconditions) + errors.Add($"a rule holds at most {AnalyzeRuleLimits.MaxPreconditions} preconditions (this one has {preconditions.Count})"); + + for (var i = 0; i < conditions.Count; i++) + { + var c = conditions[i]; + if (c == null) continue; + CheckRegexValue(errors, $"conditions[{i}].value", c.Operator, c.Value); + CheckRegexValue(errors, $"conditions[{i}].filterValue", c.FilterOperator, c.FilterValue); + CheckRegexValue(errors, $"conditions[{i}].eventAFilterValue", c.EventAFilterOperator, c.EventAFilterValue); + } + for (var i = 0; i < preconditions.Count; i++) + { + var p = preconditions[i]; + if (p == null) continue; + CheckRegexValue(errors, $"preconditions[{i}].value", p.Operator, p.Value); + } + + return errors; + } + + private static void CheckRegexValue(List errors, string label, string? op, string? value) + { + if (string.IsNullOrEmpty(value) + || !(string.Equals(op, "regex", StringComparison.OrdinalIgnoreCase) + || string.Equals(op, "not_regex", StringComparison.OrdinalIgnoreCase))) + return; + + if (value.Length > AnalyzeRuleLimits.MaxRegexLength) + { + errors.Add($"{label}: a regex may be at most {AnalyzeRuleLimits.MaxRegexLength} characters (this one has {value.Length})"); + return; + } + + try + { + _ = new Regex(value, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + } + catch (ArgumentException ex) + { + errors.Add($"{label}: value is not a valid regex: {ex.Message}"); + } + } + internal static bool IsAbsoluteHttpUrl(string? url) { if (string.IsNullOrWhiteSpace(url)) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 39c5bb81..d3cb7408 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -552,6 +552,18 @@ namespace AutopilotMonitor.Functions.Services tenantId, upn ?? "System.ReportUploadGate", new { route, limit, windowMinutes = (int)window.TotalMinutes, upn }); + /// + /// Every slot of a rule-work gate of this instance was taken and a caller got 429 + /// (RuleWorkGate: dry-run, re-analysis or pattern test, D-410). Raised for the first + /// refusal of a window only; the gate keeps the rest in the Warning trace. Recorded against + /// the refused caller's tenant. + /// + public Task RecordRuleEvaluationLimitReachedAsync(string gate, int limit, TimeSpan window, string? tenantId, string? upn) + => WriteAsync(OpsEventCategory.Security, OpsEventTypes.RuleEvaluationLimitReached, OpsEventSeverity.Warning, + $"Rule work limit reached: the {gate} gate has {limit} evaluations running on this instance and a caller got 429; further refusals in the next {window.TotalMinutes:0} min stay in the Warning trace", + tenantId, upn ?? "System.RuleWorkGate", + new { gate, limit, windowMinutes = (int)window.TotalMinutes, upn }); + /// /// The reporter's per-instance budget is spent: one Critical marker (same event type, so the same /// alert rule pushes it) that says further denials this window are trace-only. A storm is a @@ -1041,6 +1053,22 @@ namespace AutopilotMonitor.Functions.Services lift, }); + /// + /// An analysis run stopped a rule (D-410): a regex hit its match timeout, or the run used up + /// its time budget in this rule and left later rules + /// unevaluated. Warning for a tenant's custom rule, Error for a built-in or community rule. + /// RuleEvaluationStopReporter throttles it; every stop is also in the Warning trace. + /// + public Task RecordRuleEvaluationStoppedAsync( + string tenantId, string sessionId, string ruleId, string reason, bool isCatalogRule, string runKind, int skippedRules) + => WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.RuleEvaluationStopped, + isCatalogRule ? OpsEventSeverity.Error : OpsEventSeverity.Warning, + reason == EvaluationStopReasons.RegexTimeout + ? $"Rule {ruleId} stopped in the {runKind} analysis of session {sessionId}: a regex hit its 1 s match timeout; the rule kept its previous result" + : $"The {runKind} analysis of session {sessionId} used up its time budget in rule {ruleId}; {skippedRules} later rule(s) were not evaluated and kept their previous results", + tenantId, "System.RuleEngine", + new { ruleId, reason, catalogRule = isCatalogRule, sessionId, runKind, skippedRules }); + /// /// App-version duration regression radar: an app's newest version installs with a median /// duration ≥2× (and ≥5 min absolute) over the previous version's median. Fired ONCE per diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs index a4066961..182d9104 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs @@ -29,7 +29,7 @@ namespace AutopilotMonitor.Functions.Services /// (e.g. "skip when an enrollment_complete event exists"). Any other operator without a /// field has nothing to compare → fail closed. /// - private bool EvaluatePrecondition(RulePrecondition precondition, List events) + private bool EvaluatePrecondition(RulePrecondition precondition, List events, RuleEvaluationBudget budget) { if (!string.Equals(precondition.Source, "event_data", StringComparison.OrdinalIgnoreCase)) { @@ -69,34 +69,34 @@ namespace AutopilotMonitor.Functions.Services return matchingEvents.Any(e => { var fieldValue = GetDataFieldValue(e, precondition.DataField); - return fieldValue != null && MatchesOperator(fieldValue, precondition.Operator ?? string.Empty, precondition.Value ?? string.Empty); + return fieldValue != null && MatchesOperator(fieldValue, precondition.Operator ?? string.Empty, precondition.Value ?? string.Empty, budget); }); } - private (bool matched, object evidence) EvaluateCondition(RuleCondition condition, List events) + private (bool matched, object evidence) EvaluateCondition(RuleCondition condition, List events, RuleEvaluationBudget budget) { switch (condition.Source) { case "event_type": - return EvaluateEventTypeCondition(condition, events); + return EvaluateEventTypeCondition(condition, events, budget); case "event_data": - return EvaluateEventDataCondition(condition, events); + return EvaluateEventDataCondition(condition, events, budget); case "event_data_array": - return EvaluateEventDataArrayCondition(condition, events); + return EvaluateEventDataArrayCondition(condition, events, budget); case "event_count": - return EvaluateEventCountCondition(condition, events); + return EvaluateEventCountCondition(condition, events, budget); case "phase_duration": return EvaluatePhaseDurationCondition(condition, events); case "app_install_duration": - return EvaluateAppInstallDurationCondition(condition, events); + return EvaluateAppInstallDurationCondition(condition, events, budget); case "event_correlation": - return EvaluateEventCorrelationCondition(condition, events); + return EvaluateEventCorrelationCondition(condition, events, budget); case "clock_skew": return EvaluateClockSkewCondition(condition, events); @@ -106,7 +106,7 @@ namespace AutopilotMonitor.Functions.Services } } - private (bool matched, object evidence) EvaluateEventTypeCondition(RuleCondition condition, List events) + private (bool matched, object evidence) EvaluateEventTypeCondition(RuleCondition condition, List events, RuleEvaluationBudget budget) { var matchingEvents = events.Where(e => MatchesEventType(e, condition.EventType)).ToList(); @@ -148,10 +148,10 @@ namespace AutopilotMonitor.Functions.Services foreach (var evt in matchingEvents) { var fieldValue = GetDataFieldValue(evt, condition.DataField); - if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value)) + if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value, budget)) { // Check suppressByEvent: skip this match if a resolving event exists - if (IsSuppressedByEvent(evt, condition.SuppressByEvent, events)) + if (IsSuppressedByEvent(evt, condition.SuppressByEvent, events, budget)) continue; var evidence = new Dictionary @@ -174,7 +174,7 @@ namespace AutopilotMonitor.Functions.Services // Filter out suppressed events if (condition.SuppressByEvent != null) { - matchingEvents = matchingEvents.Where(e => !IsSuppressedByEvent(e, condition.SuppressByEvent, events)).ToList(); + matchingEvents = matchingEvents.Where(e => !IsSuppressedByEvent(e, condition.SuppressByEvent, events, budget)).ToList(); if (!matchingEvents.Any()) return (false, "all matches suppressed by resolving events"); } @@ -204,19 +204,19 @@ namespace AutopilotMonitor.Functions.Services /// A missing filter field stringifies to empty, so equals excludes the event /// and not_equals keeps it. No filter configured = unchanged behaviour. /// - private List ApplyValueFilter(RuleCondition condition, List matchingEvents) + private List ApplyValueFilter(RuleCondition condition, List matchingEvents, RuleEvaluationBudget budget) { if (string.IsNullOrEmpty(condition.FilterField) || string.IsNullOrEmpty(condition.FilterOperator)) return matchingEvents; return matchingEvents - .Where(e => MatchesOperator(GetDataFieldValue(e, condition.FilterField) ?? string.Empty, condition.FilterOperator, condition.FilterValue)) + .Where(e => MatchesOperator(GetDataFieldValue(e, condition.FilterField) ?? string.Empty, condition.FilterOperator, condition.FilterValue, budget)) .ToList(); } - private (bool matched, object evidence) EvaluateEventDataCondition(RuleCondition condition, List events) + private (bool matched, object evidence) EvaluateEventDataCondition(RuleCondition condition, List events, RuleEvaluationBudget budget) { - var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList()); + var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList(), budget); // Field-absence gate — see EvaluateEventTypeCondition; here it applies to the // filtered event set, so "no event with filterField=x carries dataField" is expressible. @@ -226,7 +226,7 @@ namespace AutopilotMonitor.Functions.Services foreach (var evt in matchingEvents) { var fieldValue = GetDataFieldValue(evt, condition.DataField); - if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value)) + if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value, budget)) { var evidence = new Dictionary { @@ -288,9 +288,9 @@ namespace AutopilotMonitor.Functions.Services /// fire only for array elements NOT on the list. Evidence carries the matched item value /// under field=ItemField (so {{itemField}} interpolates) plus a capped sample. /// - private (bool matched, object evidence) EvaluateEventDataArrayCondition(RuleCondition condition, List events) + private (bool matched, object evidence) EvaluateEventDataArrayCondition(RuleCondition condition, List events, RuleEvaluationBudget budget) { - var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList()); + var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList(), budget); foreach (var evt in matchingEvents) { @@ -302,7 +302,7 @@ namespace AutopilotMonitor.Functions.Services foreach (var element in items) { var itemValue = GetItemFieldValue(element, condition.ItemField); - if (itemValue != null && MatchesOperator(itemValue, condition.Operator, condition.Value)) + if (itemValue != null && MatchesOperator(itemValue, condition.Operator, condition.Value, budget)) matchedValues.Add(itemValue); } @@ -436,11 +436,11 @@ namespace AutopilotMonitor.Functions.Services } } - private (bool matched, object evidence) EvaluateEventCountCondition(RuleCondition condition, List events) + private (bool matched, object evidence) EvaluateEventCountCondition(RuleCondition condition, List events, RuleEvaluationBudget budget) { // Optional value filter: count only events whose FilterField satisfies FilterOperator/FilterValue // (e.g. only performance_snapshot events with memory_used_percent > 90). - var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList()); + var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList(), budget); // count_per_group_gte: group by DataField value (e.g. appId), fire if any group >= threshold if (condition.Operator == "count_per_group_gte" && !string.IsNullOrEmpty(condition.DataField) && int.TryParse(condition.Value, out var groupThreshold)) @@ -656,9 +656,11 @@ namespace AutopilotMonitor.Functions.Services /// /// Checks if a matched event is suppressed by a resolving event (e.g., app_install_completed - /// suppresses app_install_failed when both share the same appId). + /// suppresses app_install_failed when both share the same appId). One call scans every event, + /// so a condition that calls it per matching event is quadratic in the session size — hence + /// the budget check before the scan. /// - private bool IsSuppressedByEvent(EnrollmentEvent matchedEvent, SuppressByEventConfig? config, List allEvents) + private bool IsSuppressedByEvent(EnrollmentEvent matchedEvent, SuppressByEventConfig? config, List allEvents, RuleEvaluationBudget budget) { if (config == null || string.IsNullOrEmpty(config.EventType) || string.IsNullOrEmpty(config.JoinField)) return false; @@ -667,6 +669,7 @@ namespace AutopilotMonitor.Functions.Services if (string.IsNullOrEmpty(joinValue)) return false; + budget.ThrowIfSpent(); return allEvents.Any(e => MatchesEventType(e, config.EventType) && string.Equals(GetDataFieldValue(e, config.JoinField), joinValue, StringComparison.OrdinalIgnoreCase)); @@ -795,8 +798,18 @@ namespace AutopilotMonitor.Functions.Services }; } - private bool MatchesOperator(string fieldValue, string op, string compareValue) + /// Per-match timeout of a rule's regex. A constant: the static + /// caches by pattern, options AND timeout. + private static readonly TimeSpan RegexMatchTimeout = TimeSpan.FromSeconds(1); + + /// + /// Every operator evaluation of a rule passes through here, once per event (or array + /// element) its condition looks at — so this is where the pass's budget is checked. + /// + private static bool MatchesOperator(string fieldValue, string op, string compareValue, RuleEvaluationBudget budget) { + budget.ThrowIfSpent(); + switch (op?.ToLower()) { case "equals": @@ -812,24 +825,10 @@ namespace AutopilotMonitor.Functions.Services return fieldValue.IndexOf(compareValue, StringComparison.OrdinalIgnoreCase) < 0; case "regex": - try - { - return Regex.IsMatch(fieldValue, compareValue, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); - } - catch - { - return false; - } + return MatchesRegex(fieldValue, compareValue) == true; case "not_regex": - try - { - return !Regex.IsMatch(fieldValue, compareValue, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); - } - catch - { - return false; - } + return MatchesRegex(fieldValue, compareValue) == false; case "gt": return double.TryParse(fieldValue, out var gt1) && double.TryParse(compareValue, out var gt2) && gt1 > gt2; @@ -860,6 +859,28 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// Case-insensitive match of a rule's pattern; null for a pattern that does not parse (one + /// saved before create and update checked them), which neither regex nor not_regex counts + /// as a match. A match timeout ends the rule (D-410): it used to read as "no match", and + /// every further event paid the same second again. + /// + private static bool? MatchesRegex(string fieldValue, string pattern) + { + try + { + return Regex.IsMatch(fieldValue, pattern, RegexOptions.IgnoreCase, RegexMatchTimeout); + } + catch (RegexMatchTimeoutException) + { + throw new RuleEvaluationStoppedException(EvaluationStopReasons.RegexTimeout); + } + catch (ArgumentException) + { + return null; + } + } + private static bool MatchesInList(string fieldValue, string compareValue) { if (string.IsNullOrEmpty(fieldValue) || string.IsNullOrEmpty(compareValue)) @@ -876,7 +897,7 @@ namespace AutopilotMonitor.Functions.Services return false; } - private (bool matched, object evidence) EvaluateAppInstallDurationCondition(RuleCondition condition, List events) + private (bool matched, object evidence) EvaluateAppInstallDurationCondition(RuleCondition condition, List events, RuleEvaluationBudget budget) { // Ordering key is Sequence, never Timestamp: Sequence is a persisted, strictly // monotonic per-session counter assigned at emit time and is immune to every clock @@ -901,6 +922,10 @@ namespace AutopilotMonitor.Functions.Services foreach (var completionEvent in completionEvents) { + // Each completion scans the session for its start event; the operator below is + // skipped for unmatched apps, so the budget is checked here. + budget.ThrowIfSpent(); + var appId = GetDataFieldValue(completionEvent, "appId"); var appName = GetDataFieldValue(completionEvent, "appName"); var appKey = !string.IsNullOrWhiteSpace(appId) ? appId : appName; @@ -929,7 +954,7 @@ namespace AutopilotMonitor.Functions.Services var durationSeconds = Math.Max(0, (completionEvent.Timestamp - startEvent.Timestamp).TotalSeconds); - if (!MatchesOperator(durationSeconds.ToString(), condition.Operator, condition.Value)) + if (!MatchesOperator(durationSeconds.ToString(), condition.Operator, condition.Value, budget)) continue; return (true, new Dictionary @@ -1169,7 +1194,7 @@ namespace AutopilotMonitor.Functions.Services /// DataField/Operator/Value = optional filter on Event B /// EventAFilterField/Operator/Value = optional filter on Event A /// - private (bool matched, object evidence) EvaluateEventCorrelationCondition(RuleCondition condition, List events) + private (bool matched, object evidence) EvaluateEventCorrelationCondition(RuleCondition condition, List events, RuleEvaluationBudget budget) { if (string.IsNullOrEmpty(condition.EventType) || string.IsNullOrEmpty(condition.CorrelateEventType) || @@ -1194,7 +1219,7 @@ namespace AutopilotMonitor.Functions.Services if (string.IsNullOrEmpty(condition.EventAFilterField)) return true; var val = GetDataFieldValue(e, condition.EventAFilterField); - return val != null && MatchesOperator(val, condition.EventAFilterOperator ?? "exists", condition.EventAFilterValue ?? ""); + return val != null && MatchesOperator(val, condition.EventAFilterOperator ?? "exists", condition.EventAFilterValue ?? "", budget); }) .ToList(); @@ -1206,7 +1231,7 @@ namespace AutopilotMonitor.Functions.Services if (string.IsNullOrEmpty(condition.DataField)) return true; var val = GetDataFieldValue(e, condition.DataField); - return val != null && MatchesOperator(val, condition.Operator ?? "exists", condition.Value ?? ""); + return val != null && MatchesOperator(val, condition.Operator ?? "exists", condition.Value ?? "", budget); }) .ToList(); @@ -1226,6 +1251,10 @@ namespace AutopilotMonitor.Functions.Services foreach (var eventB in eventBList) { + // Pairing scans the A events sharing the join key, so one shared key makes this + // loop quadratic without a single operator call. + budget.ThrowIfSpent(); + var bKey = GetDataFieldValue(eventB, condition.JoinField); if (string.IsNullOrEmpty(bKey) || !eventAByJoinKey.ContainsKey(bKey)) continue; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs index 75484ff4..ff2b16af 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs @@ -21,9 +21,10 @@ namespace AutopilotMonitor.Functions.Services /// Loads the session's events (strict read — storage failures propagate) and dry-runs the /// given draft rule against them. Mirrors the event handling of AnalyzeSessionAsync /// (strict load + BackfillDerivedEventFields) so the dry-run sees exactly what a real - /// analysis pass would see. + /// analysis pass would see. The evaluation runs on + /// and ends early when is signalled (D-410). /// - public async Task DryRunRuleAsync(string tenantId, string sessionId, AnalyzeRule rule) + public async Task DryRunRuleAsync(string tenantId, string sessionId, AnalyzeRule rule, CancellationToken cancellationToken = default) { var allEvents = await _sessionRepo.GetSessionEventsStrictAsync(tenantId, sessionId); if (allEvents.Count == 0) @@ -38,7 +39,7 @@ namespace AutopilotMonitor.Functions.Services } BackfillDerivedEventFields(allEvents); - return DryRunEvaluate(rule, allEvents); + return DryRunEvaluate(rule, allEvents, new RuleEvaluationBudget(RuleEvaluationBudget.DryRun, cancellationToken)); } /// @@ -46,8 +47,11 @@ namespace AutopilotMonitor.Functions.Services /// ORDER in lock-step with EvaluateRule — factors are evaluated sequentially against the /// accumulating matchedConditions dictionary (a factor's "exists" check can observe /// earlier factor_<signal> entries), so reordering would change results. + /// A stop by the budget ends the trace with verdict "stopped", a stop reason and a note + /// naming the precondition or condition it stopped at. Confidence factors need no budget + /// check: they never run an operator and scan the events at most once each. /// - internal RuleDryRun DryRunEvaluate(AnalyzeRule rule, List events) + internal RuleDryRun DryRunEvaluate(AnalyzeRule rule, List events, RuleEvaluationBudget budget) { var dry = new RuleDryRun { @@ -56,50 +60,68 @@ namespace AutopilotMonitor.Functions.Services ConfidenceThreshold = rule.ConfidenceThreshold, }; + // The precondition or condition being evaluated — a stop note names it. + var stage = string.Empty; var preconditionsPassed = true; - if (rule.Preconditions != null) + var matchedConditions = new Dictionary(); + var allRequiredMet = true; + try { - foreach (var pre in rule.Preconditions) + if (rule.Preconditions != null) { - var passed = EvaluatePrecondition(pre, events); - if (!passed) preconditionsPassed = false; - dry.Preconditions.Add(new RuleDryRunPrecondition + for (var i = 0; i < rule.Preconditions.Count; i++) { - Source = pre.Source, - EventType = pre.EventType, - DataField = string.IsNullOrEmpty(pre.DataField) ? null : pre.DataField, - Operator = pre.Operator, - Value = pre.Value, - Passed = passed, + var pre = rule.Preconditions[i]; + stage = $"preconditions[{i}] ({pre.EventType})"; + var passed = EvaluatePrecondition(pre, events, budget); + if (!passed) preconditionsPassed = false; + dry.Preconditions.Add(new RuleDryRunPrecondition + { + Source = pre.Source, + EventType = pre.EventType, + DataField = string.IsNullOrEmpty(pre.DataField) ? null : pre.DataField, + Operator = pre.Operator, + Value = pre.Value, + Passed = passed, + }); + } + } + + // Evaluate ALL conditions (no early break — the author wants the full picture). + // Outcome-equivalent to EvaluateRule: a failed required condition makes the + // production path return null before matchedConditions is ever used. + for (var i = 0; i < rule.Conditions.Count; i++) + { + var condition = rule.Conditions[i]; + stage = $"conditions[{i}] ('{condition.Signal}')"; + var (matched, evidence) = EvaluateCondition(condition, events, budget); + if (condition.Required && !matched) allRequiredMet = false; + if (matched) matchedConditions[condition.Signal] = evidence; + + dry.Conditions.Add(new RuleDryRunCondition + { + Signal = condition.Signal, + Source = condition.Source, + EventType = string.IsNullOrEmpty(condition.EventType) ? null : condition.EventType, + Required = condition.Required, + Matched = matched, + // Matched: the evidence dictionary the production path stores. Not matched: + // the evaluator's reason string ("no matching events", …). + Evidence = evidence, + MatchingEventCount = string.IsNullOrEmpty(condition.EventType) + ? null + : events.Count(e => MatchesEventType(e, condition.EventType)), }); } } - - // Evaluate ALL conditions (no early break — the author wants the full picture). - // Outcome-equivalent to EvaluateRule: a failed required condition makes the - // production path return null before matchedConditions is ever used. - var matchedConditions = new Dictionary(); - var allRequiredMet = true; - foreach (var condition in rule.Conditions) + catch (RuleEvaluationStoppedException stop) { - var (matched, evidence) = EvaluateCondition(condition, events); - if (condition.Required && !matched) allRequiredMet = false; - if (matched) matchedConditions[condition.Signal] = evidence; - - dry.Conditions.Add(new RuleDryRunCondition - { - Signal = condition.Signal, - Source = condition.Source, - EventType = string.IsNullOrEmpty(condition.EventType) ? null : condition.EventType, - Required = condition.Required, - Matched = matched, - // Matched: the evidence dictionary the production path stores. Not matched: - // the evaluator's reason string ("no matching events", …). - Evidence = evidence, - MatchingEventCount = string.IsNullOrEmpty(condition.EventType) - ? null - : events.Count(e => MatchesEventType(e, condition.EventType)), - }); + // The trace keeps what finished before the stop; the evidence map stays unset, + // so no client previews a half-evaluated finding. + dry.Verdict = RuleDryRunVerdict.Stopped; + dry.StopReason = stop.Reason; + dry.StopNote = StopNote(stage, stop.Reason, budget, events.Count); + return dry; } dry.MatchedConditions = matchedConditions; @@ -154,6 +176,16 @@ namespace AutopilotMonitor.Functions.Services dry.WouldMarkSessionAsFailed = rule.MarkSessionAsFailed ?? rule.MarkSessionAsFailedDefault; return dry; } + + /// What the author reads when the dry-run stopped: where, why, and whether trying again helps. + private static string StopNote(string stage, string reason, RuleEvaluationBudget budget, int eventCount) => reason switch + { + EvaluationStopReasons.RegexTimeout => + $"{stage}: a regex hit the {RegexMatchTimeout.TotalSeconds:0.#} s match timeout on this session's data — the pattern backtracks catastrophically; remove nested or overlapping quantifiers. Running the test again gives the same result, and the analysis would stop this rule the same way.", + EvaluationStopReasons.TimeBudget => + $"{stage}: the dry-run used up its {budget.Limit.TotalSeconds:0.#} s time budget on this session's {eventCount} events — the rule is too expensive here; narrow its event types or simplify its conditions.", + _ => $"{stage}: the request was cancelled before the evaluation finished.", + }; } // RuleDryRun, RuleDryRunVerdict and the per-stage trace classes are wire contract and live in diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs index 1260a2ba..9a959601 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.cs @@ -1,4 +1,3 @@ -using System.Text.RegularExpressions; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; using Microsoft.Extensions.Logging; @@ -25,6 +24,9 @@ namespace AutopilotMonitor.Functions.Services _logger = logger; } + /// Time budget of one analysis run (); tests shrink it. + internal TimeSpan RunBudget { get; init; } = RuleEvaluationBudget.AnalysisRun; + /// /// Evaluates analyze rules against the full session event stream. Fetches events internally. /// Legacy overload — maps to (or @@ -71,6 +73,13 @@ namespace AutopilotMonitor.Functions.Services /// ; they remain in /// so telemetry counts the attempt. /// + /// + /// Evaluation budget (D-410): the run gets for all rules + /// together. A regex match timeout stops that rule and the run goes on; a spent budget + /// stops the rule and leaves every later rule unevaluated. A rule that does not finish + /// keeps its stored row untouched () and is + /// listed in / . + /// /// public async Task AnalyzeSessionAsync(string tenantId, string sessionId, AnalyzeRunContext context) { @@ -119,22 +128,25 @@ namespace AutopilotMonitor.Functions.Services var now = DateTime.UtcNow; - foreach (var rule in activeRules) + // One budget for the whole run, started after the storage reads. Global rules + // (built-in, community) come before the tenant's custom rules — the merge order of + // GetAllRulesForTenantAsync — so a run that spends its budget runs out on the + // custom rules. A change to that order changes who pays (D-410). + var budget = new RuleEvaluationBudget(RunBudget); + + for (var i = 0; i < activeRules.Count; i++) { + var rule = activeRules[i]; + existingByRuleId.TryGetValue(rule.RuleId, out var existing); try { - existingByRuleId.TryGetValue(rule.RuleId, out var existing); - - // Classic dedupe: a FINAL row is settled — neither Terminal nor Interim - // touches it again. Only Reanalyze (explicit user action) re-opens it. - // Interim rows are always re-evaluated (refresh / finalize / resolve). - if (context.Kind != AnalyzeRunKind.Reanalyze && existing != null && !existing.IsInterim) + if (!IsEvaluatedIn(context.Kind, existing)) continue; // Track that this rule was evaluated (for telemetry) outcome.EvaluatedRules.Add(rule); - var result = EvaluateRule(rule, allEvents); + var result = EvaluateRule(rule, allEvents, budget); if (result != null) { result.SessionId = sessionId; @@ -179,12 +191,42 @@ namespace AutopilotMonitor.Functions.Services } } } + catch (RuleEvaluationStoppedException stop) + { + // The rule did not finish: whatever is stored for it stays as it is — a stop is + // no evidence that the finding went away. + KeepStoredRow(outcome, existing); + outcome.Stops.Add(new RuleEvaluationStop(rule.RuleId, stop.Reason, rule.IsBuiltIn || rule.IsCommunity)); + + if (stop.Reason == EvaluationStopReasons.RegexTimeout) + { + _logger.LogWarning( + "Rule evaluation stopped: {Reason} in rule {RuleId} for session {SessionId} of tenant {TenantId} ({Kind}) after {ElapsedMs} ms of the run", + stop.Reason, rule.RuleId, sessionId, tenantId, context.Kind, (long)budget.Elapsed.TotalMilliseconds); + continue; + } + + // The run's budget is spent: every later rule stays unevaluated, rows kept. + foreach (var later in activeRules.Skip(i + 1)) + { + existingByRuleId.TryGetValue(later.RuleId, out var laterExisting); + if (!IsEvaluatedIn(context.Kind, laterExisting)) + continue; + KeepStoredRow(outcome, laterExisting); + outcome.SkippedRuleIds.Add(later.RuleId); + } + _logger.LogWarning( + "Rule evaluation stopped: {Reason} in rule {RuleId} for session {SessionId} of tenant {TenantId} ({Kind}) after {ElapsedMs} ms of the run; {SkippedCount} later rule(s) not evaluated", + stop.Reason, rule.RuleId, sessionId, tenantId, context.Kind, (long)budget.Elapsed.TotalMilliseconds, outcome.SkippedRuleIds.Count); + break; + } catch (Exception ex) { // Per-rule evaluation failures are isolated: a buggy rule must not kill the // whole pass. Storage-layer exceptions from the surrounding code (rule loading, // event reading, results lookup, KO-criterion side-effects) are NOT caught here // and propagate to the caller — the queue worker depends on this for retry. + KeepStoredRow(outcome, existing); _logger.LogWarning(ex, $"Error evaluating rule {rule.RuleId}"); } } @@ -192,6 +234,27 @@ namespace AutopilotMonitor.Functions.Services return outcome; } + /// + /// Classic dedupe: a FINAL row is settled — neither Terminal nor Interim touches it again. + /// Only Reanalyze (explicit user action) re-opens it. Interim rows are always re-evaluated + /// (refresh / finalize / resolve). + /// + private static bool IsEvaluatedIn(AnalyzeRunKind kind, RuleResult? existing) + => kind == AnalyzeRunKind.Reanalyze || existing == null || existing.IsInterim; + + /// + /// Hands the stored row of a rule this run could not evaluate back to the caller, unless + /// the rule already produced an outcome entry of its own. + /// + private static void KeepStoredRow(AnalysisOutcome outcome, RuleResult? existing) + { + if (existing == null + || outcome.Results.Any(r => r.RuleId == existing.RuleId) + || outcome.ResolvedResults.Contains(existing)) + return; + outcome.KeptResults.Add(existing); + } + /// True when the rule's evaluateOn matches this interim run's trigger. private static bool MatchesInterimRun(AnalyzeRule rule, AnalyzeRunContext context) { @@ -326,7 +389,7 @@ namespace AutopilotMonitor.Functions.Services /// /// Evaluates a single rule against the full session event stream /// - private RuleResult? EvaluateRule(AnalyzeRule rule, List events) + private RuleResult? EvaluateRule(AnalyzeRule rule, List events, RuleEvaluationBudget budget) { // Preconditions gate (AND-semantics, silent skip): if any precondition fails the // rule is not evaluated at all — no result row, no UI card. Distinct from @@ -335,7 +398,7 @@ namespace AutopilotMonitor.Functions.Services { foreach (var pre in rule.Preconditions) { - if (!EvaluatePrecondition(pre, events)) + if (!EvaluatePrecondition(pre, events, budget)) { _logger.LogDebug( "Rule {RuleId} skipped by precondition (eventType={EventType}, field={Field}, op={Op}, value={Value})", @@ -352,7 +415,7 @@ namespace AutopilotMonitor.Functions.Services // Evaluate each condition foreach (var condition in rule.Conditions) { - var (matched, evidence) = EvaluateCondition(condition, events); + var (matched, evidence) = EvaluateCondition(condition, events, budget); if (condition.Required && !matched) { @@ -430,8 +493,28 @@ namespace AutopilotMonitor.Functions.Services /// All rules that were evaluated in this pass (includes rules that didn't fire) public List EvaluatedRules { get; set; } = new List(); + + /// + /// Stored rows of rules this pass could not evaluate to the end (stopped, skipped after the + /// budget ran out, or thrown), unchanged. A caller that replaces a session's rows wholesale + /// (the manual re-analysis) writes them back; the queue path upserts and leaves them alone. + /// + public List KeptResults { get; set; } = new List(); + + /// Rules the evaluation budget stopped in this pass (D-410); callers report them. + public List Stops { get; set; } = new List(); + + /// Rules never started because the pass's budget ran out before them. + public List SkippedRuleIds { get; set; } = new List(); } + /// + /// One rule a pass stopped: is regex_timeout or time_budget + /// (EvaluationStopReasons); tells a built-in or community + /// rule (our pattern) from a tenant's custom rule (the author's). + /// + public sealed record RuleEvaluationStop(string RuleId, string Reason, bool IsCatalogRule); + /// Which flavor of analyze run is executing — decides rule scope, dedupe and KO behavior. public enum AnalyzeRunKind { diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEvaluationBudget.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEvaluationBudget.cs new file mode 100644 index 00000000..df124ab3 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEvaluationBudget.cs @@ -0,0 +1,58 @@ +using System.Diagnostics; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Wall-clock bound on one rule-evaluation pass: one analysis run (every active rule against a + /// session) or one dry-run of a draft. Tenant-authored rules decide how much work a pass does + /// (conditions × events × operator cost), so the pass is what gets bounded, not the single + /// regex match (D-410). The engine checks it before every operator evaluation and in the + /// evaluator loops that scan events without one; a spent budget or a signalled token throws + /// . + /// + internal sealed class RuleEvaluationBudget + { + /// One analysis run, all rules together (queue, interim, "Analyze now"). The + /// slowest production runs spend under 0.3 s outside storage I/O. + internal static readonly TimeSpan AnalysisRun = TimeSpan.FromSeconds(5); + + /// One dry-run of a draft rule; the logparser pattern test has the same 2 s (D-379). + internal static readonly TimeSpan DryRun = TimeSpan.FromSeconds(2); + + private readonly long _start = Stopwatch.GetTimestamp(); + private readonly CancellationToken _cancellationToken; + + internal RuleEvaluationBudget(TimeSpan limit, CancellationToken cancellationToken = default) + { + Limit = limit; + _cancellationToken = cancellationToken; + } + + internal TimeSpan Limit { get; } + + internal TimeSpan Elapsed => Stopwatch.GetElapsedTime(_start); + + internal void ThrowIfSpent() + { + if (_cancellationToken.IsCancellationRequested) + throw new RuleEvaluationStoppedException(EvaluationStopReasons.Cancelled); + if (Elapsed >= Limit) + throw new RuleEvaluationStoppedException(EvaluationStopReasons.TimeBudget); + } + } + + /// + /// A rule evaluation ended early. is one of : + /// a regex hit its match timeout, the pass ran out of budget, or the caller went away. + /// + internal sealed class RuleEvaluationStoppedException : Exception + { + internal RuleEvaluationStoppedException(string reason) : base($"Rule evaluation stopped: {reason}") + { + Reason = reason; + } + + internal string Reason { get; } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEvaluationStopReporter.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEvaluationStopReporter.cs new file mode 100644 index 00000000..01527b25 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEvaluationStopReporter.cs @@ -0,0 +1,70 @@ +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Turns the stops of an analysis run (, D-410) into + /// RuleEvaluationStopped ops events (singleton). A tenant decides how many custom rules it + /// has and how often its sessions are analysed, so the alarm is capped per instance: one event + /// per tenant and rule per hour, and per hour in total. The + /// engine logs every stop at Warning; this only decides which ones also reach the operators. + /// + public sealed class RuleEvaluationStopReporter + { + internal static readonly TimeSpan Window = TimeSpan.FromHours(1); + internal const int MaxEventsPerWindow = 20; + + private readonly OpsEventService _opsEvents; + private readonly object _lock = new(); + private readonly Dictionary _lastEventByRule = new(StringComparer.Ordinal); + private DateTime _windowStart = DateTime.MinValue; + private int _eventsInWindow; + + public RuleEvaluationStopReporter(OpsEventService opsEvents) + { + _opsEvents = opsEvents; + } + + public async Task ReportAsync(string tenantId, string sessionId, AnalyzeRunKind kind, AnalysisOutcome outcome) + { + foreach (var stop in outcome.Stops) + { + if (!Take(tenantId, stop.RuleId, DateTime.UtcNow)) + continue; + + // Only a stop by the spent budget leaves later rules unevaluated. + var skipped = stop.Reason == EvaluationStopReasons.RegexTimeout ? 0 : outcome.SkippedRuleIds.Count; + await _opsEvents.RecordRuleEvaluationStoppedAsync( + tenantId, sessionId, stop.RuleId, stop.Reason, stop.IsCatalogRule, kind.ToString(), skipped); + } + } + + /// True when this stop may raise an event: first of its tenant and rule within the + /// window, and the instance's window still has room. + internal bool Take(string tenantId, string ruleId, DateTime utcNow) + { + lock (_lock) + { + if (utcNow - _windowStart >= Window) + { + _windowStart = utcNow; + _eventsInWindow = 0; + // Entries come only from taken events, so the map never outgrows two windows' cap. + foreach (var stale in _lastEventByRule.Where(e => utcNow - e.Value >= Window).Select(e => e.Key).ToList()) + _lastEventByRule.Remove(stale); + } + + if (_eventsInWindow >= MaxEventsPerWindow) + return false; + + var key = tenantId + "|" + ruleId; + if (_lastEventByRule.TryGetValue(key, out var last) && utcNow - last < Window) + return false; + + _lastEventByRule[key] = utcNow; + _eventsInWindow++; + return true; + } + } + } +} diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts index 68c9e138..0d360ea4 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts @@ -93,6 +93,43 @@ describe('test_analyze_rule', () => { const res = await handlers.test_analyze_rule({ sessionId: SESSION, rule: ANALYZE_DRAFT }); expect(res.isError).toBe(true); }); + + it('waits out a full gate once and hands a stop through with its reason and note', async () => { + apiFetchMock.mockResolvedValue({ + success: true, + sessionId: SESSION, + result: { + verdict: 'stopped', + stopReason: 'regex_timeout', + stopNote: "conditions[0] ('app_failure'): a regex hit the 1 s match timeout on this session's data", + }, + }); + + const handlers = captureToolHandlers(false); + const res = await handlers.test_analyze_rule({ sessionId: SESSION, rule: ANALYZE_DRAFT }); + + // The dry-run has no side effects, so apiFetch may retry the POST once on 429 (Retry-After ≤ 10 s). + const [, init] = apiFetchMock.mock.calls[0] as [string, { retry?: boolean }]; + expect(init.retry).toBe(true); + const payload = JSON.parse(res.content[0].text); + expect(payload.result.verdict).toBe('stopped'); + expect(payload.result.stopReason).toBe('regex_timeout'); + expect(payload.result.stopNote).toContain('conditions[0]'); + }); +}); + +describe('test_log_pattern', () => { + it('waits out a full gate once: the pattern test is pure compute', async () => { + apiFetchMock.mockResolvedValue({ success: true, format: 'text', result: { matchCount: 1, parseFailureCount: 0, timeoutCount: 0, lines: [], notes: [] } }); + + const handlers = captureToolHandlers(false); + await handlers.test_log_pattern({ pattern: 'a', sampleLines: ['a'], format: 'text' }); + + const [path, init] = apiFetchMock.mock.calls[0] as [string, { method?: string; retry?: boolean }]; + expect(path).toBe('/api/rules/gather/test-pattern'); + expect(init.method).toBe('POST'); + expect(init.retry).toBe(true); + }); }); describe('registration by role', () => { diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index 39ad3707..753cf076 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -2406,6 +2406,8 @@ export interface GetRuleResultsResponse { persistFailureCount: number; /** Rule ids that failed to persist during the reanalyze — the key is omitted entirely when there were no failures (the happy-path contract stays unchanged). */ persistFailureRuleIds?: string[]; + /** Rule ids the reanalyze could not evaluate within its time limit (a regex timed out, or the run's budget ran out first); their previous results are kept. The key is omitted when every rule was evaluated. */ + stoppedRuleIds?: string[]; } /** Response of POST tenants/{tenantId}/scripts/display-names: resolved Intune script display names keyed by the canonical ref string. Always 200, possibly partial — unresolved refs stay in the dictionary with a null VALUE (dictionary values are NOT subject to WhenWritingNull, so the key is emitted with an explicit JSON null). */ @@ -3237,7 +3239,7 @@ export interface LogPatternTestResult { matchCount: number; parseFailureCount: number; timeoutCount: number; - /** Set when the test ended before the last sample line was evaluated: regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated. */ + /** Set when the test ended before the last sample line was evaluated (EvaluationStopReasons): regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated. */ stopReason?: string; readonly lines: LogPatternLineResult[]; readonly notes: string[]; @@ -4189,6 +4191,10 @@ export interface RuleDryRun { wouldMarkSessionAsFailed: boolean; /** The evidence map exactly as the production path would persist it on a RuleResult — keys are condition signals (plus factor_* markers). Clients use it to preview {{token}} interpolation of explanation/remediation. Values are heterogeneous evidence objects by design. */ matchedConditions?: Record; + /** Set with verdict "stopped": regex_timeout | time_budget | cancelled. The trace then holds only the entries evaluated before the stop. Null otherwise. */ + stopReason?: string; + /** Set with verdict "stopped": which precondition or condition the evaluation stopped at, and what to change. */ + stopNote?: string; } export interface RuleDryRunCondition { diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts index daa21341..f978bf95 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts @@ -115,6 +115,7 @@ export const OPS_EVENT_TYPES = [ "PrivilegedRouteDenied", "McpServicePrincipalFirstSeen", "ReportUploadLimitReached", + "RuleEvaluationLimitReached", "OffboardingFeedbackReceived", "FeedbackReceived", "FeedbackRatingReceived", @@ -142,6 +143,7 @@ export const OPS_EVENT_TYPES = [ "CollectLogsQuickConfigEnabled", "DiagnosticsUploadEnabled", "DiagnosticsUploadDisabled", + "RuleEvaluationStopped", "SessionActionQueued", "SessionTimeouts", "AgentEmergencyBreak", diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts index b237193e..9e6eb1ac 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts @@ -204,8 +204,9 @@ export const RULE_AUTHORING_GUIDE = { 'gt', 'lt', 'gte', 'lte', 'exists', 'not_exists', 'count_gte', 'count_per_group_gte', 'in', 'not_in', ], operatorNotes: - 'String comparisons are case-insensitive; regex runs case-insensitive with a 1s timeout ' + - '(.NET regex syntax); in/not_in take a comma-separated value list. exists/not_exists with a ' + + 'String comparisons are case-insensitive; regex runs case-insensitive (.NET regex syntax, at most ' + + '2,000 characters) — a match that runs past 1 s stops the whole rule for that session (no finding), ' + + 'so avoid nested or overlapping quantifiers; in/not_in take a comma-separated value list. exists/not_exists with a ' + 'dataField test whether ANY event of the type carries a non-empty value there — not_exists ' + 'matches when none does, also when the event type is absent (e.g. "the ESP failure carried no ' + 'errorCode"); without a dataField they test only the presence of the event type. An operator ' + diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts index a3670b7d..f8469017 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts @@ -338,7 +338,8 @@ export const ANALYZE_RULE_SCHEMA: Record = { "items": { "$ref": "#/$defs/ruleCondition" }, - "minItems": 1 + "minItems": 1, + "maxItems": 20 }, "preconditions": { "type": "array", @@ -346,6 +347,7 @@ export const ANALYZE_RULE_SCHEMA: Record = { "items": { "$ref": "#/$defs/rulePrecondition" }, + "maxItems": 10, "default": [] }, "baseConfidence": { diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts index 09ca7430..ff8c465d 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts @@ -100,6 +100,8 @@ export function registerRuleTools(server: McpServer, ga: boolean, ruleDryRun: bo const data = await apiFetch('/api/rules/gather/test-pattern', { method: 'POST', body: jsonBody({ pattern: args.pattern, format: args.format, sampleLines: args.sampleLines }), + // Pure compute without side effects: a full per-instance gate (429) is waited out once. + retry: true, }); return toolResultText(data, MAX_RESULT_SIZE_CHARS.small); } catch (error: unknown) { @@ -116,7 +118,7 @@ export function registerRuleTools(server: McpServer, ga: boolean, ruleDryRun: bo description: 'Dry-runs a DRAFT analyze rule against one real session\'s events and returns the full diagnostic trace: ' + 'verdict (fired / required_condition_not_met / below_confidence_threshold / skipped_by_precondition / ' + - 'no_conditions_matched / no_events), per-condition matched/evidence with matchingEventCount (how many session ' + + 'no_conditions_matched / no_events / stopped: stopNote says why), per-condition matched/evidence with matchingEventCount (how many session ' + 'events have that eventType at all — the first thing to check when a condition unexpectedly misses), the ' + 'confidence breakdown per factor, and the interpolated explanation and presentation preview (facts the session ' + 'did not record listed). NOTHING is persisted — safe to ' + @@ -136,6 +138,8 @@ export function registerRuleTools(server: McpServer, ga: boolean, ruleDryRun: bo // The draft arrives as a loose object; the backend validates it (ValidateDraftRule) and // answers 400 with the reasons. The cast names the wire document the endpoint deserialises. body: jsonBody({ sessionId: args.sessionId, rule: args.rule as unknown as AnalyzeRule }), + // Side-effect free: a full per-instance gate (429) is waited out once. + retry: true, }); const rule = args.rule as Record; diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs index e1a49d81..1efa41d1 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/OpsEventTypes.cs @@ -116,6 +116,13 @@ namespace AutopilotMonitor.Shared.DataAccess /// unusual load or someone aiming large bodies at the worker. /// public const string ReportUploadLimitReached = "ReportUploadLimitReached"; + /// + /// Every slot of a rule-work gate of an instance was taken and a caller was answered 429: + /// the analyze dry-run, the manual re-analysis or the logparser pattern test (details: + /// gate). Warning; at most one per instance, gate and hour, further refusals stay in + /// the Warning trace. Legitimate use never fills a gate, so this is load aimed at it. + /// + public const string RuleEvaluationLimitReached = "RuleEvaluationLimitReached"; // ── Tenant ── Tenant lifecycle: offboarding, trials, plan changes, regression radars, config toggles. public const string OffboardingFeedbackReceived = "OffboardingFeedbackReceived"; @@ -169,6 +176,14 @@ namespace AutopilotMonitor.Shared.DataAccess public const string CollectLogsQuickConfigEnabled = "CollectLogsQuickConfigEnabled"; public const string DiagnosticsUploadEnabled = "DiagnosticsUploadEnabled"; public const string DiagnosticsUploadDisabled = "DiagnosticsUploadDisabled"; + /// + /// An analysis run stopped a rule because a regex hit its match timeout, or used up its time + /// budget in it (details: reason, skippedRules). Warning for a tenant's custom + /// rule (the author's pattern), Error for a built-in or community rule (ours met data it + /// cannot handle, or the instance stalled). At most one per instance, tenant and rule per + /// hour and 20 per instance and hour; every stop is in the Warning trace. + /// + public const string RuleEvaluationStopped = "RuleEvaluationStopped"; // ── Agent ── Agent + device-side health signals. public const string SessionActionQueued = "SessionActionQueued"; @@ -215,8 +230,8 @@ namespace AutopilotMonitor.Shared.DataAccess { ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles, GraphAccessRejected, GraphAccessRecovered, GraphAppCredentialRejected, AddOnPermissionMissing, AddOnPermissionResolved, MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, OrphanSweepCompleted, OrphanSweepFailed, OrphanSweepSkippedLocked, OrphanReconcileCompleted, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed, - DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen, ReportUploadLimitReached, - OffboardingFeedbackReceived, FeedbackReceived, FeedbackRatingReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled, + DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen, ReportUploadLimitReached, RuleEvaluationLimitReached, + OffboardingFeedbackReceived, FeedbackReceived, FeedbackRatingReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled, RuleEvaluationStopped, SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected, SlaBreachNotification, SlaConsecutiveFailures, SlaEvaluationCompleted, AzureMonitorAlert, PushDeliveryFailed, PushEndpointRefused, diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Rules/AnalyzeRuleLimits.cs b/src/Shared/AutopilotMonitor.Shared/Models/Rules/AnalyzeRuleLimits.cs new file mode 100644 index 00000000..8ab77cfb --- /dev/null +++ b/src/Shared/AutopilotMonitor.Shared/Models/Rules/AnalyzeRuleLimits.cs @@ -0,0 +1,22 @@ +namespace AutopilotMonitor.Shared.Models +{ + /// + /// Size limits of a tenant-authored analyze rule, enforced on dry-run, create, update and + /// template copy. The rule's shape sets how much work every analysis of the tenant's sessions + /// does on the shared backend, so it is bounded where the rule enters. The schema carries the + /// two counts. + /// + public static class AnalyzeRuleLimits + { + /// Maximum number of conditions. + public const int MaxConditions = 20; + + /// Maximum number of preconditions. + public const int MaxPreconditions = 10; + + /// Maximum length of a value compared with the regex or not_regex operator, + /// wherever a rule can hold one (condition value, filter value, event A filter value, + /// precondition value). Same cap as the logparser pattern test. + public const int MaxRegexLength = 2000; + } +} diff --git a/src/Shared/AutopilotMonitor.Shared/Models/RulesApiModels.cs b/src/Shared/AutopilotMonitor.Shared/Models/RulesApiModels.cs index a6c0d405..55ea8e22 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/RulesApiModels.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/RulesApiModels.cs @@ -67,6 +67,27 @@ namespace AutopilotMonitor.Shared.Models public const string NoConditionsMatched = "no_conditions_matched"; public const string BelowConfidenceThreshold = "below_confidence_threshold"; public const string NoEvents = "no_events"; + + /// The evaluation hit a limit before it finished; + /// says which and where. + public const string Stopped = "stopped"; + } + + /// + /// Why an evaluation of caller-authored patterns ended early — the values of + /// and . + /// Stable API contract. + /// + public static class EvaluationStopReasons + { + /// A regex hit its 1 s match timeout (catastrophic backtracking). + public const string RegexTimeout = "regex_timeout"; + + /// The evaluation as a whole used up its time budget. + public const string TimeBudget = "time_budget"; + + /// The caller went away (client disconnect, host shutdown). + public const string Cancelled = "cancelled"; } /// Full diagnostic trace of one dry-run evaluation. Serialized camelCase to clients. @@ -102,6 +123,14 @@ namespace AutopilotMonitor.Shared.Models /// preview {{token}} interpolation of explanation/remediation. Values are heterogeneous /// evidence objects by design. public Dictionary? MatchedConditions { get; set; } + + /// Set with verdict "stopped": regex_timeout | time_budget | cancelled. The + /// trace then holds only the entries evaluated before the stop. Null otherwise. + public string? StopReason { get; set; } + + /// Set with verdict "stopped": which precondition or condition the evaluation + /// stopped at, and what to change. + public string? StopNote { get; set; } } // Declaration order == wire order. @@ -169,6 +198,11 @@ namespace AutopilotMonitor.Shared.Models /// Rule ids that failed to persist during the reanalyze — the key is omitted /// entirely when there were no failures (the happy-path contract stays unchanged). public IReadOnlyList? PersistFailureRuleIds { get; set; } + + /// Rule ids the reanalyze could not evaluate within its time limit (a regex + /// timed out, or the run's budget ran out first); their previous results are kept. The + /// key is omitted when every rule was evaluated. + public IReadOnlyList? StoppedRuleIds { get; set; } } /// Response of POST rules/ime-log-patterns/reseed (Global Admin only). @@ -204,10 +238,11 @@ namespace AutopilotMonitor.Shared.Models public int ParseFailureCount { get; set; } public int TimeoutCount { get; set; } - /// Set when the test ended before the last sample line was evaluated: - /// regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | - /// time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop - /// have no row; a note names them. Null when every line was evaluated. + /// Set when the test ended before the last sample line was evaluated + /// (EvaluationStopReasons): regex_timeout (the first line that hit the agent's 1 s match + /// timeout ends the test) | time_budget (the whole test exceeded its time budget) | + /// cancelled. Lines after the stop have no row; a note names them. Null when every line + /// was evaluated. public string? StopReason { get; set; } public List Lines { get; } = new(); diff --git a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx index 4c07acc2..087c0b70 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/OpsAlertRulesSection.tsx @@ -144,6 +144,10 @@ const OPS_EVENT_TYPES: Record = { // Every report-upload slot of an instance was taken (ReportUploadGate) and a caller got 429. // Warning, at most one per instance and hour; route it to a push channel to be told. "ReportUploadLimitReached", + // A rule-work gate of an instance was full (analyze dry-run, "Analyze now" or pattern test, + // details.gate) and a caller got 429. Warning, at most one per instance, gate and hour. + // Legitimate use never fills a gate: this is load aimed at it. + "RuleEvaluationLimitReached", "VersionBlocked", // Delivery confirmation: a Kill signal was actually SERVED to an agent (config or // telemetry channel) — as opposed to DeviceBlocked/VersionBlocked which fire on rule @@ -248,6 +252,10 @@ const OPS_EVENT_TYPES: Record = { // through the quick-config dialog (PUT ?intent=collect-logs). Replaces // DiagnosticsUploadEnabled for that path only — the targeted push signal. "CollectLogsQuickConfigEnabled", + // An analysis run stopped a rule: a regex hit its match timeout, or the run's time budget ran + // out in it. Warning for a tenant's custom rule, Error for a built-in or community rule — route + // Error to a push channel. At most one per instance, tenant and rule per hour, 20 per hour. + "RuleEvaluationStopped", ], Agent: [ "BlobStorageMissing", diff --git a/src/Web/autopilot-monitor-web/app/analyze-rules/__tests__/ruleDryRunLogic.test.ts b/src/Web/autopilot-monitor-web/app/analyze-rules/__tests__/ruleDryRunLogic.test.ts index 2ee85279..fb8fc033 100644 --- a/src/Web/autopilot-monitor-web/app/analyze-rules/__tests__/ruleDryRunLogic.test.ts +++ b/src/Web/autopilot-monitor-web/app/analyze-rules/__tests__/ruleDryRunLogic.test.ts @@ -29,12 +29,20 @@ describe("verdictLabel", () => { ["fired", "Would fire"], ["required_condition_not_met", "A required condition did not match"], ["below_confidence_threshold", "Below the confidence threshold"], + ["stopped", "Stopped at an evaluation limit"], ["some_future_verdict", "some_future_verdict"], ])("%s → %s", (verdict, label) => { expect(verdictLabel(verdict)).toBe(label); }); }); +describe("confidenceSummary of a stopped dry-run", () => { + it("names no score — the evaluation never reached the confidence stage", () => { + expect(confidenceSummary({ ...base, verdict: "stopped", finalConfidence: undefined, confidenceFactors: [], stopReason: "regex_timeout" })) + .toBe("not scored (threshold 70)"); + }); +}); + describe("confidenceSummary", () => { it("adds the matched factor weights to the base", () => { expect(confidenceSummary(base)).toBe("85 (base 70 + 15, threshold 70)"); diff --git a/src/Web/autopilot-monitor-web/app/analyze-rules/components/RuleDryRunPanel.tsx b/src/Web/autopilot-monitor-web/app/analyze-rules/components/RuleDryRunPanel.tsx index 8042954b..828d4075 100644 --- a/src/Web/autopilot-monitor-web/app/analyze-rules/components/RuleDryRunPanel.tsx +++ b/src/Web/autopilot-monitor-web/app/analyze-rules/components/RuleDryRunPanel.tsx @@ -128,6 +128,9 @@ function DryRunResult({ outcome }: { outcome: Outcome }) { )} + {/* Verdict "stopped": where the evaluation hit its limit and what to change. */} + {result.stopNote &&

{result.stopNote}

} + {result.preconditions.length > 0 && (
    {result.preconditions.map((p, i) => ( @@ -139,24 +142,26 @@ function DryRunResult({ outcome }: { outcome: Outcome }) {
)} -
    - {result.conditions.map((c) => { - const evidence = evidenceText(c); - return ( -
  • - - {c.matched ? "Matched:" : "Not matched:"} - {c.signal} - - {[c.eventType ?? c.source, c.required ? "required" : "optional", eventCountText(c)].filter(Boolean).join(" · ")} - - {evidence && {evidence}} -
    • - ); - })} -
+ {result.conditions.length > 0 && ( +
    + {result.conditions.map((c) => { + const evidence = evidenceText(c); + return ( +
  • + + {c.matched ? "Matched:" : "Not matched:"} + {c.signal} + + {[c.eventType ?? c.source, c.required ? "required" : "optional", eventCountText(c)].filter(Boolean).join(" · ")} + + {evidence && {evidence}} +
    • + ); + })} +
+ )} {fired && presentation && (
diff --git a/src/Web/autopilot-monitor-web/app/analyze-rules/ruleDryRunLogic.ts b/src/Web/autopilot-monitor-web/app/analyze-rules/ruleDryRunLogic.ts index b6ec6dbd..a1654cb0 100644 --- a/src/Web/autopilot-monitor-web/app/analyze-rules/ruleDryRunLogic.ts +++ b/src/Web/autopilot-monitor-web/app/analyze-rules/ruleDryRunLogic.ts @@ -50,6 +50,7 @@ const VERDICT_LABELS: Record = { no_conditions_matched: "No condition matched", below_confidence_threshold: "Below the confidence threshold", no_events: "The session has no events", + stopped: "Stopped at an evaluation limit", }; export function verdictLabel(verdict: string): string { diff --git a/src/Web/autopilot-monitor-web/app/sessions/components/AnalysisResultsSection.tsx b/src/Web/autopilot-monitor-web/app/sessions/components/AnalysisResultsSection.tsx index e4d875c8..4ef8fa7b 100644 --- a/src/Web/autopilot-monitor-web/app/sessions/components/AnalysisResultsSection.tsx +++ b/src/Web/autopilot-monitor-web/app/sessions/components/AnalysisResultsSection.tsx @@ -25,6 +25,9 @@ interface AnalysisResultsSectionProps { * persist" carries the underlying Storage error for diagnosis. */ persistFailureRuleIds?: string[]; + /** Rule IDs the most recent reanalyze stopped at its time limit (a regex took too long, or the run's + * budget ran out first). They keep their previous result; a yellow banner names them. */ + stoppedRuleIds?: string[]; /** Hides the "Analyze Now" control when false — a read-only cross-tenant (delegated/MSP) viewer must not * trigger a re-analysis. Defaults to true, so own-tenant / Global Admin behavior is unchanged. */ canReanalyze?: boolean; @@ -44,6 +47,7 @@ export default function AnalysisResultsSection({ setAnalysisExpanded, onReanalyze, persistFailureRuleIds, + stoppedRuleIds, canReanalyze = true, sessionTenantId, emptyState, @@ -135,7 +139,7 @@ export default function AnalysisResultsSection({ {persistFailureRuleIds && persistFailureRuleIds.length > 0 && (
{persistFailureRuleIds.length === 1 @@ -152,6 +156,24 @@ export default function AnalysisResultsSection({
)} + {stoppedRuleIds && stoppedRuleIds.length > 0 && ( +
+
+ {stoppedRuleIds.length === 1 + ? "1 rule stopped at the time limit" + : `${stoppedRuleIds.length} rules stopped at the time limit`} +
+
+ They kept their previous result. A rule stops when one of its patterns takes longer than 1 second. +
+
+ {stoppedRuleIds.join(", ")} +
+
+ )} {loadingAnalysis && analysisResults.length === 0 ? (
Running analysis...
) : openResults.length === 0 && resolvedResults.length === 0 && emptyState ? ( diff --git a/src/Web/autopilot-monitor-web/app/sessions/hooks/useSessionAnalysis.ts b/src/Web/autopilot-monitor-web/app/sessions/hooks/useSessionAnalysis.ts index ded49131..f1c56e37 100644 --- a/src/Web/autopilot-monitor-web/app/sessions/hooks/useSessionAnalysis.ts +++ b/src/Web/autopilot-monitor-web/app/sessions/hooks/useSessionAnalysis.ts @@ -10,10 +10,12 @@ export function useSessionAnalysis( sessionId: string, sessionTenantId: string | null, getAccessToken: (forceRefresh?: boolean) => Promise, - /** withVulnerabilityReport false: a page without the report (diagnosis) never reads it, also not on pushes. */ - opts?: { withVulnerabilityReport?: boolean } + /** withVulnerabilityReport false: a page without the report (diagnosis) never reads it, also not on pushes. + * notifyError: shows a failed "Analyze Now" click (busy backend, error) to the user; reads stay silent. */ + opts?: { withVulnerabilityReport?: boolean; notifyError?: (title: string, err: unknown, key?: string, fallback?: string) => void } ) { const withVulnerabilityReport = opts?.withVulnerabilityReport ?? true; + const notifyError = opts?.notifyError; const [analysisResults, setAnalysisResults] = useState([]); const [loadingAnalysis, setLoadingAnalysis] = useState(false); // True after the first successful read: before it (and after only failed reads) an empty list means @@ -24,6 +26,8 @@ export function useSessionAnalysis( // Backend reports them in /sessions/{id}/analysis response so the UI can render // a warning banner without blocking the rules that DID persist. const [persistFailureRuleIds, setPersistFailureRuleIds] = useState([]); + // Rule IDs the most recent reanalyze stopped at its time limit; they kept their previous result. + const [stoppedRuleIds, setStoppedRuleIds] = useState([]); const fetchAnalysisResults = useCallback(async (reanalyze = false) => { if (!sessionTenantId || !isGuid(sessionTenantId)) return; @@ -41,13 +45,16 @@ export function useSessionAnalysis( // more StoreRuleResultAsync calls returned false during the reanalyze loop. We always // set it so a successful retry clears a previous warning. setPersistFailureRuleIds(Array.isArray(data.persistFailureRuleIds) ? data.persistFailureRuleIds : []); + setStoppedRuleIds(Array.isArray(data.stoppedRuleIds) ? data.stoppedRuleIds : []); setAnalysisLoaded(true); } catch (error) { console.error("Failed to fetch analysis results:", error); + // A click deserves an answer: the server's text says whether trying again in a moment helps. + if (reanalyze) notifyError?.("Re-analysis failed", error, "reanalyze-error", "The analysis could not be started."); } finally { setLoadingAnalysis(false); } - }, [sessionId, sessionTenantId, getAccessToken]); + }, [sessionId, sessionTenantId, getAccessToken, notifyError]); const fetchVulnerabilityReport = useCallback(async (rescan = false) => { if (!withVulnerabilityReport || !sessionTenantId || !isGuid(sessionTenantId)) return; @@ -85,5 +92,6 @@ export function useSessionAnalysis( fetchAnalysisResults, fetchVulnerabilityReport, persistFailureRuleIds, + stoppedRuleIds, }; } diff --git a/src/Web/autopilot-monitor-web/app/sessions/page.tsx b/src/Web/autopilot-monitor-web/app/sessions/page.tsx index 0942c362..fc7da464 100644 --- a/src/Web/autopilot-monitor-web/app/sessions/page.tsx +++ b/src/Web/autopilot-monitor-web/app/sessions/page.tsx @@ -116,7 +116,7 @@ function SessionDetailContent() { // Session-scoped hooks (data/SignalR/derivations) const detail = useSessionDetail({ sessionId, tenantId, globalAdminMode, tenantIdOverride, user, getAccessToken, addNotification }); - const analysis = useSessionAnalysis(sessionId, detail.sessionTenantId, getAccessToken); + const analysis = useSessionAnalysis(sessionId, detail.sessionTenantId, getAccessToken, { notifyError }); const tenantConfig = useSessionTenantConfig(detail.sessionTenantId, getAccessToken); const eventsApi = useSessionEvents({ sessionId, @@ -183,7 +183,7 @@ function SessionDetailContent() { // Returns an empty map when the tenant hasn't granted DeviceManagementScripts.Read.All. // Map keys are "{Kind}:{Id}" -- the renderer uses lookupScriptDisplayName to read them. const scriptDisplayNamesByRefKey = useScriptDisplayNames(sessionTenantId ?? tenantId, events, getAccessToken); - const { analysisResults, loadingAnalysis, analysisLoaded, vulnerabilityReport, fetchAnalysisResults, fetchVulnerabilityReport, persistFailureRuleIds } = analysis; + const { analysisResults, loadingAnalysis, analysisLoaded, vulnerabilityReport, fetchAnalysisResults, fetchVulnerabilityReport, persistFailureRuleIds, stoppedRuleIds } = analysis; const { showScriptOutput, enableSoftwareInventoryAnalyzer, enableIntegrityBypassAnalyzer } = tenantConfig; const { filteredEvents, @@ -698,6 +698,7 @@ function SessionDetailContent() { // so it stays with Admin/GA/Operator (the backend serves it read-tier regardless). canReanalyze={!isReadOnlyView && ((user?.isTenantAdmin ?? false) || (user?.isGlobalAdmin ?? false) || user?.role === "Operator")} persistFailureRuleIds={persistFailureRuleIds} + stoppedRuleIds={stoppedRuleIds} sessionTenantId={sessionTenantId || undefined} emptyState={showInvestigateHint({ status: session?.status, diff --git a/src/Web/autopilot-monitor-web/lib/generated/shared-manifests.generated.ts b/src/Web/autopilot-monitor-web/lib/generated/shared-manifests.generated.ts index 9368dac4..83572bee 100644 --- a/src/Web/autopilot-monitor-web/lib/generated/shared-manifests.generated.ts +++ b/src/Web/autopilot-monitor-web/lib/generated/shared-manifests.generated.ts @@ -664,6 +664,7 @@ export const SHARED_MANIFEST = { "PrivilegedRouteDenied", "McpServicePrincipalFirstSeen", "ReportUploadLimitReached", + "RuleEvaluationLimitReached", "OffboardingFeedbackReceived", "FeedbackReceived", "FeedbackRatingReceived", @@ -691,6 +692,7 @@ export const SHARED_MANIFEST = { "CollectLogsQuickConfigEnabled", "DiagnosticsUploadEnabled", "DiagnosticsUploadDisabled", + "RuleEvaluationStopped", "SessionActionQueued", "SessionTimeouts", "AgentEmergencyBreak", diff --git a/src/Web/autopilot-monitor-web/lib/generated/shared-manifests.json b/src/Web/autopilot-monitor-web/lib/generated/shared-manifests.json index 0241c42c..14c69d11 100644 --- a/src/Web/autopilot-monitor-web/lib/generated/shared-manifests.json +++ b/src/Web/autopilot-monitor-web/lib/generated/shared-manifests.json @@ -660,6 +660,7 @@ "PrivilegedRouteDenied", "McpServicePrincipalFirstSeen", "ReportUploadLimitReached", + "RuleEvaluationLimitReached", "OffboardingFeedbackReceived", "FeedbackReceived", "FeedbackRatingReceived", @@ -687,6 +688,7 @@ "CollectLogsQuickConfigEnabled", "DiagnosticsUploadEnabled", "DiagnosticsUploadDisabled", + "RuleEvaluationStopped", "SessionActionQueued", "SessionTimeouts", "AgentEmergencyBreak", @@ -11471,6 +11473,18 @@ }, "optional": true, "doc": "Rule ids that failed to persist during the reanalyze — the key is omitted entirely when there were no failures (the happy-path contract stays unchanged)." + }, + { + "name": "stoppedRuleIds", + "type": { + "kind": "array", + "element": { + "kind": "primitive", + "name": "string" + } + }, + "optional": true, + "doc": "Rule ids the reanalyze could not evaluate within its time limit (a regex timed out, or the run's budget ran out first); their previous results are kept. The key is omitted when every rule was evaluated." } ] }, @@ -15328,7 +15342,7 @@ "name": "string" }, "optional": true, - "doc": "Set when the test ended before the last sample line was evaluated: regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated." + "doc": "Set when the test ended before the last sample line was evaluated (EvaluationStopReasons): regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated." }, { "name": "lines", @@ -19379,6 +19393,24 @@ }, "optional": true, "doc": "The evidence map exactly as the production path would persist it on a RuleResult — keys are condition signals (plus factor_* markers). Clients use it to preview {{token}} interpolation of explanation/remediation. Values are heterogeneous evidence objects by design." + }, + { + "name": "stopReason", + "type": { + "kind": "primitive", + "name": "string" + }, + "optional": true, + "doc": "Set with verdict \"stopped\": regex_timeout | time_budget | cancelled. The trace then holds only the entries evaluated before the stop. Null otherwise." + }, + { + "name": "stopNote", + "type": { + "kind": "primitive", + "name": "string" + }, + "optional": true, + "doc": "Set with verdict \"stopped\": which precondition or condition the evaluation stopped at, and what to change." } ] }, diff --git a/src/Web/autopilot-monitor-web/lib/generated/wire-types.generated.ts b/src/Web/autopilot-monitor-web/lib/generated/wire-types.generated.ts index 273f21ff..178b2944 100644 --- a/src/Web/autopilot-monitor-web/lib/generated/wire-types.generated.ts +++ b/src/Web/autopilot-monitor-web/lib/generated/wire-types.generated.ts @@ -2406,6 +2406,8 @@ export interface GetRuleResultsResponse { persistFailureCount: number; /** Rule ids that failed to persist during the reanalyze — the key is omitted entirely when there were no failures (the happy-path contract stays unchanged). */ persistFailureRuleIds?: string[]; + /** Rule ids the reanalyze could not evaluate within its time limit (a regex timed out, or the run's budget ran out first); their previous results are kept. The key is omitted when every rule was evaluated. */ + stoppedRuleIds?: string[]; } /** Response of POST tenants/{tenantId}/scripts/display-names: resolved Intune script display names keyed by the canonical ref string. Always 200, possibly partial — unresolved refs stay in the dictionary with a null VALUE (dictionary values are NOT subject to WhenWritingNull, so the key is emitted with an explicit JSON null). */ @@ -3237,7 +3239,7 @@ export interface LogPatternTestResult { matchCount: number; parseFailureCount: number; timeoutCount: number; - /** Set when the test ended before the last sample line was evaluated: regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated. */ + /** Set when the test ended before the last sample line was evaluated (EvaluationStopReasons): regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated. */ stopReason?: string; readonly lines: LogPatternLineResult[]; readonly notes: string[]; @@ -4189,6 +4191,10 @@ export interface RuleDryRun { wouldMarkSessionAsFailed: boolean; /** The evidence map exactly as the production path would persist it on a RuleResult — keys are condition signals (plus factor_* markers). Clients use it to preview {{token}} interpolation of explanation/remediation. Values are heterogeneous evidence objects by design. */ matchedConditions?: Record; + /** Set with verdict "stopped": regex_timeout | time_budget | cancelled. The trace then holds only the entries evaluated before the stop. Null otherwise. */ + stopReason?: string; + /** Set with verdict "stopped": which precondition or condition the evaluation stopped at, and what to change. */ + stopNote?: string; } export interface RuleDryRunCondition {