mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-11-05 18:07:23 +00:00
Analyze rules and hardware whitelist: tenant patterns run on a budget
Tenant-authored regex conditions could make the shared backend spend one second per event and condition: a match timeout read as "no match" and the next event paid again, with no limit per request or per analysis run. Analyze rules (D-410): - Every evaluation pass has a time budget: 5 s for all rules of an analysis run (queue, interim, Analyze now), 2 s plus the request token for a dry-run. It is checked before every operator and in the evaluator scans that run without one. - The first regex match timeout stops that rule; a spent budget stops the run. A rule that does not finish keeps its stored result; Analyze now writes those rows back after its rebuild and names them (stoppedRuleIds). - The dry-run answers verdict "stopped" with stopReason and stopNote. - Dry-run, Analyze now and the pattern test take a slot per instance (2 / 4 / 2) and answer 429 with Retry-After when full. A full gate and a stopped rule raise throttled ops events (RuleEvaluationLimitReached, RuleEvaluationStopped). - Rules are capped at 20 conditions, 10 preconditions and 2,000-character regex values (condition, filter, event A filter and precondition values); every regex must parse. Checked on dry-run, create, update and template copy; the schema carries both counts. - Portal: the dry-run panel shows the stop, a failed Analyze now click is reported, and a banner lists rules that stopped. MCP: both dry-run tools wait out a full gate once. Hardware whitelist (D-411): wildcard entries are matched without building a regex per entry and request; a device value over 256 characters only meets "*". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -89,12 +89,14 @@
|
||||
"type": "array",
|
||||
"description": "Conditions evaluated against the event stream. All required conditions must match for the rule to fire.",
|
||||
"items": { "$ref": "#/$defs/ruleCondition" },
|
||||
"minItems": 1
|
||||
"minItems": 1,
|
||||
"maxItems": 20
|
||||
},
|
||||
"preconditions": {
|
||||
"type": "array",
|
||||
"description": "Optional device-fact gates evaluated BEFORE conditions. ALL preconditions must pass; if any fails the rule is silently skipped — no result, no UI card. Useful for filtering out hardware/OS profiles where the rule does not apply (e.g. virtual machines).",
|
||||
"items": { "$ref": "#/$defs/rulePrecondition" },
|
||||
"maxItems": 10,
|
||||
"default": []
|
||||
},
|
||||
"baseConfidence": {
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
using AutopilotMonitor.Functions.Functions.Rules;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The size limits of a tenant-authored analyze rule (D-410): at most 20 conditions and 10
|
||||
/// preconditions, and every value a regex or not_regex operator compares — condition value,
|
||||
/// filter value, event A filter value, precondition value — at most 2,000 characters and
|
||||
/// parseable. One implementation feeds the dry-run validation and every write path (create,
|
||||
/// update, template copy); the boundary itself is always allowed.
|
||||
/// </summary>
|
||||
public class AnalyzeRuleShapeLimitsTests
|
||||
{
|
||||
private const string TenantId = "tenant-limits";
|
||||
|
||||
[Fact]
|
||||
public void A_rule_at_every_limit_passes()
|
||||
{
|
||||
var rule = Rule(AnalyzeRuleLimits.MaxConditions, AnalyzeRuleLimits.MaxPreconditions);
|
||||
var atLimit = new string('a', AnalyzeRuleLimits.MaxRegexLength);
|
||||
rule.Conditions[0].Operator = "regex";
|
||||
rule.Conditions[0].Value = atLimit;
|
||||
rule.Conditions[1].FilterField = "f";
|
||||
rule.Conditions[1].FilterOperator = "not_regex";
|
||||
rule.Conditions[1].FilterValue = atLimit;
|
||||
rule.Conditions[2].EventAFilterOperator = "REGEX";
|
||||
rule.Conditions[2].EventAFilterValue = atLimit;
|
||||
rule.Preconditions[0].Operator = "regex";
|
||||
rule.Preconditions[0].Value = atLimit;
|
||||
|
||||
Assert.Empty(AnalyzeRuleService.ShapeLimitErrors(rule));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void One_condition_or_precondition_over_the_limit_is_named()
|
||||
{
|
||||
var errors = AnalyzeRuleService.ShapeLimitErrors(Rule(AnalyzeRuleLimits.MaxConditions + 1, AnalyzeRuleLimits.MaxPreconditions + 1));
|
||||
|
||||
Assert.Contains(errors, e => e.Contains("at most 20 conditions (this one has 21)"));
|
||||
Assert.Contains(errors, e => e.Contains("at most 10 preconditions (this one has 11)"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("conditions[0].value")]
|
||||
[InlineData("conditions[0].filterValue")]
|
||||
[InlineData("conditions[0].eventAFilterValue")]
|
||||
[InlineData("preconditions[0].value")]
|
||||
public void A_regex_over_the_length_limit_is_rejected_wherever_it_sits(string field)
|
||||
{
|
||||
var rule = Rule(1, 1);
|
||||
Place(rule, field, "regex", new string('a', AnalyzeRuleLimits.MaxRegexLength + 1));
|
||||
|
||||
var error = Assert.Single(AnalyzeRuleService.ShapeLimitErrors(rule));
|
||||
Assert.StartsWith($"{field}: a regex may be at most 2000 characters (this one has 2001)", error);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("conditions[0].filterValue")]
|
||||
[InlineData("conditions[0].eventAFilterValue")]
|
||||
[InlineData("preconditions[0].value")]
|
||||
public void An_unparseable_regex_is_rejected_beyond_the_condition_value(string field)
|
||||
{
|
||||
var rule = Rule(1, 1);
|
||||
Place(rule, field, "not_regex", "([unclosed");
|
||||
|
||||
var error = Assert.Single(AnalyzeRuleService.ShapeLimitErrors(rule));
|
||||
Assert.StartsWith($"{field}: value is not a valid regex", error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_long_value_of_any_other_operator_is_not_limited()
|
||||
{
|
||||
var rule = Rule(1, 0);
|
||||
rule.Conditions[0].Operator = "not_in";
|
||||
rule.Conditions[0].Value = string.Join(",", Enumerable.Range(0, 1000).Select(i => $"app-{i:D5}"));
|
||||
|
||||
Assert.Empty(AnalyzeRuleService.ShapeLimitErrors(rule));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_dry_run_validation_carries_the_limits()
|
||||
{
|
||||
var rule = Rule(AnalyzeRuleLimits.MaxConditions + 1, 0);
|
||||
|
||||
Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("at most 20 conditions"));
|
||||
}
|
||||
|
||||
// ===== write paths =====
|
||||
|
||||
[Fact]
|
||||
public async Task Create_rejects_an_oversized_rule_before_anything_is_stored()
|
||||
{
|
||||
var repo = new Mock<IRuleRepository>();
|
||||
var service = new AnalyzeRuleService(repo.Object, NullLogger<AnalyzeRuleService>.Instance);
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => service.CreateRuleAsync(TenantId, Rule(AnalyzeRuleLimits.MaxConditions + 1, 0)));
|
||||
repo.Verify(r => r.StoreAnalyzeRuleAsync(It.IsAny<AnalyzeRule>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Update_of_a_custom_rule_rejects_an_oversized_regex_before_anything_is_stored()
|
||||
{
|
||||
var repo = new Mock<IRuleRepository>();
|
||||
var service = new AnalyzeRuleService(repo.Object, NullLogger<AnalyzeRuleService>.Instance);
|
||||
var rule = Rule(1, 0);
|
||||
Place(rule, "conditions[0].value", "not_regex", new string('a', AnalyzeRuleLimits.MaxRegexLength + 1));
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => service.UpdateRuleAsync(TenantId, rule));
|
||||
repo.Verify(r => r.StoreAnalyzeRuleAsync(It.IsAny<AnalyzeRule>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Template_copy_rejects_an_oversized_regex_from_its_variable()
|
||||
{
|
||||
// ANALYZE-SEC-006 shape: the template variable IS the tenant's not_regex allow-list.
|
||||
var template = Rule(1, 0);
|
||||
template.RuleId = "ANALYZE-SEC-006";
|
||||
template.IsBuiltIn = true;
|
||||
template.Conditions[0].Operator = "not_regex";
|
||||
template.TemplateVariables = new List<TemplateVariable> { new() { Name = "allowList", ConditionIndex = 0, Field = "value" } };
|
||||
|
||||
var repo = new Mock<IRuleRepository>();
|
||||
repo.Setup(r => r.GetAnalyzeRulesAsync("global")).ReturnsAsync(new List<AnalyzeRule> { template });
|
||||
repo.Setup(r => r.GetAnalyzeRulesAsync(TenantId)).ReturnsAsync(new List<AnalyzeRule>());
|
||||
repo.Setup(r => r.GetRuleStatesAsync(TenantId)).ReturnsAsync(new Dictionary<string, RuleState>());
|
||||
var service = new AnalyzeRuleService(repo.Object, NullLogger<AnalyzeRuleService>.Instance);
|
||||
|
||||
await Assert.ThrowsAsync<ArgumentException>(() => service.CreateFromTemplateAsync(TenantId, "ANALYZE-SEC-006",
|
||||
new Dictionary<string, string> { ["allowList"] = new string('a', AnalyzeRuleLimits.MaxRegexLength + 1) }));
|
||||
repo.Verify(r => r.StoreAnalyzeRuleAsync(It.IsAny<AnalyzeRule>(), TenantId), Times.Never);
|
||||
}
|
||||
|
||||
// ===== fixtures =====
|
||||
|
||||
private static AnalyzeRule Rule(int conditions, int preconditions) => new()
|
||||
{
|
||||
RuleId = "CUSTOM-LIMITS",
|
||||
IsBuiltIn = false, // the model defaults to a built-in rule
|
||||
Title = "limits",
|
||||
Severity = "warning",
|
||||
Category = "test",
|
||||
BaseConfidence = 80,
|
||||
ConfidenceThreshold = 40,
|
||||
Conditions = Enumerable.Range(0, conditions).Select(i => new RuleCondition
|
||||
{
|
||||
Signal = $"s{i}", Source = "event_data", EventType = "app_install_failed", DataField = "errorCode",
|
||||
Operator = "equals", Value = "1", Required = true,
|
||||
}).ToList(),
|
||||
Preconditions = Enumerable.Range(0, preconditions).Select(_ => new RulePrecondition
|
||||
{
|
||||
Source = "event_data", EventType = "os_info", DataField = "edition", Operator = "equals", Value = "Enterprise",
|
||||
}).ToList(),
|
||||
};
|
||||
|
||||
private static void Place(AnalyzeRule rule, string field, string op, string value)
|
||||
{
|
||||
switch (field)
|
||||
{
|
||||
case "conditions[0].value":
|
||||
rule.Conditions[0].Operator = op;
|
||||
rule.Conditions[0].Value = value;
|
||||
break;
|
||||
case "conditions[0].filterValue":
|
||||
rule.Conditions[0].FilterField = "f";
|
||||
rule.Conditions[0].FilterOperator = op;
|
||||
rule.Conditions[0].FilterValue = value;
|
||||
break;
|
||||
case "conditions[0].eventAFilterValue":
|
||||
rule.Conditions[0].EventAFilterField = "f";
|
||||
rule.Conditions[0].EventAFilterOperator = op;
|
||||
rule.Conditions[0].EventAFilterValue = value;
|
||||
break;
|
||||
default:
|
||||
rule.Preconditions[0].Operator = op;
|
||||
rule.Preconditions[0].Value = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -95,6 +95,7 @@ public class DerivedWriterLivenessTests
|
||||
configService: null!,
|
||||
channelDispatcher: null!,
|
||||
GuardFor(row),
|
||||
stopReporter: null!,
|
||||
NullLogger<AnalyzeOnEnrollmentEndHandler>.Instance);
|
||||
|
||||
await handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope
|
||||
@@ -116,6 +117,7 @@ public class DerivedWriterLivenessTests
|
||||
ruleService: null!, Mock.Of<IRuleRepository>(), Mock.Of<ISessionRepository>(), Mock.Of<IMetricsRepository>(),
|
||||
signalRNotification: null!, configService: null!, channelDispatcher: null!,
|
||||
new SessionDeletionGuard(reader.Object, NullLogger<SessionDeletionGuard>.Instance),
|
||||
stopReporter: null!,
|
||||
NullLogger<AnalyzeOnEnrollmentEndHandler>.Instance);
|
||||
|
||||
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.RegularExpressions;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The hardware whitelist matches tenant wildcards against device headers on every agent request
|
||||
/// without building a Regex (D-411): same results as the former regex construction for every shape
|
||||
/// an entry can take, linear work for the worst entry a tenant can write, and a device value longer
|
||||
/// than any real hardware string only meets "*".
|
||||
/// </summary>
|
||||
public class HardwareWhitelistValidatorTests
|
||||
{
|
||||
/// <summary>The construction this replaced, kept as the oracle: '*' → '.*', '?' → '.', the rest escaped.</summary>
|
||||
private static bool FormerRegexMatch(string text, string pattern)
|
||||
{
|
||||
if (pattern == "*")
|
||||
return true;
|
||||
var regex = "^" + Regex.Escape(pattern).Replace("\\*", ".*").Replace("\\?", ".") + "$";
|
||||
return Regex.IsMatch(text, regex, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Dell Inc.", "Dell*")]
|
||||
[InlineData("dell inc.", "DELL*")]
|
||||
[InlineData("Hewlett-Packard", "HP*")]
|
||||
[InlineData("HP", "HP*")]
|
||||
[InlineData("LENOVO", "Lenovo*")]
|
||||
[InlineData("Microsoft Corporation", "Microsoft Corporation")]
|
||||
[InlineData("Microsoft Corporation ", "Microsoft Corporation")]
|
||||
[InlineData("Microsoft Corp", "Microsoft Corporation")]
|
||||
[InlineData("ThinkPad T14 Gen 6", "*T14*")]
|
||||
[InlineData("ThinkPad T14 Gen 6", "*Gen ?")]
|
||||
[InlineData("ThinkPad T14 Gen 16", "*Gen ?")]
|
||||
[InlineData("ThinkPad T14 Gen 6", "ThinkPad*Gen*")]
|
||||
[InlineData("VMware20,1", "VMware??,?")]
|
||||
[InlineData("VMware20,1", "VMware?,?")]
|
||||
[InlineData("a.b", "a.b")]
|
||||
[InlineData("axb", "a.b")]
|
||||
[InlineData("f(x)", "f(x)")]
|
||||
[InlineData("f(x)", "f(*)")]
|
||||
[InlineData("a\\b", "a\\b")]
|
||||
[InlineData("a\\xyzb", "a\\*b")]
|
||||
[InlineData("a*b", "a\\*b")]
|
||||
[InlineData("[abc]", "[abc]")]
|
||||
[InlineData("a", "[abc]")]
|
||||
[InlineData("x+y", "x+y")]
|
||||
[InlineData("xxy", "x+y")]
|
||||
[InlineData("$1.00", "$1.00")]
|
||||
[InlineData("^start", "^start")]
|
||||
[InlineData("abc", "**")]
|
||||
[InlineData("abc", "*?*")]
|
||||
[InlineData("ab", "???")]
|
||||
[InlineData("abc", "???")]
|
||||
[InlineData("abcc", "*c")]
|
||||
[InlineData("ÄÖÜ GmbH", "äöü*")]
|
||||
[InlineData("straße", "STRASSE")]
|
||||
public void Matches_like_the_former_regex_construction(string value, string pattern)
|
||||
{
|
||||
Assert.Equal(FormerRegexMatch(value, pattern), HardwareWhitelistValidator.MatchesWildcard(value, pattern));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Case_folding_is_ordinal_the_one_deliberate_difference()
|
||||
{
|
||||
// The regex engine folded Turkish dotted 'İ' to 'i'; ordinal folding does not. SMBIOS strings are ASCII.
|
||||
Assert.True(FormerRegexMatch("İstanbul", "istanbul*"));
|
||||
Assert.False(HardwareWhitelistValidator.MatchesWildcard("İstanbul", "istanbul*"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_worst_whitelist_a_tenant_can_write_costs_milliseconds_per_request()
|
||||
{
|
||||
// 200 entries of 128 characters, each a chain of wildcards that never matches, against the
|
||||
// longest value still compared (the cap) and a 32 KB header (past the cap).
|
||||
var entry = string.Concat(Enumerable.Repeat("*a", 63)) + "*b";
|
||||
var whitelist = Enumerable.Repeat(entry, 200).ToArray();
|
||||
var atCap = new string('a', HardwareWhitelistValidator.MaxMatchedValueLength);
|
||||
var huge = new string('a', 32 * 1024);
|
||||
|
||||
var clock = Stopwatch.StartNew();
|
||||
var atCapResult = HardwareWhitelistValidator.ValidateHardware(atCap, "Model", whitelist, new[] { "*" });
|
||||
var hugeResult = HardwareWhitelistValidator.ValidateHardware(huge, "Model", whitelist, new[] { "*" });
|
||||
clock.Stop();
|
||||
|
||||
Assert.False(atCapResult.IsValid);
|
||||
Assert.False(hugeResult.IsValid);
|
||||
// The former construction spent up to 200 × 100 ms per list on this.
|
||||
Assert.True(clock.ElapsedMilliseconds < 500, $"took {clock.ElapsedMilliseconds} ms");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_value_longer_than_any_hardware_string_only_meets_the_star()
|
||||
{
|
||||
var longValue = "Dell" + new string('x', HardwareWhitelistValidator.MaxMatchedValueLength);
|
||||
|
||||
Assert.False(HardwareWhitelistValidator.MatchesWildcard(longValue, "Dell*"));
|
||||
Assert.True(HardwareWhitelistValidator.MatchesWildcard(longValue, "*"));
|
||||
|
||||
var rejected = HardwareWhitelistValidator.ValidateHardware(longValue, "Model", new[] { "Dell*" }, new[] { "*" });
|
||||
Assert.False(rejected.IsValid);
|
||||
Assert.True(rejected.ErrorMessage!.Length < HardwareWhitelistValidator.MaxMatchedValueLength + 64, "the rejection message carries a shortened value");
|
||||
}
|
||||
}
|
||||
@@ -56,10 +56,11 @@ public class ReportUploadGateTests
|
||||
var gate = CreateGate(limit: 1);
|
||||
var start = new DateTime(2026, 10, 29, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
Assert.True(gate.TakeAlert(start));
|
||||
Assert.False(gate.TakeAlert(start.AddMinutes(59)));
|
||||
Assert.True(gate.TakeAlert(start + ReportUploadGate.AlertWindow));
|
||||
Assert.False(gate.TakeAlert(start + ReportUploadGate.AlertWindow));
|
||||
var window = ReportUploadGate.AlertWindow;
|
||||
Assert.True(gate.TakeAlert(start, window));
|
||||
Assert.False(gate.TakeAlert(start.AddMinutes(59), window));
|
||||
Assert.True(gate.TakeAlert(start + window, window));
|
||||
Assert.False(gate.TakeAlert(start + window, window));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Functions.Rules;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The evaluation budget of tenant-authored rules (D-410): the first regex match timeout stops the
|
||||
/// rule instead of costing one second per event, a spent run budget leaves the later rules
|
||||
/// unevaluated, every scan without an operator call is checked too, a rule that did not finish keeps
|
||||
/// its stored row, and the dry-run reports the stop as verdict "stopped".
|
||||
/// </summary>
|
||||
public class RuleEvaluationBudgetTests
|
||||
{
|
||||
private const string TenantId = "tenant-budget";
|
||||
private const string SessionId = "session-budget";
|
||||
|
||||
/// <summary>Nested quantifier over input it cannot match: catastrophic backtracking, ends at the 1 s match timeout.</summary>
|
||||
private const string CatastrophicPattern = @"^(\w+\s?)+$";
|
||||
private static readonly string NonMatchingInput = new string('a', 60) + "!";
|
||||
|
||||
// ===== analysis run =====
|
||||
|
||||
[Fact]
|
||||
public async Task Regex_timeout_stops_the_rule_after_one_match_and_the_next_rule_still_runs()
|
||||
{
|
||||
var events = Enumerable.Range(1, 200).Select(i => Event("app_install_failed", i, ("errorDetail", NonMatchingInput))).ToList();
|
||||
var redos = CustomRule("TEST-REDOS", new RuleCondition
|
||||
{
|
||||
Signal = "slow", Source = "event_data", EventType = "app_install_failed",
|
||||
DataField = "errorDetail", Operator = "regex", Value = CatastrophicPattern, Required = true,
|
||||
});
|
||||
var next = CustomRule("TEST-NEXT", ExistsCondition("app_install_failed"));
|
||||
|
||||
var clock = Stopwatch.StartNew();
|
||||
var outcome = await Harness(events, redos, next).Engine().AnalyzeSessionAsync(TenantId, SessionId);
|
||||
clock.Stop();
|
||||
|
||||
// One timed-out match, not 200 of them.
|
||||
Assert.True(clock.ElapsedMilliseconds < 4000, $"took {clock.ElapsedMilliseconds} ms");
|
||||
var stop = Assert.Single(outcome.Stops);
|
||||
Assert.Equal(new RuleEvaluationStop("TEST-REDOS", EvaluationStopReasons.RegexTimeout, IsCatalogRule: false), stop);
|
||||
Assert.Empty(outcome.SkippedRuleIds);
|
||||
Assert.Equal(new[] { "TEST-NEXT" }, outcome.Results.Select(r => r.RuleId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Spent_budget_stops_the_run_keeps_every_unfinished_row_and_resolves_nothing()
|
||||
{
|
||||
var events = new List<EnrollmentEvent> { Event("app_install_failed", 1, ("appId", "app-1")) };
|
||||
var first = CustomRule("TEST-A", new RuleCondition
|
||||
{
|
||||
Signal = "a", Source = "event_data", EventType = "app_install_failed",
|
||||
DataField = "appId", Operator = "equals", Value = "app-1", Required = true,
|
||||
});
|
||||
var second = CustomRule("TEST-B", ExistsCondition("app_install_failed"));
|
||||
var third = CustomRule("TEST-C", ExistsCondition("app_install_failed"));
|
||||
var interimA = StoredRow("TEST-A", isInterim: true);
|
||||
var interimB = StoredRow("TEST-B", isInterim: true);
|
||||
|
||||
var harness = Harness(events, first, second, third);
|
||||
harness.ExistingResults.AddRange(new[] { interimA, interimB });
|
||||
var engine = harness.Engine(TimeSpan.Zero);
|
||||
|
||||
var outcome = await engine.AnalyzeSessionAsync(TenantId, SessionId, AnalyzeRunContext.Terminal());
|
||||
|
||||
var stop = Assert.Single(outcome.Stops);
|
||||
Assert.Equal("TEST-A", stop.RuleId);
|
||||
Assert.Equal(EvaluationStopReasons.TimeBudget, stop.Reason);
|
||||
Assert.Equal(new[] { "TEST-B", "TEST-C" }, outcome.SkippedRuleIds);
|
||||
Assert.Empty(outcome.Results);
|
||||
Assert.Empty(outcome.ResolvedResults); // a stop is no evidence that a finding went away
|
||||
Assert.Equal(new[] { interimA, interimB }, outcome.KeptResults);
|
||||
Assert.Null(interimA.ResolvedAt);
|
||||
Assert.True(interimA.IsInterim);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Each of these evaluators scans the session per matching event without calling an operator
|
||||
/// (quadratic in the session size); without its own check a spent budget would not stop it, and
|
||||
/// the rule would simply be evaluated.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData("event_correlation pairing")]
|
||||
[InlineData("suppressByEvent scan")]
|
||||
[InlineData("app_install_duration start search")]
|
||||
public async Task Scans_without_an_operator_call_check_the_budget_too(string scan)
|
||||
{
|
||||
var (condition, events) = scan switch
|
||||
{
|
||||
"event_correlation pairing" => (
|
||||
new RuleCondition
|
||||
{
|
||||
Signal = "pair", Source = "event_correlation", EventType = "app_install_started",
|
||||
CorrelateEventType = "app_install_failed", JoinField = "appId", Required = true,
|
||||
},
|
||||
new[] { Event("app_install_started", 1, ("appId", "x")), Event("app_install_failed", 2, ("appId", "x")) }),
|
||||
"suppressByEvent scan" => (
|
||||
new RuleCondition
|
||||
{
|
||||
Signal = "failed", Source = "event_type", EventType = "app_install_failed", Operator = "exists", Required = true,
|
||||
SuppressByEvent = new SuppressByEventConfig { EventType = "app_install_completed", JoinField = "appId" },
|
||||
},
|
||||
new[] { Event("app_install_failed", 1, ("appId", "x")) }),
|
||||
_ => (
|
||||
new RuleCondition { Signal = "slow", Source = "app_install_duration", Operator = "gt", Value = "0", Required = true },
|
||||
new[] { Event("app_install_completed", 1, ("appId", "x")) }),
|
||||
};
|
||||
|
||||
var outcome = await Harness(events.ToList(), CustomRule("TEST-SCAN", condition)).Engine(TimeSpan.Zero)
|
||||
.AnalyzeSessionAsync(TenantId, SessionId);
|
||||
|
||||
var stop = Assert.Single(outcome.Stops);
|
||||
Assert.Equal(EvaluationStopReasons.TimeBudget, stop.Reason);
|
||||
Assert.Empty(outcome.Results);
|
||||
}
|
||||
|
||||
// ===== dry-run =====
|
||||
|
||||
[Fact]
|
||||
public void Dry_run_regex_timeout_ends_with_verdict_stopped_and_a_note_naming_the_condition()
|
||||
{
|
||||
var rule = CustomRule("TEST-DRY",
|
||||
ExistsCondition("app_install_failed"),
|
||||
new RuleCondition
|
||||
{
|
||||
Signal = "slow", Source = "event_data", EventType = "app_install_failed",
|
||||
DataField = "errorDetail", Operator = "regex", Value = CatastrophicPattern, Required = true,
|
||||
});
|
||||
var events = Enumerable.Range(1, 50).Select(i => Event("app_install_failed", i, ("errorDetail", NonMatchingInput))).ToList();
|
||||
|
||||
var dry = Harness(events).Engine().DryRunEvaluate(rule, events, new RuleEvaluationBudget(RuleEvaluationBudget.DryRun));
|
||||
|
||||
Assert.Equal(RuleDryRunVerdict.Stopped, dry.Verdict);
|
||||
Assert.Equal(EvaluationStopReasons.RegexTimeout, dry.StopReason);
|
||||
Assert.StartsWith("conditions[1] ('slow'): a regex hit the 1 s match timeout", dry.StopNote);
|
||||
// Only what finished before the stop; no half-evaluated evidence map.
|
||||
Assert.Equal(new[] { "exists_app_install_failed" }, dry.Conditions.Select(c => c.Signal));
|
||||
Assert.Null(dry.MatchedConditions);
|
||||
Assert.Null(dry.FinalConfidence);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dry_run_spent_budget_stops_at_the_first_operator()
|
||||
{
|
||||
var rule = CustomRule("TEST-DRY", ExistsCondition("app_install_failed"));
|
||||
rule.Preconditions = new List<RulePrecondition>
|
||||
{
|
||||
new() { Source = "event_data", EventType = "os_info", DataField = "edition", Operator = "equals", Value = "Enterprise" },
|
||||
};
|
||||
var events = new List<EnrollmentEvent> { Event("os_info", 1, ("edition", "Enterprise")) };
|
||||
|
||||
var dry = Harness(events).Engine().DryRunEvaluate(rule, events, new RuleEvaluationBudget(TimeSpan.Zero));
|
||||
|
||||
Assert.Equal(RuleDryRunVerdict.Stopped, dry.Verdict);
|
||||
Assert.Equal(EvaluationStopReasons.TimeBudget, dry.StopReason);
|
||||
Assert.StartsWith("preconditions[0] (os_info): the dry-run used up its 0 s time budget on this session's 1 events", dry.StopNote);
|
||||
Assert.Empty(dry.Preconditions);
|
||||
Assert.Empty(dry.Conditions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Dry_run_of_a_caller_who_went_away_stops_as_cancelled()
|
||||
{
|
||||
var events = new List<EnrollmentEvent> { Event("app_install_failed", 1, ("appId", "x")) };
|
||||
var rule = CustomRule("TEST-DRY", new RuleCondition
|
||||
{
|
||||
Signal = "a", Source = "event_data", EventType = "app_install_failed", DataField = "appId", Operator = "equals", Value = "x", Required = true,
|
||||
});
|
||||
using var gone = new CancellationTokenSource();
|
||||
gone.Cancel();
|
||||
|
||||
var dry = await Harness(events).Engine().DryRunRuleAsync(TenantId, SessionId, rule, gone.Token);
|
||||
|
||||
Assert.Equal(RuleDryRunVerdict.Stopped, dry.Verdict);
|
||||
Assert.Equal(EvaluationStopReasons.Cancelled, dry.StopReason);
|
||||
}
|
||||
|
||||
// ===== "Analyze now" =====
|
||||
|
||||
[Fact]
|
||||
public async Task Reanalyze_with_a_full_gate_answers_429_and_raises_one_ops_event()
|
||||
{
|
||||
var opsEvents = new List<OpsEventEntry>();
|
||||
var harness = Harness(new List<EnrollmentEvent>());
|
||||
var gate = new ReanalyzeGate(1, TestNotifications.CapturingOpsEvents(opsEvents), NullLogger<ReanalyzeGate>.Instance);
|
||||
using var busy = gate.TryEnter();
|
||||
var function = harness.Function(gate, opsEvents);
|
||||
|
||||
var (req, _) = EndpointHarness.Request(TenantId, queryString: "?reanalyze=true");
|
||||
var response = await function.Run(req, SessionId);
|
||||
|
||||
Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode);
|
||||
Assert.Equal(Constants.ApiErrorCodes.RateLimited, EndpointHarness.ErrorCode(response));
|
||||
Assert.True(response.Headers.TryGetValues("Retry-After", out var retryAfter));
|
||||
Assert.Equal("5", retryAfter!.Single());
|
||||
var alert = Assert.Single(opsEvents);
|
||||
Assert.Equal(OpsEventTypes.RuleEvaluationLimitReached, alert.EventType);
|
||||
Assert.Contains("re-analysis", alert.Details);
|
||||
harness.Maintenance.Verify(m => m.DeleteSessionRuleResultsAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reanalyze_writes_back_the_row_of_a_stopped_rule_and_names_it()
|
||||
{
|
||||
var events = Enumerable.Range(1, 20).Select(i => Event("app_install_failed", i, ("errorDetail", NonMatchingInput))).ToList();
|
||||
var redos = CustomRule("TEST-REDOS", new RuleCondition
|
||||
{
|
||||
Signal = "slow", Source = "event_data", EventType = "app_install_failed",
|
||||
DataField = "errorDetail", Operator = "regex", Value = CatastrophicPattern, Required = true,
|
||||
});
|
||||
var earlierFinding = StoredRow("TEST-REDOS", isInterim: false);
|
||||
var harness = Harness(events, redos);
|
||||
harness.ExistingResults.Add(earlierFinding);
|
||||
var opsEvents = new List<OpsEventEntry>();
|
||||
var function = harness.Function(
|
||||
new ReanalyzeGate(TestNotifications.CapturingOpsEvents(opsEvents), NullLogger<ReanalyzeGate>.Instance), opsEvents);
|
||||
|
||||
var (req, _) = EndpointHarness.Request(TenantId, queryString: "?reanalyze=true");
|
||||
var response = await function.Run(req, SessionId);
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
harness.Maintenance.Verify(m => m.DeleteSessionRuleResultsAsync(TenantId, SessionId), Times.Once);
|
||||
harness.Rules.Verify(r => r.StoreRuleResultAsync(earlierFinding), Times.Once);
|
||||
|
||||
response.Body.Position = 0;
|
||||
using var body = JsonDocument.Parse(response.Body);
|
||||
Assert.Equal(new[] { "TEST-REDOS" }, body.RootElement.GetProperty("stoppedRuleIds").EnumerateArray().Select(e => e.GetString()));
|
||||
|
||||
var stopped = Assert.Single(opsEvents);
|
||||
Assert.Equal(OpsEventTypes.RuleEvaluationStopped, stopped.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Warning, stopped.Severity);
|
||||
}
|
||||
|
||||
// ===== fixtures =====
|
||||
|
||||
private static RuleCondition ExistsCondition(string eventType) => new()
|
||||
{
|
||||
Signal = "exists_" + eventType, Source = "event_type", EventType = eventType, Operator = "exists", Required = true,
|
||||
};
|
||||
|
||||
private static AnalyzeRule CustomRule(string ruleId, params RuleCondition[] conditions) => new()
|
||||
{
|
||||
RuleId = ruleId,
|
||||
IsBuiltIn = false, // the model defaults to a built-in rule
|
||||
Title = ruleId,
|
||||
Severity = "warning",
|
||||
Category = "test",
|
||||
Enabled = true,
|
||||
BaseConfidence = 80,
|
||||
ConfidenceThreshold = 40,
|
||||
Conditions = conditions.ToList(),
|
||||
Explanation = "test",
|
||||
};
|
||||
|
||||
private static RuleResult StoredRow(string ruleId, bool isInterim) => new()
|
||||
{
|
||||
RuleId = ruleId,
|
||||
ResultId = "result-" + ruleId,
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
IsInterim = isInterim,
|
||||
DetectedAt = DateTime.UtcNow.AddMinutes(-30),
|
||||
};
|
||||
|
||||
private static EnrollmentEvent Event(string eventType, int sequence, params (string Key, object Value)[] data) => new()
|
||||
{
|
||||
EventId = $"evt-{sequence}",
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
EventType = eventType,
|
||||
Sequence = sequence,
|
||||
Timestamp = DateTime.UtcNow.AddMinutes(sequence),
|
||||
Data = data.ToDictionary(d => d.Key, d => d.Value),
|
||||
};
|
||||
|
||||
private static RuleHarness Harness(List<EnrollmentEvent> events, params AnalyzeRule[] customRules) => new(events, customRules);
|
||||
|
||||
/// <summary>The tenant's custom rules behind a real AnalyzeRuleService, mocked storage.</summary>
|
||||
private sealed class RuleHarness
|
||||
{
|
||||
public Mock<IRuleRepository> Rules { get; } = new();
|
||||
public Mock<ISessionRepository> Sessions { get; } = new();
|
||||
public Mock<IMaintenanceRepository> Maintenance { get; } = new();
|
||||
public List<RuleResult> ExistingResults { get; } = new();
|
||||
|
||||
public RuleHarness(List<EnrollmentEvent> events, AnalyzeRule[] customRules)
|
||||
{
|
||||
Rules.Setup(r => r.GetAnalyzeRulesAsync("global")).ReturnsAsync(new List<AnalyzeRule>());
|
||||
Rules.Setup(r => r.GetAnalyzeRulesAsync(TenantId)).ReturnsAsync(customRules.ToList());
|
||||
Rules.Setup(r => r.GetRuleStatesAsync(It.IsAny<string>())).ReturnsAsync(new Dictionary<string, RuleState>());
|
||||
Rules.Setup(r => r.GetRuleResultsAsync(TenantId, SessionId)).ReturnsAsync(() => ExistingResults.ToList());
|
||||
Rules.Setup(r => r.StoreRuleResultAsync(It.IsAny<RuleResult>())).ReturnsAsync(true);
|
||||
Sessions.Setup(s => s.GetSessionEventsStrictAsync(TenantId, SessionId, It.IsAny<int>())).ReturnsAsync(events);
|
||||
}
|
||||
|
||||
private AnalyzeRuleService Service() => new(Rules.Object, NullLogger<AnalyzeRuleService>.Instance);
|
||||
|
||||
public RuleEngine Engine(TimeSpan? runBudget = null) =>
|
||||
new(Service(), Rules.Object, Sessions.Object, NullLogger<RuleEvaluationBudgetTests>.Instance)
|
||||
{
|
||||
RunBudget = runBudget ?? RuleEvaluationBudget.AnalysisRun,
|
||||
};
|
||||
|
||||
public GetRuleResultsFunction Function(ReanalyzeGate gate, List<OpsEventEntry> opsEvents) =>
|
||||
new(NullLogger<GetRuleResultsFunction>.Instance, Rules.Object, Maintenance.Object, Sessions.Object, Service(),
|
||||
gate, new RuleEvaluationStopReporter(TestNotifications.CapturingOpsEvents(opsEvents)));
|
||||
}
|
||||
}
|
||||
@@ -69,7 +69,7 @@ public class RuleEvidenceKeysTests
|
||||
Required = true,
|
||||
};
|
||||
|
||||
var dry = Engine().DryRunEvaluate(Rule(condition), Events());
|
||||
var dry = Engine().DryRunEvaluate(Rule(condition), Events(), new RuleEvaluationBudget(RuleEvaluationBudget.DryRun));
|
||||
|
||||
var matched = Assert.IsAssignableFrom<IDictionary<string, object>>(dry.MatchedConditions);
|
||||
Assert.True(matched.ContainsKey("sig"), $"{source}/{op} did not match the fixture");
|
||||
|
||||
@@ -344,6 +344,11 @@ public class RulePresentationTests
|
||||
Assert.Equal(RulePresentationLimits.TemplateMaxLength, fact.GetProperty("value").GetProperty("maxLength").GetInt32());
|
||||
Assert.Equal(RulePresentationLimits.TemplateMaxLength, subject.GetProperty("key").GetProperty("maxLength").GetInt32());
|
||||
Assert.Equal(RulePresentationLimits.TemplateMaxLength, subject.GetProperty("name").GetProperty("maxLength").GetInt32());
|
||||
|
||||
// The rule-shape counts (D-410): the schema lets validate_rule refuse what the backend refuses.
|
||||
var analyzeRule = defs.GetProperty("analyzeRule").GetProperty("properties");
|
||||
Assert.Equal(AnalyzeRuleLimits.MaxConditions, analyzeRule.GetProperty("conditions").GetProperty("maxItems").GetInt32());
|
||||
Assert.Equal(AnalyzeRuleLimits.MaxPreconditions, analyzeRule.GetProperty("preconditions").GetProperty("maxItems").GetInt32());
|
||||
}
|
||||
|
||||
private static string[] Strings(JsonElement array) => array.EnumerateArray().Select(e => e.GetString()!).ToArray();
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Functions.Rules;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The per-instance caps on synchronous rule work (D-379, D-410) and the ops events behind them: a
|
||||
/// caller past a gate's limit gets 429 with a retry hint, the first refusal of a window raises one
|
||||
/// RuleEvaluationLimitReached naming the gate, and stopped rules reach the operators through a
|
||||
/// reporter that a tenant's rule count cannot turn into an alarm storm.
|
||||
/// </summary>
|
||||
public class RuleWorkGateTests
|
||||
{
|
||||
private const string Upn = "admin@example.com";
|
||||
|
||||
private readonly List<OpsEventEntry> _opsEvents = new();
|
||||
|
||||
[Theory]
|
||||
[InlineData("dry-run", 5)]
|
||||
[InlineData("re-analysis", 5)]
|
||||
[InlineData("pattern-test", 3)]
|
||||
public async Task Caller_past_the_limit_gets_429_with_a_retry_hint_and_one_ops_event_per_window(string gateName, int retryAfter)
|
||||
{
|
||||
var gate = Gate(gateName, limit: 1);
|
||||
using var busy = gate.TryEnter();
|
||||
Assert.NotNull(busy);
|
||||
Assert.Null(gate.TryEnter());
|
||||
|
||||
var (req, _) = EndpointHarness.Request("tenant-a", Upn);
|
||||
var refused = await gate.RefuseAsync(req, "tenant-a", Upn);
|
||||
await gate.RefuseAsync(req, "tenant-a", Upn);
|
||||
|
||||
Assert.Equal(HttpStatusCode.TooManyRequests, refused.StatusCode);
|
||||
Assert.Equal(Constants.ApiErrorCodes.RateLimited, EndpointHarness.ErrorCode(refused));
|
||||
Assert.True(refused.Headers.TryGetValues("Retry-After", out var header));
|
||||
Assert.Equal(retryAfter.ToString(), header!.Single());
|
||||
// The situation only: the portal and the MCP phrase the retry from Retry-After.
|
||||
Assert.EndsWith("running right now.", ErrorText(refused));
|
||||
|
||||
var alert = Assert.Single(_opsEvents);
|
||||
Assert.Equal(OpsEventTypes.RuleEvaluationLimitReached, alert.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Warning, alert.Severity);
|
||||
Assert.Equal("tenant-a", alert.TenantId);
|
||||
using var details = JsonDocument.Parse(alert.Details!);
|
||||
Assert.Equal(gateName, details.RootElement.GetProperty("gate").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Default_limits_never_refuse_a_person()
|
||||
{
|
||||
var ops = TestNotifications.CapturingOpsEvents(_opsEvents);
|
||||
Assert.Equal(2, new RuleDryRunGate(ops, NullLogger<RuleDryRunGate>.Instance).Limit);
|
||||
Assert.Equal(4, new ReanalyzeGate(ops, NullLogger<ReanalyzeGate>.Instance).Limit);
|
||||
}
|
||||
|
||||
// ===== RuleEvaluationStopReporter =====
|
||||
|
||||
[Fact]
|
||||
public async Task A_stopped_custom_rule_is_a_warning_and_a_stopped_catalog_rule_an_error()
|
||||
{
|
||||
var reporter = new RuleEvaluationStopReporter(TestNotifications.CapturingOpsEvents(_opsEvents));
|
||||
var outcome = new AnalysisOutcome();
|
||||
outcome.Stops.Add(new RuleEvaluationStop("CUSTOM-1", EvaluationStopReasons.RegexTimeout, IsCatalogRule: false));
|
||||
outcome.Stops.Add(new RuleEvaluationStop("ANALYZE-APP-002", EvaluationStopReasons.TimeBudget, IsCatalogRule: true));
|
||||
outcome.SkippedRuleIds.AddRange(new[] { "CUSTOM-2", "CUSTOM-3" });
|
||||
|
||||
await reporter.ReportAsync("tenant-a", "session-1", AnalyzeRunKind.Terminal, outcome);
|
||||
|
||||
Assert.Equal(2, _opsEvents.Count);
|
||||
Assert.All(_opsEvents, e => Assert.Equal(OpsEventTypes.RuleEvaluationStopped, e.EventType));
|
||||
Assert.Equal(OpsEventSeverity.Warning, _opsEvents[0].Severity);
|
||||
Assert.Equal(OpsEventSeverity.Error, _opsEvents[1].Severity);
|
||||
using var budgetStop = JsonDocument.Parse(_opsEvents[1].Details!);
|
||||
Assert.Equal(2, budgetStop.RootElement.GetProperty("skippedRules").GetInt32());
|
||||
using var regexStop = JsonDocument.Parse(_opsEvents[0].Details!);
|
||||
Assert.Equal(0, regexStop.RootElement.GetProperty("skippedRules").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void One_event_per_tenant_and_rule_per_window()
|
||||
{
|
||||
var reporter = new RuleEvaluationStopReporter(TestNotifications.CapturingOpsEvents(_opsEvents));
|
||||
var start = new DateTime(2026, 11, 5, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
Assert.True(reporter.Take("tenant-a", "CUSTOM-1", start));
|
||||
Assert.False(reporter.Take("tenant-a", "CUSTOM-1", start.AddMinutes(59)));
|
||||
Assert.True(reporter.Take("tenant-b", "CUSTOM-1", start.AddMinutes(1)));
|
||||
Assert.True(reporter.Take("tenant-a", "CUSTOM-1", start + RuleEvaluationStopReporter.Window));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_tenant_with_many_rules_cannot_flood_the_alarm()
|
||||
{
|
||||
var reporter = new RuleEvaluationStopReporter(TestNotifications.CapturingOpsEvents(_opsEvents));
|
||||
var start = new DateTime(2026, 11, 5, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
var taken = Enumerable.Range(0, 1000).Count(i => reporter.Take("tenant-a", $"CUSTOM-{i}", start.AddSeconds(i)));
|
||||
|
||||
Assert.Equal(RuleEvaluationStopReporter.MaxEventsPerWindow, taken);
|
||||
Assert.True(reporter.Take("tenant-a", "CUSTOM-new", start + RuleEvaluationStopReporter.Window));
|
||||
}
|
||||
|
||||
private RuleWorkGate Gate(string name, int limit)
|
||||
{
|
||||
var ops = TestNotifications.CapturingOpsEvents(_opsEvents);
|
||||
return name switch
|
||||
{
|
||||
"dry-run" => new RuleDryRunGate(limit, ops, NullLogger<RuleDryRunGate>.Instance),
|
||||
"re-analysis" => new ReanalyzeGate(limit, ops, NullLogger<ReanalyzeGate>.Instance),
|
||||
_ => new TestLogPatternGate(limit, ops, NullLogger<TestLogPatternGate>.Instance),
|
||||
};
|
||||
}
|
||||
|
||||
private static string ErrorText(Microsoft.Azure.Functions.Worker.Http.HttpResponseData response)
|
||||
{
|
||||
response.Body.Position = 0;
|
||||
using var doc = JsonDocument.Parse(response.Body);
|
||||
return doc.RootElement.GetProperty("error").GetString()!;
|
||||
}
|
||||
}
|
||||
@@ -140,6 +140,26 @@ public class RulesWireParityTests
|
||||
new DryRunAnalyzeRuleResponse { Success = true, SessionId = sessionId, Result = result });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DryRunAnalyzeRuleResponse_carries_a_stop_with_its_reason_and_note()
|
||||
{
|
||||
var sessionId = "7c1e2a90-1111-4b7c-a1d2-52f3bbbb0006";
|
||||
var result = new RuleDryRun
|
||||
{
|
||||
Verdict = RuleDryRunVerdict.Stopped,
|
||||
EventCount = 4200,
|
||||
BaseConfidence = 50,
|
||||
ConfidenceThreshold = 60,
|
||||
// D-410: verdict "stopped" — the trace keeps only the entries evaluated before the stop.
|
||||
StopReason = EvaluationStopReasons.RegexTimeout,
|
||||
StopNote = "conditions[0] ('slow'): a regex hit the 1 s match timeout on this session's data",
|
||||
};
|
||||
|
||||
AssertParity(
|
||||
new { success = true, sessionId, result },
|
||||
new DryRunAnalyzeRuleResponse { Success = true, SessionId = sessionId, Result = result });
|
||||
}
|
||||
|
||||
// ---- GetRuleResults ------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
@@ -178,6 +198,8 @@ public class RulesWireParityTests
|
||||
};
|
||||
var openResults = results.Where(r => r.ResolvedAt == null).ToList();
|
||||
var persistFailureRuleIds = new List<string> { "ANALYZE-NET-001" };
|
||||
// D-410: rules the reanalyze stopped at its evaluation budget, omitted when there are none.
|
||||
var stoppedRuleIds = new List<string> { "CUSTOM-SLOW-001" };
|
||||
|
||||
AssertParity(
|
||||
new
|
||||
@@ -190,7 +212,8 @@ public class RulesWireParityTests
|
||||
highCount = openResults.Count(r => r.Severity == "high"),
|
||||
warningCount = openResults.Count(r => r.Severity == "warning"),
|
||||
persistFailureCount = persistFailureRuleIds.Count,
|
||||
persistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null
|
||||
persistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null,
|
||||
stoppedRuleIds = stoppedRuleIds.Count > 0 ? stoppedRuleIds : null
|
||||
},
|
||||
new GetRuleResultsResponse
|
||||
{
|
||||
@@ -202,7 +225,8 @@ public class RulesWireParityTests
|
||||
HighCount = openResults.Count(r => r.Severity == "high"),
|
||||
WarningCount = openResults.Count(r => r.Severity == "warning"),
|
||||
PersistFailureCount = persistFailureRuleIds.Count,
|
||||
PersistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null
|
||||
PersistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null,
|
||||
StoppedRuleIds = stoppedRuleIds.Count > 0 ? stoppedRuleIds : null
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -227,7 +227,7 @@ public class TestLogPatternFunctionTests
|
||||
Assert.Equal(1, result.TimeoutCount);
|
||||
Assert.Single(result.Lines);
|
||||
Assert.Equal("regex_timeout", result.Lines[0].Outcome);
|
||||
Assert.Equal(TestLogPatternFunction.StopReasonRegexTimeout, result.StopReason);
|
||||
Assert.Equal(EvaluationStopReasons.RegexTimeout, result.StopReason);
|
||||
Assert.Contains(result.Notes, n => n.Contains("line 1 hit") && n.Contains("lines 2-200 were not evaluated"));
|
||||
Assert.DoesNotContain(result.Notes, n => n.Contains("case-SENSITIVE"));
|
||||
Assert.True(sw.ElapsedMilliseconds < 4000, $"catastrophic request took {sw.ElapsedMilliseconds} ms");
|
||||
@@ -259,7 +259,7 @@ public class TestLogPatternFunctionTests
|
||||
|
||||
Assert.Single(result.Lines);
|
||||
Assert.Equal("matched", result.Lines[0].Outcome);
|
||||
Assert.Equal(TestLogPatternFunction.StopReasonTimeBudget, result.StopReason);
|
||||
Assert.Equal(EvaluationStopReasons.TimeBudget, result.StopReason);
|
||||
Assert.Contains(result.Notes, n => n.Contains("after line 1") && n.Contains("lines 2-3 were not evaluated"));
|
||||
|
||||
var twoLines = TestLogPatternFunction.EvaluatePattern(
|
||||
@@ -282,16 +282,19 @@ public class TestLogPatternFunctionTests
|
||||
TestLogPatternFunction.EvaluationBudget, cts.Token);
|
||||
|
||||
Assert.Empty(result.Lines);
|
||||
Assert.Equal(TestLogPatternFunction.StopReasonCancelled, result.StopReason);
|
||||
Assert.Equal(EvaluationStopReasons.Cancelled, result.StopReason);
|
||||
Assert.Equal(0, result.MatchCount);
|
||||
}
|
||||
|
||||
// ===== the per-instance gate =====
|
||||
|
||||
private static TestLogPatternGate Gate(int limit)
|
||||
=> new(limit, TestNotifications.CapturingOpsEvents(new()), Microsoft.Extensions.Logging.Abstractions.NullLogger<TestLogPatternGate>.Instance);
|
||||
|
||||
[Fact]
|
||||
public void Gate_HandsOutLimitSlots_AndTakesThemBackOnDispose()
|
||||
{
|
||||
var gate = new TestLogPatternGate(1);
|
||||
var gate = Gate(1);
|
||||
Assert.Equal(1, gate.Limit);
|
||||
|
||||
var first = gate.TryEnter();
|
||||
@@ -309,7 +312,7 @@ public class TestLogPatternFunctionTests
|
||||
[Fact]
|
||||
public void Gate_DefaultLimit_IsTwoPerInstance()
|
||||
{
|
||||
var gate = new TestLogPatternGate();
|
||||
var gate = new TestLogPatternGate(TestNotifications.CapturingOpsEvents(new()), Microsoft.Extensions.Logging.Abstractions.NullLogger<TestLogPatternGate>.Instance);
|
||||
Assert.Equal(2, gate.Limit);
|
||||
Assert.Equal(TestLogPatternGate.DefaultLimit, gate.Limit);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using System.Net.Http;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
@@ -24,4 +26,15 @@ internal static class TestNotifications
|
||||
|
||||
internal static OpsAlertDispatchService InertOpsAlertDispatch(AdminConfigurationService adminConfigService)
|
||||
=> new(adminConfigService, InertDispatcher(), NullLogger<OpsAlertDispatchService>.Instance);
|
||||
|
||||
/// <summary>An <see cref="OpsEventService"/> that records every saved event in <paramref name="sink"/> and sends nothing.</summary>
|
||||
internal static OpsEventService CapturingOpsEvents(List<OpsEventEntry> sink)
|
||||
{
|
||||
var repo = new Mock<IOpsEventRepository>();
|
||||
repo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Returns<OpsEventEntry>(e => { lock (sink) sink.Add(e); return Task.CompletedTask; });
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
return new OpsEventService(repo.Object, NullLogger<OpsEventService>.Instance, InertOpsAlertDispatch(adminConfig.Object));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ILogger<ReportUploadGate> _logger;
|
||||
private long _nextAlertTicks;
|
||||
|
||||
public ReportUploadGate(OpsEventService opsEvents, ILogger<ReportUploadGate> logger)
|
||||
: this(DefaultLimit, opsEvents, logger)
|
||||
@@ -44,20 +43,12 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
"Report upload refused on {Route}: {Limit} uploads already in progress on this instance (tenant {TenantId})",
|
||||
route, Limit, tenantId);
|
||||
|
||||
if (TakeAlert(DateTime.UtcNow))
|
||||
if (TakeAlert(DateTime.UtcNow, AlertWindow))
|
||||
await _opsEvents.RecordReportUploadLimitReachedAsync(route, Limit, AlertWindow, tenantId, upn);
|
||||
|
||||
return await req.ErrorAsync(HttpStatusCode.TooManyRequests, Constants.ApiErrorCodes.RateLimited,
|
||||
"Too many reports are being uploaded right now. Try again in a few seconds.",
|
||||
"Too many reports are being uploaded right now.",
|
||||
retryAfterSeconds: RetryAfterSeconds);
|
||||
}
|
||||
|
||||
/// <summary>True for the first refusal of an alert window, the one that raises the ops event.</summary>
|
||||
internal bool TakeAlert(DateTime utcNow)
|
||||
{
|
||||
var next = Interlocked.Read(ref _nextAlertTicks);
|
||||
return utcNow.Ticks >= next
|
||||
&& Interlocked.CompareExchange(ref _nextAlertTicks, utcNow.Add(AlertWindow).Ticks, next) == next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+61
-22
@@ -19,6 +19,11 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
/// modified (the engine's dry-run path has no MarkSessionAsFailed side effect by construction).
|
||||
/// Global-scope callers may dry-run against any tenant's session — the tenant is resolved from
|
||||
/// the session, same as GetRuleResultsFunction.
|
||||
///
|
||||
/// The draft is caller-authored work on the shared app, so it is bounded (D-410): the draft's
|
||||
/// size is capped (<see cref="AnalyzeRuleService.ShapeLimitErrors"/>), the evaluation stops at
|
||||
/// the first regex match timeout, at the 2 s budget or when the caller goes away (verdict
|
||||
/// "stopped"), and <see cref="RuleDryRunGate"/> caps how many dry-runs run per instance.
|
||||
/// </summary>
|
||||
public class DryRunAnalyzeRuleFunction
|
||||
{
|
||||
@@ -26,21 +31,25 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
private readonly IRuleRepository _ruleRepo;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly AnalyzeRuleService _analyzeRuleService;
|
||||
private readonly RuleDryRunGate _gate;
|
||||
|
||||
public DryRunAnalyzeRuleFunction(
|
||||
ILogger<DryRunAnalyzeRuleFunction> logger,
|
||||
IRuleRepository ruleRepo,
|
||||
ISessionRepository sessionRepo,
|
||||
AnalyzeRuleService analyzeRuleService)
|
||||
AnalyzeRuleService analyzeRuleService,
|
||||
RuleDryRunGate gate)
|
||||
{
|
||||
_logger = logger;
|
||||
_ruleRepo = ruleRepo;
|
||||
_sessionRepo = sessionRepo;
|
||||
_analyzeRuleService = analyzeRuleService;
|
||||
_gate = gate;
|
||||
}
|
||||
[Function("DryRunAnalyzeRule")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "rules/analyze/dryrun")] HttpRequestData req)
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "rules/analyze/dryrun")] HttpRequestData req,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Authentication + TenantAdminOrGlobalReader authorization enforced by PolicyEnforcementMiddleware.
|
||||
// Read-only diagnostic, so the read-only Global Reader is admitted alongside Tenant Admins and GAs.
|
||||
@@ -66,6 +75,11 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
var sessionId = request.SessionId!.Trim();
|
||||
var rule = request.Rule!;
|
||||
|
||||
// A slot of this instance before any tenant data is read; held to the response.
|
||||
using var slot = _gate.TryEnter();
|
||||
if (slot == null)
|
||||
return await _gate.RefuseAsync(req, requestCtx.TenantId, requestCtx.UserPrincipalName);
|
||||
|
||||
// Global-scope cross-tenant fallback: resolve the session's actual tenant so a GA /
|
||||
// Global Reader can dry-run against any tenant's session (mirrors GetRuleResultsFunction).
|
||||
effectiveTenantId = await requestCtx.ResolveSessionScopeAsync(_sessionRepo, sessionId);
|
||||
@@ -77,11 +91,23 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
}
|
||||
|
||||
var ruleEngine = new RuleEngine(_analyzeRuleService, _ruleRepo, _sessionRepo, _logger);
|
||||
var result = await ruleEngine.DryRunRuleAsync(effectiveTenantId, sessionId, rule);
|
||||
var result = await ruleEngine.DryRunRuleAsync(effectiveTenantId, sessionId, rule, cancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Dry-run of draft rule {RuleId} against session {SessionId}: verdict={Verdict}, confidence={Confidence}",
|
||||
string.IsNullOrEmpty(rule.RuleId) ? "(no id)" : rule.RuleId, sessionId, result.Verdict, result.FinalConfidence);
|
||||
var ruleLabel = string.IsNullOrEmpty(rule.RuleId) ? "(no id)" : rule.RuleId;
|
||||
if (result.Verdict == RuleDryRunVerdict.Stopped)
|
||||
{
|
||||
// Warning so it reaches App Insights: one hit is an author with a broken rule, a
|
||||
// cluster from one caller is load aimed at this endpoint (the gate reports that).
|
||||
_logger.LogWarning(
|
||||
"Analyze dry-run stopped: {Reason} for draft rule {RuleId} against session {SessionId} ({EventCount} events): {Note}",
|
||||
result.StopReason, ruleLabel, sessionId, result.EventCount, result.StopNote);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Dry-run of draft rule {RuleId} against session {SessionId}: verdict={Verdict}, confidence={Confidence}",
|
||||
ruleLabel, sessionId, result.Verdict, result.FinalConfidence);
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new DryRunAnalyzeRuleResponse { Success = true, SessionId = sessionId, Result = result });
|
||||
@@ -124,8 +150,9 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
/// <summary>
|
||||
/// Structural validation of a draft rule before evaluation. Returns human-readable errors
|
||||
/// (empty list = valid). Deliberately limited to what would make the dry-run silently
|
||||
/// meaningless or throw — full authoring lint (schema, guardrails, event-type catalog)
|
||||
/// lives client-side in the MCP `validate_rule` tool.
|
||||
/// meaningless or throw, plus the size limits every write path enforces too — full
|
||||
/// authoring lint (schema, guardrails, event-type catalog) lives client-side in the MCP
|
||||
/// `validate_rule` tool. Also the gate of community rule submissions.
|
||||
/// </summary>
|
||||
internal static List<string> ValidateDraftRule(AnalyzeRule? rule)
|
||||
{
|
||||
@@ -185,22 +212,11 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
System.Globalization.CultureInfo.InvariantCulture, out var skewThreshold) || skewThreshold <= 0)
|
||||
errors.Add($"{label}: clock_skew requires a positive numeric value (threshold in seconds)");
|
||||
}
|
||||
|
||||
if ((string.Equals(c.Operator, "regex", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(c.Operator, "not_regex", StringComparison.OrdinalIgnoreCase))
|
||||
&& !string.IsNullOrEmpty(c.Value))
|
||||
{
|
||||
try
|
||||
{
|
||||
_ = new Regex(c.Value, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
errors.Add($"{label}: value is not a valid regex: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Counts and every regex-bearing value (length first, then it must parse).
|
||||
errors.AddRange(AnalyzeRuleService.ShapeLimitErrors(rule));
|
||||
|
||||
for (var i = 0; i < (rule.ConfidenceFactors?.Count ?? 0); i++)
|
||||
{
|
||||
var f = rule.ConfidenceFactors![i];
|
||||
@@ -214,4 +230,27 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-instance cap on concurrent analyze dry-runs (singleton). A slot is held from before the
|
||||
/// session is read to the response, so this endpoint never occupies more than
|
||||
/// <see cref="ConcurrencyGate.Limit"/> threads of an instance, whatever the request rate.
|
||||
/// Retry-After 5 s: an event read plus the 2 s budget plus one running match.
|
||||
/// </summary>
|
||||
public sealed class RuleDryRunGate : RuleWorkGate
|
||||
{
|
||||
internal const int DefaultLimit = 2;
|
||||
|
||||
public RuleDryRunGate(OpsEventService opsEvents, ILogger<RuleDryRunGate> logger)
|
||||
: this(DefaultLimit, opsEvents, logger)
|
||||
{
|
||||
}
|
||||
|
||||
internal RuleDryRunGate(int limit, OpsEventService opsEvents, ILogger<RuleDryRunGate> logger)
|
||||
: base(limit, "dry-run",
|
||||
"Too many rule tests are running right now.",
|
||||
retryAfterSeconds: 5, opsEvents, logger)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
/// Returns analysis results (rule evaluations) for a session.
|
||||
/// Supports on-demand re-analysis via ?reanalyze=true query parameter.
|
||||
/// Global admins can request analysis for any tenant by passing ?tenantId=...
|
||||
/// The re-analysis runs the tenant's rules synchronously, so it takes a slot of
|
||||
/// <see cref="ReanalyzeGate"/> and its run is bounded by the evaluation budget (D-410).
|
||||
/// </summary>
|
||||
public class GetRuleResultsFunction
|
||||
{
|
||||
@@ -21,19 +23,25 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly AnalyzeRuleService _analyzeRuleService;
|
||||
private readonly ReanalyzeGate _gate;
|
||||
private readonly RuleEvaluationStopReporter _stopReporter;
|
||||
|
||||
public GetRuleResultsFunction(
|
||||
ILogger<GetRuleResultsFunction> logger,
|
||||
IRuleRepository ruleRepo,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
ISessionRepository sessionRepo,
|
||||
AnalyzeRuleService analyzeRuleService)
|
||||
AnalyzeRuleService analyzeRuleService,
|
||||
ReanalyzeGate gate,
|
||||
RuleEvaluationStopReporter stopReporter)
|
||||
{
|
||||
_logger = logger;
|
||||
_ruleRepo = ruleRepo;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_sessionRepo = sessionRepo;
|
||||
_analyzeRuleService = analyzeRuleService;
|
||||
_gate = gate;
|
||||
_stopReporter = stopReporter;
|
||||
}
|
||||
|
||||
[Function("GetRuleResults")]
|
||||
@@ -65,6 +73,14 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
// and a 500 would blank the page even when most rules persisted fine.
|
||||
var persistFailureRuleIds = new List<string>();
|
||||
|
||||
// Rules the re-analysis could not evaluate within its budget; they keep their rows.
|
||||
var stoppedRuleIds = new List<string>();
|
||||
|
||||
// A slot of this instance for the whole rebuild (analysis, delete, rewrite).
|
||||
using var slot = reanalyze ? _gate.TryEnter() : null;
|
||||
if (reanalyze && slot == null)
|
||||
return await _gate.RefuseAsync(req, requestCtx.TenantId, requestCtx.UserPrincipalName);
|
||||
|
||||
if (reanalyze)
|
||||
{
|
||||
try
|
||||
@@ -76,12 +92,16 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
// channel notification. Rules that no longer fire come back as ResolvedResults
|
||||
// (kept for audit) instead of silently vanishing.
|
||||
var outcome = await ruleEngine.AnalyzeSessionAsync(effectiveTenantId, sessionId, reanalyze: true);
|
||||
stoppedRuleIds.AddRange(outcome.Stops.Select(s => s.RuleId).Concat(outcome.SkippedRuleIds));
|
||||
await _stopReporter.ReportAsync(effectiveTenantId, sessionId, AnalyzeRunKind.Reanalyze, outcome);
|
||||
|
||||
// Delete existing results so stale entries (e.g. rows from since-deleted or
|
||||
// disabled rules) don't persist after re-analysis
|
||||
// disabled rules) don't persist after re-analysis. Rows of rules the run could
|
||||
// not evaluate (KeptResults) go back unchanged — a stop is no evidence that a
|
||||
// finding went away.
|
||||
await _maintenanceRepo.DeleteSessionRuleResultsAsync(effectiveTenantId, sessionId);
|
||||
|
||||
foreach (var result in outcome.Results.Concat(outcome.ResolvedResults))
|
||||
foreach (var result in outcome.Results.Concat(outcome.ResolvedResults).Concat(outcome.KeptResults))
|
||||
{
|
||||
var stored = await _ruleRepo.StoreRuleResultAsync(result);
|
||||
if (!stored)
|
||||
@@ -133,9 +153,33 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
HighCount = openResults.Count(r => r.Severity == "high"),
|
||||
WarningCount = openResults.Count(r => r.Severity == "warning"),
|
||||
PersistFailureCount = persistFailureRuleIds.Count,
|
||||
PersistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null
|
||||
PersistFailureRuleIds = persistFailureRuleIds.Count > 0 ? persistFailureRuleIds : null,
|
||||
StoppedRuleIds = stoppedRuleIds.Count > 0 ? stoppedRuleIds : null
|
||||
});
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-instance cap on concurrent "Analyze now" rebuilds (singleton). A slot is held from the
|
||||
/// analysis to the last rewritten row. The button is the only caller and a rebuild takes a
|
||||
/// fraction of a second, so four slots never refuse a person; they stop a request flood from
|
||||
/// holding every thread of an instance. Retry-After 5 s: one run's budget.
|
||||
/// </summary>
|
||||
public sealed class ReanalyzeGate : RuleWorkGate
|
||||
{
|
||||
internal const int DefaultLimit = 4;
|
||||
|
||||
public ReanalyzeGate(OpsEventService opsEvents, ILogger<ReanalyzeGate> logger)
|
||||
: this(DefaultLimit, opsEvents, logger)
|
||||
{
|
||||
}
|
||||
|
||||
internal ReanalyzeGate(int limit, OpsEventService opsEvents, ILogger<ReanalyzeGate> logger)
|
||||
: base(limit, "re-analysis",
|
||||
"Too many analyses are running right now.",
|
||||
retryAfterSeconds: 5, opsEvents, logger)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Logging;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -61,16 +62,10 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
/// <summary>
|
||||
/// Wall-clock budget for one whole test. A legitimate pattern finishes 200 lines in
|
||||
/// milliseconds; the worst case per request is this budget plus one running match
|
||||
/// (≤ <see cref="MatchTimeout"/>), which is what <see cref="RetryAfterSeconds"/> promises.
|
||||
/// (≤ <see cref="MatchTimeout"/>), which is what the gate's Retry-After promises.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan EvaluationBudget = TimeSpan.FromSeconds(2);
|
||||
|
||||
private const int RetryAfterSeconds = 3;
|
||||
|
||||
internal const string StopReasonRegexTimeout = "regex_timeout";
|
||||
internal const string StopReasonTimeBudget = "time_budget";
|
||||
internal const string StopReasonCancelled = "cancelled";
|
||||
|
||||
[Function("TestLogPattern")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "rules/gather/test-pattern")] HttpRequestData req,
|
||||
@@ -94,12 +89,8 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
var slot = _gate.TryEnter();
|
||||
if (slot == null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Log-pattern test refused: {Limit} evaluations already running on this instance",
|
||||
_gate.Limit);
|
||||
return await req.ErrorAsync(HttpStatusCode.TooManyRequests, Constants.ApiErrorCodes.RateLimited,
|
||||
"too many pattern tests are running right now — retry in a few seconds",
|
||||
retryAfterSeconds: RetryAfterSeconds);
|
||||
var requestCtx = req.GetRequestContext();
|
||||
return await _gate.RefuseAsync(req, requestCtx.TenantId, requestCtx.UserPrincipalName);
|
||||
}
|
||||
|
||||
using (slot)
|
||||
@@ -190,7 +181,7 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
Stop(result, StopReasonCancelled,
|
||||
Stop(result, EvaluationStopReasons.Cancelled,
|
||||
i == 0 ? "evaluation cancelled before the first line" : $"evaluation cancelled after line {i}",
|
||||
i, lines.Count);
|
||||
break;
|
||||
@@ -198,7 +189,7 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
// Checked before every line but the first: the previous line spent the budget.
|
||||
if (i > 0 && clock.Elapsed >= budget)
|
||||
{
|
||||
Stop(result, StopReasonTimeBudget,
|
||||
Stop(result, EvaluationStopReasons.TimeBudget,
|
||||
$"evaluation stopped after line {i}: the test exceeded its {budget.TotalSeconds:0.#} s budget — a pattern this slow would strain the device too; anchor it (^) and avoid unbounded .* scans",
|
||||
i, lines.Count);
|
||||
break;
|
||||
@@ -236,7 +227,7 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
{
|
||||
row.Outcome = "regex_timeout";
|
||||
result.TimeoutCount++;
|
||||
Stop(result, StopReasonRegexTimeout,
|
||||
Stop(result, EvaluationStopReasons.RegexTimeout,
|
||||
$"line {i + 1} hit the agent's {MatchTimeout.TotalSeconds:0.#} s match timeout (catastrophic backtracking) — on the device such lines are skipped and only counted as regex timeouts; remove nested or overlapping quantifiers before deploying",
|
||||
i + 1, lines.Count);
|
||||
break;
|
||||
@@ -308,17 +299,21 @@ namespace AutopilotMonitor.Functions.Functions.Rules
|
||||
/// construction to the end of the evaluation; a caller finding no free slot is answered 429
|
||||
/// at once, so this endpoint can never occupy more than <see cref="ConcurrencyGate.Limit"/>
|
||||
/// worker threads of an instance, whatever the request rate. Nothing but this test waits on
|
||||
/// this gate.
|
||||
/// this gate. Retry-After 3 s: the 2 s budget plus one running match.
|
||||
/// </summary>
|
||||
public sealed class TestLogPatternGate : ConcurrencyGate
|
||||
public sealed class TestLogPatternGate : RuleWorkGate
|
||||
{
|
||||
internal const int DefaultLimit = 2;
|
||||
|
||||
public TestLogPatternGate() : this(DefaultLimit)
|
||||
public TestLogPatternGate(OpsEventService opsEvents, ILogger<TestLogPatternGate> logger)
|
||||
: this(DefaultLimit, opsEvents, logger)
|
||||
{
|
||||
}
|
||||
|
||||
internal TestLogPatternGate(int limit) : base(limit)
|
||||
internal TestLogPatternGate(int limit, OpsEventService opsEvents, ILogger<TestLogPatternGate> logger)
|
||||
: base(limit, "pattern-test",
|
||||
"Too many pattern tests are running right now.",
|
||||
retryAfterSeconds: 3, opsEvents, logger)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ namespace AutopilotMonitor.Functions.Helpers;
|
||||
public abstract class ConcurrencyGate
|
||||
{
|
||||
private readonly SemaphoreSlim _slots;
|
||||
private long _nextAlertTicks;
|
||||
|
||||
protected ConcurrencyGate(int limit)
|
||||
{
|
||||
@@ -22,6 +23,17 @@ public abstract class ConcurrencyGate
|
||||
/// <summary>A slot to dispose when the work ends, or null when every slot is taken.</summary>
|
||||
public IDisposable? TryEnter() => _slots.Wait(0) ? new Slot(_slots) : null;
|
||||
|
||||
/// <summary>
|
||||
/// True for the first refusal of an alert window, the one that raises the gate's ops event;
|
||||
/// later refusals in that window get false, so a full gate can never flood the alarm.
|
||||
/// </summary>
|
||||
internal bool TakeAlert(DateTime utcNow, TimeSpan window)
|
||||
{
|
||||
var next = Interlocked.Read(ref _nextAlertTicks);
|
||||
return utcNow.Ticks >= next
|
||||
&& Interlocked.CompareExchange(ref _nextAlertTicks, utcNow.Add(window).Ticks, next) == next;
|
||||
}
|
||||
|
||||
private sealed class Slot : IDisposable
|
||||
{
|
||||
private SemaphoreSlim? _slots;
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Helpers;
|
||||
|
||||
/// <summary>
|
||||
/// Per-instance cap on rule work a caller starts synchronously: the analyze dry-run, the manual
|
||||
/// re-analysis and the logparser pattern test (D-379, D-410), each its own singleton subclass. The
|
||||
/// work runs caller-authored patterns on the shared app, so the request rate must not decide how
|
||||
/// many threads it holds. A caller who finds every slot taken is answered 429 at once with a retry
|
||||
/// hint; the first refusal per instance, gate and hour raises <c>RuleEvaluationLimitReached</c> for
|
||||
/// the operators, later refusals in that window stay in the Warning trace. Legitimate use never
|
||||
/// fills a gate, so a refusal is load aimed at it.
|
||||
/// </summary>
|
||||
public abstract class RuleWorkGate : ConcurrencyGate
|
||||
{
|
||||
internal static readonly TimeSpan AlertWindow = TimeSpan.FromHours(1);
|
||||
|
||||
private readonly string _gate;
|
||||
private readonly string _busyMessage;
|
||||
private readonly int _retryAfterSeconds;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ILogger _logger;
|
||||
|
||||
protected RuleWorkGate(int limit, string gate, string busyMessage, int retryAfterSeconds, OpsEventService opsEvents, ILogger logger)
|
||||
: base(limit)
|
||||
{
|
||||
_gate = gate;
|
||||
_busyMessage = busyMessage;
|
||||
_retryAfterSeconds = retryAfterSeconds;
|
||||
_opsEvents = opsEvents;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The 429 for a caller who found every slot taken. The message names the situation only;
|
||||
/// clients phrase the retry from <c>Retry-After</c> (the portal appends "Try again in N s.").
|
||||
/// </summary>
|
||||
public async Task<HttpResponseData> RefuseAsync(HttpRequestData req, string? tenantId, string? upn)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Rule work refused: the {Gate} gate has {Limit} evaluations running on this instance (tenant {TenantId})",
|
||||
_gate, Limit, tenantId);
|
||||
|
||||
if (TakeAlert(DateTime.UtcNow, AlertWindow))
|
||||
await _opsEvents.RecordRuleEvaluationLimitReachedAsync(_gate, Limit, AlertWindow, tenantId, upn);
|
||||
|
||||
return await req.ErrorAsync(HttpStatusCode.TooManyRequests, Constants.ApiErrorCodes.RateLimited,
|
||||
_busyMessage, retryAfterSeconds: _retryAfterSeconds);
|
||||
}
|
||||
}
|
||||
@@ -158,8 +158,13 @@ builder.Services.AddMemoryCache();
|
||||
builder.Services.AddHttpClient();
|
||||
builder.Services.AddSingleton<ResiliencePolicies>();
|
||||
builder.Services.AddSingleton<TableStorageService>();
|
||||
// Per-instance cap on concurrent logparser pattern tests (caller-controlled regex work, D-379).
|
||||
// Per-instance caps on rule work a caller starts synchronously, each with an ops event when it is
|
||||
// hit: logparser pattern tests (D-379), analyze dry-runs and "Analyze now" rebuilds (D-410).
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Functions.Rules.TestLogPatternGate>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Functions.Rules.RuleDryRunGate>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Functions.Rules.ReanalyzeGate>();
|
||||
// Throttled ops events for rules the analysis budget stopped (D-410).
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.RuleEvaluationStopReporter>();
|
||||
// Per-instance cap on concurrent report uploads (20 MiB bodies), with an ops event when it is hit.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Functions.Reports.ReportUploadGate>();
|
||||
// Shared QueueClient bootstrap (code-quality audit D2): single Managed-Identity/connection-string
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using System;
|
||||
using System.IO.Enumeration;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Security
|
||||
@@ -11,6 +11,13 @@ namespace AutopilotMonitor.Functions.Security
|
||||
/// </summary>
|
||||
public static class HardwareWhitelistValidator
|
||||
{
|
||||
/// <summary>
|
||||
/// Longest device value an entry other than "*" is compared with. Real SMBIOS manufacturer
|
||||
/// and model strings are a few dozen characters; a longer header is not one, and the
|
||||
/// comparison costs value × entry length on every agent request (D-411).
|
||||
/// </summary>
|
||||
internal const int MaxMatchedValueLength = 256;
|
||||
|
||||
/// <summary>
|
||||
/// Validates hardware against whitelist
|
||||
/// </summary>
|
||||
@@ -50,11 +57,11 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var manufacturerMatch = manufacturerWhitelist.Any(pattern => MatchesWildcard(manufacturer, pattern));
|
||||
if (!manufacturerMatch)
|
||||
{
|
||||
logger?.LogWarning($"Hardware rejected: Manufacturer '{manufacturer}' not in whitelist");
|
||||
logger?.LogWarning($"Hardware rejected: Manufacturer '{Shorten(manufacturer)}' not in whitelist");
|
||||
return new HardwareValidationResult
|
||||
{
|
||||
IsValid = false,
|
||||
ErrorMessage = $"Manufacturer '{manufacturer}' is not authorized",
|
||||
ErrorMessage = $"Manufacturer '{Shorten(manufacturer)}' is not authorized",
|
||||
Manufacturer = manufacturer,
|
||||
Model = model
|
||||
};
|
||||
@@ -64,11 +71,11 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var modelMatch = modelWhitelist.Any(pattern => MatchesWildcard(model, pattern));
|
||||
if (!modelMatch)
|
||||
{
|
||||
logger?.LogWarning($"Hardware rejected: Model '{model}' not in whitelist");
|
||||
logger?.LogWarning($"Hardware rejected: Model '{Shorten(model)}' not in whitelist");
|
||||
return new HardwareValidationResult
|
||||
{
|
||||
IsValid = false,
|
||||
ErrorMessage = $"Model '{model}' is not authorized",
|
||||
ErrorMessage = $"Model '{Shorten(model)}' is not authorized",
|
||||
Manufacturer = manufacturer,
|
||||
Model = model
|
||||
};
|
||||
@@ -84,30 +91,26 @@ namespace AutopilotMonitor.Functions.Security
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Matches a string against a wildcard pattern (supports * and ?)
|
||||
/// Whole-value wildcard match, case-insensitive: '*' is any run of characters (also none),
|
||||
/// '?' exactly one, everything else literal. The tenant writes the entries and the device
|
||||
/// the value, so no Regex is built from them: the BCL's simple-expression matcher needs at
|
||||
/// most value × entry steps and never backtracks (D-411). Its escape character is '\',
|
||||
/// which is literal in an entry, so it is doubled first. Case folding is ordinal — SMBIOS
|
||||
/// strings are ASCII.
|
||||
/// </summary>
|
||||
private static bool MatchesWildcard(string text, string pattern)
|
||||
internal static bool MatchesWildcard(string text, string pattern)
|
||||
{
|
||||
if (pattern == "*")
|
||||
return true;
|
||||
|
||||
// Convert wildcard pattern to regex
|
||||
// Escape regex special chars, then replace * and ?
|
||||
var regexPattern = "^" + Regex.Escape(pattern)
|
||||
.Replace("\\*", ".*")
|
||||
.Replace("\\?", ".") + "$";
|
||||
|
||||
try
|
||||
{
|
||||
// Use timeout to prevent ReDoS attacks with pathological input
|
||||
return Regex.IsMatch(text, regexPattern, RegexOptions.IgnoreCase, TimeSpan.FromMilliseconds(100));
|
||||
}
|
||||
catch (RegexMatchTimeoutException)
|
||||
{
|
||||
// Timeout indicates potential ReDoS attempt - treat as non-match
|
||||
if (text.Length > MaxMatchedValueLength)
|
||||
return false;
|
||||
}
|
||||
|
||||
return FileSystemName.MatchesSimpleExpression(pattern.Replace("\\", "\\\\"), text, ignoreCase: true);
|
||||
}
|
||||
|
||||
/// <summary>The value as a rejection message shows it: never longer than a real hardware string.</summary>
|
||||
private static string Shorten(string value)
|
||||
=> value.Length <= MaxMatchedValueLength ? value : value.Substring(0, MaxMatchedValueLength) + "…";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+7
@@ -51,6 +51,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
|
||||
private readonly TenantConfigurationService _configService;
|
||||
private readonly Notifications.NotificationChannelDispatcher _channelDispatcher;
|
||||
private readonly Deletion.SessionDeletionGuard _deletionGuard;
|
||||
private readonly RuleEvaluationStopReporter _stopReporter;
|
||||
private readonly ILogger<AnalyzeOnEnrollmentEndHandler> _logger;
|
||||
|
||||
public const string ReasonEnrollmentComplete = "enrollment_complete";
|
||||
@@ -68,6 +69,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
|
||||
TenantConfigurationService configService,
|
||||
Notifications.NotificationChannelDispatcher channelDispatcher,
|
||||
Deletion.SessionDeletionGuard deletionGuard,
|
||||
RuleEvaluationStopReporter stopReporter,
|
||||
ILogger<AnalyzeOnEnrollmentEndHandler> logger)
|
||||
{
|
||||
_ruleService = ruleService;
|
||||
@@ -78,6 +80,7 @@ namespace AutopilotMonitor.Functions.Services.Analyze
|
||||
_configService = configService;
|
||||
_channelDispatcher = channelDispatcher;
|
||||
_deletionGuard = deletionGuard;
|
||||
_stopReporter = stopReporter;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -131,6 +134,10 @@ namespace AutopilotMonitor.Functions.Services.Analyze
|
||||
var ruleEngine = new RuleEngine(_ruleService, _ruleRepo, _sessionRepo, _logger);
|
||||
var outcome = await ruleEngine.AnalyzeSessionAsync(envelope.TenantId, envelope.SessionId, context).ConfigureAwait(false);
|
||||
|
||||
// Rules the evaluation budget stopped (D-410) become throttled ops events. Their stored
|
||||
// rows were left alone by the engine; this path only upserts, so KeptResults is moot.
|
||||
await _stopReporter.ReportAsync(envelope.TenantId, envelope.SessionId, context.Kind, outcome).ConfigureAwait(false);
|
||||
|
||||
foreach (var result in outcome.Results.Concat(outcome.ResolvedResults))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using AutopilotMonitor.Functions.Services.Caching;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -334,6 +335,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
ValidateEvaluateOn(rule);
|
||||
ValidateRelatedDocs(rule);
|
||||
ValidatePresentation(rule);
|
||||
ValidateShapeLimits(rule);
|
||||
|
||||
rule.IsBuiltIn = false;
|
||||
rule.IsCommunity = false;
|
||||
@@ -373,6 +375,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
ValidateEvaluateOn(rule);
|
||||
ValidateRelatedDocs(rule);
|
||||
ValidatePresentation(rule);
|
||||
ValidateShapeLimits(rule);
|
||||
|
||||
// Custom rules are fully tenant-owned: fold a notify override into the row default
|
||||
// (there is no separate RuleState for them) — mirrors how the UI edits Enabled.
|
||||
@@ -473,6 +476,9 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
// A template variable can carry the tenant's own regex (ANALYZE-SEC-006's allow-list).
|
||||
ValidateShapeLimits(customRule);
|
||||
|
||||
// Store the custom rule in the tenant partition
|
||||
var success = await _ruleRepo.StoreAnalyzeRuleAsync(customRule, tenantId);
|
||||
if (!success)
|
||||
@@ -825,6 +831,77 @@ namespace AutopilotMonitor.Functions.Services
|
||||
private static bool IsTemplateWithin(string? text, int maxLength)
|
||||
=> !string.IsNullOrWhiteSpace(text) && text.Length <= maxLength;
|
||||
|
||||
/// <summary>
|
||||
/// Rejects a rule over the <see cref="AnalyzeRuleLimits"/> with <see cref="ArgumentException"/>
|
||||
/// (mapped to 400) naming every offending field. Runs on create, custom-rule update (also the
|
||||
/// portal's enable/notify toggles, which send the whole rule) and template copy.
|
||||
/// </summary>
|
||||
internal static void ValidateShapeLimits(AnalyzeRule rule)
|
||||
{
|
||||
var errors = ShapeLimitErrors(rule);
|
||||
if (errors.Count > 0)
|
||||
throw new ArgumentException("Invalid rule: " + string.Join("; ", errors));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The size limits of a tenant-authored rule (D-410): the number of conditions and
|
||||
/// preconditions, and every value compared with regex or not_regex (condition value,
|
||||
/// filter value, event A filter value, precondition value). The length is checked before a
|
||||
/// Regex is built; a value within it must then parse. Returns user-facing errors, empty when
|
||||
/// the rule fits. One implementation for the dry-run validation and every write path.
|
||||
/// </summary>
|
||||
internal static List<string> ShapeLimitErrors(AnalyzeRule rule)
|
||||
{
|
||||
var errors = new List<string>();
|
||||
var conditions = rule.Conditions ?? new List<RuleCondition>();
|
||||
var preconditions = rule.Preconditions ?? new List<RulePrecondition>();
|
||||
|
||||
if (conditions.Count > AnalyzeRuleLimits.MaxConditions)
|
||||
errors.Add($"a rule holds at most {AnalyzeRuleLimits.MaxConditions} conditions (this one has {conditions.Count})");
|
||||
if (preconditions.Count > AnalyzeRuleLimits.MaxPreconditions)
|
||||
errors.Add($"a rule holds at most {AnalyzeRuleLimits.MaxPreconditions} preconditions (this one has {preconditions.Count})");
|
||||
|
||||
for (var i = 0; i < conditions.Count; i++)
|
||||
{
|
||||
var c = conditions[i];
|
||||
if (c == null) continue;
|
||||
CheckRegexValue(errors, $"conditions[{i}].value", c.Operator, c.Value);
|
||||
CheckRegexValue(errors, $"conditions[{i}].filterValue", c.FilterOperator, c.FilterValue);
|
||||
CheckRegexValue(errors, $"conditions[{i}].eventAFilterValue", c.EventAFilterOperator, c.EventAFilterValue);
|
||||
}
|
||||
for (var i = 0; i < preconditions.Count; i++)
|
||||
{
|
||||
var p = preconditions[i];
|
||||
if (p == null) continue;
|
||||
CheckRegexValue(errors, $"preconditions[{i}].value", p.Operator, p.Value);
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
private static void CheckRegexValue(List<string> errors, string label, string? op, string? value)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value)
|
||||
|| !(string.Equals(op, "regex", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(op, "not_regex", StringComparison.OrdinalIgnoreCase)))
|
||||
return;
|
||||
|
||||
if (value.Length > AnalyzeRuleLimits.MaxRegexLength)
|
||||
{
|
||||
errors.Add($"{label}: a regex may be at most {AnalyzeRuleLimits.MaxRegexLength} characters (this one has {value.Length})");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_ = new Regex(value, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
errors.Add($"{label}: value is not a valid regex: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
internal static bool IsAbsoluteHttpUrl(string? url)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
|
||||
@@ -552,6 +552,18 @@ namespace AutopilotMonitor.Functions.Services
|
||||
tenantId, upn ?? "System.ReportUploadGate",
|
||||
new { route, limit, windowMinutes = (int)window.TotalMinutes, upn });
|
||||
|
||||
/// <summary>
|
||||
/// Every slot of a rule-work gate of this instance was taken and a caller got 429
|
||||
/// (<c>RuleWorkGate</c>: dry-run, re-analysis or pattern test, D-410). Raised for the first
|
||||
/// refusal of a window only; the gate keeps the rest in the Warning trace. Recorded against
|
||||
/// the refused caller's tenant.
|
||||
/// </summary>
|
||||
public Task RecordRuleEvaluationLimitReachedAsync(string gate, int limit, TimeSpan window, string? tenantId, string? upn)
|
||||
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.RuleEvaluationLimitReached, OpsEventSeverity.Warning,
|
||||
$"Rule work limit reached: the {gate} gate has {limit} evaluations running on this instance and a caller got 429; further refusals in the next {window.TotalMinutes:0} min stay in the Warning trace",
|
||||
tenantId, upn ?? "System.RuleWorkGate",
|
||||
new { gate, limit, windowMinutes = (int)window.TotalMinutes, upn });
|
||||
|
||||
/// <summary>
|
||||
/// The reporter's per-instance budget is spent: one Critical marker (same event type, so the same
|
||||
/// alert rule pushes it) that says further denials this window are trace-only. A storm is a
|
||||
@@ -1041,6 +1053,22 @@ namespace AutopilotMonitor.Functions.Services
|
||||
lift,
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
/// An analysis run stopped a rule (D-410): a regex hit its match timeout, or the run used up
|
||||
/// its time budget in this rule and left <paramref name="skippedRules"/> later rules
|
||||
/// unevaluated. Warning for a tenant's custom rule, Error for a built-in or community rule.
|
||||
/// <c>RuleEvaluationStopReporter</c> throttles it; every stop is also in the Warning trace.
|
||||
/// </summary>
|
||||
public Task RecordRuleEvaluationStoppedAsync(
|
||||
string tenantId, string sessionId, string ruleId, string reason, bool isCatalogRule, string runKind, int skippedRules)
|
||||
=> WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.RuleEvaluationStopped,
|
||||
isCatalogRule ? OpsEventSeverity.Error : OpsEventSeverity.Warning,
|
||||
reason == EvaluationStopReasons.RegexTimeout
|
||||
? $"Rule {ruleId} stopped in the {runKind} analysis of session {sessionId}: a regex hit its 1 s match timeout; the rule kept its previous result"
|
||||
: $"The {runKind} analysis of session {sessionId} used up its time budget in rule {ruleId}; {skippedRules} later rule(s) were not evaluated and kept their previous results",
|
||||
tenantId, "System.RuleEngine",
|
||||
new { ruleId, reason, catalogRule = isCatalogRule, sessionId, runKind, skippedRules });
|
||||
|
||||
/// <summary>
|
||||
/// App-version duration regression radar: an app's newest version installs with a median
|
||||
/// duration ≥2× (and ≥5 min absolute) over the previous version's median. Fired ONCE per
|
||||
|
||||
@@ -29,7 +29,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// (e.g. "skip when an enrollment_complete event exists"). Any other operator without a
|
||||
/// field has nothing to compare → fail closed.
|
||||
/// </summary>
|
||||
private bool EvaluatePrecondition(RulePrecondition precondition, List<EnrollmentEvent> events)
|
||||
private bool EvaluatePrecondition(RulePrecondition precondition, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
if (!string.Equals(precondition.Source, "event_data", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
@@ -69,34 +69,34 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return matchingEvents.Any(e =>
|
||||
{
|
||||
var fieldValue = GetDataFieldValue(e, precondition.DataField);
|
||||
return fieldValue != null && MatchesOperator(fieldValue, precondition.Operator ?? string.Empty, precondition.Value ?? string.Empty);
|
||||
return fieldValue != null && MatchesOperator(fieldValue, precondition.Operator ?? string.Empty, precondition.Value ?? string.Empty, budget);
|
||||
});
|
||||
}
|
||||
|
||||
private (bool matched, object evidence) EvaluateCondition(RuleCondition condition, List<EnrollmentEvent> events)
|
||||
private (bool matched, object evidence) EvaluateCondition(RuleCondition condition, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
switch (condition.Source)
|
||||
{
|
||||
case "event_type":
|
||||
return EvaluateEventTypeCondition(condition, events);
|
||||
return EvaluateEventTypeCondition(condition, events, budget);
|
||||
|
||||
case "event_data":
|
||||
return EvaluateEventDataCondition(condition, events);
|
||||
return EvaluateEventDataCondition(condition, events, budget);
|
||||
|
||||
case "event_data_array":
|
||||
return EvaluateEventDataArrayCondition(condition, events);
|
||||
return EvaluateEventDataArrayCondition(condition, events, budget);
|
||||
|
||||
case "event_count":
|
||||
return EvaluateEventCountCondition(condition, events);
|
||||
return EvaluateEventCountCondition(condition, events, budget);
|
||||
|
||||
case "phase_duration":
|
||||
return EvaluatePhaseDurationCondition(condition, events);
|
||||
|
||||
case "app_install_duration":
|
||||
return EvaluateAppInstallDurationCondition(condition, events);
|
||||
return EvaluateAppInstallDurationCondition(condition, events, budget);
|
||||
|
||||
case "event_correlation":
|
||||
return EvaluateEventCorrelationCondition(condition, events);
|
||||
return EvaluateEventCorrelationCondition(condition, events, budget);
|
||||
|
||||
case "clock_skew":
|
||||
return EvaluateClockSkewCondition(condition, events);
|
||||
@@ -106,7 +106,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
private (bool matched, object evidence) EvaluateEventTypeCondition(RuleCondition condition, List<EnrollmentEvent> events)
|
||||
private (bool matched, object evidence) EvaluateEventTypeCondition(RuleCondition condition, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
var matchingEvents = events.Where(e => MatchesEventType(e, condition.EventType)).ToList();
|
||||
|
||||
@@ -148,10 +148,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
foreach (var evt in matchingEvents)
|
||||
{
|
||||
var fieldValue = GetDataFieldValue(evt, condition.DataField);
|
||||
if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value))
|
||||
if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value, budget))
|
||||
{
|
||||
// Check suppressByEvent: skip this match if a resolving event exists
|
||||
if (IsSuppressedByEvent(evt, condition.SuppressByEvent, events))
|
||||
if (IsSuppressedByEvent(evt, condition.SuppressByEvent, events, budget))
|
||||
continue;
|
||||
|
||||
var evidence = new Dictionary<string, object>
|
||||
@@ -174,7 +174,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// Filter out suppressed events
|
||||
if (condition.SuppressByEvent != null)
|
||||
{
|
||||
matchingEvents = matchingEvents.Where(e => !IsSuppressedByEvent(e, condition.SuppressByEvent, events)).ToList();
|
||||
matchingEvents = matchingEvents.Where(e => !IsSuppressedByEvent(e, condition.SuppressByEvent, events, budget)).ToList();
|
||||
if (!matchingEvents.Any())
|
||||
return (false, "all matches suppressed by resolving events");
|
||||
}
|
||||
@@ -204,19 +204,19 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// A missing filter field stringifies to empty, so <c>equals</c> excludes the event
|
||||
/// and <c>not_equals</c> keeps it. No filter configured = unchanged behaviour.
|
||||
/// </summary>
|
||||
private List<EnrollmentEvent> ApplyValueFilter(RuleCondition condition, List<EnrollmentEvent> matchingEvents)
|
||||
private List<EnrollmentEvent> ApplyValueFilter(RuleCondition condition, List<EnrollmentEvent> matchingEvents, RuleEvaluationBudget budget)
|
||||
{
|
||||
if (string.IsNullOrEmpty(condition.FilterField) || string.IsNullOrEmpty(condition.FilterOperator))
|
||||
return matchingEvents;
|
||||
|
||||
return matchingEvents
|
||||
.Where(e => MatchesOperator(GetDataFieldValue(e, condition.FilterField) ?? string.Empty, condition.FilterOperator, condition.FilterValue))
|
||||
.Where(e => MatchesOperator(GetDataFieldValue(e, condition.FilterField) ?? string.Empty, condition.FilterOperator, condition.FilterValue, budget))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private (bool matched, object evidence) EvaluateEventDataCondition(RuleCondition condition, List<EnrollmentEvent> events)
|
||||
private (bool matched, object evidence) EvaluateEventDataCondition(RuleCondition condition, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList());
|
||||
var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList(), budget);
|
||||
|
||||
// Field-absence gate — see EvaluateEventTypeCondition; here it applies to the
|
||||
// filtered event set, so "no event with filterField=x carries dataField" is expressible.
|
||||
@@ -226,7 +226,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
foreach (var evt in matchingEvents)
|
||||
{
|
||||
var fieldValue = GetDataFieldValue(evt, condition.DataField);
|
||||
if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value))
|
||||
if (fieldValue != null && MatchesOperator(fieldValue, condition.Operator, condition.Value, budget))
|
||||
{
|
||||
var evidence = new Dictionary<string, object>
|
||||
{
|
||||
@@ -288,9 +288,9 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// fire only for array elements NOT on the list. Evidence carries the matched item value
|
||||
/// under <c>field</c>=ItemField (so <c>{{itemField}}</c> interpolates) plus a capped sample.
|
||||
/// </summary>
|
||||
private (bool matched, object evidence) EvaluateEventDataArrayCondition(RuleCondition condition, List<EnrollmentEvent> events)
|
||||
private (bool matched, object evidence) EvaluateEventDataArrayCondition(RuleCondition condition, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList());
|
||||
var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList(), budget);
|
||||
|
||||
foreach (var evt in matchingEvents)
|
||||
{
|
||||
@@ -302,7 +302,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
foreach (var element in items)
|
||||
{
|
||||
var itemValue = GetItemFieldValue(element, condition.ItemField);
|
||||
if (itemValue != null && MatchesOperator(itemValue, condition.Operator, condition.Value))
|
||||
if (itemValue != null && MatchesOperator(itemValue, condition.Operator, condition.Value, budget))
|
||||
matchedValues.Add(itemValue);
|
||||
}
|
||||
|
||||
@@ -436,11 +436,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
private (bool matched, object evidence) EvaluateEventCountCondition(RuleCondition condition, List<EnrollmentEvent> events)
|
||||
private (bool matched, object evidence) EvaluateEventCountCondition(RuleCondition condition, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
// Optional value filter: count only events whose FilterField satisfies FilterOperator/FilterValue
|
||||
// (e.g. only performance_snapshot events with memory_used_percent > 90).
|
||||
var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList());
|
||||
var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList(), budget);
|
||||
|
||||
// count_per_group_gte: group by DataField value (e.g. appId), fire if any group >= threshold
|
||||
if (condition.Operator == "count_per_group_gte" && !string.IsNullOrEmpty(condition.DataField) && int.TryParse(condition.Value, out var groupThreshold))
|
||||
@@ -656,9 +656,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a matched event is suppressed by a resolving event (e.g., app_install_completed
|
||||
/// suppresses app_install_failed when both share the same appId).
|
||||
/// suppresses app_install_failed when both share the same appId). One call scans every event,
|
||||
/// so a condition that calls it per matching event is quadratic in the session size — hence
|
||||
/// the budget check before the scan.
|
||||
/// </summary>
|
||||
private bool IsSuppressedByEvent(EnrollmentEvent matchedEvent, SuppressByEventConfig? config, List<EnrollmentEvent> allEvents)
|
||||
private bool IsSuppressedByEvent(EnrollmentEvent matchedEvent, SuppressByEventConfig? config, List<EnrollmentEvent> allEvents, RuleEvaluationBudget budget)
|
||||
{
|
||||
if (config == null || string.IsNullOrEmpty(config.EventType) || string.IsNullOrEmpty(config.JoinField))
|
||||
return false;
|
||||
@@ -667,6 +669,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (string.IsNullOrEmpty(joinValue))
|
||||
return false;
|
||||
|
||||
budget.ThrowIfSpent();
|
||||
return allEvents.Any(e =>
|
||||
MatchesEventType(e, config.EventType) &&
|
||||
string.Equals(GetDataFieldValue(e, config.JoinField), joinValue, StringComparison.OrdinalIgnoreCase));
|
||||
@@ -795,8 +798,18 @@ namespace AutopilotMonitor.Functions.Services
|
||||
};
|
||||
}
|
||||
|
||||
private bool MatchesOperator(string fieldValue, string op, string compareValue)
|
||||
/// <summary>Per-match timeout of a rule's regex. A constant: the static <see cref="Regex.IsMatch(string, string, RegexOptions, TimeSpan)"/>
|
||||
/// caches by pattern, options AND timeout.</summary>
|
||||
private static readonly TimeSpan RegexMatchTimeout = TimeSpan.FromSeconds(1);
|
||||
|
||||
/// <summary>
|
||||
/// Every operator evaluation of a rule passes through here, once per event (or array
|
||||
/// element) its condition looks at — so this is where the pass's budget is checked.
|
||||
/// </summary>
|
||||
private static bool MatchesOperator(string fieldValue, string op, string compareValue, RuleEvaluationBudget budget)
|
||||
{
|
||||
budget.ThrowIfSpent();
|
||||
|
||||
switch (op?.ToLower())
|
||||
{
|
||||
case "equals":
|
||||
@@ -812,24 +825,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return fieldValue.IndexOf(compareValue, StringComparison.OrdinalIgnoreCase) < 0;
|
||||
|
||||
case "regex":
|
||||
try
|
||||
{
|
||||
return Regex.IsMatch(fieldValue, compareValue, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return MatchesRegex(fieldValue, compareValue) == true;
|
||||
|
||||
case "not_regex":
|
||||
try
|
||||
{
|
||||
return !Regex.IsMatch(fieldValue, compareValue, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return MatchesRegex(fieldValue, compareValue) == false;
|
||||
|
||||
case "gt":
|
||||
return double.TryParse(fieldValue, out var gt1) && double.TryParse(compareValue, out var gt2) && gt1 > gt2;
|
||||
@@ -860,6 +859,28 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Case-insensitive match of a rule's pattern; null for a pattern that does not parse (one
|
||||
/// saved before create and update checked them), which neither regex nor not_regex counts
|
||||
/// as a match. A match timeout ends the rule (D-410): it used to read as "no match", and
|
||||
/// every further event paid the same second again.
|
||||
/// </summary>
|
||||
private static bool? MatchesRegex(string fieldValue, string pattern)
|
||||
{
|
||||
try
|
||||
{
|
||||
return Regex.IsMatch(fieldValue, pattern, RegexOptions.IgnoreCase, RegexMatchTimeout);
|
||||
}
|
||||
catch (RegexMatchTimeoutException)
|
||||
{
|
||||
throw new RuleEvaluationStoppedException(EvaluationStopReasons.RegexTimeout);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool MatchesInList(string fieldValue, string compareValue)
|
||||
{
|
||||
if (string.IsNullOrEmpty(fieldValue) || string.IsNullOrEmpty(compareValue))
|
||||
@@ -876,7 +897,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return false;
|
||||
}
|
||||
|
||||
private (bool matched, object evidence) EvaluateAppInstallDurationCondition(RuleCondition condition, List<EnrollmentEvent> events)
|
||||
private (bool matched, object evidence) EvaluateAppInstallDurationCondition(RuleCondition condition, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
// Ordering key is Sequence, never Timestamp: Sequence is a persisted, strictly
|
||||
// monotonic per-session counter assigned at emit time and is immune to every clock
|
||||
@@ -901,6 +922,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
foreach (var completionEvent in completionEvents)
|
||||
{
|
||||
// Each completion scans the session for its start event; the operator below is
|
||||
// skipped for unmatched apps, so the budget is checked here.
|
||||
budget.ThrowIfSpent();
|
||||
|
||||
var appId = GetDataFieldValue(completionEvent, "appId");
|
||||
var appName = GetDataFieldValue(completionEvent, "appName");
|
||||
var appKey = !string.IsNullOrWhiteSpace(appId) ? appId : appName;
|
||||
@@ -929,7 +954,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
var durationSeconds = Math.Max(0, (completionEvent.Timestamp - startEvent.Timestamp).TotalSeconds);
|
||||
|
||||
if (!MatchesOperator(durationSeconds.ToString(), condition.Operator, condition.Value))
|
||||
if (!MatchesOperator(durationSeconds.ToString(), condition.Operator, condition.Value, budget))
|
||||
continue;
|
||||
|
||||
return (true, new Dictionary<string, object>
|
||||
@@ -1169,7 +1194,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// DataField/Operator/Value = optional filter on Event B
|
||||
/// EventAFilterField/Operator/Value = optional filter on Event A
|
||||
/// </summary>
|
||||
private (bool matched, object evidence) EvaluateEventCorrelationCondition(RuleCondition condition, List<EnrollmentEvent> events)
|
||||
private (bool matched, object evidence) EvaluateEventCorrelationCondition(RuleCondition condition, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
if (string.IsNullOrEmpty(condition.EventType) ||
|
||||
string.IsNullOrEmpty(condition.CorrelateEventType) ||
|
||||
@@ -1194,7 +1219,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (string.IsNullOrEmpty(condition.EventAFilterField))
|
||||
return true;
|
||||
var val = GetDataFieldValue(e, condition.EventAFilterField);
|
||||
return val != null && MatchesOperator(val, condition.EventAFilterOperator ?? "exists", condition.EventAFilterValue ?? "");
|
||||
return val != null && MatchesOperator(val, condition.EventAFilterOperator ?? "exists", condition.EventAFilterValue ?? "", budget);
|
||||
})
|
||||
.ToList();
|
||||
|
||||
@@ -1206,7 +1231,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (string.IsNullOrEmpty(condition.DataField))
|
||||
return true;
|
||||
var val = GetDataFieldValue(e, condition.DataField);
|
||||
return val != null && MatchesOperator(val, condition.Operator ?? "exists", condition.Value ?? "");
|
||||
return val != null && MatchesOperator(val, condition.Operator ?? "exists", condition.Value ?? "", budget);
|
||||
})
|
||||
.ToList();
|
||||
|
||||
@@ -1226,6 +1251,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
foreach (var eventB in eventBList)
|
||||
{
|
||||
// Pairing scans the A events sharing the join key, so one shared key makes this
|
||||
// loop quadratic without a single operator call.
|
||||
budget.ThrowIfSpent();
|
||||
|
||||
var bKey = GetDataFieldValue(eventB, condition.JoinField);
|
||||
if (string.IsNullOrEmpty(bKey) || !eventAByJoinKey.ContainsKey(bKey))
|
||||
continue;
|
||||
|
||||
@@ -21,9 +21,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// Loads the session's events (strict read — storage failures propagate) and dry-runs the
|
||||
/// given draft rule against them. Mirrors the event handling of AnalyzeSessionAsync
|
||||
/// (strict load + BackfillDerivedEventFields) so the dry-run sees exactly what a real
|
||||
/// analysis pass would see.
|
||||
/// analysis pass would see. The evaluation runs on <see cref="RuleEvaluationBudget.DryRun"/>
|
||||
/// and ends early when <paramref name="cancellationToken"/> is signalled (D-410).
|
||||
/// </summary>
|
||||
public async Task<RuleDryRun> DryRunRuleAsync(string tenantId, string sessionId, AnalyzeRule rule)
|
||||
public async Task<RuleDryRun> DryRunRuleAsync(string tenantId, string sessionId, AnalyzeRule rule, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var allEvents = await _sessionRepo.GetSessionEventsStrictAsync(tenantId, sessionId);
|
||||
if (allEvents.Count == 0)
|
||||
@@ -38,7 +39,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
BackfillDerivedEventFields(allEvents);
|
||||
return DryRunEvaluate(rule, allEvents);
|
||||
return DryRunEvaluate(rule, allEvents, new RuleEvaluationBudget(RuleEvaluationBudget.DryRun, cancellationToken));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -46,8 +47,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// ORDER in lock-step with EvaluateRule — factors are evaluated sequentially against the
|
||||
/// accumulating matchedConditions dictionary (a factor's "exists" check can observe
|
||||
/// earlier factor_<signal> entries), so reordering would change results.
|
||||
/// A stop by the budget ends the trace with verdict "stopped", a stop reason and a note
|
||||
/// naming the precondition or condition it stopped at. Confidence factors need no budget
|
||||
/// check: they never run an operator and scan the events at most once each.
|
||||
/// </summary>
|
||||
internal RuleDryRun DryRunEvaluate(AnalyzeRule rule, List<EnrollmentEvent> events)
|
||||
internal RuleDryRun DryRunEvaluate(AnalyzeRule rule, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
var dry = new RuleDryRun
|
||||
{
|
||||
@@ -56,50 +60,68 @@ namespace AutopilotMonitor.Functions.Services
|
||||
ConfidenceThreshold = rule.ConfidenceThreshold,
|
||||
};
|
||||
|
||||
// The precondition or condition being evaluated — a stop note names it.
|
||||
var stage = string.Empty;
|
||||
var preconditionsPassed = true;
|
||||
if (rule.Preconditions != null)
|
||||
var matchedConditions = new Dictionary<string, object>();
|
||||
var allRequiredMet = true;
|
||||
try
|
||||
{
|
||||
foreach (var pre in rule.Preconditions)
|
||||
if (rule.Preconditions != null)
|
||||
{
|
||||
var passed = EvaluatePrecondition(pre, events);
|
||||
if (!passed) preconditionsPassed = false;
|
||||
dry.Preconditions.Add(new RuleDryRunPrecondition
|
||||
for (var i = 0; i < rule.Preconditions.Count; i++)
|
||||
{
|
||||
Source = pre.Source,
|
||||
EventType = pre.EventType,
|
||||
DataField = string.IsNullOrEmpty(pre.DataField) ? null : pre.DataField,
|
||||
Operator = pre.Operator,
|
||||
Value = pre.Value,
|
||||
Passed = passed,
|
||||
var pre = rule.Preconditions[i];
|
||||
stage = $"preconditions[{i}] ({pre.EventType})";
|
||||
var passed = EvaluatePrecondition(pre, events, budget);
|
||||
if (!passed) preconditionsPassed = false;
|
||||
dry.Preconditions.Add(new RuleDryRunPrecondition
|
||||
{
|
||||
Source = pre.Source,
|
||||
EventType = pre.EventType,
|
||||
DataField = string.IsNullOrEmpty(pre.DataField) ? null : pre.DataField,
|
||||
Operator = pre.Operator,
|
||||
Value = pre.Value,
|
||||
Passed = passed,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluate ALL conditions (no early break — the author wants the full picture).
|
||||
// Outcome-equivalent to EvaluateRule: a failed required condition makes the
|
||||
// production path return null before matchedConditions is ever used.
|
||||
for (var i = 0; i < rule.Conditions.Count; i++)
|
||||
{
|
||||
var condition = rule.Conditions[i];
|
||||
stage = $"conditions[{i}] ('{condition.Signal}')";
|
||||
var (matched, evidence) = EvaluateCondition(condition, events, budget);
|
||||
if (condition.Required && !matched) allRequiredMet = false;
|
||||
if (matched) matchedConditions[condition.Signal] = evidence;
|
||||
|
||||
dry.Conditions.Add(new RuleDryRunCondition
|
||||
{
|
||||
Signal = condition.Signal,
|
||||
Source = condition.Source,
|
||||
EventType = string.IsNullOrEmpty(condition.EventType) ? null : condition.EventType,
|
||||
Required = condition.Required,
|
||||
Matched = matched,
|
||||
// Matched: the evidence dictionary the production path stores. Not matched:
|
||||
// the evaluator's reason string ("no matching events", …).
|
||||
Evidence = evidence,
|
||||
MatchingEventCount = string.IsNullOrEmpty(condition.EventType)
|
||||
? null
|
||||
: events.Count(e => MatchesEventType(e, condition.EventType)),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluate ALL conditions (no early break — the author wants the full picture).
|
||||
// Outcome-equivalent to EvaluateRule: a failed required condition makes the
|
||||
// production path return null before matchedConditions is ever used.
|
||||
var matchedConditions = new Dictionary<string, object>();
|
||||
var allRequiredMet = true;
|
||||
foreach (var condition in rule.Conditions)
|
||||
catch (RuleEvaluationStoppedException stop)
|
||||
{
|
||||
var (matched, evidence) = EvaluateCondition(condition, events);
|
||||
if (condition.Required && !matched) allRequiredMet = false;
|
||||
if (matched) matchedConditions[condition.Signal] = evidence;
|
||||
|
||||
dry.Conditions.Add(new RuleDryRunCondition
|
||||
{
|
||||
Signal = condition.Signal,
|
||||
Source = condition.Source,
|
||||
EventType = string.IsNullOrEmpty(condition.EventType) ? null : condition.EventType,
|
||||
Required = condition.Required,
|
||||
Matched = matched,
|
||||
// Matched: the evidence dictionary the production path stores. Not matched:
|
||||
// the evaluator's reason string ("no matching events", …).
|
||||
Evidence = evidence,
|
||||
MatchingEventCount = string.IsNullOrEmpty(condition.EventType)
|
||||
? null
|
||||
: events.Count(e => MatchesEventType(e, condition.EventType)),
|
||||
});
|
||||
// The trace keeps what finished before the stop; the evidence map stays unset,
|
||||
// so no client previews a half-evaluated finding.
|
||||
dry.Verdict = RuleDryRunVerdict.Stopped;
|
||||
dry.StopReason = stop.Reason;
|
||||
dry.StopNote = StopNote(stage, stop.Reason, budget, events.Count);
|
||||
return dry;
|
||||
}
|
||||
|
||||
dry.MatchedConditions = matchedConditions;
|
||||
@@ -154,6 +176,16 @@ namespace AutopilotMonitor.Functions.Services
|
||||
dry.WouldMarkSessionAsFailed = rule.MarkSessionAsFailed ?? rule.MarkSessionAsFailedDefault;
|
||||
return dry;
|
||||
}
|
||||
|
||||
/// <summary>What the author reads when the dry-run stopped: where, why, and whether trying again helps.</summary>
|
||||
private static string StopNote(string stage, string reason, RuleEvaluationBudget budget, int eventCount) => reason switch
|
||||
{
|
||||
EvaluationStopReasons.RegexTimeout =>
|
||||
$"{stage}: a regex hit the {RegexMatchTimeout.TotalSeconds:0.#} s match timeout on this session's data — the pattern backtracks catastrophically; remove nested or overlapping quantifiers. Running the test again gives the same result, and the analysis would stop this rule the same way.",
|
||||
EvaluationStopReasons.TimeBudget =>
|
||||
$"{stage}: the dry-run used up its {budget.Limit.TotalSeconds:0.#} s time budget on this session's {eventCount} events — the rule is too expensive here; narrow its event types or simplify its conditions.",
|
||||
_ => $"{stage}: the request was cancelled before the evaluation finished.",
|
||||
};
|
||||
}
|
||||
|
||||
// RuleDryRun, RuleDryRunVerdict and the per-stage trace classes are wire contract and live in
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -25,6 +24,9 @@ namespace AutopilotMonitor.Functions.Services
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>Time budget of one analysis run (<see cref="RuleEvaluationBudget.AnalysisRun"/>); tests shrink it.</summary>
|
||||
internal TimeSpan RunBudget { get; init; } = RuleEvaluationBudget.AnalysisRun;
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates analyze rules against the full session event stream. Fetches events internally.
|
||||
/// Legacy overload — maps to <see cref="AnalyzeRunKind.Terminal"/> (or
|
||||
@@ -71,6 +73,13 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <see cref="AnalysisOutcome.Results"/>; they remain in <see cref="AnalysisOutcome.EvaluatedRules"/>
|
||||
/// so telemetry counts the attempt.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Evaluation budget (D-410):</b> the run gets <see cref="RunBudget"/> for all rules
|
||||
/// together. A regex match timeout stops that rule and the run goes on; a spent budget
|
||||
/// stops the rule and leaves every later rule unevaluated. A rule that does not finish
|
||||
/// keeps its stored row untouched (<see cref="AnalysisOutcome.KeptResults"/>) and is
|
||||
/// listed in <see cref="AnalysisOutcome.Stops"/> / <see cref="AnalysisOutcome.SkippedRuleIds"/>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public async Task<AnalysisOutcome> AnalyzeSessionAsync(string tenantId, string sessionId, AnalyzeRunContext context)
|
||||
{
|
||||
@@ -119,22 +128,25 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
foreach (var rule in activeRules)
|
||||
// One budget for the whole run, started after the storage reads. Global rules
|
||||
// (built-in, community) come before the tenant's custom rules — the merge order of
|
||||
// GetAllRulesForTenantAsync — so a run that spends its budget runs out on the
|
||||
// custom rules. A change to that order changes who pays (D-410).
|
||||
var budget = new RuleEvaluationBudget(RunBudget);
|
||||
|
||||
for (var i = 0; i < activeRules.Count; i++)
|
||||
{
|
||||
var rule = activeRules[i];
|
||||
existingByRuleId.TryGetValue(rule.RuleId, out var existing);
|
||||
try
|
||||
{
|
||||
existingByRuleId.TryGetValue(rule.RuleId, out var existing);
|
||||
|
||||
// Classic dedupe: a FINAL row is settled — neither Terminal nor Interim
|
||||
// touches it again. Only Reanalyze (explicit user action) re-opens it.
|
||||
// Interim rows are always re-evaluated (refresh / finalize / resolve).
|
||||
if (context.Kind != AnalyzeRunKind.Reanalyze && existing != null && !existing.IsInterim)
|
||||
if (!IsEvaluatedIn(context.Kind, existing))
|
||||
continue;
|
||||
|
||||
// Track that this rule was evaluated (for telemetry)
|
||||
outcome.EvaluatedRules.Add(rule);
|
||||
|
||||
var result = EvaluateRule(rule, allEvents);
|
||||
var result = EvaluateRule(rule, allEvents, budget);
|
||||
if (result != null)
|
||||
{
|
||||
result.SessionId = sessionId;
|
||||
@@ -179,12 +191,42 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (RuleEvaluationStoppedException stop)
|
||||
{
|
||||
// The rule did not finish: whatever is stored for it stays as it is — a stop is
|
||||
// no evidence that the finding went away.
|
||||
KeepStoredRow(outcome, existing);
|
||||
outcome.Stops.Add(new RuleEvaluationStop(rule.RuleId, stop.Reason, rule.IsBuiltIn || rule.IsCommunity));
|
||||
|
||||
if (stop.Reason == EvaluationStopReasons.RegexTimeout)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Rule evaluation stopped: {Reason} in rule {RuleId} for session {SessionId} of tenant {TenantId} ({Kind}) after {ElapsedMs} ms of the run",
|
||||
stop.Reason, rule.RuleId, sessionId, tenantId, context.Kind, (long)budget.Elapsed.TotalMilliseconds);
|
||||
continue;
|
||||
}
|
||||
|
||||
// The run's budget is spent: every later rule stays unevaluated, rows kept.
|
||||
foreach (var later in activeRules.Skip(i + 1))
|
||||
{
|
||||
existingByRuleId.TryGetValue(later.RuleId, out var laterExisting);
|
||||
if (!IsEvaluatedIn(context.Kind, laterExisting))
|
||||
continue;
|
||||
KeepStoredRow(outcome, laterExisting);
|
||||
outcome.SkippedRuleIds.Add(later.RuleId);
|
||||
}
|
||||
_logger.LogWarning(
|
||||
"Rule evaluation stopped: {Reason} in rule {RuleId} for session {SessionId} of tenant {TenantId} ({Kind}) after {ElapsedMs} ms of the run; {SkippedCount} later rule(s) not evaluated",
|
||||
stop.Reason, rule.RuleId, sessionId, tenantId, context.Kind, (long)budget.Elapsed.TotalMilliseconds, outcome.SkippedRuleIds.Count);
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Per-rule evaluation failures are isolated: a buggy rule must not kill the
|
||||
// whole pass. Storage-layer exceptions from the surrounding code (rule loading,
|
||||
// event reading, results lookup, KO-criterion side-effects) are NOT caught here
|
||||
// and propagate to the caller — the queue worker depends on this for retry.
|
||||
KeepStoredRow(outcome, existing);
|
||||
_logger.LogWarning(ex, $"Error evaluating rule {rule.RuleId}");
|
||||
}
|
||||
}
|
||||
@@ -192,6 +234,27 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Classic dedupe: a FINAL row is settled — neither Terminal nor Interim touches it again.
|
||||
/// Only Reanalyze (explicit user action) re-opens it. Interim rows are always re-evaluated
|
||||
/// (refresh / finalize / resolve).
|
||||
/// </summary>
|
||||
private static bool IsEvaluatedIn(AnalyzeRunKind kind, RuleResult? existing)
|
||||
=> kind == AnalyzeRunKind.Reanalyze || existing == null || existing.IsInterim;
|
||||
|
||||
/// <summary>
|
||||
/// Hands the stored row of a rule this run could not evaluate back to the caller, unless
|
||||
/// the rule already produced an outcome entry of its own.
|
||||
/// </summary>
|
||||
private static void KeepStoredRow(AnalysisOutcome outcome, RuleResult? existing)
|
||||
{
|
||||
if (existing == null
|
||||
|| outcome.Results.Any(r => r.RuleId == existing.RuleId)
|
||||
|| outcome.ResolvedResults.Contains(existing))
|
||||
return;
|
||||
outcome.KeptResults.Add(existing);
|
||||
}
|
||||
|
||||
/// <summary>True when the rule's evaluateOn matches this interim run's trigger.</summary>
|
||||
private static bool MatchesInterimRun(AnalyzeRule rule, AnalyzeRunContext context)
|
||||
{
|
||||
@@ -326,7 +389,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <summary>
|
||||
/// Evaluates a single rule against the full session event stream
|
||||
/// </summary>
|
||||
private RuleResult? EvaluateRule(AnalyzeRule rule, List<EnrollmentEvent> events)
|
||||
private RuleResult? EvaluateRule(AnalyzeRule rule, List<EnrollmentEvent> events, RuleEvaluationBudget budget)
|
||||
{
|
||||
// Preconditions gate (AND-semantics, silent skip): if any precondition fails the
|
||||
// rule is not evaluated at all — no result row, no UI card. Distinct from
|
||||
@@ -335,7 +398,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
foreach (var pre in rule.Preconditions)
|
||||
{
|
||||
if (!EvaluatePrecondition(pre, events))
|
||||
if (!EvaluatePrecondition(pre, events, budget))
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Rule {RuleId} skipped by precondition (eventType={EventType}, field={Field}, op={Op}, value={Value})",
|
||||
@@ -352,7 +415,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// Evaluate each condition
|
||||
foreach (var condition in rule.Conditions)
|
||||
{
|
||||
var (matched, evidence) = EvaluateCondition(condition, events);
|
||||
var (matched, evidence) = EvaluateCondition(condition, events, budget);
|
||||
|
||||
if (condition.Required && !matched)
|
||||
{
|
||||
@@ -430,8 +493,28 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
/// <summary>All rules that were evaluated in this pass (includes rules that didn't fire)</summary>
|
||||
public List<AnalyzeRule> EvaluatedRules { get; set; } = new List<AnalyzeRule>();
|
||||
|
||||
/// <summary>
|
||||
/// Stored rows of rules this pass could not evaluate to the end (stopped, skipped after the
|
||||
/// budget ran out, or thrown), unchanged. A caller that replaces a session's rows wholesale
|
||||
/// (the manual re-analysis) writes them back; the queue path upserts and leaves them alone.
|
||||
/// </summary>
|
||||
public List<RuleResult> KeptResults { get; set; } = new List<RuleResult>();
|
||||
|
||||
/// <summary>Rules the evaluation budget stopped in this pass (D-410); callers report them.</summary>
|
||||
public List<RuleEvaluationStop> Stops { get; set; } = new List<RuleEvaluationStop>();
|
||||
|
||||
/// <summary>Rules never started because the pass's budget ran out before them.</summary>
|
||||
public List<string> SkippedRuleIds { get; set; } = new List<string>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One rule a pass stopped: <paramref name="Reason"/> is regex_timeout or time_budget
|
||||
/// (EvaluationStopReasons); <paramref name="IsCatalogRule"/> tells a built-in or community
|
||||
/// rule (our pattern) from a tenant's custom rule (the author's).
|
||||
/// </summary>
|
||||
public sealed record RuleEvaluationStop(string RuleId, string Reason, bool IsCatalogRule);
|
||||
|
||||
/// <summary>Which flavor of analyze run is executing — decides rule scope, dedupe and KO behavior.</summary>
|
||||
public enum AnalyzeRunKind
|
||||
{
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Diagnostics;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Wall-clock bound on one rule-evaluation pass: one analysis run (every active rule against a
|
||||
/// session) or one dry-run of a draft. Tenant-authored rules decide how much work a pass does
|
||||
/// (conditions × events × operator cost), so the pass is what gets bounded, not the single
|
||||
/// regex match (D-410). The engine checks it before every operator evaluation and in the
|
||||
/// evaluator loops that scan events without one; a spent budget or a signalled token throws
|
||||
/// <see cref="RuleEvaluationStoppedException"/>.
|
||||
/// </summary>
|
||||
internal sealed class RuleEvaluationBudget
|
||||
{
|
||||
/// <summary>One analysis run, all rules together (queue, interim, "Analyze now"). The
|
||||
/// slowest production runs spend under 0.3 s outside storage I/O.</summary>
|
||||
internal static readonly TimeSpan AnalysisRun = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>One dry-run of a draft rule; the logparser pattern test has the same 2 s (D-379).</summary>
|
||||
internal static readonly TimeSpan DryRun = TimeSpan.FromSeconds(2);
|
||||
|
||||
private readonly long _start = Stopwatch.GetTimestamp();
|
||||
private readonly CancellationToken _cancellationToken;
|
||||
|
||||
internal RuleEvaluationBudget(TimeSpan limit, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Limit = limit;
|
||||
_cancellationToken = cancellationToken;
|
||||
}
|
||||
|
||||
internal TimeSpan Limit { get; }
|
||||
|
||||
internal TimeSpan Elapsed => Stopwatch.GetElapsedTime(_start);
|
||||
|
||||
internal void ThrowIfSpent()
|
||||
{
|
||||
if (_cancellationToken.IsCancellationRequested)
|
||||
throw new RuleEvaluationStoppedException(EvaluationStopReasons.Cancelled);
|
||||
if (Elapsed >= Limit)
|
||||
throw new RuleEvaluationStoppedException(EvaluationStopReasons.TimeBudget);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A rule evaluation ended early. <see cref="Reason"/> is one of <see cref="EvaluationStopReasons"/>:
|
||||
/// a regex hit its match timeout, the pass ran out of budget, or the caller went away.
|
||||
/// </summary>
|
||||
internal sealed class RuleEvaluationStoppedException : Exception
|
||||
{
|
||||
internal RuleEvaluationStoppedException(string reason) : base($"Rule evaluation stopped: {reason}")
|
||||
{
|
||||
Reason = reason;
|
||||
}
|
||||
|
||||
internal string Reason { get; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Turns the stops of an analysis run (<see cref="AnalysisOutcome.Stops"/>, D-410) into
|
||||
/// <c>RuleEvaluationStopped</c> ops events (singleton). A tenant decides how many custom rules it
|
||||
/// has and how often its sessions are analysed, so the alarm is capped per instance: one event
|
||||
/// per tenant and rule per hour, and <see cref="MaxEventsPerWindow"/> per hour in total. The
|
||||
/// engine logs every stop at Warning; this only decides which ones also reach the operators.
|
||||
/// </summary>
|
||||
public sealed class RuleEvaluationStopReporter
|
||||
{
|
||||
internal static readonly TimeSpan Window = TimeSpan.FromHours(1);
|
||||
internal const int MaxEventsPerWindow = 20;
|
||||
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly object _lock = new();
|
||||
private readonly Dictionary<string, DateTime> _lastEventByRule = new(StringComparer.Ordinal);
|
||||
private DateTime _windowStart = DateTime.MinValue;
|
||||
private int _eventsInWindow;
|
||||
|
||||
public RuleEvaluationStopReporter(OpsEventService opsEvents)
|
||||
{
|
||||
_opsEvents = opsEvents;
|
||||
}
|
||||
|
||||
public async Task ReportAsync(string tenantId, string sessionId, AnalyzeRunKind kind, AnalysisOutcome outcome)
|
||||
{
|
||||
foreach (var stop in outcome.Stops)
|
||||
{
|
||||
if (!Take(tenantId, stop.RuleId, DateTime.UtcNow))
|
||||
continue;
|
||||
|
||||
// Only a stop by the spent budget leaves later rules unevaluated.
|
||||
var skipped = stop.Reason == EvaluationStopReasons.RegexTimeout ? 0 : outcome.SkippedRuleIds.Count;
|
||||
await _opsEvents.RecordRuleEvaluationStoppedAsync(
|
||||
tenantId, sessionId, stop.RuleId, stop.Reason, stop.IsCatalogRule, kind.ToString(), skipped);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>True when this stop may raise an event: first of its tenant and rule within the
|
||||
/// window, and the instance's window still has room.</summary>
|
||||
internal bool Take(string tenantId, string ruleId, DateTime utcNow)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (utcNow - _windowStart >= Window)
|
||||
{
|
||||
_windowStart = utcNow;
|
||||
_eventsInWindow = 0;
|
||||
// Entries come only from taken events, so the map never outgrows two windows' cap.
|
||||
foreach (var stale in _lastEventByRule.Where(e => utcNow - e.Value >= Window).Select(e => e.Key).ToList())
|
||||
_lastEventByRule.Remove(stale);
|
||||
}
|
||||
|
||||
if (_eventsInWindow >= MaxEventsPerWindow)
|
||||
return false;
|
||||
|
||||
var key = tenantId + "|" + ruleId;
|
||||
if (_lastEventByRule.TryGetValue(key, out var last) && utcNow - last < Window)
|
||||
return false;
|
||||
|
||||
_lastEventByRule[key] = utcNow;
|
||||
_eventsInWindow++;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -93,6 +93,43 @@ describe('test_analyze_rule', () => {
|
||||
const res = await handlers.test_analyze_rule({ sessionId: SESSION, rule: ANALYZE_DRAFT });
|
||||
expect(res.isError).toBe(true);
|
||||
});
|
||||
|
||||
it('waits out a full gate once and hands a stop through with its reason and note', async () => {
|
||||
apiFetchMock.mockResolvedValue({
|
||||
success: true,
|
||||
sessionId: SESSION,
|
||||
result: {
|
||||
verdict: 'stopped',
|
||||
stopReason: 'regex_timeout',
|
||||
stopNote: "conditions[0] ('app_failure'): a regex hit the 1 s match timeout on this session's data",
|
||||
},
|
||||
});
|
||||
|
||||
const handlers = captureToolHandlers(false);
|
||||
const res = await handlers.test_analyze_rule({ sessionId: SESSION, rule: ANALYZE_DRAFT });
|
||||
|
||||
// The dry-run has no side effects, so apiFetch may retry the POST once on 429 (Retry-After ≤ 10 s).
|
||||
const [, init] = apiFetchMock.mock.calls[0] as [string, { retry?: boolean }];
|
||||
expect(init.retry).toBe(true);
|
||||
const payload = JSON.parse(res.content[0].text);
|
||||
expect(payload.result.verdict).toBe('stopped');
|
||||
expect(payload.result.stopReason).toBe('regex_timeout');
|
||||
expect(payload.result.stopNote).toContain('conditions[0]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('test_log_pattern', () => {
|
||||
it('waits out a full gate once: the pattern test is pure compute', async () => {
|
||||
apiFetchMock.mockResolvedValue({ success: true, format: 'text', result: { matchCount: 1, parseFailureCount: 0, timeoutCount: 0, lines: [], notes: [] } });
|
||||
|
||||
const handlers = captureToolHandlers(false);
|
||||
await handlers.test_log_pattern({ pattern: 'a', sampleLines: ['a'], format: 'text' });
|
||||
|
||||
const [path, init] = apiFetchMock.mock.calls[0] as [string, { method?: string; retry?: boolean }];
|
||||
expect(path).toBe('/api/rules/gather/test-pattern');
|
||||
expect(init.method).toBe('POST');
|
||||
expect(init.retry).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('registration by role', () => {
|
||||
|
||||
@@ -2406,6 +2406,8 @@ export interface GetRuleResultsResponse {
|
||||
persistFailureCount: number;
|
||||
/** Rule ids that failed to persist during the reanalyze — the key is omitted entirely when there were no failures (the happy-path contract stays unchanged). */
|
||||
persistFailureRuleIds?: string[];
|
||||
/** Rule ids the reanalyze could not evaluate within its time limit (a regex timed out, or the run's budget ran out first); their previous results are kept. The key is omitted when every rule was evaluated. */
|
||||
stoppedRuleIds?: string[];
|
||||
}
|
||||
|
||||
/** Response of POST tenants/{tenantId}/scripts/display-names: resolved Intune script display names keyed by the canonical ref string. Always 200, possibly partial — unresolved refs stay in the dictionary with a null VALUE (dictionary values are NOT subject to WhenWritingNull, so the key is emitted with an explicit JSON null). */
|
||||
@@ -3237,7 +3239,7 @@ export interface LogPatternTestResult {
|
||||
matchCount: number;
|
||||
parseFailureCount: number;
|
||||
timeoutCount: number;
|
||||
/** Set when the test ended before the last sample line was evaluated: regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated. */
|
||||
/** Set when the test ended before the last sample line was evaluated (EvaluationStopReasons): regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated. */
|
||||
stopReason?: string;
|
||||
readonly lines: LogPatternLineResult[];
|
||||
readonly notes: string[];
|
||||
@@ -4189,6 +4191,10 @@ export interface RuleDryRun {
|
||||
wouldMarkSessionAsFailed: boolean;
|
||||
/** The evidence map exactly as the production path would persist it on a RuleResult — keys are condition signals (plus factor_* markers). Clients use it to preview {{token}} interpolation of explanation/remediation. Values are heterogeneous evidence objects by design. */
|
||||
matchedConditions?: Record<string, unknown>;
|
||||
/** Set with verdict "stopped": regex_timeout | time_budget | cancelled. The trace then holds only the entries evaluated before the stop. Null otherwise. */
|
||||
stopReason?: string;
|
||||
/** Set with verdict "stopped": which precondition or condition the evaluation stopped at, and what to change. */
|
||||
stopNote?: string;
|
||||
}
|
||||
|
||||
export interface RuleDryRunCondition {
|
||||
|
||||
@@ -115,6 +115,7 @@ export const OPS_EVENT_TYPES = [
|
||||
"PrivilegedRouteDenied",
|
||||
"McpServicePrincipalFirstSeen",
|
||||
"ReportUploadLimitReached",
|
||||
"RuleEvaluationLimitReached",
|
||||
"OffboardingFeedbackReceived",
|
||||
"FeedbackReceived",
|
||||
"FeedbackRatingReceived",
|
||||
@@ -142,6 +143,7 @@ export const OPS_EVENT_TYPES = [
|
||||
"CollectLogsQuickConfigEnabled",
|
||||
"DiagnosticsUploadEnabled",
|
||||
"DiagnosticsUploadDisabled",
|
||||
"RuleEvaluationStopped",
|
||||
"SessionActionQueued",
|
||||
"SessionTimeouts",
|
||||
"AgentEmergencyBreak",
|
||||
|
||||
@@ -204,8 +204,9 @@ export const RULE_AUTHORING_GUIDE = {
|
||||
'gt', 'lt', 'gte', 'lte', 'exists', 'not_exists', 'count_gte', 'count_per_group_gte', 'in', 'not_in',
|
||||
],
|
||||
operatorNotes:
|
||||
'String comparisons are case-insensitive; regex runs case-insensitive with a 1s timeout ' +
|
||||
'(.NET regex syntax); in/not_in take a comma-separated value list. exists/not_exists with a ' +
|
||||
'String comparisons are case-insensitive; regex runs case-insensitive (.NET regex syntax, at most ' +
|
||||
'2,000 characters) — a match that runs past 1 s stops the whole rule for that session (no finding), ' +
|
||||
'so avoid nested or overlapping quantifiers; in/not_in take a comma-separated value list. exists/not_exists with a ' +
|
||||
'dataField test whether ANY event of the type carries a non-empty value there — not_exists ' +
|
||||
'matches when none does, also when the event type is absent (e.g. "the ESP failure carried no ' +
|
||||
'errorCode"); without a dataField they test only the presence of the event type. An operator ' +
|
||||
|
||||
@@ -338,7 +338,8 @@ export const ANALYZE_RULE_SCHEMA: Record<string, unknown> = {
|
||||
"items": {
|
||||
"$ref": "#/$defs/ruleCondition"
|
||||
},
|
||||
"minItems": 1
|
||||
"minItems": 1,
|
||||
"maxItems": 20
|
||||
},
|
||||
"preconditions": {
|
||||
"type": "array",
|
||||
@@ -346,6 +347,7 @@ export const ANALYZE_RULE_SCHEMA: Record<string, unknown> = {
|
||||
"items": {
|
||||
"$ref": "#/$defs/rulePrecondition"
|
||||
},
|
||||
"maxItems": 10,
|
||||
"default": []
|
||||
},
|
||||
"baseConfidence": {
|
||||
|
||||
@@ -100,6 +100,8 @@ export function registerRuleTools(server: McpServer, ga: boolean, ruleDryRun: bo
|
||||
const data = await apiFetch<TestLogPatternResponse>('/api/rules/gather/test-pattern', {
|
||||
method: 'POST',
|
||||
body: jsonBody<TestLogPatternRequest>({ pattern: args.pattern, format: args.format, sampleLines: args.sampleLines }),
|
||||
// Pure compute without side effects: a full per-instance gate (429) is waited out once.
|
||||
retry: true,
|
||||
});
|
||||
return toolResultText(data, MAX_RESULT_SIZE_CHARS.small);
|
||||
} catch (error: unknown) {
|
||||
@@ -116,7 +118,7 @@ export function registerRuleTools(server: McpServer, ga: boolean, ruleDryRun: bo
|
||||
description:
|
||||
'Dry-runs a DRAFT analyze rule against one real session\'s events and returns the full diagnostic trace: ' +
|
||||
'verdict (fired / required_condition_not_met / below_confidence_threshold / skipped_by_precondition / ' +
|
||||
'no_conditions_matched / no_events), per-condition matched/evidence with matchingEventCount (how many session ' +
|
||||
'no_conditions_matched / no_events / stopped: stopNote says why), per-condition matched/evidence with matchingEventCount (how many session ' +
|
||||
'events have that eventType at all — the first thing to check when a condition unexpectedly misses), the ' +
|
||||
'confidence breakdown per factor, and the interpolated explanation and presentation preview (facts the session ' +
|
||||
'did not record listed). NOTHING is persisted — safe to ' +
|
||||
@@ -136,6 +138,8 @@ export function registerRuleTools(server: McpServer, ga: boolean, ruleDryRun: bo
|
||||
// The draft arrives as a loose object; the backend validates it (ValidateDraftRule) and
|
||||
// answers 400 with the reasons. The cast names the wire document the endpoint deserialises.
|
||||
body: jsonBody<DryRunAnalyzeRuleRequest>({ sessionId: args.sessionId, rule: args.rule as unknown as AnalyzeRule }),
|
||||
// Side-effect free: a full per-instance gate (429) is waited out once.
|
||||
retry: true,
|
||||
});
|
||||
|
||||
const rule = args.rule as Record<string, unknown>;
|
||||
|
||||
@@ -116,6 +116,13 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// unusual load or someone aiming large bodies at the worker.
|
||||
/// </summary>
|
||||
public const string ReportUploadLimitReached = "ReportUploadLimitReached";
|
||||
/// <summary>
|
||||
/// Every slot of a rule-work gate of an instance was taken and a caller was answered 429:
|
||||
/// the analyze dry-run, the manual re-analysis or the logparser pattern test (details:
|
||||
/// <c>gate</c>). Warning; at most one per instance, gate and hour, further refusals stay in
|
||||
/// the Warning trace. Legitimate use never fills a gate, so this is load aimed at it.
|
||||
/// </summary>
|
||||
public const string RuleEvaluationLimitReached = "RuleEvaluationLimitReached";
|
||||
|
||||
// ── Tenant ── Tenant lifecycle: offboarding, trials, plan changes, regression radars, config toggles.
|
||||
public const string OffboardingFeedbackReceived = "OffboardingFeedbackReceived";
|
||||
@@ -169,6 +176,14 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
public const string CollectLogsQuickConfigEnabled = "CollectLogsQuickConfigEnabled";
|
||||
public const string DiagnosticsUploadEnabled = "DiagnosticsUploadEnabled";
|
||||
public const string DiagnosticsUploadDisabled = "DiagnosticsUploadDisabled";
|
||||
/// <summary>
|
||||
/// An analysis run stopped a rule because a regex hit its match timeout, or used up its time
|
||||
/// budget in it (details: <c>reason</c>, <c>skippedRules</c>). Warning for a tenant's custom
|
||||
/// rule (the author's pattern), Error for a built-in or community rule (ours met data it
|
||||
/// cannot handle, or the instance stalled). At most one per instance, tenant and rule per
|
||||
/// hour and 20 per instance and hour; every stop is in the Warning trace.
|
||||
/// </summary>
|
||||
public const string RuleEvaluationStopped = "RuleEvaluationStopped";
|
||||
|
||||
// ── Agent ── Agent + device-side health signals.
|
||||
public const string SessionActionQueued = "SessionActionQueued";
|
||||
@@ -215,8 +230,8 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
{
|
||||
ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles, GraphAccessRejected, GraphAccessRecovered, GraphAppCredentialRejected, AddOnPermissionMissing, AddOnPermissionResolved,
|
||||
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, OrphanSweepCompleted, OrphanSweepFailed, OrphanSweepSkippedLocked, OrphanReconcileCompleted, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed,
|
||||
DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen, ReportUploadLimitReached,
|
||||
OffboardingFeedbackReceived, FeedbackReceived, FeedbackRatingReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled,
|
||||
DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen, ReportUploadLimitReached, RuleEvaluationLimitReached,
|
||||
OffboardingFeedbackReceived, FeedbackReceived, FeedbackRatingReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled, RuleEvaluationStopped,
|
||||
SessionActionQueued, SessionTimeouts, AgentEmergencyBreak, AgentBinaryIntegrityMismatch, CmTraceTimeSkewRegression, ExcessiveSessionEvents, NewImeVersionDetected, ImePatternDriftSuspected, BlobStorageMissing, BlobStorageUnreachable, TelemetryItemsRejected,
|
||||
SlaBreachNotification, SlaConsecutiveFailures, SlaEvaluationCompleted,
|
||||
AzureMonitorAlert, PushDeliveryFailed, PushEndpointRefused,
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AutopilotMonitor.Shared.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Size limits of a tenant-authored analyze rule, enforced on dry-run, create, update and
|
||||
/// template copy. The rule's shape sets how much work every analysis of the tenant's sessions
|
||||
/// does on the shared backend, so it is bounded where the rule enters. The schema carries the
|
||||
/// two counts.
|
||||
/// </summary>
|
||||
public static class AnalyzeRuleLimits
|
||||
{
|
||||
/// <summary>Maximum number of conditions.</summary>
|
||||
public const int MaxConditions = 20;
|
||||
|
||||
/// <summary>Maximum number of preconditions.</summary>
|
||||
public const int MaxPreconditions = 10;
|
||||
|
||||
/// <summary>Maximum length of a value compared with the regex or not_regex operator,
|
||||
/// wherever a rule can hold one (condition value, filter value, event A filter value,
|
||||
/// precondition value). Same cap as the logparser pattern test.</summary>
|
||||
public const int MaxRegexLength = 2000;
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,27 @@ namespace AutopilotMonitor.Shared.Models
|
||||
public const string NoConditionsMatched = "no_conditions_matched";
|
||||
public const string BelowConfidenceThreshold = "below_confidence_threshold";
|
||||
public const string NoEvents = "no_events";
|
||||
|
||||
/// <summary>The evaluation hit a limit before it finished; <see cref="RuleDryRun.StopReason"/>
|
||||
/// says which and <see cref="RuleDryRun.StopNote"/> where.</summary>
|
||||
public const string Stopped = "stopped";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Why an evaluation of caller-authored patterns ended early — the values of
|
||||
/// <see cref="RuleDryRun.StopReason"/> and <see cref="LogPatternTestResult.StopReason"/>.
|
||||
/// Stable API contract.
|
||||
/// </summary>
|
||||
public static class EvaluationStopReasons
|
||||
{
|
||||
/// <summary>A regex hit its 1 s match timeout (catastrophic backtracking).</summary>
|
||||
public const string RegexTimeout = "regex_timeout";
|
||||
|
||||
/// <summary>The evaluation as a whole used up its time budget.</summary>
|
||||
public const string TimeBudget = "time_budget";
|
||||
|
||||
/// <summary>The caller went away (client disconnect, host shutdown).</summary>
|
||||
public const string Cancelled = "cancelled";
|
||||
}
|
||||
|
||||
/// <summary>Full diagnostic trace of one dry-run evaluation. Serialized camelCase to clients.</summary>
|
||||
@@ -102,6 +123,14 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// preview {{token}} interpolation of explanation/remediation. Values are heterogeneous
|
||||
/// evidence objects by design.</summary>
|
||||
public Dictionary<string, object>? MatchedConditions { get; set; }
|
||||
|
||||
/// <summary>Set with verdict "stopped": regex_timeout | time_budget | cancelled. The
|
||||
/// trace then holds only the entries evaluated before the stop. Null otherwise.</summary>
|
||||
public string? StopReason { get; set; }
|
||||
|
||||
/// <summary>Set with verdict "stopped": which precondition or condition the evaluation
|
||||
/// stopped at, and what to change.</summary>
|
||||
public string? StopNote { get; set; }
|
||||
}
|
||||
|
||||
// Declaration order == wire order.
|
||||
@@ -169,6 +198,11 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// <summary>Rule ids that failed to persist during the reanalyze — the key is omitted
|
||||
/// entirely when there were no failures (the happy-path contract stays unchanged).</summary>
|
||||
public IReadOnlyList<string>? PersistFailureRuleIds { get; set; }
|
||||
|
||||
/// <summary>Rule ids the reanalyze could not evaluate within its time limit (a regex
|
||||
/// timed out, or the run's budget ran out first); their previous results are kept. The
|
||||
/// key is omitted when every rule was evaluated.</summary>
|
||||
public IReadOnlyList<string>? StoppedRuleIds { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Response of POST rules/ime-log-patterns/reseed (Global Admin only).</summary>
|
||||
@@ -204,10 +238,11 @@ namespace AutopilotMonitor.Shared.Models
|
||||
public int ParseFailureCount { get; set; }
|
||||
public int TimeoutCount { get; set; }
|
||||
|
||||
/// <summary>Set when the test ended before the last sample line was evaluated:
|
||||
/// regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) |
|
||||
/// time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop
|
||||
/// have no row; a note names them. Null when every line was evaluated.</summary>
|
||||
/// <summary>Set when the test ended before the last sample line was evaluated
|
||||
/// (EvaluationStopReasons): regex_timeout (the first line that hit the agent's 1 s match
|
||||
/// timeout ends the test) | time_budget (the whole test exceeded its time budget) |
|
||||
/// cancelled. Lines after the stop have no row; a note names them. Null when every line
|
||||
/// was evaluated.</summary>
|
||||
public string? StopReason { get; set; }
|
||||
|
||||
public List<LogPatternLineResult> Lines { get; } = new();
|
||||
|
||||
@@ -144,6 +144,10 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
// Every report-upload slot of an instance was taken (ReportUploadGate) and a caller got 429.
|
||||
// Warning, at most one per instance and hour; route it to a push channel to be told.
|
||||
"ReportUploadLimitReached",
|
||||
// A rule-work gate of an instance was full (analyze dry-run, "Analyze now" or pattern test,
|
||||
// details.gate) and a caller got 429. Warning, at most one per instance, gate and hour.
|
||||
// Legitimate use never fills a gate: this is load aimed at it.
|
||||
"RuleEvaluationLimitReached",
|
||||
"VersionBlocked",
|
||||
// Delivery confirmation: a Kill signal was actually SERVED to an agent (config or
|
||||
// telemetry channel) — as opposed to DeviceBlocked/VersionBlocked which fire on rule
|
||||
@@ -248,6 +252,10 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
// through the quick-config dialog (PUT ?intent=collect-logs). Replaces
|
||||
// DiagnosticsUploadEnabled for that path only — the targeted push signal.
|
||||
"CollectLogsQuickConfigEnabled",
|
||||
// An analysis run stopped a rule: a regex hit its match timeout, or the run's time budget ran
|
||||
// out in it. Warning for a tenant's custom rule, Error for a built-in or community rule — route
|
||||
// Error to a push channel. At most one per instance, tenant and rule per hour, 20 per hour.
|
||||
"RuleEvaluationStopped",
|
||||
],
|
||||
Agent: [
|
||||
"BlobStorageMissing",
|
||||
|
||||
@@ -29,12 +29,20 @@ describe("verdictLabel", () => {
|
||||
["fired", "Would fire"],
|
||||
["required_condition_not_met", "A required condition did not match"],
|
||||
["below_confidence_threshold", "Below the confidence threshold"],
|
||||
["stopped", "Stopped at an evaluation limit"],
|
||||
["some_future_verdict", "some_future_verdict"],
|
||||
])("%s → %s", (verdict, label) => {
|
||||
expect(verdictLabel(verdict)).toBe(label);
|
||||
});
|
||||
});
|
||||
|
||||
describe("confidenceSummary of a stopped dry-run", () => {
|
||||
it("names no score — the evaluation never reached the confidence stage", () => {
|
||||
expect(confidenceSummary({ ...base, verdict: "stopped", finalConfidence: undefined, confidenceFactors: [], stopReason: "regex_timeout" }))
|
||||
.toBe("not scored (threshold 70)");
|
||||
});
|
||||
});
|
||||
|
||||
describe("confidenceSummary", () => {
|
||||
it("adds the matched factor weights to the base", () => {
|
||||
expect(confidenceSummary(base)).toBe("85 (base 70 + 15, threshold 70)");
|
||||
|
||||
@@ -128,6 +128,9 @@ function DryRunResult({ outcome }: { outcome: Outcome }) {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Verdict "stopped": where the evaluation hit its limit and what to change. */}
|
||||
{result.stopNote && <p className="mt-2 text-xs text-gray-700 break-words">{result.stopNote}</p>}
|
||||
|
||||
{result.preconditions.length > 0 && (
|
||||
<ul className="mt-3 space-y-1 text-xs text-gray-600">
|
||||
{result.preconditions.map((p, i) => (
|
||||
@@ -139,24 +142,26 @@ function DryRunResult({ outcome }: { outcome: Outcome }) {
|
||||
</ul>
|
||||
)}
|
||||
|
||||
<ul className="mt-3 divide-y divide-gray-200 rounded-md border border-gray-200 bg-white text-xs">
|
||||
{result.conditions.map((c) => {
|
||||
const evidence = evidenceText(c);
|
||||
return (
|
||||
<li key={c.signal} className="flex flex-wrap items-baseline gap-x-3 gap-y-1 px-3 py-2">
|
||||
<span className={`font-bold ${c.matched ? "text-green-700" : "text-gray-400"}`} aria-hidden="true">
|
||||
{c.matched ? "✓" : "✗"}
|
||||
</span>
|
||||
<span className="sr-only">{c.matched ? "Matched:" : "Not matched:"}</span>
|
||||
<span className="font-mono font-medium text-gray-900">{c.signal}</span>
|
||||
<span className="text-gray-500">
|
||||
{[c.eventType ?? c.source, c.required ? "required" : "optional", eventCountText(c)].filter(Boolean).join(" · ")}
|
||||
</span>
|
||||
{evidence && <span className="basis-full pl-5 text-gray-600 break-words">{evidence}</span>}
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
{result.conditions.length > 0 && (
|
||||
<ul className="mt-3 divide-y divide-gray-200 rounded-md border border-gray-200 bg-white text-xs">
|
||||
{result.conditions.map((c) => {
|
||||
const evidence = evidenceText(c);
|
||||
return (
|
||||
<li key={c.signal} className="flex flex-wrap items-baseline gap-x-3 gap-y-1 px-3 py-2">
|
||||
<span className={`font-bold ${c.matched ? "text-green-700" : "text-gray-400"}`} aria-hidden="true">
|
||||
{c.matched ? "✓" : "✗"}
|
||||
</span>
|
||||
<span className="sr-only">{c.matched ? "Matched:" : "Not matched:"}</span>
|
||||
<span className="font-mono font-medium text-gray-900">{c.signal}</span>
|
||||
<span className="text-gray-500">
|
||||
{[c.eventType ?? c.source, c.required ? "required" : "optional", eventCountText(c)].filter(Boolean).join(" · ")}
|
||||
</span>
|
||||
{evidence && <span className="basis-full pl-5 text-gray-600 break-words">{evidence}</span>}
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
)}
|
||||
|
||||
{fired && presentation && (
|
||||
<div className="mt-3">
|
||||
|
||||
@@ -50,6 +50,7 @@ const VERDICT_LABELS: Record<string, string> = {
|
||||
no_conditions_matched: "No condition matched",
|
||||
below_confidence_threshold: "Below the confidence threshold",
|
||||
no_events: "The session has no events",
|
||||
stopped: "Stopped at an evaluation limit",
|
||||
};
|
||||
|
||||
export function verdictLabel(verdict: string): string {
|
||||
|
||||
@@ -25,6 +25,9 @@ interface AnalysisResultsSectionProps {
|
||||
* persist" carries the underlying Storage error for diagnosis.
|
||||
*/
|
||||
persistFailureRuleIds?: string[];
|
||||
/** Rule IDs the most recent reanalyze stopped at its time limit (a regex took too long, or the run's
|
||||
* budget ran out first). They keep their previous result; a yellow banner names them. */
|
||||
stoppedRuleIds?: string[];
|
||||
/** Hides the "Analyze Now" control when false — a read-only cross-tenant (delegated/MSP) viewer must not
|
||||
* trigger a re-analysis. Defaults to true, so own-tenant / Global Admin behavior is unchanged. */
|
||||
canReanalyze?: boolean;
|
||||
@@ -44,6 +47,7 @@ export default function AnalysisResultsSection({
|
||||
setAnalysisExpanded,
|
||||
onReanalyze,
|
||||
persistFailureRuleIds,
|
||||
stoppedRuleIds,
|
||||
canReanalyze = true,
|
||||
sessionTenantId,
|
||||
emptyState,
|
||||
@@ -135,7 +139,7 @@ export default function AnalysisResultsSection({
|
||||
{persistFailureRuleIds && persistFailureRuleIds.length > 0 && (
|
||||
<div
|
||||
role="alert"
|
||||
className="mb-3 rounded-md border border-yellow-300 bg-yellow-50 p-3 text-sm text-yellow-900"
|
||||
className="mb-3 rounded-md border border-yellow-300 bg-yellow-50 p-3 text-sm text-yellow-900 dark:text-yellow-100"
|
||||
>
|
||||
<div className="font-medium">
|
||||
{persistFailureRuleIds.length === 1
|
||||
@@ -152,6 +156,24 @@ export default function AnalysisResultsSection({
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{stoppedRuleIds && stoppedRuleIds.length > 0 && (
|
||||
<div
|
||||
role="alert"
|
||||
className="mb-3 rounded-md border border-yellow-300 bg-yellow-50 p-3 text-sm text-yellow-900 dark:text-yellow-100"
|
||||
>
|
||||
<div className="font-medium">
|
||||
{stoppedRuleIds.length === 1
|
||||
? "1 rule stopped at the time limit"
|
||||
: `${stoppedRuleIds.length} rules stopped at the time limit`}
|
||||
</div>
|
||||
<div className="mt-1 text-xs text-yellow-800">
|
||||
They kept their previous result. A rule stops when one of its patterns takes longer than 1 second.
|
||||
</div>
|
||||
<div className="mt-1 font-mono text-xs">
|
||||
{stoppedRuleIds.join(", ")}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{loadingAnalysis && analysisResults.length === 0 ? (
|
||||
<div className="text-center py-4 text-gray-500">Running analysis...</div>
|
||||
) : openResults.length === 0 && resolvedResults.length === 0 && emptyState ? (
|
||||
|
||||
@@ -10,10 +10,12 @@ export function useSessionAnalysis(
|
||||
sessionId: string,
|
||||
sessionTenantId: string | null,
|
||||
getAccessToken: (forceRefresh?: boolean) => Promise<string | null>,
|
||||
/** withVulnerabilityReport false: a page without the report (diagnosis) never reads it, also not on pushes. */
|
||||
opts?: { withVulnerabilityReport?: boolean }
|
||||
/** withVulnerabilityReport false: a page without the report (diagnosis) never reads it, also not on pushes.
|
||||
* notifyError: shows a failed "Analyze Now" click (busy backend, error) to the user; reads stay silent. */
|
||||
opts?: { withVulnerabilityReport?: boolean; notifyError?: (title: string, err: unknown, key?: string, fallback?: string) => void }
|
||||
) {
|
||||
const withVulnerabilityReport = opts?.withVulnerabilityReport ?? true;
|
||||
const notifyError = opts?.notifyError;
|
||||
const [analysisResults, setAnalysisResults] = useState<RuleResult[]>([]);
|
||||
const [loadingAnalysis, setLoadingAnalysis] = useState(false);
|
||||
// True after the first successful read: before it (and after only failed reads) an empty list means
|
||||
@@ -24,6 +26,8 @@ export function useSessionAnalysis(
|
||||
// Backend reports them in /sessions/{id}/analysis response so the UI can render
|
||||
// a warning banner without blocking the rules that DID persist.
|
||||
const [persistFailureRuleIds, setPersistFailureRuleIds] = useState<string[]>([]);
|
||||
// Rule IDs the most recent reanalyze stopped at its time limit; they kept their previous result.
|
||||
const [stoppedRuleIds, setStoppedRuleIds] = useState<string[]>([]);
|
||||
|
||||
const fetchAnalysisResults = useCallback(async (reanalyze = false) => {
|
||||
if (!sessionTenantId || !isGuid(sessionTenantId)) return;
|
||||
@@ -41,13 +45,16 @@ export function useSessionAnalysis(
|
||||
// more StoreRuleResultAsync calls returned false during the reanalyze loop. We always
|
||||
// set it so a successful retry clears a previous warning.
|
||||
setPersistFailureRuleIds(Array.isArray(data.persistFailureRuleIds) ? data.persistFailureRuleIds : []);
|
||||
setStoppedRuleIds(Array.isArray(data.stoppedRuleIds) ? data.stoppedRuleIds : []);
|
||||
setAnalysisLoaded(true);
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch analysis results:", error);
|
||||
// A click deserves an answer: the server's text says whether trying again in a moment helps.
|
||||
if (reanalyze) notifyError?.("Re-analysis failed", error, "reanalyze-error", "The analysis could not be started.");
|
||||
} finally {
|
||||
setLoadingAnalysis(false);
|
||||
}
|
||||
}, [sessionId, sessionTenantId, getAccessToken]);
|
||||
}, [sessionId, sessionTenantId, getAccessToken, notifyError]);
|
||||
|
||||
const fetchVulnerabilityReport = useCallback(async (rescan = false) => {
|
||||
if (!withVulnerabilityReport || !sessionTenantId || !isGuid(sessionTenantId)) return;
|
||||
@@ -85,5 +92,6 @@ export function useSessionAnalysis(
|
||||
fetchAnalysisResults,
|
||||
fetchVulnerabilityReport,
|
||||
persistFailureRuleIds,
|
||||
stoppedRuleIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -116,7 +116,7 @@ function SessionDetailContent() {
|
||||
|
||||
// Session-scoped hooks (data/SignalR/derivations)
|
||||
const detail = useSessionDetail({ sessionId, tenantId, globalAdminMode, tenantIdOverride, user, getAccessToken, addNotification });
|
||||
const analysis = useSessionAnalysis(sessionId, detail.sessionTenantId, getAccessToken);
|
||||
const analysis = useSessionAnalysis(sessionId, detail.sessionTenantId, getAccessToken, { notifyError });
|
||||
const tenantConfig = useSessionTenantConfig(detail.sessionTenantId, getAccessToken);
|
||||
const eventsApi = useSessionEvents({
|
||||
sessionId,
|
||||
@@ -183,7 +183,7 @@ function SessionDetailContent() {
|
||||
// Returns an empty map when the tenant hasn't granted DeviceManagementScripts.Read.All.
|
||||
// Map keys are "{Kind}:{Id}" -- the renderer uses lookupScriptDisplayName to read them.
|
||||
const scriptDisplayNamesByRefKey = useScriptDisplayNames(sessionTenantId ?? tenantId, events, getAccessToken);
|
||||
const { analysisResults, loadingAnalysis, analysisLoaded, vulnerabilityReport, fetchAnalysisResults, fetchVulnerabilityReport, persistFailureRuleIds } = analysis;
|
||||
const { analysisResults, loadingAnalysis, analysisLoaded, vulnerabilityReport, fetchAnalysisResults, fetchVulnerabilityReport, persistFailureRuleIds, stoppedRuleIds } = analysis;
|
||||
const { showScriptOutput, enableSoftwareInventoryAnalyzer, enableIntegrityBypassAnalyzer } = tenantConfig;
|
||||
const {
|
||||
filteredEvents,
|
||||
@@ -698,6 +698,7 @@ function SessionDetailContent() {
|
||||
// so it stays with Admin/GA/Operator (the backend serves it read-tier regardless).
|
||||
canReanalyze={!isReadOnlyView && ((user?.isTenantAdmin ?? false) || (user?.isGlobalAdmin ?? false) || user?.role === "Operator")}
|
||||
persistFailureRuleIds={persistFailureRuleIds}
|
||||
stoppedRuleIds={stoppedRuleIds}
|
||||
sessionTenantId={sessionTenantId || undefined}
|
||||
emptyState={showInvestigateHint({
|
||||
status: session?.status,
|
||||
|
||||
@@ -664,6 +664,7 @@ export const SHARED_MANIFEST = {
|
||||
"PrivilegedRouteDenied",
|
||||
"McpServicePrincipalFirstSeen",
|
||||
"ReportUploadLimitReached",
|
||||
"RuleEvaluationLimitReached",
|
||||
"OffboardingFeedbackReceived",
|
||||
"FeedbackReceived",
|
||||
"FeedbackRatingReceived",
|
||||
@@ -691,6 +692,7 @@ export const SHARED_MANIFEST = {
|
||||
"CollectLogsQuickConfigEnabled",
|
||||
"DiagnosticsUploadEnabled",
|
||||
"DiagnosticsUploadDisabled",
|
||||
"RuleEvaluationStopped",
|
||||
"SessionActionQueued",
|
||||
"SessionTimeouts",
|
||||
"AgentEmergencyBreak",
|
||||
|
||||
@@ -660,6 +660,7 @@
|
||||
"PrivilegedRouteDenied",
|
||||
"McpServicePrincipalFirstSeen",
|
||||
"ReportUploadLimitReached",
|
||||
"RuleEvaluationLimitReached",
|
||||
"OffboardingFeedbackReceived",
|
||||
"FeedbackReceived",
|
||||
"FeedbackRatingReceived",
|
||||
@@ -687,6 +688,7 @@
|
||||
"CollectLogsQuickConfigEnabled",
|
||||
"DiagnosticsUploadEnabled",
|
||||
"DiagnosticsUploadDisabled",
|
||||
"RuleEvaluationStopped",
|
||||
"SessionActionQueued",
|
||||
"SessionTimeouts",
|
||||
"AgentEmergencyBreak",
|
||||
@@ -11471,6 +11473,18 @@
|
||||
},
|
||||
"optional": true,
|
||||
"doc": "Rule ids that failed to persist during the reanalyze — the key is omitted entirely when there were no failures (the happy-path contract stays unchanged)."
|
||||
},
|
||||
{
|
||||
"name": "stoppedRuleIds",
|
||||
"type": {
|
||||
"kind": "array",
|
||||
"element": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
}
|
||||
},
|
||||
"optional": true,
|
||||
"doc": "Rule ids the reanalyze could not evaluate within its time limit (a regex timed out, or the run's budget ran out first); their previous results are kept. The key is omitted when every rule was evaluated."
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -15328,7 +15342,7 @@
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true,
|
||||
"doc": "Set when the test ended before the last sample line was evaluated: regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated."
|
||||
"doc": "Set when the test ended before the last sample line was evaluated (EvaluationStopReasons): regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated."
|
||||
},
|
||||
{
|
||||
"name": "lines",
|
||||
@@ -19379,6 +19393,24 @@
|
||||
},
|
||||
"optional": true,
|
||||
"doc": "The evidence map exactly as the production path would persist it on a RuleResult — keys are condition signals (plus factor_* markers). Clients use it to preview {{token}} interpolation of explanation/remediation. Values are heterogeneous evidence objects by design."
|
||||
},
|
||||
{
|
||||
"name": "stopReason",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true,
|
||||
"doc": "Set with verdict \"stopped\": regex_timeout | time_budget | cancelled. The trace then holds only the entries evaluated before the stop. Null otherwise."
|
||||
},
|
||||
{
|
||||
"name": "stopNote",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "string"
|
||||
},
|
||||
"optional": true,
|
||||
"doc": "Set with verdict \"stopped\": which precondition or condition the evaluation stopped at, and what to change."
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -2406,6 +2406,8 @@ export interface GetRuleResultsResponse {
|
||||
persistFailureCount: number;
|
||||
/** Rule ids that failed to persist during the reanalyze — the key is omitted entirely when there were no failures (the happy-path contract stays unchanged). */
|
||||
persistFailureRuleIds?: string[];
|
||||
/** Rule ids the reanalyze could not evaluate within its time limit (a regex timed out, or the run's budget ran out first); their previous results are kept. The key is omitted when every rule was evaluated. */
|
||||
stoppedRuleIds?: string[];
|
||||
}
|
||||
|
||||
/** Response of POST tenants/{tenantId}/scripts/display-names: resolved Intune script display names keyed by the canonical ref string. Always 200, possibly partial — unresolved refs stay in the dictionary with a null VALUE (dictionary values are NOT subject to WhenWritingNull, so the key is emitted with an explicit JSON null). */
|
||||
@@ -3237,7 +3239,7 @@ export interface LogPatternTestResult {
|
||||
matchCount: number;
|
||||
parseFailureCount: number;
|
||||
timeoutCount: number;
|
||||
/** Set when the test ended before the last sample line was evaluated: regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated. */
|
||||
/** Set when the test ended before the last sample line was evaluated (EvaluationStopReasons): regex_timeout (the first line that hit the agent's 1 s match timeout ends the test) | time_budget (the whole test exceeded its time budget) | cancelled. Lines after the stop have no row; a note names them. Null when every line was evaluated. */
|
||||
stopReason?: string;
|
||||
readonly lines: LogPatternLineResult[];
|
||||
readonly notes: string[];
|
||||
@@ -4189,6 +4191,10 @@ export interface RuleDryRun {
|
||||
wouldMarkSessionAsFailed: boolean;
|
||||
/** The evidence map exactly as the production path would persist it on a RuleResult — keys are condition signals (plus factor_* markers). Clients use it to preview {{token}} interpolation of explanation/remediation. Values are heterogeneous evidence objects by design. */
|
||||
matchedConditions?: Record<string, unknown>;
|
||||
/** Set with verdict "stopped": regex_timeout | time_budget | cancelled. The trace then holds only the entries evaluated before the stop. Null otherwise. */
|
||||
stopReason?: string;
|
||||
/** Set with verdict "stopped": which precondition or condition the evaluation stopped at, and what to change. */
|
||||
stopNote?: string;
|
||||
}
|
||||
|
||||
export interface RuleDryRunCondition {
|
||||
|
||||
Reference in New Issue
Block a user