From 3c1229a301311a5d1cb801b50665117fe2ed09b8 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Wed, 5 Aug 2026 17:24:28 +0200 Subject: [PATCH] MCP rule authoring: deterministic context, validate_rule, and session dry-run (#124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(backend): dry-run endpoint for draft analyze rules (rule authoring) POST /api/rules/analyze/dryrun evaluates a DRAFT analyze rule against one session's events and returns a full diagnostic trace: per-condition matched/evidence (incl. per-eventType occurrence counts), per-factor confidence breakdown, verdict, and the would-be MatchedConditions map for {{token}} interpolation preview. Persists nothing: no RuleResult row, no session-status change, no notifications. - RuleEngine.DryRun.cs: DryRunRuleAsync / DryRunEvaluate share the exact evaluators (EvaluatePrecondition/EvaluateCondition/EvaluateConfidenceFactor) with the production path but evaluate ALL conditions instead of breaking on the first failed required one; parity with EvaluateRule is pinned by tests. - DryRunAnalyzeRuleFunction: TenantAdminOrGlobalReader (read-in-effect POST), GA/GlobalReader cross-tenant via session tenant resolution, 1 MB body cap, structural draft validation (unknown source/operator, correlation without joinField, invalid regex, non-evaluable confidenceFactor DSL) with actionable error messages. - EndpointAccessPolicyCatalog: register the new route. - RuleEngineDryRunTests: trace semantics, production parity matrix, side-effect freedom (VerifyNoOtherCalls), draft validation matrix. Co-Authored-By: Claude Fable 5 * feat(mcp): rule-authoring resources — guide, schemas, guardrails Deterministic authoring context for customer AIs, replacing chunked search_docs retrieval for rule creation: get_resource (and the MCP-protocol resources) now serve - rule_authoring_guide: hand-written, complete authoring knowledge (collector types, triggers, condition sources, operators, confidence model incl. the exact factor-condition DSL shapes, {{token}} interpolation order, guardrail matching semantics, pitfalls, validate->dry-run workflow) - rule_schemas: the JSON Schemas (2020-12) for gather + analyze rules - rule_guardrails: the on-device collection allowlists Schemas + guardrails are generated into rule-authoring.generated.ts by a new combine.js emitter (same pattern as the web guardrails mirror; rules/ stays the single source of truth and CI guardrails-in-sync guards the file). rule-authoring-drift.test.ts additionally pins generated == repo source, guide enums == schema enums, and the get_resource size cap. Co-Authored-By: Claude Fable 5 * feat(mcp): validate_rule + test_analyze_rule tools and author-rule prompt Closes the authoring loop for customer AIs: draft -> validate -> dry-run against a real session -> iterate -> create in the portal. - validate_rule (MCP-local, no backend call): ajv 2020-12 validation against the baked schemas, guardrail pre-flight mirroring the agent matching semantics (hive-stripped segment-bounded registry/file prefixes, whitespace-bounded WMI, exact commands, /-bounded event log channels, hard blocks), and semantic lint for schema-valid-but-dead rules (unreachable threshold, non-evaluable factor DSL, unknown eventTypes, unanchored not_regex allowlists, unresolvable {{tokens}}, trigger cross-fields, outputEventType conventions/collisions). - test_analyze_rule: calls the new POST /api/rules/analyze/dryrun and adds the {{token}}-interpolated explanation/remediation preview via the existing interpolate-rule-template. - author-rule MCP prompt guiding scenario -> resources -> draft -> validate -> dry-run. - Both tools are visible to every role (backend enforces TenantAdminOrGlobalReader on the dryrun route); catalog snapshot updated. - deps: ajv + ajv-formats promoted to direct dependencies. Co-Authored-By: Claude Fable 5 * docs(okf): rule authoring surface concept doc docs/rules/rule-authoring-surface.md: the AI-assisted authoring stack (deterministic MCP resources via combine.js baking, MCP-local validate_rule layers, the dry-run trace twin with its parity + side-effect invariants, and what is deliberately absent); registered in index.md, logged 2026-08-05. Customer docs (rules/ai-assisted-rule-authoring.md + SUMMARY + FAQ entry) are committed in the autopilotmonitor-docs repository separately. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- docs/index.md | 1 + docs/log.md | 4 + docs/rules/rule-authoring-surface.md | 117 ++ rules/scripts/combine.js | 42 + .../RuleEngineDryRunTests.cs | 418 +++++++ .../Rules/DryRunAnalyzeRuleFunction.cs | 233 ++++ .../Security/EndpointAccessPolicyCatalog.cs | 11 +- .../Services/RuleEngine.DryRun.cs | 238 ++++ .../autopilot-monitor-mcp/package-lock.json | 8 +- .../autopilot-monitor-mcp/package.json | 2 + .../__tests__/rule-authoring-drift.test.ts | 114 ++ .../src/__tests__/rule-validation.test.ts | 274 +++++ .../src/__tests__/tools-catalog-order.test.ts | 5 + .../src/__tests__/tools-rules.test.ts | 117 ++ .../autopilot-monitor-mcp/src/prompts.ts | 39 + .../src/resource-catalog.ts | 19 +- .../autopilot-monitor-mcp/src/resources.ts | 70 +- .../src/rule-authoring-guide.ts | 254 +++++ .../src/rule-authoring.generated.ts | 1005 +++++++++++++++++ .../src/rule-validation.ts | 400 +++++++ .../autopilot-monitor-mcp/src/tools.ts | 2 + .../autopilot-monitor-mcp/src/tools/admin.ts | 9 +- .../autopilot-monitor-mcp/src/tools/rules.ts | 114 ++ 23 files changed, 3486 insertions(+), 10 deletions(-) create mode 100644 docs/rules/rule-authoring-surface.md create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineDryRunTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs create mode 100644 src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-authoring-drift.test.ts create mode 100644 src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts create mode 100644 src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts create mode 100644 src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts create mode 100644 src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts create mode 100644 src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts create mode 100644 src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts diff --git a/docs/index.md b/docs/index.md index fe137a86..f772724a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,7 @@ https://docs.autopilotmonitor.com (separate repository). * [Gather Rule Phase Scoping, One-Shot Triggers & Emit-on-Change](rules/gather-rule-phase-scoping.md) - Restricting gather rules to enrollment phases (activePhases / activeFromPhase sticky latch), one-shot collection at a phase's start/end (phase_change / phase_exit), and emitMode "on_change" result-dedup. * [Gather Rule Guardrails](rules/gather-rule-guardrails.md) - Why the allowlists in rules/guardrails.json are enforced on the agent and nowhere else, which hard blocks survive unrestricted mode, and the rule that every collector must call a guard and emit security_warning on refusal. +* [Rule Authoring Surface](rules/rule-authoring-surface.md) - How customer AIs author rules against the real contract: the baked MCP resources (guide/schemas/guardrails via combine.js), the MCP-local validate_rule pre-flight, and the side-effect-free POST rules/analyze/dryrun whose trace twin shares the production evaluators (parity + side-effect-freedom pinned by tests). # Backend diff --git a/docs/log.md b/docs/log.md index 10040f69..58591734 100644 --- a/docs/log.md +++ b/docs/log.md @@ -1,5 +1,9 @@ # Log +## 2026-08-05 + +* **Creation**: Added `rules/rule-authoring-surface.md` — the AI-assisted rule-authoring stack: three deterministic MCP resources (`rule_authoring_guide` hand-written, `rule_schemas` + `rule_guardrails` generated into `rule-authoring.generated.ts` by a new combine.js emitter, drift-guarded twice), the MCP-local `validate_rule` (ajv 2020-12 against the single-rule $defs, guardrail pre-flight mirroring GatherRuleGuards matching semantics incl. hard blocks, semantic lint for schema-valid-but-dead rules), and the side-effect-free `POST /api/rules/analyze/dryrun` (TenantAdminOrGlobalReader; `RuleEngine.DryRun.cs` trace twin sharing the production evaluators, all-conditions trace with per-condition evidence + matchingEventCount, parity and side-effect-freedom pinned by RuleEngineDryRunTests). MCP `test_analyze_rule` wraps the endpoint with `{{token}}`-interpolated previews; `author-rule` prompt drives the draft → validate → dry-run loop. Deliberately absent: gather-rule remote execution, rule writes via MCP. + ## 2026-08-04 * **Update**: `rules/gather-rule-phase-scoping.md` — new section "Which phase feeds the triggers": `GatherRuleExecutorHost` now drives `phase_change`/`phase_exit`/`on_event` gather triggers (and the phase-scope gates) from the post-reduce `TimelineEventStream` (every emitted event's `(eventType, phase)` published by `EventTimelineEmitter`) instead of the pre-reduce `SignalIngress.SignalPosted` payloads. Trigger: session `32312a32` (RealmJoin tenant rsneuffen.de) — a Phase-Start rule on FinalizingSetup fired at the raw ESP-exit `EspPhaseChanged(FinalizingSetup)` signal, 7 minutes before the engine's RealmJoin-gated `phase_transition(FinalizingSetup)`, reading `HKLM\SOFTWARE\RealmJoin\Custom\BIOS` before the RealmJoin package created it. Phase triggers now fire exactly when the timeline shows the phase; `on_event` additionally sees engine-emitted event types (`enrollment_complete`, `realmjoin_resolved`, …) for the first time in V2, restoring V1 semantics and the documented end-of-enrollment pattern. Other raw-signal hosts (DeviceInfo, keep-awake, PPKG) deliberately unchanged. diff --git a/docs/rules/rule-authoring-surface.md b/docs/rules/rule-authoring-surface.md new file mode 100644 index 00000000..4efe54e5 --- /dev/null +++ b/docs/rules/rule-authoring-surface.md @@ -0,0 +1,117 @@ +--- +type: Concept +title: Rule Authoring Surface — MCP Resources, validate_rule, and the Analyze Dry-Run +description: >- + How customer AIs author gather/analyze rules against the real contract: the baked + authoring resources (guide, schemas, guardrails) generated by combine.js, the + MCP-local validate_rule pre-flight, and the side-effect-free backend dry-run that + traces a draft analyze rule against one session's events. +resource: src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts +tags: [rules, mcp, analyze-rules, gather-rules, dry-run, authoring] +timestamp: 2026-08-05 +--- + +# Concept + +Customers author rules with an AI connected to the MCP server. Retrieval +(`search_docs`) is deliberately NOT the knowledge source for authoring: chunked +top-K retrieval can silently omit an operator, a guardrail prefix, or an enum, +and the AI then produces a plausible-but-dead rule. The authoring surface is +therefore **deterministic**: three `get_resource` entries carry the complete +contract, a local validator rejects what production would reject or silently +ignore, and a backend dry-run evaluates a draft against real session events with +a full trace. + +The intended loop: draft → `validate_rule` → `test_analyze_rule` against a real +session (one that should fire, one that should not) → iterate → create in the +portal. The `author-rule` MCP prompt walks a client through exactly this order. + +# Schema + +## Resources (deterministic authoring context) + +| Resource | Content | Source | +| --- | --- | --- | +| `rule_authoring_guide` | Hand-written complete authoring knowledge: collector types, triggers, condition sources, operators, confidence model incl. the exact factor DSL, `{{token}}` interpolation order, guardrail matching semantics, pitfalls | `src/rule-authoring-guide.ts` | +| `rule_schemas` | Both JSON Schemas (2020-12) verbatim | generated | +| `rule_guardrails` | `rules/guardrails.json` verbatim | generated | + +Generation: `node rules/scripts/combine.js` bakes schemas + guardrails into +`src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts` (same +pattern as the web's `utils/guardrails.generated.ts`; the tsconfig `rootDir` and +the Docker build's exclusion of `rules/schema` rule out importing the JSON +directly). Drift is closed twice: CI's `guardrails-in-sync` job re-runs +combine.js and fails on any diff, and `rule-authoring-drift.test.ts` pins +generated == repo source plus guide enums == schema enums. + +## validate_rule (MCP-local) + +`src/rule-validation.ts`, no backend call. Three layers: + +1. **Schema** — ajv (draft 2020-12) against the baked schemas, validating the + single-rule `$defs` entry directly (the top-level oneOf wrapper would double + every error). +2. **Guardrails** — mirrors the AGENT's matching semantics from + `GatherRuleGuards.cs` (see [gather-rule-guardrails.md](gather-rule-guardrails.md)): + hive-stripped segment-bounded registry/file prefixes, whitespace-bounded WMI + prefixes, EXACT command matching, `/`-bounded event-log channels, and the + code-enforced hard blocks (`Security` etc. channels, `C:\Users`, + `C:\Windows\System32\config`). +3. **Semantic lint** — schema-valid-but-dead rules: unreachable + confidenceThreshold, factor conditions outside the three parseable shapes + (`exists`, `count >= N`, `phase_duration > N`), unknown event types (warning; + `gather_*` softened to info), unanchored `not_regex` allow-lists, + unresolvable `{{token}}`s, trigger cross-fields, outputEventType + convention/collision checks. + +The validator is advisory pre-flight only — the agent re-enforces guardrails +on-device and the backend re-validates structure. It must never be stricter +than production. + +## POST /api/rules/analyze/dryrun (backend) + +`DryRunAnalyzeRuleFunction` → `RuleEngine.DryRunRuleAsync`. Body +`{ sessionId, rule }` (draft `AnalyzeRule`, 1 MB cap). Policy tier +`TenantAdminOrGlobalReader` — a POST by shape but a pure read in effect, so the +read-only Global Reader is admitted; platform-scope callers reach any tenant's +session via `FindSessionTenantIdAsync` (same pattern as `GetRuleResultsFunction`). + +`RuleEngine.DryRun.cs` is a partial-class trace twin of `EvaluateRule`: it +shares the exact evaluators (`EvaluatePrecondition` / `EvaluateCondition` / +`EvaluateConfidenceFactor`) but evaluates **all** conditions instead of breaking +on the first failed required one — the full picture is the point of a dry run. +Verdicts: `fired`, `skipped_by_precondition`, `required_condition_not_met`, +`no_conditions_matched`, `below_confidence_threshold`, `no_events`. Each +condition entry carries `matched`, the production evidence dictionary (or the +evaluator's reason string) and `matchingEventCount` — how many session events +even have that eventType, the first thing to check on an unexpected miss. + +Invariants, pinned by `RuleEngineDryRunTests`: + +* **Parity**: verdict `fired` ⇔ `EvaluateRule` produces a result, with identical + confidence and evidence keys. Factor evaluation stays sequential against the + accumulating `matchedConditions` (a factor can observe earlier `factor_*` + entries) — reordering would change results. +* **Side-effect freedom**: exactly one strict event read; never a RuleResult + write, never `TryMarkSessionFailedFromRuleAsync`, never a notification. + `wouldMarkSessionAsFailed` is reported, not acted on. + +`test_analyze_rule` (MCP) wraps the endpoint and adds the interpolated +explanation/remediation preview via the existing `interpolate-rule-template.ts` +against the returned `matchedConditions`. + +## What is deliberately absent + +* **No gather-rule remote execution** — gather rules run as SYSTEM on devices; + the dry-run surface is analyze-only. Gather authors get guardrail pre-flight + plus the on-device debug log ([gather-rule-debug-log](../agent/gather-rule-debug-log.md)). +* **No rule write/deploy via MCP** — creating rules stays in the portal. + +# Citations + +* `src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs` — trace twin + DTOs +* `src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs` — endpoint + draft validation +* `src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts` — validator +* `src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts` — validate_rule / test_analyze_rule +* `src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts` — guide resource +* `rules/scripts/combine.js` — MCP module generation diff --git a/rules/scripts/combine.js b/rules/scripts/combine.js index 2efb1c5a..ade90a7a 100644 --- a/rules/scripts/combine.js +++ b/rules/scripts/combine.js @@ -161,3 +161,45 @@ ${flatArray(guardrails.blockedEventLogChannels)} } else { console.warn('guardrails.json not found — skipping guardrails generation'); } + +// ── MCP rule-authoring module generation ──────────────────────────────────── +// The MCP server exposes the rule schemas + guardrails as get_resource content +// so a customer's AI can author rules against the real contract instead of +// retrieval fragments. The server's tsconfig (rootDir: src, no resolveJsonModule) +// and the Docker build (rules/schema not copied) rule out importing the JSON +// directly — so we bake it into a TS module, exactly like the web mirror above. +// CI's guardrails-in-sync job re-runs this script and fails on any diff, which +// makes drift between rules/* and this generated module impossible to merge. + +const mcpAuthoringOutput = path.resolve( + rulesRoot, '..', 'src', 'McpServer', 'autopilot-monitor-mcp', 'src', 'rule-authoring.generated.ts' +); + +{ + const guardrails = JSON.parse(fs.readFileSync(guardrailsPath, 'utf8')); + const gatherSchema = JSON.parse(fs.readFileSync(path.join(rulesRoot, 'schema', 'gather-rule.schema.json'), 'utf8')); + const analyzeSchema = JSON.parse(fs.readFileSync(path.join(rulesRoot, 'schema', 'analyze-rule.schema.json'), 'utf8')); + + const ts = `/** + * AUTO-GENERATED from rules/guardrails.json + rules/schema/*.schema.json — DO NOT EDIT. + * Run: node rules/scripts/combine.js + * + * Consumed by the MCP server's rule-authoring surface (get_resource + + * validate_rule): the JSON Schemas are the validation contract, the guardrails + * are the agent-side collection allowlists. Single source of truth is rules/; + * the CI guardrails-in-sync job guards this file against drift. + */ + +/** JSON Schema (2020-12) for gather rules — rules/schema/gather-rule.schema.json verbatim. */ +export const GATHER_RULE_SCHEMA: Record = ${JSON.stringify(gatherSchema, null, 2)}; + +/** JSON Schema (2020-12) for analyze rules — rules/schema/analyze-rule.schema.json verbatim. */ +export const ANALYZE_RULE_SCHEMA: Record = ${JSON.stringify(analyzeSchema, null, 2)}; + +/** Gather-rule collection guardrails — rules/guardrails.json verbatim. */ +export const RULE_GUARDRAILS = ${JSON.stringify(guardrails, null, 2)} as const; +`; + + fs.writeFileSync(mcpAuthoringOutput, ts, 'utf8'); + console.log('rule-authoring.generated.ts: schemas + guardrails baked for MCP server'); +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineDryRunTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineDryRunTests.cs new file mode 100644 index 00000000..946a95fc --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineDryRunTests.cs @@ -0,0 +1,418 @@ +using AutopilotMonitor.Functions.Functions.Rules; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Pins the dry-run (rule-authoring) evaluation path: +/// 1. The trace evaluates EVERY condition (no early break) so authors see the full picture. +/// 2. The verdict/confidence is outcome-equivalent to the production EvaluateRule path +/// (parity tests via AnalyzeSessionAsync). +/// 3. The dry-run is side-effect free: only the strict event read touches storage — never +/// UpdateSessionStatusAsync, never a rule-result write. +/// Plus the draft validation surface of . +/// +public class RuleEngineDryRunTests +{ + private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901"; + + // ===== Trace semantics ===== + + [Fact] + public async Task Fired_FullTrace_WithFactorAndInterpolationEvidence() + { + var rule = MakeRule(); + rule.ConfidenceFactors = new List + { + new() { Signal = "app_install_failed", Condition = "count >= 2", Weight = 15 }, + }; + var events = new List + { + AppFailedEvent("app-1", "0x80070005"), + AppFailedEvent("app-1", "0x80070005"), + }; + + var (dry, sessionRepo, ruleRepo) = await DryRunAsync(rule, events); + + Assert.Equal(RuleDryRunVerdict.Fired, dry.Verdict); + Assert.Equal(2, dry.EventCount); + Assert.Single(dry.Conditions); + Assert.True(dry.Conditions[0].Matched); + Assert.Equal(2, dry.Conditions[0].MatchingEventCount); + Assert.IsType>(dry.Conditions[0].Evidence); + + Assert.Single(dry.ConfidenceFactors); + Assert.True(dry.ConfidenceFactors[0].Matched); + Assert.Equal(50 + 15, dry.FinalConfidence); + + // Evidence map matches what production would persist — including the factor marker — + // so clients can preview {{token}} interpolation against it. + Assert.NotNull(dry.MatchedConditions); + Assert.True(dry.MatchedConditions!.ContainsKey("app_failure")); + Assert.True(dry.MatchedConditions.ContainsKey("factor_app_install_failed")); + + // Side-effect freedom: exactly one strict event read, nothing else. + sessionRepo.Verify(s => s.GetSessionEventsStrictAsync(TenantId, SessionId, It.IsAny()), Times.Once); + sessionRepo.VerifyNoOtherCalls(); + ruleRepo.VerifyNoOtherCalls(); + } + + [Fact] + public async Task RequiredConditionNotMet_StillEvaluatesAllConditions() + { + var rule = MakeRule(); + rule.Conditions.Add(new RuleCondition + { + Signal = "os_edition", Source = "event_data", EventType = "os_info", + DataField = "edition", Operator = "equals", Value = "Enterprise", Required = false, + }); + var events = new List { OsInfoEvent("Enterprise") }; // no app_install_failed + + var (dry, _, _) = await DryRunAsync(rule, events); + + Assert.Equal(RuleDryRunVerdict.RequiredConditionNotMet, dry.Verdict); + // Production breaks on the first failed required condition; the dry-run keeps going — + // the second (optional) condition must appear in the trace, matched. + Assert.Equal(2, dry.Conditions.Count); + Assert.False(dry.Conditions[0].Matched); + Assert.Equal("no matching events", dry.Conditions[0].Evidence); + Assert.Equal(0, dry.Conditions[0].MatchingEventCount); + Assert.True(dry.Conditions[1].Matched); + + Assert.Null(dry.FinalConfidence); + Assert.Empty(dry.ConfidenceFactors); // factor stage never reached, mirrors production + Assert.False(dry.WouldMarkSessionAsFailed); + } + + [Fact] + public async Task PreconditionFails_VerdictSkipped_ConditionsStillTraced() + { + var rule = MakeRule(); + rule.Preconditions = new List + { + new() { Source = "event_data", EventType = "hardware_spec", DataField = "isVirtualMachine", Operator = "equals", Value = "false" }, + }; + var events = new List + { + HardwareSpecEvent(isVirtualMachine: "true"), + AppFailedEvent("app-1", "0x1"), + }; + + var (dry, _, _) = await DryRunAsync(rule, events); + + Assert.Equal(RuleDryRunVerdict.SkippedByPrecondition, dry.Verdict); + Assert.Single(dry.Preconditions); + Assert.False(dry.Preconditions[0].Passed); + // Author still sees that the condition WOULD have matched. + Assert.Single(dry.Conditions); + Assert.True(dry.Conditions[0].Matched); + Assert.Null(dry.FinalConfidence); + } + + [Fact] + public async Task BelowThreshold_ReportsFinalConfidence() + { + var rule = MakeRule(); + rule.BaseConfidence = 30; + rule.ConfidenceThreshold = 60; + var events = new List { AppFailedEvent("app-1", "0x1") }; + + var (dry, _, _) = await DryRunAsync(rule, events); + + Assert.Equal(RuleDryRunVerdict.BelowConfidenceThreshold, dry.Verdict); + Assert.Equal(30, dry.FinalConfidence); + Assert.Equal(60, dry.ConfidenceThreshold); + Assert.False(dry.WouldMarkSessionAsFailed); + } + + [Fact] + public async Task NoEvents_ShortCircuits() + { + var (dry, _, _) = await DryRunAsync(MakeRule(), new List()); + + Assert.Equal(RuleDryRunVerdict.NoEvents, dry.Verdict); + Assert.Equal(0, dry.EventCount); + Assert.Empty(dry.Conditions); + } + + [Fact] + public async Task AllOptionalNoneMatch_VerdictNoConditionsMatched() + { + var rule = MakeRule(); + rule.Conditions[0].Required = false; + var events = new List { OsInfoEvent("Pro") }; + + var (dry, _, _) = await DryRunAsync(rule, events); + + Assert.Equal(RuleDryRunVerdict.NoConditionsMatched, dry.Verdict); + Assert.Null(dry.FinalConfidence); + } + + [Fact] + public async Task Fired_WithMarkSessionAsFailed_OnlyReportsFlag() + { + var rule = MakeRule(); + rule.MarkSessionAsFailedDefault = true; + var events = new List { AppFailedEvent("app-1", "0x1") }; + + var (dry, sessionRepo, _) = await DryRunAsync(rule, events); + + Assert.Equal(RuleDryRunVerdict.Fired, dry.Verdict); + Assert.True(dry.WouldMarkSessionAsFailed); + // The flag is REPORTED, never acted upon: no session read, no status write. + sessionRepo.Verify(s => s.GetSessionEventsStrictAsync(TenantId, SessionId, It.IsAny()), Times.Once); + sessionRepo.VerifyNoOtherCalls(); + } + + // ===== Parity with the production path ===== + + public static TheoryData ParityScenarios => new() { "fires", "required_miss", "below_threshold", "precondition_skip" }; + + /// + /// The dry-run verdict must be outcome-equivalent to EvaluateRule (via AnalyzeSessionAsync): + /// fired ⇔ a RuleResult is produced, and on fire the confidence and evidence keys are identical. + /// + [Theory] + [MemberData(nameof(ParityScenarios))] + public async Task DryRunVerdict_MatchesProductionOutcome(string scenario) + { + var rule = MakeRule(); + rule.ConfidenceFactors = new List + { + new() { Signal = "app_install_failed", Condition = "count >= 2", Weight = 15 }, + }; + List events = scenario switch + { + "fires" => new() { AppFailedEvent("a", "0x1"), AppFailedEvent("a", "0x1") }, + "required_miss" => new() { OsInfoEvent("Pro") }, + "below_threshold" => new() { AppFailedEvent("a", "0x1") }, + "precondition_skip" => new() { HardwareSpecEvent("true"), AppFailedEvent("a", "0x1") }, + _ => throw new ArgumentOutOfRangeException(nameof(scenario)), + }; + if (scenario == "below_threshold") { rule.BaseConfidence = 30; rule.ConfidenceThreshold = 60; } + if (scenario == "precondition_skip") + { + rule.Preconditions = new List + { + new() { Source = "event_data", EventType = "hardware_spec", DataField = "isVirtualMachine", Operator = "equals", Value = "false" }, + }; + } + + var production = await RunProductionAsync(rule, events); + var (dry, _, _) = await DryRunAsync(rule, events); + + var productionFired = production.Results.Count == 1; + Assert.Equal(productionFired, dry.Verdict == RuleDryRunVerdict.Fired); + + if (productionFired) + { + var result = production.Results[0]; + Assert.Equal(result.ConfidenceScore, dry.FinalConfidence); + Assert.Equal( + result.MatchedConditions.Keys.OrderBy(k => k), + dry.MatchedConditions!.Keys.OrderBy(k => k)); + } + } + + // ===== Draft validation (DryRunAnalyzeRuleFunction.ValidateDraftRule) ===== + + [Fact] + public void Validate_ValidRule_NoErrors() + { + Assert.Empty(DryRunAnalyzeRuleFunction.ValidateDraftRule(MakeRule())); + } + + [Fact] + public void Validate_NullRule_Error() + { + Assert.Contains("rule is required", DryRunAnalyzeRuleFunction.ValidateDraftRule(null)); + } + + [Fact] + public void Validate_NoConditions_Error() + { + var rule = MakeRule(); + rule.Conditions.Clear(); + Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("at least one condition")); + } + + [Theory] + [InlineData("Event_Type")] // wrong case — the evaluator switch is case-sensitive + [InlineData("eventtype")] + [InlineData("")] + public void Validate_UnknownOrMissingSource_Error(string source) + { + var rule = MakeRule(); + rule.Conditions[0].Source = source; + Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("source")); + } + + [Fact] + public void Validate_UnknownOperator_Error() + { + var rule = MakeRule(); + rule.Conditions[0].Operator = "starts_with"; + Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("unknown operator 'starts_with'")); + } + + [Fact] + public void Validate_CorrelationWithoutJoinField_Error() + { + var rule = MakeRule(); + rule.Conditions[0].Source = "event_correlation"; + rule.Conditions[0].CorrelateEventType = "app_install_completed"; + rule.Conditions[0].JoinField = ""; + Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("joinField")); + } + + [Fact] + public void Validate_InvalidRegex_Error() + { + var rule = MakeRule(); + rule.Conditions[0].Operator = "regex"; + rule.Conditions[0].Value = "([unclosed"; + Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("not a valid regex")); + } + + [Theory] + [InlineData("count > 3")] // wrong comparator + [InlineData("Count >= 3")] // wrong case + [InlineData("phase_duration >= 3")] // wrong comparator + [InlineData("always")] + public void Validate_UnsupportedFactorCondition_Error(string condition) + { + var rule = MakeRule(); + rule.ConfidenceFactors = new List { new() { Signal = "x", Condition = condition, Weight = 10 } }; + Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("not evaluable")); + } + + [Theory] + [InlineData("exists")] + [InlineData("count >= 3")] + [InlineData("count >=3")] + [InlineData("phase_duration > 300")] + public void Validate_SupportedFactorCondition_NoError(string condition) + { + var rule = MakeRule(); + rule.ConfidenceFactors = new List { new() { Signal = "x", Condition = condition, Weight = 10 } }; + Assert.Empty(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule)); + } + + [Fact] + public void Validate_DuplicateSignals_Error() + { + var rule = MakeRule(); + rule.Conditions.Add(new RuleCondition + { + Signal = "app_failure", Source = "event_type", EventType = "os_info", Operator = "exists", Required = false, + }); + Assert.Contains(DryRunAnalyzeRuleFunction.ValidateDraftRule(rule), e => e.Contains("duplicate signal")); + } + + [Fact] + public void Validate_ConfidenceOutOfRange_Error() + { + var rule = MakeRule(); + rule.BaseConfidence = 120; + rule.ConfidenceThreshold = -5; + var errors = DryRunAnalyzeRuleFunction.ValidateDraftRule(rule); + Assert.Contains(errors, e => e.Contains("baseConfidence")); + Assert.Contains(errors, e => e.Contains("confidenceThreshold")); + } + + // ===== Helpers ===== + + /// Simple single-condition rule: required event_type match on app_install_failed, + /// base 50 / threshold 40 — fires whenever the event exists. + private static AnalyzeRule MakeRule() => new() + { + RuleId = "ANALYZE-TEST-001", + Title = "Test rule", + Severity = "warning", + Category = "apps", + // Drafts behave like tenant custom rules; also keeps the parity path clear of the + // live-catalog sunset filter in GetAllRulesForTenantAsync (built-in ids only). + IsBuiltIn = false, + BaseConfidence = 50, + ConfidenceThreshold = 40, + Conditions = new List + { + new() + { + Signal = "app_failure", Source = "event_type", EventType = "app_install_failed", + Operator = "exists", Value = "", Required = true, + }, + }, + Explanation = "App {{appId}} failed with {{errorCode}}.", + }; + + private static EnrollmentEvent AppFailedEvent(string appId, string errorCode) => new() + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "app_install_failed", + Timestamp = DateTime.UtcNow, + Sequence = 1, + Data = new Dictionary { ["appId"] = appId, ["errorCode"] = errorCode }, + }; + + private static EnrollmentEvent OsInfoEvent(string edition) => new() + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "os_info", + Timestamp = DateTime.UtcNow, + Sequence = 2, + Data = new Dictionary { ["edition"] = edition }, + }; + + private static EnrollmentEvent HardwareSpecEvent(string isVirtualMachine) => new() + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "hardware_spec", + Timestamp = DateTime.UtcNow, + Sequence = 3, + Data = new Dictionary { ["isVirtualMachine"] = isVirtualMachine }, + }; + + private static (RuleEngine engine, Mock sessionRepo, Mock ruleRepo) MakeEngine(List events) + { + var ruleRepo = new Mock(); + var sessionRepo = new Mock(); + sessionRepo.Setup(s => s.GetSessionEventsStrictAsync(TenantId, SessionId, It.IsAny())).ReturnsAsync(events); + var ruleService = new AnalyzeRuleService(ruleRepo.Object, NullLogger.Instance); + var engine = new RuleEngine(ruleService, ruleRepo.Object, sessionRepo.Object, NullLogger.Instance); + return (engine, sessionRepo, ruleRepo); + } + + private static async Task<(RuleDryRun dry, Mock sessionRepo, Mock ruleRepo)> DryRunAsync( + AnalyzeRule rule, List events) + { + var (engine, sessionRepo, ruleRepo) = MakeEngine(events); + var dry = await engine.DryRunRuleAsync(TenantId, SessionId, rule); + return (dry, sessionRepo, ruleRepo); + } + + private static async Task RunProductionAsync(AnalyzeRule rule, List events) + { + var ruleRepo = new Mock(); + ruleRepo.Setup(r => r.GetAnalyzeRulesAsync("global")).ReturnsAsync(new List()); + ruleRepo.Setup(r => r.GetAnalyzeRulesAsync(TenantId)).ReturnsAsync(new List { rule }); + ruleRepo.Setup(r => r.GetRuleStatesAsync(It.IsAny())).ReturnsAsync(new Dictionary()); + ruleRepo.Setup(r => r.GetRuleResultsAsync(It.IsAny(), It.IsAny())).ReturnsAsync(new List()); + var sessionRepo = new Mock(); + sessionRepo.Setup(s => s.GetSessionEventsStrictAsync(TenantId, SessionId, It.IsAny())).ReturnsAsync(events); + var ruleService = new AnalyzeRuleService(ruleRepo.Object, NullLogger.Instance); + var engine = new RuleEngine(ruleService, ruleRepo.Object, sessionRepo.Object, NullLogger.Instance); + return await engine.AnalyzeSessionAsync(TenantId, SessionId); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs new file mode 100644 index 00000000..48c501b9 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs @@ -0,0 +1,233 @@ +using System.Net; +using System.Text.RegularExpressions; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Services; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Functions.Functions.Rules +{ + /// + /// Dry-runs a DRAFT analyze rule against one session's events and returns a full diagnostic + /// trace (per-condition matched/evidence, confidence breakdown, verdict) WITHOUT persisting + /// anything. Powers the rule-authoring loop in the portal and the MCP `test_analyze_rule` tool. + /// + /// The rule arrives in the request body and never touches storage; the session status is never + /// modified (the engine's dry-run path has no MarkSessionAsFailed side effect by construction). + /// Global-scope callers may dry-run against any tenant's session — the tenant is resolved from + /// the session, same as GetRuleResultsFunction. + /// + public class DryRunAnalyzeRuleFunction + { + private readonly ILogger _logger; + private readonly IRuleRepository _ruleRepo; + private readonly ISessionRepository _sessionRepo; + private readonly AnalyzeRuleService _analyzeRuleService; + + public DryRunAnalyzeRuleFunction( + ILogger logger, + IRuleRepository ruleRepo, + ISessionRepository sessionRepo, + AnalyzeRuleService analyzeRuleService) + { + _logger = logger; + _ruleRepo = ruleRepo; + _sessionRepo = sessionRepo; + _analyzeRuleService = analyzeRuleService; + } + + public sealed class DryRunRequest + { + public string? SessionId { get; set; } + public AnalyzeRule? Rule { get; set; } + } + + [Function("DryRunAnalyzeRule")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "rules/analyze/dryrun")] HttpRequestData req) + { + // Authentication + TenantAdminOrGlobalReader authorization enforced by PolicyEnforcementMiddleware. + // Read-only diagnostic, so the read-only Global Reader is admitted alongside Tenant Admins and GAs. + var requestCtx = req.GetRequestContext(); + var effectiveTenantId = requestCtx.TargetTenantId; + + if (req.Headers.TryGetValues("Content-Length", out var clValues) + && long.TryParse(clValues.FirstOrDefault(), out var contentLength) + && contentLength > 1_048_576) // 1 MB limit + { + return await BadRequestAsync(req, new[] { "Request body too large" }); + } + + var body = await new StreamReader(req.Body).ReadToEndAsync(); + DryRunRequest? request; + try + { + request = JsonConvert.DeserializeObject(body); + } + catch (JsonException ex) + { + return await BadRequestAsync(req, new[] { $"Request body is not valid JSON: {ex.Message}" }); + } + + if (request == null || string.IsNullOrWhiteSpace(request.SessionId)) + { + return await BadRequestAsync(req, new[] { "sessionId is required" }); + } + + var errors = ValidateDraftRule(request.Rule); + if (errors.Count > 0) + { + return await BadRequestAsync(req, errors); + } + + var sessionId = request.SessionId!.Trim(); + var rule = request.Rule!; + + // Global-scope cross-tenant fallback: resolve the session's actual tenant so a GA / + // Global Reader can dry-run against any tenant's session (mirrors GetRuleResultsFunction). + if (requestCtx.HasGlobalScope) + { + var resolvedTenantId = await _sessionRepo.FindSessionTenantIdAsync(sessionId); + if (resolvedTenantId != null && !string.Equals(resolvedTenantId, effectiveTenantId, StringComparison.OrdinalIgnoreCase)) + { + effectiveTenantId = resolvedTenantId; + } + } + + var session = await _sessionRepo.GetSessionAsync(effectiveTenantId, sessionId); + if (session == null) + { + var notFound = req.CreateResponse(HttpStatusCode.NotFound); + await notFound.WriteAsJsonAsync(new { success = false, message = "Session not found", sessionId }); + return notFound; + } + + var ruleEngine = new RuleEngine(_analyzeRuleService, _ruleRepo, _sessionRepo, _logger); + var result = await ruleEngine.DryRunRuleAsync(effectiveTenantId, sessionId, rule); + + _logger.LogInformation( + "Dry-run of draft rule {RuleId} against session {SessionId}: verdict={Verdict}, confidence={Confidence}", + string.IsNullOrEmpty(rule.RuleId) ? "(no id)" : rule.RuleId, sessionId, result.Verdict, result.FinalConfidence); + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new { success = true, sessionId, result }); + return response; + } + + private static async Task BadRequestAsync(HttpRequestData req, IReadOnlyList errors) + { + var badRequest = req.CreateResponse(HttpStatusCode.BadRequest); + await badRequest.WriteAsJsonAsync(new { success = false, message = errors[0], errors }); + return badRequest; + } + + // ===== Draft validation ===== + + // Exact-match sets mirroring what the RuleEngine actually dispatches on. The evaluator + // switch is case-sensitive ("event_type", not "Event_Type"), and an unknown source/operator + // silently evaluates to false in production — for a dry-run that silence is exactly the + // confusion we want to catch up front. + private static readonly HashSet KnownSources = new(StringComparer.Ordinal) + { + "event_type", "event_data", "event_data_array", "event_count", + "phase_duration", "app_install_duration", "event_correlation", + }; + + private static readonly HashSet KnownOperators = new(StringComparer.OrdinalIgnoreCase) + { + "equals", "not_equals", "contains", "not_contains", "regex", "not_regex", + "gt", "lt", "gte", "lte", "exists", "not_exists", + "count_gte", "count_per_group_gte", "in", "not_in", + }; + + // The three shapes EvaluateConfidenceFactor actually parses. Anything else is silently + // false in production ("count > 3", "Count >= 3", "phase_duration>=300", …). + private static readonly Regex FactorConditionShape = new( + @"^(exists|count >= ?\d+|phase_duration > ?\d+)$", RegexOptions.Compiled); + + /// + /// Structural validation of a draft rule before evaluation. Returns human-readable errors + /// (empty list = valid). Deliberately limited to what would make the dry-run silently + /// meaningless or throw — full authoring lint (schema, guardrails, event-type catalog) + /// lives client-side in the MCP `validate_rule` tool. + /// + internal static List ValidateDraftRule(AnalyzeRule? rule) + { + var errors = new List(); + if (rule == null) + { + errors.Add("rule is required"); + return errors; + } + + if (rule.Conditions == null || rule.Conditions.Count == 0) + { + errors.Add("rule must contain at least one condition"); + return errors; + } + + if (rule.BaseConfidence < 0 || rule.BaseConfidence > 100) + errors.Add($"baseConfidence must be 0-100 (was {rule.BaseConfidence})"); + if (rule.ConfidenceThreshold < 0 || rule.ConfidenceThreshold > 100) + errors.Add($"confidenceThreshold must be 0-100 (was {rule.ConfidenceThreshold})"); + + var seenSignals = new HashSet(StringComparer.Ordinal); + for (var i = 0; i < rule.Conditions.Count; i++) + { + var c = rule.Conditions[i]; + var label = $"conditions[{i}]"; + + if (string.IsNullOrWhiteSpace(c.Signal)) + errors.Add($"{label}: signal is required"); + else if (!seenSignals.Add(c.Signal)) + errors.Add($"{label}: duplicate signal '{c.Signal}' — signals must be unique (evidence is keyed by signal)"); + + if (string.IsNullOrWhiteSpace(c.Source)) + errors.Add($"{label}: source is required"); + else if (!KnownSources.Contains(c.Source)) + errors.Add($"{label}: unknown source '{c.Source}' (must be one of: {string.Join(", ", KnownSources)}; exact lowercase)"); + + if (!string.IsNullOrEmpty(c.Operator) && !KnownOperators.Contains(c.Operator)) + errors.Add($"{label}: unknown operator '{c.Operator}'"); + + if (string.Equals(c.Source, "event_correlation", StringComparison.Ordinal)) + { + if (string.IsNullOrWhiteSpace(c.CorrelateEventType)) + errors.Add($"{label}: event_correlation requires correlateEventType"); + if (string.IsNullOrWhiteSpace(c.JoinField)) + errors.Add($"{label}: event_correlation requires joinField"); + } + + if ((string.Equals(c.Operator, "regex", StringComparison.OrdinalIgnoreCase) + || string.Equals(c.Operator, "not_regex", StringComparison.OrdinalIgnoreCase)) + && !string.IsNullOrEmpty(c.Value)) + { + try + { + _ = new Regex(c.Value, RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + } + catch (ArgumentException ex) + { + errors.Add($"{label}: value is not a valid regex: {ex.Message}"); + } + } + } + + for (var i = 0; i < (rule.ConfidenceFactors?.Count ?? 0); i++) + { + var f = rule.ConfidenceFactors![i]; + var label = $"confidenceFactors[{i}]"; + if (string.IsNullOrWhiteSpace(f.Signal)) + errors.Add($"{label}: signal is required"); + if (string.IsNullOrEmpty(f.Condition) || !FactorConditionShape.IsMatch(f.Condition)) + errors.Add($"{label}: condition '{f.Condition}' is not evaluable — supported shapes: \"exists\", \"count >= N\", \"phase_duration > N\" (exact spacing)"); + } + + return errors; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index 2a46fbd4..55e2a610 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -67,9 +67,9 @@ public enum EndpointPolicy /// /// Own-tenant Admin, OR a platform role (Global Admin / Global Reader). Tenant-scoped admin-tier /// READ access (e.g. GET tenants/{tenantId}/graph-permissions/status). Like - /// but additionally admits the read-only Global Reader — used only - /// on GET routes so the reader gets visibility without write power, while tenant - /// Operators/Viewers stay excluded. (GET config/{tenantId} moved OFF this tier to MemberRead: + /// but additionally admits the read-only Global Reader — used on + /// GET routes (and read-in-effect POSTs like rules/analyze/dryrun) so the reader gets + /// visibility without write power, while tenant Operators/Viewers stay excluded. (GET config/{tenantId} moved OFF this tier to MemberRead: /// its handler redacts secrets for every non-admin caller, so member-tier read is safe there.) /// TenantAdminOrGlobalReader, @@ -264,6 +264,11 @@ public static class EndpointAccessPolicyCatalog new("GET", "rules/gather", EndpointPolicy.MemberRead), new("GET", "rules/analyze", EndpointPolicy.MemberRead), new("GET", "rules/ime-log-patterns", EndpointPolicy.MemberRead), + // POST by shape (draft rule in the body) but a pure READ in effect: evaluates the draft + // against a session's events and persists NOTHING (DryRunAnalyzeRuleFunction). Admin-read + // tier so the read-only Global Reader can help debug rules while tenant Operators/Viewers + // stay excluded from the authoring surface. + new("POST", "rules/analyze/dryrun", EndpointPolicy.TenantAdminOrGlobalReader), // Member tier so Operators/Viewers can see their tenant's configuration read-only in the // Settings UI. Safe because the handler redacts by default: only a Global Admin or the // tenant's OWN admin gets the unredacted secrets — every other admitted caller (Operator, diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs new file mode 100644 index 00000000..d9b85b35 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs @@ -0,0 +1,238 @@ +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// Dry-run (diagnostic) evaluation of a single draft analyze rule against a session's events. + /// Powers the rule-authoring loop (portal / MCP `test_analyze_rule`): the author gets a full + /// per-condition trace instead of the production path's fire-or-silent-null. + /// + /// Deliberately side-effect free: nothing is persisted, the session status is never touched, + /// no notifications are emitted. The trace loop mirrors .EvaluateRule + /// but does NOT short-circuit on the first failed required condition — evaluating every + /// condition is the whole point of a dry run. Both paths share the same evaluators + /// (EvaluatePrecondition / EvaluateCondition / EvaluateConfidenceFactor), so a verdict of + /// "fired" here is equivalent to EvaluateRule returning a result; the parity test + /// (RuleEngineDryRunTests) pins that equivalence. + /// + public partial class RuleEngine + { + /// + /// Loads the session's events (strict read — storage failures propagate) and dry-runs the + /// given draft rule against them. Mirrors the event handling of AnalyzeSessionAsync + /// (strict load + BackfillDerivedEventFields) so the dry-run sees exactly what a real + /// analysis pass would see. + /// + public async Task DryRunRuleAsync(string tenantId, string sessionId, AnalyzeRule rule) + { + var allEvents = await _sessionRepo.GetSessionEventsStrictAsync(tenantId, sessionId); + if (allEvents.Count == 0) + { + return new RuleDryRun + { + Verdict = RuleDryRunVerdict.NoEvents, + EventCount = 0, + BaseConfidence = rule.BaseConfidence, + ConfidenceThreshold = rule.ConfidenceThreshold, + }; + } + + BackfillDerivedEventFields(allEvents); + return DryRunEvaluate(rule, allEvents); + } + + /// + /// Trace twin of EvaluateRule. Keep the confidence arithmetic and the factor evaluation + /// ORDER in lock-step with EvaluateRule — factors are evaluated sequentially against the + /// accumulating matchedConditions dictionary (a factor's "exists" check can observe + /// earlier factor_<signal> entries), so reordering would change results. + /// + internal RuleDryRun DryRunEvaluate(AnalyzeRule rule, List events) + { + var dry = new RuleDryRun + { + EventCount = events.Count, + BaseConfidence = rule.BaseConfidence, + ConfidenceThreshold = rule.ConfidenceThreshold, + }; + + var preconditionsPassed = true; + if (rule.Preconditions != null) + { + foreach (var pre in rule.Preconditions) + { + var passed = EvaluatePrecondition(pre, events); + if (!passed) preconditionsPassed = false; + dry.Preconditions.Add(new RuleDryRunPrecondition + { + Source = pre.Source, + EventType = pre.EventType, + DataField = string.IsNullOrEmpty(pre.DataField) ? null : pre.DataField, + Operator = pre.Operator, + Value = pre.Value, + Passed = passed, + }); + } + } + + // Evaluate ALL conditions (no early break — the author wants the full picture). + // Outcome-equivalent to EvaluateRule: a failed required condition makes the + // production path return null before matchedConditions is ever used. + var matchedConditions = new Dictionary(); + var allRequiredMet = true; + foreach (var condition in rule.Conditions) + { + var (matched, evidence) = EvaluateCondition(condition, events); + if (condition.Required && !matched) allRequiredMet = false; + if (matched) matchedConditions[condition.Signal] = evidence; + + dry.Conditions.Add(new RuleDryRunCondition + { + Signal = condition.Signal, + Source = condition.Source, + EventType = string.IsNullOrEmpty(condition.EventType) ? null : condition.EventType, + Required = condition.Required, + Matched = matched, + // Matched: the evidence dictionary the production path stores. Not matched: + // the evaluator's reason string ("no matching events", …). + Evidence = evidence, + MatchingEventCount = string.IsNullOrEmpty(condition.EventType) + ? null + : events.Count(e => MatchesEventType(e, condition.EventType)), + }); + } + + dry.MatchedConditions = matchedConditions; + + if (!preconditionsPassed) + { + dry.Verdict = RuleDryRunVerdict.SkippedByPrecondition; + return dry; + } + + if (!allRequiredMet) + { + dry.Verdict = RuleDryRunVerdict.RequiredConditionNotMet; + return dry; + } + + if (matchedConditions.Count == 0) + { + dry.Verdict = RuleDryRunVerdict.NoConditionsMatched; + return dry; + } + + var confidence = rule.BaseConfidence; + foreach (var factor in rule.ConfidenceFactors) + { + var factorMatched = EvaluateConfidenceFactor(factor, events, matchedConditions); + if (factorMatched) + { + confidence += factor.Weight; + matchedConditions[$"factor_{factor.Signal}"] = true; + } + + dry.ConfidenceFactors.Add(new RuleDryRunFactor + { + Signal = factor.Signal, + Condition = factor.Condition, + Weight = factor.Weight, + Matched = factorMatched, + }); + } + + confidence = Math.Min(confidence, 100); + dry.FinalConfidence = confidence; + + if (confidence < rule.ConfidenceThreshold) + { + dry.Verdict = RuleDryRunVerdict.BelowConfidenceThreshold; + return dry; + } + + dry.Verdict = RuleDryRunVerdict.Fired; + dry.WouldMarkSessionAsFailed = rule.MarkSessionAsFailed ?? rule.MarkSessionAsFailedDefault; + return dry; + } + } + + /// Verdict strings for . Stable API contract. + public static class RuleDryRunVerdict + { + public const string Fired = "fired"; + public const string SkippedByPrecondition = "skipped_by_precondition"; + public const string RequiredConditionNotMet = "required_condition_not_met"; + public const string NoConditionsMatched = "no_conditions_matched"; + public const string BelowConfidenceThreshold = "below_confidence_threshold"; + public const string NoEvents = "no_events"; + } + + /// Full diagnostic trace of one dry-run evaluation. Serialized camelCase to clients. + public sealed class RuleDryRun + { + public string Verdict { get; set; } = string.Empty; + + /// Number of events in the session the rule was evaluated against. + public int EventCount { get; set; } + + public List Preconditions { get; } = new(); + public List Conditions { get; } = new(); + + /// Empty unless all required conditions were met (mirrors the production path, + /// which never reaches factor evaluation otherwise). + public List ConfidenceFactors { get; } = new(); + + public int BaseConfidence { get; set; } + + /// base + matched factor weights, capped at 100. Null when the evaluation ended + /// before the confidence stage (precondition skip / required miss / nothing matched). + public int? FinalConfidence { get; set; } + + public int ConfidenceThreshold { get; set; } + + /// True only for verdict "fired" AND the rule's effective MarkSessionAsFailed flag. + /// The dry-run itself never touches the session. + public bool WouldMarkSessionAsFailed { get; set; } + + /// The evidence map exactly as the production path would persist it on a + /// RuleResult — keys are condition signals (plus factor_* markers). Clients use it to + /// preview {{token}} interpolation of explanation/remediation. + public Dictionary? MatchedConditions { get; set; } + } + + public sealed class RuleDryRunPrecondition + { + public string Source { get; set; } = string.Empty; + public string EventType { get; set; } = string.Empty; + public string? DataField { get; set; } + public string? Operator { get; set; } + public string? Value { get; set; } + public bool Passed { get; set; } + } + + public sealed class RuleDryRunCondition + { + public string Signal { get; set; } = string.Empty; + public string Source { get; set; } = string.Empty; + public string? EventType { get; set; } + public bool Required { get; set; } + public bool Matched { get; set; } + + /// Matched: the evidence dictionary (eventId, timestamp, field, value, …). + /// Not matched: the evaluator's reason string (e.g. "no matching events"). + public object? Evidence { get; set; } + + /// How many session events have this condition's eventType at all — the first + /// thing an author checks when a condition unexpectedly doesn't match. Null when the + /// condition has no eventType. + public int? MatchingEventCount { get; set; } + } + + public sealed class RuleDryRunFactor + { + public string Signal { get; set; } = string.Empty; + public string Condition { get; set; } = string.Empty; + public int Weight { get; set; } + public bool Matched { get; set; } + } +} diff --git a/src/McpServer/autopilot-monitor-mcp/package-lock.json b/src/McpServer/autopilot-monitor-mcp/package-lock.json index f33a1d59..824e2781 100644 --- a/src/McpServer/autopilot-monitor-mcp/package-lock.json +++ b/src/McpServer/autopilot-monitor-mcp/package-lock.json @@ -10,6 +10,8 @@ "dependencies": { "@huggingface/transformers": "^3.8.1", "@modelcontextprotocol/sdk": "^1.30.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "compression": "^1.8.1", "express": "^5.2.1", "fuse.js": "^7.5.0", @@ -1736,9 +1738,9 @@ } }, "node_modules/ajv": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", - "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", diff --git a/src/McpServer/autopilot-monitor-mcp/package.json b/src/McpServer/autopilot-monitor-mcp/package.json index 2ae9d14f..f9e8f6da 100644 --- a/src/McpServer/autopilot-monitor-mcp/package.json +++ b/src/McpServer/autopilot-monitor-mcp/package.json @@ -21,6 +21,8 @@ "dependencies": { "@huggingface/transformers": "^3.8.1", "@modelcontextprotocol/sdk": "^1.30.0", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "compression": "^1.8.1", "express": "^5.2.1", "fuse.js": "^7.5.0", diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-authoring-drift.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-authoring-drift.test.ts new file mode 100644 index 00000000..049a6cd3 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-authoring-drift.test.ts @@ -0,0 +1,114 @@ +/** + * Drift guards for the rule-authoring surface: + * + * 1. rule-authoring.generated.ts MUST equal the repo source of truth + * (rules/guardrails.json + rules/schema/*.schema.json). CI's guardrails-in-sync + * job re-runs combine.js, but this test also catches a hand-edited generated + * file locally. Same-repo only; skipped in an isolated package. + * 2. The hand-written RULE_AUTHORING_GUIDE enum lists MUST equal the schema enums — + * the guide is prose, the schema is contract; they may never diverge. + * 3. Every rule-authoring resource must fit the get_resource size cap. + */ +import { describe, it, expect } from 'vitest'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { existsSync, readFileSync } from 'node:fs'; +import { GATHER_RULE_SCHEMA, ANALYZE_RULE_SCHEMA, RULE_GUARDRAILS } from '../rule-authoring.generated.js'; +import { RULE_AUTHORING_GUIDE } from '../rule-authoring-guide.js'; +import { getResourceContent } from '../resource-catalog.js'; +import { MAX_RESULT_SIZE_CHARS } from '../tools/shared.js'; + +/** Walk up from this test file to the repo root (marked by AutopilotMonitor.sln). */ +function findRulesDir(): string | null { + let dir = dirname(fileURLToPath(import.meta.url)); + for (let i = 0; i < 12; i++) { + if (existsSync(join(dir, 'AutopilotMonitor.sln'))) { + const p = join(dir, 'rules'); + return existsSync(p) ? p : null; + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return null; +} + +function schemaEnum(schema: Record, defName: string | null, propName: string): string[] { + // Navigate $defs..properties..enum (or items.enum for arrays). + const defs = (schema['$defs'] ?? {}) as Record; + const root = defName ? (defs[defName] as Record) : schema; + expect(root, `$defs.${defName} missing`).toBeTruthy(); + const props = root['properties'] as Record; + const prop = props[propName] as Record | undefined; + expect(prop, `property ${propName} missing`).toBeTruthy(); + const en = (prop!['enum'] ?? (prop!['items'] as Record | undefined)?.['enum']) as string[] | undefined; + expect(en, `enum for ${propName} missing`).toBeTruthy(); + return en!; +} + +describe('rule-authoring.generated.ts drift vs rules/ source of truth', () => { + const rulesDir = findRulesDir(); + const itOrSkip = rulesDir ? it : it.skip; + + itOrSkip('guardrails match rules/guardrails.json', () => { + const source = JSON.parse(readFileSync(join(rulesDir!, 'guardrails.json'), 'utf8')); + expect(RULE_GUARDRAILS).toEqual(source); + }); + + itOrSkip('gather schema matches rules/schema/gather-rule.schema.json', () => { + const source = JSON.parse(readFileSync(join(rulesDir!, 'schema', 'gather-rule.schema.json'), 'utf8')); + expect(GATHER_RULE_SCHEMA).toEqual(source); + }); + + itOrSkip('analyze schema matches rules/schema/analyze-rule.schema.json', () => { + const source = JSON.parse(readFileSync(join(rulesDir!, 'schema', 'analyze-rule.schema.json'), 'utf8')); + expect(ANALYZE_RULE_SCHEMA).toEqual(source); + }); +}); + +describe('RULE_AUTHORING_GUIDE enums vs schema contract', () => { + it('gather collector types, triggers, categories, severities match the gather schema', () => { + expect([...RULE_AUTHORING_GUIDE.enums.gatherCollectorTypes]).toEqual( + schemaEnum(GATHER_RULE_SCHEMA, 'gatherRule', 'collectorType')); + expect([...RULE_AUTHORING_GUIDE.enums.gatherTriggers]).toEqual( + schemaEnum(GATHER_RULE_SCHEMA, 'gatherRule', 'trigger')); + expect([...RULE_AUTHORING_GUIDE.enums.gatherCategories]).toEqual( + schemaEnum(GATHER_RULE_SCHEMA, 'gatherRule', 'category')); + expect([...RULE_AUTHORING_GUIDE.enums.gatherOutputSeverities]).toEqual( + schemaEnum(GATHER_RULE_SCHEMA, 'gatherRule', 'outputSeverity')); + // triggerPhase itself is a free string in the schema (empty = every transition); + // the canonical phase list lives on activePhases/activeFromPhase. + expect([...RULE_AUTHORING_GUIDE.gatherRules.phases]).toEqual( + schemaEnum(GATHER_RULE_SCHEMA, 'gatherRule', 'activePhases')); + }); + + it('analyze condition sources, operators, categories, severities match the analyze schema', () => { + expect([...RULE_AUTHORING_GUIDE.enums.analyzeConditionSources]).toEqual( + schemaEnum(ANALYZE_RULE_SCHEMA, 'ruleCondition', 'source')); + expect([...RULE_AUTHORING_GUIDE.analyzeRules.operators]).toEqual( + schemaEnum(ANALYZE_RULE_SCHEMA, 'ruleCondition', 'operator')); + expect([...RULE_AUTHORING_GUIDE.enums.analyzeCategories]).toEqual( + schemaEnum(ANALYZE_RULE_SCHEMA, 'analyzeRule', 'category')); + expect([...RULE_AUTHORING_GUIDE.enums.analyzeSeverities]).toEqual( + schemaEnum(ANALYZE_RULE_SCHEMA, 'analyzeRule', 'severity')); + }); + + it('guide condition-source prose covers every schema source (and no phantom)', () => { + const documented = Object.keys(RULE_AUTHORING_GUIDE.analyzeRules.conditionSources).sort(); + const schema = [...schemaEnum(ANALYZE_RULE_SCHEMA, 'ruleCondition', 'source')].sort(); + expect(documented).toEqual(schema); + }); + + it('guide collector-type prose covers every schema collector type (and no phantom)', () => { + const documented = Object.keys(RULE_AUTHORING_GUIDE.gatherRules.collectorTypes).sort(); + const schema = [...schemaEnum(GATHER_RULE_SCHEMA, 'gatherRule', 'collectorType')].sort(); + expect(documented).toEqual(schema); + }); +}); + +describe('rule-authoring resources fit the get_resource size cap', () => { + it.each(['rule_authoring_guide', 'rule_schemas', 'rule_guardrails'] as const)('%s stays under the cap', (name) => { + const size = JSON.stringify(getResourceContent(name), null, 2).length; + expect(size).toBeLessThan(MAX_RESULT_SIZE_CHARS.small); + }); +}); diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts new file mode 100644 index 00000000..f88a7cd9 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts @@ -0,0 +1,274 @@ +/** + * Matrix tests for the MCP-local rule draft validator: schema layer, guardrail + * layer (agent matching semantics), and the semantic lint that catches + * schema-valid-but-silently-dead rules. + */ +import { describe, it, expect } from 'vitest'; +import { validateRuleDraft, type ValidationFinding } from '../rule-validation.js'; + +const errors = (findings: ValidationFinding[]) => findings.filter((f) => f.level === 'error').map((f) => f.message); +const warnings = (findings: ValidationFinding[]) => findings.filter((f) => f.level === 'warning').map((f) => f.message); + +function validGather(): Record { + return { + ruleId: 'GATHER-DEVICE-101', + title: 'Pending file renames', + collectorType: 'registry', + target: 'HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Component Based Servicing\\RebootPending', + trigger: 'startup', + outputEventType: 'gather_pending_file_renames', + }; +} + +function validAnalyze(): Record { + return { + ruleId: 'ANALYZE-APP-101', + title: 'Repeated install failures', + severity: 'high', + category: 'apps', + baseConfidence: 55, + confidenceThreshold: 50, + conditions: [ + { + signal: 'app_retry_storm', + source: 'event_count', + eventType: 'app_install_failed', + dataField: 'appId', + operator: 'count_per_group_gte', + value: '3', + required: true, + }, + ], + explanation: 'App {{appName}} failed repeatedly (last error {{errorCode}}).', + }; +} + +describe('type detection', () => { + it('detects gather via collectorType and analyze via conditions', () => { + expect(validateRuleDraft(validGather()).ruleType).toBe('gather'); + expect(validateRuleDraft(validAnalyze()).ruleType).toBe('analyze'); + }); + + it('unknown shape yields a single explanatory error', () => { + const r = validateRuleDraft({ title: 'what am I' }); + expect(r.ruleType).toBe('unknown'); + expect(r.valid).toBe(false); + expect(r.findings).toHaveLength(1); + }); +}); + +describe('schema layer', () => { + it('valid drafts pass', () => { + expect(validateRuleDraft(validGather()).valid).toBe(true); + expect(validateRuleDraft(validAnalyze()).valid).toBe(true); + }); + + it('missing required fields fail with schema errors', () => { + const gather = validGather(); + delete gather.outputEventType; + expect(errors(validateRuleDraft(gather).findings).some((m) => m.includes('outputEventType'))).toBe(true); + + const analyze = validAnalyze(); + delete analyze.explanation; + expect(errors(validateRuleDraft(analyze).findings).some((m) => m.includes('explanation'))).toBe(true); + }); + + it('enum violations and unknown properties fail', () => { + const bad = validAnalyze(); + bad.severity = 'catastrophic'; + (bad as Record).madeUpField = 1; + const msgs = errors(validateRuleDraft(bad).findings); + expect(msgs.some((m) => m.includes('/severity'))).toBe(true); + expect(msgs.some((m) => m.includes('additional'))).toBe(true); + }); + + it('bad ruleId pattern fails', () => { + const bad = validGather(); + bad.ruleId = 'GATHER-device-1'; + expect(errors(validateRuleDraft(bad).findings).some((m) => m.includes('/ruleId'))).toBe(true); + }); +}); + +describe('gather guardrails (agent matching semantics)', () => { + it('accepts an allow-listed registry target with hive prefix', () => { + expect(validateRuleDraft(validGather()).valid).toBe(true); + }); + + it('rejects a registry target outside the allowlist', () => { + const bad = validGather(); + bad.target = 'HKLM\\SOFTWARE\\Contoso\\Secrets'; + expect(errors(validateRuleDraft(bad).findings).some((m) => m.includes('registry target'))).toBe(true); + }); + + it('rejects prefix spoofing (segment boundary)', () => { + const bad = validGather(); + bad.target = 'HKLM\\SOFTWARE\\Microsoft\\EnrollmentsEvil\\x'; // "...\\Enrollments" is allowed, "EnrollmentsEvil" is not + expect(validateRuleDraft(bad).valid).toBe(false); + }); + + it('rejects hard-blocked file paths even though C:\\Users is not in filePrefixes', () => { + const bad = validGather(); + bad.collectorType = 'file'; + bad.target = 'C:\\Users\\admin\\secret.txt'; + expect(errors(validateRuleDraft(bad).findings).some((m) => m.includes('hard-blocked'))).toBe(true); + }); + + it('accepts an allow-listed file target and rejects others', () => { + const ok = validGather(); + ok.collectorType = 'file'; + ok.target = 'C:\\ProgramData\\Microsoft\\IntuneManagementExtension\\Logs\\IntuneManagementExtension.log'; + expect(validateRuleDraft(ok).valid).toBe(true); + + const bad = validGather(); + bad.collectorType = 'file'; + bad.target = 'C:\\Temp\\anything.log'; + expect(validateRuleDraft(bad).valid).toBe(false); + }); + + it('WMI: prefix must be whitespace-bounded', () => { + const ok = validGather(); + ok.collectorType = 'wmi'; + ok.target = 'SELECT * FROM Win32_BIOS WHERE PrimaryBIOS = TRUE'; + expect(validateRuleDraft(ok).valid).toBe(true); + + const spoof = validGather(); + spoof.collectorType = 'wmi'; + spoof.target = 'SELECT * FROM Win32_BIOSX'; + expect(validateRuleDraft(spoof).valid).toBe(false); + }); + + it('commands match exactly — extra arguments are rejected', () => { + const ok = validGather(); + ok.collectorType = 'command_allowlisted'; + ok.target = 'dsregcmd /status'; + expect(validateRuleDraft(ok).valid).toBe(true); + + const bad = validGather(); + bad.collectorType = 'command_allowlisted'; + bad.target = 'dsregcmd /status; whoami'; + expect(validateRuleDraft(bad).valid).toBe(false); + }); + + it('event log: hard-blocked channels rejected, allow-listed channel with /Admin suffix accepted', () => { + const blocked = validGather(); + blocked.collectorType = 'eventlog'; + blocked.target = 'Security'; + expect(errors(validateRuleDraft(blocked).findings).some((m) => m.includes('hard-blocked'))).toBe(true); + + const ok = validGather(); + ok.collectorType = 'eventlog'; + ok.target = 'Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Admin'; + expect(validateRuleDraft(ok).valid).toBe(true); + }); +}); + +describe('gather semantic lint', () => { + it('interval without intervalSeconds is an error; on_event without triggerEventType is an error', () => { + const a = validGather(); + a.trigger = 'interval'; + expect(errors(validateRuleDraft(a).findings).some((m) => m.includes('intervalSeconds'))).toBe(true); + + const b = validGather(); + b.trigger = 'on_event'; + expect(errors(validateRuleDraft(b).findings).some((m) => m.includes('triggerEventType'))).toBe(true); + }); + + it('outputEventType colliding with a built-in event type warns', () => { + const bad = validGather(); + bad.outputEventType = 'app_install_failed'; + expect(warnings(validateRuleDraft(bad).findings).some((m) => m.includes('collides'))).toBe(true); + }); + + it('outputEventType without gather_ prefix warns', () => { + const r = validGather(); + r.outputEventType = 'pending_file_renames'; + expect(warnings(validateRuleDraft(r).findings).some((m) => m.includes('gather_'))).toBe(true); + }); +}); + +describe('analyze semantic lint', () => { + it('unreachable threshold is an error, above-base-with-factors is info', () => { + const dead = validAnalyze(); + dead.confidenceThreshold = 90; + dead.baseConfidence = 50; + expect(errors(validateRuleDraft(dead).findings).some((m) => m.includes('never fire'))).toBe(true); + + const corroborated = validAnalyze(); + corroborated.confidenceThreshold = 60; + corroborated.confidenceFactors = [{ signal: 'enrollment_failed', condition: 'count >= 1', weight: 15 }]; + const r = validateRuleDraft(corroborated); + expect(r.valid).toBe(true); + expect(r.findings.some((f) => f.level === 'info' && f.message.includes('confidence factors'))).toBe(true); + }); + + it('non-evaluable factor condition shapes are errors', () => { + for (const condition of ['count > 3', 'Count >= 3', 'phase_duration >= 30', 'always']) { + const bad = validAnalyze(); + bad.confidenceFactors = [{ signal: 'x', condition, weight: 10 }]; + expect(errors(validateRuleDraft(bad).findings).some((m) => m.includes('not evaluable')), condition).toBe(true); + } + }); + + it('"exists" factor pointing at a non-existent signal warns', () => { + const bad = validAnalyze(); + bad.confidenceFactors = [{ signal: 'no_such_signal', condition: 'exists', weight: 10 }]; + expect(warnings(validateRuleDraft(bad).findings).some((m) => m.includes('no_such_signal'))).toBe(true); + }); + + it('unknown event types warn; gather_ event types are info', () => { + const unknown = validAnalyze(); + (unknown.conditions as Array>)[0].eventType = 'made_up_event'; + expect(warnings(validateRuleDraft(unknown).findings).some((m) => m.includes('made_up_event'))).toBe(true); + + const gatherRef = validAnalyze(); + (gatherRef.conditions as Array>)[0].eventType = 'gather_pending_file_renames'; + const r = validateRuleDraft(gatherRef); + expect(r.valid).toBe(true); + expect(r.findings.some((f) => f.level === 'info' && f.message.includes('gather_'))).toBe(true); + }); + + it('duplicate signals and correlation without joinField are errors', () => { + const dup = validAnalyze(); + (dup.conditions as Array>).push({ + signal: 'app_retry_storm', source: 'event_type', eventType: 'os_info', operator: 'exists', value: '', + }); + expect(errors(validateRuleDraft(dup).findings).some((m) => m.includes('duplicate signal'))).toBe(true); + + const corr = validAnalyze(); + (corr.conditions as Array>)[0] = { + signal: 'pair', source: 'event_correlation', eventType: 'app_install_started', correlateEventType: 'app_install_failed', + }; + expect(errors(validateRuleDraft(corr).findings).some((m) => m.includes('joinField'))).toBe(true); + }); + + it('invalid regex is an error; unanchored not_regex warns', () => { + const bad = validAnalyze(); + (bad.conditions as Array>)[0] = { + signal: 's', source: 'event_data', eventType: 'app_install_failed', dataField: 'errorCode', operator: 'regex', value: '([unclosed', + }; + expect(errors(validateRuleDraft(bad).findings).some((m) => m.includes('not a valid regex'))).toBe(true); + + const unanchored = validAnalyze(); + (unanchored.conditions as Array>)[0] = { + signal: 's', source: 'event_data_array', eventType: 'provisioning_package_scan', dataField: 'artifacts', + itemField: 'identity', operator: 'not_regex', value: 'Microsoft\\.Windows', + }; + expect(warnings(validateRuleDraft(unanchored).findings).some((m) => m.includes('anchored'))).toBe(true); + }); + + it('unresolvable {{tokens}} warn, resolvable ones (dataField/auto/signal) do not', () => { + const r = validateRuleDraft(validAnalyze()); + // {{appName}} + {{errorCode}} are auto-captured fields → no warning. + expect(warnings(r.findings).filter((m) => m.includes('{{'))).toEqual([]); + + const bad = validAnalyze(); + bad.explanation = 'Value is {{nonexistent_token}}.'; + expect(warnings(validateRuleDraft(bad).findings).some((m) => m.includes('nonexistent_token'))).toBe(true); + }); + + it('markSessionAsFailedDefault=true yields an info finding', () => { + const ko = validAnalyze(); + ko.markSessionAsFailedDefault = true; + expect(validateRuleDraft(ko).findings.some((f) => f.level === 'info' && f.message.includes('KO criterion'))).toBe(true); + }); +}); diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts index 0816664b..4d7f37d2 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts @@ -150,7 +150,12 @@ describe('role catalog snapshot — privilege-leak guard', () => { 'search_sessions', 'search_sessions_by_cve', 'search_sessions_by_event', + // Rule-authoring pair, available to every role: validate_rule is MCP-local + // (no backend call at all) and test_analyze_rule's dryrun endpoint enforces + // TenantAdminOrGlobalReader server-side (a Viewer's call 403s there). + 'test_analyze_rule', 'update_tenant_config', + 'validate_rule', ]; // ── Named difference lists (each a deliberate role-boundary decision). ── diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts new file mode 100644 index 00000000..44e23118 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts @@ -0,0 +1,117 @@ +/** + * Handler tests for the rule-authoring tools (tools/rules.ts): request shape of the + * dry-run POST, interpolation of the returned trace, and validate_rule's local + * (no-backend) behaviour. + */ +import { describe, it, expect, beforeEach } from 'vitest'; +import { vi } from 'vitest'; +import { registerRuleTools } from '../tools/rules.js'; + +const { apiFetchMock } = vi.hoisted(() => ({ apiFetchMock: vi.fn() })); +vi.mock('../client.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, apiFetch: apiFetchMock }; +}); + +type Handler = (args: Record) => Promise<{ content: Array<{ text: string }>; isError?: boolean }>; + +function captureToolHandlers(ga: boolean): Record { + const handlers: Record = {}; + const fake = { registerTool: (name: string, _def: unknown, handler: Handler) => { handlers[name] = handler; } }; + registerRuleTools(fake as never, ga); + return handlers; +} + +const SESSION = 'b2c3d4e5-f6a7-8901-bcde-f12345678901'; + +const ANALYZE_DRAFT = { + ruleId: 'ANALYZE-APP-101', + title: 'Repeated install failures', + severity: 'high', + category: 'apps', + conditions: [ + { signal: 'app_failure', source: 'event_type', eventType: 'app_install_failed', operator: 'exists', value: '', required: true }, + ], + explanation: 'App {{appName}} failed with {{errorCode}}.', + remediation: [{ title: 'Check {{appName}}', steps: ['Inspect error {{errorCode}}'] }], +}; + +beforeEach(() => { + apiFetchMock.mockReset(); +}); + +describe('test_analyze_rule', () => { + it('POSTs sessionId + rule to the dryrun endpoint and interpolates the response', async () => { + apiFetchMock.mockResolvedValue({ + success: true, + sessionId: SESSION, + result: { + verdict: 'fired', + finalConfidence: 65, + matchedConditions: { + app_failure: { eventType: 'app_install_failed', appName: 'Contoso App', errorCode: '0x80070005' }, + }, + }, + }); + + const handlers = captureToolHandlers(false); + const res = await handlers.test_analyze_rule({ sessionId: SESSION, rule: ANALYZE_DRAFT }); + + expect(apiFetchMock).toHaveBeenCalledTimes(1); + const [path, init] = apiFetchMock.mock.calls[0] as [string, RequestInit]; + expect(path).toBe('/api/rules/analyze/dryrun'); + expect(init.method).toBe('POST'); + const body = JSON.parse(String(init.body)); + expect(body.sessionId).toBe(SESSION); + expect(body.rule.ruleId).toBe('ANALYZE-APP-101'); + + expect(res.isError).toBeFalsy(); + const payload = JSON.parse(res.content[0].text); + expect(payload.result.verdict).toBe('fired'); + expect(payload.interpolatedExplanation).toBe('App Contoso App failed with 0x80070005.'); + expect(payload.interpolatedRemediation[0].title).toBe('Check Contoso App'); + expect(payload.interpolatedRemediation[0].steps[0]).toBe('Inspect error 0x80070005'); + }); + + it('leaves tokens untouched when the dry-run produced no evidence', async () => { + apiFetchMock.mockResolvedValue({ + success: true, + sessionId: SESSION, + result: { verdict: 'required_condition_not_met', matchedConditions: {} }, + }); + + const handlers = captureToolHandlers(false); + const res = await handlers.test_analyze_rule({ sessionId: SESSION, rule: ANALYZE_DRAFT }); + const payload = JSON.parse(res.content[0].text); + expect(payload.interpolatedExplanation).toBe('App {{appName}} failed with {{errorCode}}.'); + }); + + it('surfaces backend errors via toolError (no throw)', async () => { + apiFetchMock.mockRejectedValue(new Error('boom')); + const handlers = captureToolHandlers(false); + const res = await handlers.test_analyze_rule({ sessionId: SESSION, rule: ANALYZE_DRAFT }); + expect(res.isError).toBe(true); + }); +}); + +describe('validate_rule', () => { + it('is fully MCP-local (never calls the backend) and reports findings', async () => { + const handlers = captureToolHandlers(false); + const res = await handlers.validate_rule({ rule: { ...ANALYZE_DRAFT, confidenceThreshold: 99, baseConfidence: 50 } }); + + expect(apiFetchMock).not.toHaveBeenCalled(); + const payload = JSON.parse(res.content[0].text); + expect(payload.ruleType).toBe('analyze'); + expect(payload.valid).toBe(false); + expect(payload.errorCount).toBeGreaterThan(0); + expect(payload.findings.some((f: { message: string }) => f.message.includes('never fire'))).toBe(true); + }); + + it('valid analyze draft points to test_analyze_rule as the next step', async () => { + const handlers = captureToolHandlers(false); + const res = await handlers.validate_rule({ rule: ANALYZE_DRAFT }); + const payload = JSON.parse(res.content[0].text); + expect(payload.valid).toBe(true); + expect(payload.nextStep).toContain('test_analyze_rule'); + }); +}); diff --git a/src/McpServer/autopilot-monitor-mcp/src/prompts.ts b/src/McpServer/autopilot-monitor-mcp/src/prompts.ts index 201e01d1..3e647e54 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/prompts.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/prompts.ts @@ -141,6 +141,45 @@ export function registerPrompts(server: McpServer, ga: boolean): void { }), ); + server.registerPrompt( + 'author-rule', + { + title: 'Author a Gather / Analyze Rule', + description: + 'Guided creation of a custom gather or analyze rule from a scenario description, ' + + 'with validation and a session dry-run before anything is deployed.', + argsSchema: { + scenario: z.string().describe('What should be detected or collected, in plain language'), + sessionId: z.string().optional().describe('Optional: a recent session UUID to dry-run the draft analyze rule against'), + }, + }, + ({ scenario, sessionId }) => ({ + messages: [ + { + role: 'user', + content: { + type: 'text', + text: + `Create an Autopilot Monitor rule for this scenario: ${scenario}\n\n` + + 'Work through this order:\n' + + '1. Read get_resource(name="rule_authoring_guide") plus get_resource(name="rule_schemas") ' + + 'and get_resource(name="event_types"). For gather rules also get_resource(name="rule_guardrails").\n' + + '2. Decide: does this need a gather rule (new data from the device), an analyze rule ' + + '(evaluate existing events), or both? If both, draft the gather rule first and reference ' + + 'its outputEventType from the analyze rule.\n' + + '3. Draft the rule JSON and call validate_rule with it. Fix every error; explain warnings you keep.\n' + + (sessionId + ? `4. Dry-run the analyze rule: test_analyze_rule(sessionId="${sessionId}", rule=). ` + + 'Walk through the per-condition trace; iterate until the verdict matches the expectation, and check the interpolated explanation reads well.\n' + : '4. Ask me for a recent sessionId (one where the rule SHOULD fire and ideally one where it should not) and dry-run the analyze rule with test_analyze_rule.\n') + + '5. Present the final rule JSON with a short explanation of when it fires, its confidence ' + + 'model, and how to create it in the portal (Settings → Gather Rules / Analyze Rules).', + }, + }, + ], + }), + ); + // Global Admin only — relies on get_platform_metrics (a GA-only tool that is // not registered for normal users). Hidden from non-GA so it never references // a tool they cannot see. diff --git a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts index 89a17606..77381bf1 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts @@ -5,6 +5,8 @@ * stateless-HTTP MCP clients whose resource discovery is broken). */ import { DIAG_ZIP_MAP } from './diag-zip-map.js'; +import { RULE_AUTHORING_GUIDE } from './rule-authoring-guide.js'; +import { GATHER_RULE_SCHEMA, ANALYZE_RULE_SCHEMA, RULE_GUARDRAILS } from './rule-authoring.generated.js'; // Single source of truth for the model-facing event-type catalog. MUST stay in // sync with C# `Constants.EventTypes` (Shared) — enforced by the vitest drift test @@ -378,7 +380,13 @@ export function assertKnownDevicePropertyKeys(keys: string[]): void { ); } -export type ResourceName = 'event_types' | 'device_properties' | 'diag_zip_layout'; +export type ResourceName = + | 'event_types' + | 'device_properties' + | 'diag_zip_layout' + | 'rule_authoring_guide' + | 'rule_schemas' + | 'rule_guardrails'; export function getResourceContent(name: ResourceName): unknown { switch (name) { @@ -388,5 +396,14 @@ export function getResourceContent(name: ResourceName): unknown { return DEVICE_PROPERTIES_CATALOG; case 'diag_zip_layout': return DIAG_ZIP_MAP; + case 'rule_authoring_guide': + return RULE_AUTHORING_GUIDE; + case 'rule_schemas': + return { + gatherRuleSchema: GATHER_RULE_SCHEMA, + analyzeRuleSchema: ANALYZE_RULE_SCHEMA, + }; + case 'rule_guardrails': + return RULE_GUARDRAILS; } } diff --git a/src/McpServer/autopilot-monitor-mcp/src/resources.ts b/src/McpServer/autopilot-monitor-mcp/src/resources.ts index 28022f32..0659d84f 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/resources.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/resources.ts @@ -1,5 +1,5 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { EVENT_TYPES_CATALOG, DEVICE_PROPERTIES_CATALOG } from './resource-catalog.js'; +import { EVENT_TYPES_CATALOG, DEVICE_PROPERTIES_CATALOG, getResourceContent } from './resource-catalog.js'; import { DIAG_ZIP_MAP } from './diag-zip-map.js'; /** @@ -71,4 +71,72 @@ export function registerResources(server: McpServer): void { ], }) ); + + server.registerResource( + 'rule_authoring_guide', + 'autopilot://rule-authoring-guide', + { + title: 'Rule Authoring Guide', + mimeType: 'application/json', + description: + 'Complete guide for authoring gather and analyze rules: collector types, triggers, ' + + 'condition sources, operators, confidence model, {{token}} interpolation, guardrail ' + + 'semantics and common pitfalls. Read together with rule_schemas and rule_guardrails; ' + + 'validate drafts with validate_rule and dry-run analyze rules with test_analyze_rule.', + }, + async () => ({ + contents: [ + { + uri: 'autopilot://rule-authoring-guide', + mimeType: 'application/json', + text: JSON.stringify(getResourceContent('rule_authoring_guide'), null, 2), + }, + ], + }) + ); + + server.registerResource( + 'rule_schemas', + 'autopilot://rule-schemas', + { + title: 'Rule JSON Schemas', + mimeType: 'application/json', + description: + 'The JSON Schemas (2020-12) for gather rules and analyze rules — the exact validation ' + + 'contract (required fields, enums, patterns). Generated from rules/schema/ in the ' + + 'product repository.', + }, + async () => ({ + contents: [ + { + uri: 'autopilot://rule-schemas', + mimeType: 'application/json', + text: JSON.stringify(getResourceContent('rule_schemas'), null, 2), + }, + ], + }) + ); + + server.registerResource( + 'rule_guardrails', + 'autopilot://rule-guardrails', + { + title: 'Gather Rule Guardrails', + mimeType: 'application/json', + description: + 'The collection allowlists enforced on-device for gather rules: registry/file/WMI/' + + 'diagnostics path prefixes, event log channels (plus hard-blocked channels) and the ' + + 'exact allowed command list. A gather rule whose target is not covered here will be ' + + 'blocked by the agent.', + }, + async () => ({ + contents: [ + { + uri: 'autopilot://rule-guardrails', + mimeType: 'application/json', + text: JSON.stringify(getResourceContent('rule_guardrails'), null, 2), + }, + ], + }) + ); } diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts new file mode 100644 index 00000000..2bb16112 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts @@ -0,0 +1,254 @@ +/** + * The rule-authoring guide served via get_resource(name="rule_authoring_guide"). + * + * This is the complete, deterministic knowledge an AI needs to author gather + * and analyze rules — the counterpart to retrieval via search_docs, which only + * returns fragments. Everything here mirrors verified engine behaviour: + * - condition/factor semantics: Backend RuleEngine.ConditionEvaluators.cs + * - guardrail matching: Agent GatherRuleGuards.cs + * - schema contract: rules/schema/*.schema.json (served as rule_schemas, + * generated into rule-authoring.generated.ts — the drift test pins the two). + * + * Keep enum lists in sync with the schemas — __tests__/rule-authoring-drift.test.ts + * fails the build when they diverge. + */ + +export const RULE_AUTHORING_GUIDE = { + overview: + 'Autopilot Monitor has two customer-authorable rule families:\n' + + '- GATHER rules run ON the device during enrollment and collect extra data ' + + '(registry values, event log entries, WMI queries, files, allow-listed commands, parsed logs). ' + + 'Each execution emits an event with the rule\'s outputEventType.\n' + + '- ANALYZE rules run in the backend AFTER enrollment (and on-demand) against the session\'s ' + + 'event stream and produce findings with explanation, remediation and a confidence score.\n' + + 'They compose: a gather rule collects a signal, an analyze rule turns it into a finding.', + + authoringWorkflow: [ + '1. Read this guide plus get_resource(name="rule_schemas") (the JSON Schema contract), ' + + 'get_resource(name="rule_guardrails") (what gather rules may touch) and ' + + 'get_resource(name="event_types") (which event types exist).', + '2. Draft the rule as JSON. If the analysis needs data no existing event carries, draft the ' + + 'gather rule FIRST, then reference its outputEventType in the analyze rule.', + '3. Call validate_rule with the draft — fix every error, take warnings seriously.', + '4. For analyze rules: call test_analyze_rule with the draft plus a sessionId of a real ' + + 'session from your tenant (ideally one where the rule SHOULD fire and one where it should ' + + 'NOT). The dry-run returns a per-condition trace with evidence — iterate until the verdict ' + + 'matches your expectation. Nothing is persisted by a dry-run.', + '5. Gather rules cannot be dry-run server-side (they execute on devices). Validate them, then ' + + 'deploy to a test device/VM; the agent debug log setting for gather rules shows per-rule ' + + 'execution detail on-device.', + '6. Create the finished rule in the portal (Settings → Gather Rules / Analyze Rules). Custom ' + + 'rules are tenant-scoped. Well-tested rules can be proposed as community rules via GitHub.', + ], + + ids: { + gather: 'GATHER-{CATEGORY}-{NNN}, matching ^GATHER-[A-Z]+-\\d{3}$ (e.g. GATHER-NET-011)', + analyze: 'ANALYZE-{CATEGORY}-{NNN}, matching ^ANALYZE-[A-Z]+-\\d{3}$ (e.g. ANALYZE-APP-101)', + note: + 'Pick an ID that does not collide with built-in rules (the portal rejects collisions). ' + + 'Common category shorthands: NET, ID, APPS/APP, DEVICE/DEV, ESP, ENRL/ENROLL, SEC, CORR.', + }, + + gatherRules: { + collectorTypes: { + registry: + 'Read a registry value or enumerate subkeys. target = key path WITHOUT the HKLM\\ prefix ' + + 'semantics of your choice — use the exact prefixes from rule_guardrails (they are rooted at ' + + 'HKLM). parameters: valueName (single value), listSubkeys ("true"), severityIfExists / ' + + 'severityIfNotExists to escalate the emitted event severity.', + eventlog: + 'Query a Windows event log channel. target = channel name (must be in the guardrails ' + + 'eventLogChannels allowlist; Security/PowerShell/Sysmon channels are hard-blocked). ' + + 'parameters: eventId, source, maxEntries.', + wmi: 'Run a WMI query. target = the query; it must start with one of the allow-listed prefixes ' + + '(e.g. "SELECT * FROM Win32_BIOS").', + file: + 'Check file/directory existence, optionally read content. target = path under an allow-listed ' + + 'prefix. parameters: readContent ("true"). C:\\Users is blocked.', + command_allowlisted: + 'Run one of the exactly allow-listed commands (see rule_guardrails allowedCommands). ' + + 'Matching is EXACT (trimmed, case-insensitive) — no arguments can be added or removed.', + logparser: + 'Parse log files with a regex. target = file/glob under an allow-listed prefix. parameters: ' + + 'pattern (regex, named groups become event data), format ("cmtrace" for IME-style logs), ' + + 'maxLines, trackPosition ("true" = only new lines on re-runs).', + }, + triggers: { + startup: 'Once when the agent starts.', + phase_change: + 'When the enrollment reaches a phase (set triggerPhase). Phases follow the engine-reduced ' + + 'timeline — the same phases the portal timeline shows, not raw ESP registry flips.', + phase_exit: 'When the enrollment leaves a phase (set triggerPhase).', + interval: 'Periodically (set intervalSeconds).', + on_event: + 'When a specific event type is emitted (set triggerEventType). This sees every emitted ' + + 'event including engine events such as enrollment_complete.', + }, + phases: ['Start', 'DevicePreparation', 'DeviceSetup', 'AppsDevice', 'AccountSetup', 'AppsUser', 'FinalizingSetup', 'Complete'], + scoping: + 'Optional activePhases (array) or activeFromPhase restrict when the rule may run at all; ' + + 'emitMode "on_change" emits only when the collected payload differs from the previous run ' + + '(default "always").', + output: + 'outputEventType names the emitted event; use the gather_ prefix convention (e.g. ' + + '"gather_pending_reboot") so analyze rules and humans can tell collected events from ' + + 'built-in agent events. outputSeverity: Info | Warning | Error | Critical.', + guardrails: + 'Every target is checked on-device against rules/guardrails.json (served as ' + + 'rule_guardrails). Matching semantics: registry/file prefixes are segment-bounded (the next ' + + 'character after the prefix must be "\\"), WMI prefixes bound on whitespace, commands match ' + + 'exactly, event log channels bound on "/". Hard blocks (Security event log, PowerShell ' + + 'channels, C:\\Users, download/exec-style commands) are enforced in agent code and cannot be ' + + 'lifted by config. If a needed target is not allow-listed, the allowlist itself has to be ' + + 'extended via a GitHub contribution to rules/guardrails.json — a rule alone cannot widen it.', + template: { + ruleId: 'GATHER-DEVICE-101', + title: 'Collect pending file rename operations', + description: 'Reads PendingFileRenameOperations to detect a pending reboot blocking installs.', + category: 'device', + version: '1.0.0', + author: 'Contoso IT', + enabled: true, + collectorType: 'registry', + target: 'SYSTEM\\CurrentControlSet\\Control\\Session Manager', + parameters: { valueName: 'PendingFileRenameOperations' }, + trigger: 'phase_change', + triggerPhase: 'DeviceSetup', + outputEventType: 'gather_pending_file_renames', + outputSeverity: 'Info', + tags: ['device', 'reboot'], + }, + }, + + analyzeRules: { + conditionSources: { + event_type: + 'Does an event of eventType exist? With dataField+operator+value it additionally filters ' + + 'on a field inside those events. Most common source.', + event_data: + 'Compare a data field inside events of eventType (dataField + operator + value). Supports ' + + 'dot-paths into nested objects; "message" resolves to the event message.', + event_data_array: + 'Iterate an ARRAY field (dataField) inside ONE event and test a sub-field of each element ' + + '(itemField; empty = element itself). Matches if ANY element passes. Allow-list pattern: ' + + 'operator not_regex against an ANCHORED allow-regex (start with ^ and use \\b boundaries) ' + + 'fires only for elements NOT on the list — an unanchored regex would let ' + + '"Evil.Prefix.AllowedName.Suffix" impostors through.', + event_count: + 'Count events of eventType: operator count_gte (global) or count_per_group_gte (per ' + + 'distinct dataField value, e.g. 3+ failures of the SAME app via dataField=appId). Optional ' + + 'filterField/filterOperator/filterValue restrict which events are counted.', + phase_duration: + 'Match a phase (eventType=esp_phase_changed, dataField=espPhase, operator=equals, ' + + 'value=); the evidence carries durationSeconds. Combine with a confidenceFactor ' + + '"phase_duration > N" for the actual duration check.', + app_install_duration: 'Compare app install duration in seconds (operator gt/lt/gte/lte, value=seconds).', + event_correlation: + 'Join two event types over a shared field: eventType + correlateEventType + joinField, ' + + 'optional timeWindowSeconds, optional eventAFilterField/-Operator/-Value to pre-filter the ' + + 'first event, optional dataField+operator+value tested on the correlated pair.', + }, + caseSensitivity: + 'source values are matched CASE-SENSITIVELY by the engine — write them exactly as listed ' + + '(lowercase with underscores). Event types match case-insensitively.', + operators: [ + 'equals', 'not_equals', 'contains', 'not_contains', 'regex', 'not_regex', + 'gt', 'lt', 'gte', 'lte', 'exists', 'not_exists', 'count_gte', 'count_per_group_gte', 'in', 'not_in', + ], + operatorNotes: + 'String comparisons are case-insensitive; regex runs case-insensitive with a 1s timeout ' + + '(.NET regex syntax); in/not_in take a comma-separated value list. An operator the engine ' + + 'does not know silently evaluates to FALSE in production — validate_rule catches this.', + required: + 'required=true conditions are the firing core: ALL of them must match or the rule produces ' + + 'nothing. required=false conditions are optional reinforcers whose evidence feeds ' + + 'confidenceFactors. A rule where NO condition matched never fires, even if all conditions ' + + 'are optional.', + preconditions: + 'Optional gate BEFORE conditions: source must be "event_data"; all preconditions are ' + + 'AND-combined and a failing one silently skips the rule (no finding, no UI card). Use for ' + + 'applicability ("skip on virtual machines", "only when marker event absent via not_exists").', + suppressByEvent: + 'On a condition, suppressByEvent {eventType, joinField} drops matches that a resolving event ' + + 'shares the joinField value with — e.g. app_install_failed suppressed by a later ' + + 'app_install_completed of the same appId.', + confidence: { + model: + 'If all required conditions match (and at least one condition matched), score = ' + + 'baseConfidence + sum of weights of matched confidenceFactors, capped at 100. The rule ' + + 'fires only if score >= confidenceThreshold.', + factorConditions: + 'A factor\'s condition must be EXACTLY one of these shapes (anything else silently never ' + + 'matches): "exists" (the condition signal named by factor.signal matched), "count >= N" ' + + '(factor.signal is an EVENT TYPE here — counts events of that type, case-sensitive), ' + + '"phase_duration > N" (seconds, requires a matched phase_duration condition).', + bestPractices: + 'baseConfidence 40-60 for single-signal rules, 50-70 for correlations; set ' + + 'confidenceThreshold at or slightly below baseConfidence so the rule can fire without ' + + 'factors; weight optional reinforcers 10-20 each.', + }, + interpolation: + 'explanation and remediation text may contain {{token}} placeholders, resolved from the ' + + 'matched evidence at display time: first a matched condition whose dataField equals the ' + + 'token, then the auto-captured fields appId / appName / errorPatternId / errorCode / ' + + 'exitCode / status from the matched events, then a condition SIGNAL name. Unresolved tokens ' + + 'stay visible as {{token}} so typos are noticeable. test_analyze_rule returns the ' + + 'interpolated preview for a real session.', + markSessionAsFailed: + 'markSessionAsFailedDefault=true escalates a firing rule to a terminal Failed session ' + + 'status (KO criterion). Use sparingly — only for findings that genuinely mean the ' + + 'enrollment failed.', + template: { + ruleId: 'ANALYZE-APP-101', + title: 'Repeated install failures of the same app', + description: 'Detects an app failing to install 3+ times in one enrollment.', + severity: 'high', + category: 'apps', + version: '1.0.0', + author: 'Contoso IT', + enabled: true, + trigger: 'single', + baseConfidence: 55, + conditions: [ + { + signal: 'app_retry_storm', + source: 'event_count', + eventType: 'app_install_failed', + dataField: 'appId', + operator: 'count_per_group_gte', + value: '3', + required: true, + }, + ], + confidenceFactors: [ + { signal: 'enrollment_failed', condition: 'count >= 1', weight: 15 }, + ], + confidenceThreshold: 50, + explanation: 'App {{appName}} failed to install repeatedly (last error {{errorCode}}).', + remediation: [ + { title: 'Check the app package', steps: ['Review the install command line', 'Test the package on a reference device'] }, + ], + tags: ['apps', 'retry'], + }, + }, + + enums: { + gatherCategories: ['network', 'identity', 'apps', 'device', 'esp', 'enrollment'], + analyzeCategories: ['network', 'identity', 'enrollment', 'apps', 'esp', 'device', 'security'], + analyzeSeverities: ['info', 'warning', 'high', 'critical'], + gatherOutputSeverities: ['Info', 'Warning', 'Error', 'Critical'], + gatherCollectorTypes: ['registry', 'eventlog', 'wmi', 'file', 'command_allowlisted', 'logparser'], + gatherTriggers: ['startup', 'phase_change', 'phase_exit', 'interval', 'on_event'], + analyzeConditionSources: ['event_type', 'event_data', 'event_data_array', 'phase_duration', 'event_count', 'app_install_duration', 'event_correlation'], + }, + + commonPitfalls: [ + 'Referencing an event type that never occurs in real sessions — check get_resource(name="event_types") and use test_analyze_rule\'s per-condition matchingEventCount to verify.', + 'confidenceThreshold higher than baseConfidence plus all factor weights — the rule can mathematically never fire.', + 'Unanchored allow-list regex with not_regex — anchors (^, \\b) are mandatory or impostor values pass.', + 'A confidenceFactor condition like "count > 3" or "Count >= 3" — only the exact shapes "exists", "count >= N", "phase_duration > N" evaluate.', + 'Gather targets outside the guardrail allowlists — the agent blocks them at run time; validate_rule checks this up front.', + 'Duplicate condition signals — evidence is keyed by signal, the second overwrites the first.', + 'Expecting gather events in sessions that ran BEFORE the gather rule was deployed — dry-run against a recent session.', + ], +} as const; diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts new file mode 100644 index 00000000..b6127be9 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts @@ -0,0 +1,1005 @@ +/** + * AUTO-GENERATED from rules/guardrails.json + rules/schema/*.schema.json — DO NOT EDIT. + * Run: node rules/scripts/combine.js + * + * Consumed by the MCP server's rule-authoring surface (get_resource + + * validate_rule): the JSON Schemas are the validation contract, the guardrails + * are the agent-side collection allowlists. Single source of truth is rules/; + * the CI guardrails-in-sync job guards this file against drift. + */ + +/** JSON Schema (2020-12) for gather rules — rules/schema/gather-rule.schema.json verbatim. */ +export const GATHER_RULE_SCHEMA: Record = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/OliverKieselbach/Autopilot-Monitor/main/rules/schema/gather-rule.schema.json", + "title": "Autopilot Monitor Gather Rule", + "description": "Defines what data the agent should collect from a Windows device during Autopilot enrollment.", + "oneOf": [ + { + "$ref": "#/$defs/gatherRule" + }, + { + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/gatherRule" + } + } + }, + "required": [ + "rules" + ], + "additionalProperties": false + } + ], + "$defs": { + "gatherRule": { + "type": "object", + "properties": { + "$schema": { + "type": "string", + "description": "JSON Schema reference for editor IntelliSense." + }, + "ruleId": { + "type": "string", + "description": "Unique rule identifier (e.g., \"GATHER-NET-001\").", + "pattern": "^GATHER-[A-Z]+-\\d{3}$" + }, + "title": { + "type": "string", + "description": "Human-readable rule title (e.g., \"Collect WinHTTP Proxy Settings\")." + }, + "description": { + "type": "string", + "description": "Detailed description of what this rule collects and why." + }, + "category": { + "type": "string", + "description": "Rule category.", + "enum": [ + "network", + "identity", + "apps", + "device", + "esp", + "enrollment" + ] + }, + "version": { + "type": "string", + "description": "Semantic version of this rule (e.g., \"1.0.0\").", + "default": "1.0.0" + }, + "author": { + "type": "string", + "description": "Author of this rule.", + "default": "Autopilot Monitor" + }, + "enabled": { + "type": "boolean", + "description": "Whether this rule is enabled. Set to false to add a rule without activating it immediately.", + "default": true + }, + "isCommunity": { + "type": "boolean", + "description": "Whether this is a community-contributed rule. Community rules are stored in the global partition alongside built-in rules.", + "default": false + }, + "collectorType": { + "type": "string", + "description": "Type of data collection.", + "enum": [ + "registry", + "eventlog", + "wmi", + "file", + "command_allowlisted", + "logparser" + ] + }, + "target": { + "type": "string", + "description": "Target for collection. Registry path, event log name, WMI query, file path, command string, or log file path depending on collectorType." + }, + "parameters": { + "type": "object", + "description": "Additional parameters for the collector (all values are strings).", + "additionalProperties": { + "type": "string" + } + }, + "trigger": { + "type": "string", + "description": "When to collect data. \"phase_change\" fires once when triggerPhase is entered, \"phase_exit\" once when it is left.", + "enum": [ + "startup", + "phase_change", + "phase_exit", + "interval", + "on_event" + ] + }, + "intervalSeconds": { + "type": "integer", + "description": "Interval in seconds (only used when trigger = \"interval\").", + "minimum": 1 + }, + "triggerPhase": { + "type": "string", + "description": "Phase to trigger on (used when trigger = \"phase_change\" or \"phase_exit\"). Empty = every phase transition." + }, + "triggerEventType": { + "type": "string", + "description": "Event type to trigger on (only used when trigger = \"on_event\")." + }, + "activePhases": { + "type": "array", + "description": "Run the rule only while the current enrollment phase is one of these phases (EnrollmentPhase enum names). Empty/absent = unrestricted. Mutually exclusive with activeFromPhase.", + "items": { + "type": "string", + "enum": [ + "Start", + "DevicePreparation", + "DeviceSetup", + "AppsDevice", + "AccountSetup", + "AppsUser", + "FinalizingSetup", + "Complete" + ] + } + }, + "activeFromPhase": { + "type": "string", + "description": "Activate the rule once the enrollment phase first reaches this phase, then keep it active for the rest of the session (sticky latch). Absent = unrestricted. Mutually exclusive with activePhases.", + "enum": [ + "Start", + "DevicePreparation", + "DeviceSetup", + "AppsDevice", + "AccountSetup", + "AppsUser", + "FinalizingSetup", + "Complete" + ] + }, + "emitMode": { + "type": "string", + "description": "Emit behavior: \"always\" (default — emit on every collection) or \"on_change\" (poll on the trigger cadence but emit only when the collected result changes; the first in-scope result always emits).", + "enum": [ + "always", + "on_change" + ] + }, + "outputEventType": { + "type": "string", + "description": "EventType for the emitted event (e.g., \"gather_proxy_settings\")." + }, + "outputSeverity": { + "type": "string", + "description": "Severity for the emitted event.", + "enum": [ + "Info", + "Warning", + "Error", + "Critical" + ], + "default": "Info" + }, + "tags": { + "type": "array", + "description": "Tags for filtering and categorization.", + "items": { + "type": "string" + }, + "default": [] + } + }, + "required": [ + "ruleId", + "title", + "collectorType", + "target", + "trigger", + "outputEventType" + ], + "additionalProperties": false + } + } +}; + +/** JSON Schema (2020-12) for analyze rules — rules/schema/analyze-rule.schema.json verbatim. */ +export const ANALYZE_RULE_SCHEMA: Record = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/OliverKieselbach/Autopilot-Monitor/main/rules/schema/analyze-rule.schema.json", + "title": "Autopilot Monitor Analyze Rule", + "description": "Defines how to analyze collected events to detect issues during Autopilot enrollment.", + "oneOf": [ + { + "$ref": "#/$defs/analyzeRule" + }, + { + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "rules": { + "type": "array", + "items": { + "$ref": "#/$defs/analyzeRule" + } + } + }, + "required": [ + "rules" + ], + "additionalProperties": false + } + ], + "$defs": { + "analyzeRule": { + "type": "object", + "properties": { + "$schema": { + "type": "string", + "description": "JSON Schema reference for editor IntelliSense." + }, + "ruleId": { + "type": "string", + "description": "Unique rule identifier (e.g., \"ANALYZE-APP-001\").", + "pattern": "^ANALYZE-[A-Z]+-\\d{3}$" + }, + "title": { + "type": "string", + "description": "Human-readable rule title (e.g., \"Proxy Authentication Required\")." + }, + "description": { + "type": "string", + "description": "Detailed description of what this rule detects." + }, + "severity": { + "type": "string", + "description": "Severity level of the detected issue.", + "enum": [ + "info", + "warning", + "high", + "critical" + ] + }, + "category": { + "type": "string", + "description": "Rule category.", + "enum": [ + "network", + "identity", + "enrollment", + "apps", + "esp", + "device", + "security" + ] + }, + "version": { + "type": "string", + "description": "Semantic version of this rule (e.g., \"1.0.0\").", + "default": "1.0.0" + }, + "author": { + "type": "string", + "description": "Author of this rule.", + "default": "Autopilot Monitor" + }, + "enabled": { + "type": "boolean", + "description": "Whether this rule is enabled. Set to false to add a rule without activating it immediately.", + "default": true + }, + "isCommunity": { + "type": "boolean", + "description": "Whether this is a community-contributed rule. Community rules are stored in the global partition alongside built-in rules.", + "default": false + }, + "trigger": { + "type": "string", + "description": "Rule trigger type: \"single\" matches individual events, \"correlation\" combines multiple event types.", + "enum": [ + "single", + "correlation" + ], + "default": "single" + }, + "conditions": { + "type": "array", + "description": "Conditions evaluated against the event stream. All required conditions must match for the rule to fire.", + "items": { + "$ref": "#/$defs/ruleCondition" + }, + "minItems": 1 + }, + "preconditions": { + "type": "array", + "description": "Optional device-fact gates evaluated BEFORE conditions. ALL preconditions must pass; if any fails the rule is silently skipped — no result, no UI card. Useful for filtering out hardware/OS profiles where the rule does not apply (e.g. virtual machines).", + "items": { + "$ref": "#/$defs/rulePrecondition" + }, + "default": [] + }, + "baseConfidence": { + "type": "integer", + "description": "Base confidence score (0-100) when required conditions match.", + "minimum": 0, + "maximum": 100, + "default": 50 + }, + "confidenceFactors": { + "type": "array", + "description": "Additional factors that increase confidence when matched.", + "items": { + "$ref": "#/$defs/confidenceFactor" + }, + "default": [] + }, + "confidenceThreshold": { + "type": "integer", + "description": "Minimum confidence score (0-100) to create a RuleResult.", + "minimum": 0, + "maximum": 100, + "default": 40 + }, + "explanation": { + "type": "string", + "description": "Detailed explanation of the detected issue. Supports markdown formatting." + }, + "remediation": { + "type": "array", + "description": "Steps to remediate the detected issue.", + "items": { + "$ref": "#/$defs/remediationStep" + }, + "default": [] + }, + "relatedDocs": { + "type": "array", + "description": "Links to relevant documentation.", + "items": { + "$ref": "#/$defs/relatedDoc" + }, + "default": [] + }, + "tags": { + "type": "array", + "description": "Tags for filtering and categorization.", + "items": { + "type": "string" + }, + "default": [] + }, + "markSessionAsFailedDefault": { + "type": "boolean", + "description": "When true, a firing of this rule causes the entire enrollment session to be marked as failed (KO criterion). Tenants can override this via the portal.", + "default": false + }, + "templateVariables": { + "type": "array", + "description": "Template variables that must be customized per-tenant. If non-empty, the rule is a template: enabling it creates a tenant custom copy with the user's values substituted.", + "items": { + "$ref": "#/$defs/templateVariable" + }, + "default": [] + }, + "derivedFromTemplateRuleId": { + "type": "string", + "description": "If this custom rule was created from a template, the original template rule's ID. Used to track lineage and prevent duplicate copies." + } + }, + "required": [ + "ruleId", + "title", + "severity", + "category", + "conditions", + "explanation" + ], + "additionalProperties": false + }, + "ruleCondition": { + "type": "object", + "description": "A condition evaluated against the event stream.", + "properties": { + "signal": { + "type": "string", + "description": "Descriptive name for this signal (e.g., \"proxy_407_error\")." + }, + "source": { + "type": "string", + "description": "Source of the signal.", + "enum": [ + "event_type", + "event_data", + "event_data_array", + "phase_duration", + "event_count", + "app_install_duration", + "event_correlation" + ] + }, + "eventType": { + "type": "string", + "description": "Event type to match on. For event_correlation: the first event type (Event A)." + }, + "dataField": { + "type": "string", + "description": "Data field to match on. Uses dot notation for nested fields (e.g., \"data.errorCode\"). For event_data_array: the array field to iterate (e.g., \"artifacts\")." + }, + "itemField": { + "type": "string", + "description": "For event_data_array only: the sub-field on each array element to test with operator/value (e.g., \"identity\"). Empty = treat each element as a scalar. The condition matches when ANY element satisfies the operator." + }, + "operator": { + "type": "string", + "description": "Comparison operator.", + "enum": [ + "equals", + "not_equals", + "contains", + "not_contains", + "regex", + "not_regex", + "gt", + "lt", + "gte", + "lte", + "exists", + "not_exists", + "count_gte", + "count_per_group_gte", + "in", + "not_in" + ] + }, + "value": { + "type": "string", + "description": "Value to compare against." + }, + "required": { + "type": "boolean", + "description": "Whether this condition must match for the rule to fire. If false, it only contributes to confidence scoring.", + "default": false + }, + "filterField": { + "type": "string", + "description": "For event_count only: optional value filter — only events whose filterField satisfies filterOperator/filterValue are counted (e.g. count only performance_snapshot events with memory_used_percent > 90). Applies before counting for both count_gte and count_per_group_gte." + }, + "filterOperator": { + "type": "string", + "description": "Operator for the event_count value filter.", + "enum": [ + "equals", + "not_equals", + "contains", + "not_contains", + "regex", + "not_regex", + "gt", + "lt", + "gte", + "lte", + "exists", + "not_exists", + "in", + "not_in" + ] + }, + "filterValue": { + "type": "string", + "description": "Value for the event_count value filter." + }, + "correlateEventType": { + "type": "string", + "description": "The second event type to correlate with (Event B). Only used when source = \"event_correlation\"." + }, + "joinField": { + "type": "string", + "description": "Data field to join on — must have the same value in both Event A and Event B." + }, + "timeWindowSeconds": { + "type": "integer", + "description": "Maximum time in seconds between Event A and Event B. Null or 0 means no time limit.", + "minimum": 0 + }, + "suppressByEvent": { + "type": "object", + "description": "Optional suppression: if an event of the specified type exists with the same joinField value as the matched event, the match is skipped. Used to prevent rules from firing when a subsequent event resolved the issue (e.g., app_install_completed suppresses app_install_failed for the same appId).", + "properties": { + "eventType": { + "type": "string", + "description": "The event type that resolves/suppresses the matched event (e.g., \"app_install_completed\")." + }, + "joinField": { + "type": "string", + "description": "The data field to join on — must have the same value in both the matched event and the suppressing event (e.g., \"appId\")." + } + }, + "required": [ + "eventType", + "joinField" + ], + "additionalProperties": false + }, + "eventAFilterField": { + "type": "string", + "description": "Optional filter field on Event A (the first event)." + }, + "eventAFilterOperator": { + "type": "string", + "description": "Operator for the Event A filter.", + "enum": [ + "equals", + "not_equals", + "contains", + "not_contains", + "regex", + "not_regex", + "gt", + "lt", + "gte", + "lte", + "exists", + "not_exists", + "count_gte", + "count_per_group_gte", + "in", + "not_in" + ] + }, + "eventAFilterValue": { + "type": "string", + "description": "Value for the Event A filter." + } + }, + "required": [ + "signal", + "source" + ], + "additionalProperties": false + }, + "rulePrecondition": { + "type": "object", + "description": "A device-fact gate evaluated before conditions. Pure boolean filter — does not contribute to evidence or confidence.", + "properties": { + "source": { + "type": "string", + "description": "Currently only event_data is supported.", + "enum": [ + "event_data" + ], + "default": "event_data" + }, + "eventType": { + "type": "string", + "description": "Event type carrying the field to test (e.g., \"hardware_spec\", \"os_info\", \"tpm_status\")." + }, + "dataField": { + "type": "string", + "description": "Data field to test. Uses dot notation for nested fields." + }, + "operator": { + "type": "string", + "description": "Comparison operator. Same vocabulary as conditions.", + "enum": [ + "equals", + "not_equals", + "contains", + "not_contains", + "regex", + "not_regex", + "gt", + "lt", + "gte", + "lte", + "exists", + "not_exists", + "in", + "not_in" + ] + }, + "value": { + "type": "string", + "description": "Value to compare against. Boolean values are stringified (\"true\"/\"false\")." + }, + "description": { + "type": "string", + "description": "Human-readable note explaining the intent (e.g., \"skip on virtual machines\")." + } + }, + "required": [ + "source", + "operator" + ], + "additionalProperties": false + }, + "confidenceFactor": { + "type": "object", + "description": "A factor that increases confidence when matched.", + "properties": { + "signal": { + "type": "string", + "description": "Descriptive name for this factor." + }, + "condition": { + "type": "string", + "description": "Condition expression (e.g., \"count >= 5\", \"exists\", \"duration > 300\")." + }, + "weight": { + "type": "integer", + "description": "Confidence weight to add when this factor matches (0-100).", + "minimum": 0, + "maximum": 100 + } + }, + "required": [ + "signal", + "condition", + "weight" + ], + "additionalProperties": false + }, + "remediationStep": { + "type": "object", + "description": "A remediation step with title and sub-steps.", + "properties": { + "title": { + "type": "string", + "description": "Title of the remediation approach." + }, + "steps": { + "type": "array", + "description": "Ordered steps to execute.", + "items": { + "type": "string" + } + } + }, + "required": [ + "title", + "steps" + ], + "additionalProperties": false + }, + "relatedDoc": { + "type": "object", + "description": "A link to related documentation.", + "properties": { + "title": { + "type": "string", + "description": "Display title for the link." + }, + "url": { + "type": "string", + "description": "URL to the documentation.", + "format": "uri" + } + }, + "required": [ + "title", + "url" + ], + "additionalProperties": false + }, + "templateVariable": { + "type": "object", + "description": "A variable in a rule condition that must be customized per-tenant before the rule can be used.", + "properties": { + "name": { + "type": "string", + "description": "Machine name for this variable (e.g., \"cert_subject\")." + }, + "label": { + "type": "string", + "description": "Human-readable label shown in the configuration UI." + }, + "description": { + "type": "string", + "description": "Help text explaining what value is expected." + }, + "conditionIndex": { + "type": "integer", + "description": "Zero-based index into the conditions array where this variable lives.", + "minimum": 0 + }, + "field": { + "type": "string", + "description": "Which field on the condition to customize.", + "enum": [ + "value", + "eventType", + "dataField", + "eventAFilterValue" + ], + "default": "value" + }, + "placeholder": { + "type": "string", + "description": "The placeholder value that ships with the template (e.g., \"CN=YOUR-CERTIFICATE-SUBJECT\")." + }, + "validation": { + "type": "string", + "description": "Optional regex pattern to validate user input.", + "format": "regex" + } + }, + "required": [ + "name", + "label", + "conditionIndex", + "field", + "placeholder" + ], + "additionalProperties": false + } + } +}; + +/** Gather-rule collection guardrails — rules/guardrails.json verbatim. */ +export const RULE_GUARDRAILS = { + "registryPrefixes": [ + { + "category": "MDM / Enrollment", + "prefixes": [ + "SOFTWARE\\Microsoft\\Enrollments", + "SOFTWARE\\Microsoft\\EnterpriseDesktopAppManagement", + "SOFTWARE\\Microsoft\\Provisioning", + "SOFTWARE\\Microsoft\\PolicyManager", + "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\MDM" + ] + }, + { + "category": "AAD / Entra Join", + "prefixes": [ + "SOFTWARE\\Microsoft\\IdentityStore", + "SYSTEM\\CurrentControlSet\\Control\\CloudDomainJoin" + ] + }, + { + "category": "Windows Update / WUfB", + "prefixes": [ + "SOFTWARE\\Microsoft\\WindowsUpdate", + "SOFTWARE\\Policies\\Microsoft\\Windows\\WindowsUpdate" + ] + }, + { + "category": "BitLocker", + "prefixes": [ + "SOFTWARE\\Microsoft\\BitLocker", + "SYSTEM\\CurrentControlSet\\Control\\BitLockerStatus" + ] + }, + { + "category": "Network / Proxy", + "prefixes": [ + "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Internet Settings", + "SYSTEM\\CurrentControlSet\\Services\\Tcpip" + ] + }, + { + "category": "Autopilot / OOBE / Setup", + "prefixes": [ + "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Setup", + "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\OOBE", + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon", + "SOFTWARE\\Microsoft\\Windows\\Autopilot" + ] + }, + { + "category": "TPM", + "prefixes": [ + "SYSTEM\\CurrentControlSet\\Services\\TPM", + "SOFTWARE\\Microsoft\\Tpm" + ] + }, + { + "category": "Secure Boot", + "prefixes": [ + "SYSTEM\\CurrentControlSet\\Control\\SecureBoot" + ] + }, + { + "category": "Intune IME", + "prefixes": [ + "SOFTWARE\\Microsoft\\IntuneManagementExtension" + ] + }, + { + "category": "Certificates (SCEP)", + "prefixes": [ + "SOFTWARE\\Microsoft\\SystemCertificates", + "SOFTWARE\\Policies\\Microsoft\\SystemCertificates" + ] + }, + { + "category": "Servicing", + "prefixes": [ + "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Component Based Servicing" + ] + }, + { + "category": "RealmJoin", + "prefixes": [ + "SYSTEM\\CurrentControlSet\\Services\\realmjoin", + "SOFTWARE\\RealmJoin" + ] + } + ], + "filePrefixes": [ + "C:\\ProgramData\\Microsoft\\IntuneManagementExtension\\Logs", + "C:\\Windows\\CCM\\Logs", + "C:\\Windows\\Logs", + "C:\\Windows\\Panther", + "C:\\Windows\\SetupDiag", + "C:\\ProgramData\\Microsoft\\DiagnosticLogCSP", + "C:\\Windows\\SoftwareDistribution\\ReportingEvents.log" + ], + "wmiQueryPrefixes": [ + "SELECT * FROM Win32_OperatingSystem", + "SELECT * FROM Win32_ComputerSystem", + "SELECT * FROM Win32_BIOS", + "SELECT * FROM Win32_Processor", + "SELECT * FROM Win32_BaseBoard", + "SELECT * FROM Win32_Battery", + "SELECT * FROM Win32_TPM", + "SELECT * FROM Win32_NetworkAdapter", + "SELECT * FROM Win32_NetworkAdapterConfiguration", + "SELECT * FROM Win32_DiskDrive", + "SELECT * FROM Win32_LogicalDisk", + "SELECT * FROM SoftwareLicensingProduct" + ], + "eventLogChannels": [ + { + "category": "Core Windows logs", + "channels": [ + "Application", + "System", + "Setup", + "Microsoft-Windows-Kernel-Boot", + "Microsoft-Windows-Diagnostics-Performance" + ] + }, + { + "category": "MDM / Enrollment", + "channels": [ + "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", + "Microsoft-Windows-ModernDeployment-Diagnostics-Provider", + "Microsoft-Windows-Provisioning-Diagnostics-Provider", + "Microsoft-Windows-AAD", + "Microsoft-Windows-User Device Registration" + ] + }, + { + "category": "ESP / Shell / Apps", + "channels": [ + "Microsoft-Windows-Shell-Core", + "Microsoft-Windows-AppXDeployment", + "Microsoft-Windows-AppXDeploymentServer", + "Microsoft-Windows-AppReadiness", + "Microsoft-Windows-Store" + ] + }, + { + "category": "Security posture / Crypto", + "channels": [ + "Microsoft-Windows-BitLocker-API", + "Microsoft-Windows-BitLocker-DrivePreparationTool", + "Microsoft-Windows-TPM-WMI", + "Microsoft-Windows-CertificateServicesClient-Lifecycle-System" + ] + }, + { + "category": "Update / Servicing", + "channels": [ + "Microsoft-Windows-WindowsUpdateClient" + ] + }, + { + "category": "Logon / Session", + "channels": [ + "Microsoft-Windows-Winlogon", + "Microsoft-Windows-User Profile Service", + "Microsoft-Windows-GroupPolicy", + "Microsoft-Windows-TaskScheduler" + ] + }, + { + "category": "Network", + "channels": [ + "Microsoft-Windows-NetworkProfile", + "Microsoft-Windows-Dhcp-Client", + "Microsoft-Windows-DNS-Client", + "Microsoft-Windows-NCSI", + "Microsoft-Windows-WLAN-AutoConfig", + "Microsoft-Windows-Time-Service" + ] + } + ], + "blockedEventLogChannels": [ + "Security", + "Microsoft-Windows-PowerShell", + "Windows PowerShell", + "Microsoft-Windows-Sysmon" + ], + "allowedCommands": [ + { + "category": "TPM and Security", + "commands": [ + "Get-Tpm", + "Get-SecureBootPolicy", + "Get-SecureBootUEFI -Name SetupMode" + ] + }, + { + "category": "BitLocker", + "commands": [ + "Get-BitLockerVolume -MountPoint C:" + ] + }, + { + "category": "Network", + "commands": [ + "Get-NetAdapter | Select-Object Name, Status, InterfaceDescription, MacAddress, LinkSpeed", + "Get-DnsClientServerAddress | Select-Object InterfaceAlias, ServerAddresses", + "Get-NetIPConfiguration | Select-Object InterfaceAlias, IPv4Address, IPv4DefaultGateway, DNSServer", + "netsh winhttp show proxy", + "ipconfig /all" + ] + }, + { + "category": "Domain / Identity", + "commands": [ + "nltest /dsgetdc:", + "dsregcmd /status" + ] + }, + { + "category": "Certificate", + "commands": [ + "certutil -store My", + "$c = Get-ChildItem Cert:\\LocalMachine\\My; if ($c) { $c | Select-Object Subject, Thumbprint, Issuer, NotBefore, NotAfter, HasPrivateKey | ConvertTo-Json } else { '{\"CertificateCount\": 0}' }" + ] + }, + { + "category": "Windows Update", + "commands": [ + "Get-HotFix | Select-Object -First 10 HotFixID, InstalledOn, Description" + ] + }, + { + "category": "Autopilot / Hardware Identity", + "commands": [ + "try { $cs = Get-CimInstance -ClassName Win32_ComputerSystem; $bios = Get-CimInstance -ClassName Win32_BIOS; $hash = $null; try { $hash = (Get-CimInstance -Namespace root/cimv2/mdm/dmmap -ClassName MDM_DevDetail_Ext01 -Filter \"InstanceID='Ext' AND ParentID='./DevDetail'\" -ErrorAction Stop).DeviceHardwareData } catch { $hash = \"ERROR: $($_.Exception.Message)\" }; [pscustomobject]@{ Manufacturer = $cs.Manufacturer; Model = $cs.Model; SystemSKU = $cs.SystemSKUNumber; SerialNumber = $bios.SerialNumber; BiosVersion = $bios.SMBIOSBIOSVersion; HardwareHash = $hash } | ConvertTo-Json -Compress } catch { @{ error = $_.Exception.Message } | ConvertTo-Json -Compress }" + ] + } + ], + "diagnosticsPathPrefixes": [ + "C:\\ProgramData\\AutopilotMonitor", + "C:\\ProgramData\\Microsoft\\IntuneManagementExtension\\Logs", + "C:\\Windows\\Panther", + "C:\\Windows\\Logs", + "C:\\Windows\\SetupDiag", + "C:\\Windows\\SoftwareDistribution\\ReportingEvents.log", + "C:\\Windows\\System32\\winevt\\Logs", + "C:\\Windows\\CCM\\Logs", + "C:\\ProgramData\\Microsoft\\DiagnosticLogCSP", + "C:\\ProgramData\\Microsoft\\Windows\\WER", + "C:\\Windows\\Logs\\CBS", + "C:\\Install\\Log" + ], + "blockedFilePrefixes": [ + "C:\\Users" + ] +} as const; diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts new file mode 100644 index 00000000..cd1df756 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts @@ -0,0 +1,400 @@ +/** + * MCP-local validation of draft gather/analyze rules for the `validate_rule` tool. + * + * Three layers: + * 1. JSON-Schema validation (ajv, draft 2020-12) against the baked schemas — the same + * contract CI enforces on the built-in rule catalog. + * 2. Guardrail checks for gather targets, mirroring the AGENT's enforcement semantics + * (GatherRuleGuards.cs): hive-stripped, segment-bounded prefix matching for + * registry/file paths, whitespace-bounded prefixes for WMI, EXACT matching for + * commands, '/'-bounded channel matching for event logs, plus the hard blocks that + * config can never lift. + * 3. Semantic lint that catches rules which would be schema-valid but silently dead or + * misleading in production (unreachable confidence threshold, non-evaluable factor + * DSL, unknown event types, unanchored allow-list regex, unresolvable {{tokens}}). + * + * Everything here is advisory pre-flight: the agent re-enforces guardrails on-device + * and the backend re-validates structure on dry-run. This module must therefore only + * ever REJECT things production would reject or silently ignore — never invent its + * own stricter contract. + */ +import { Ajv2020 } from 'ajv/dist/2020.js'; +import addFormatsModule from 'ajv-formats'; + +// ajv-formats ships CJS with `export =` typings that Node16 module resolution +// refuses to treat as callable through the default import — at runtime the +// default IS the plugin function. Narrow cast, no behaviour involved. +const addFormats = addFormatsModule as unknown as (ajv: Ajv2020) => void; +import { GATHER_RULE_SCHEMA, ANALYZE_RULE_SCHEMA, RULE_GUARDRAILS } from './rule-authoring.generated.js'; +import { isKnownEventType } from './resource-catalog.js'; + +export type FindingLevel = 'error' | 'warning' | 'info'; + +export interface ValidationFinding { + level: FindingLevel; + message: string; +} + +export interface ValidationResult { + ruleType: 'gather' | 'analyze' | 'unknown'; + /** true when no error-level findings exist (warnings/info allowed). */ + valid: boolean; + findings: ValidationFinding[]; +} + +// ── ajv setup (module-level: compiled once per process) ───────────────────── + +const ajv = new Ajv2020({ allErrors: true, strict: false }); +addFormats(ajv); +ajv.addSchema(GATHER_RULE_SCHEMA); +ajv.addSchema(ANALYZE_RULE_SCHEMA); + +/** Validate against the single-rule $def directly — the top-level oneOf (rule | + * {rules:[...]} wrapper) would produce confusing double errors for drafts. */ +function schemaValidate(schemaId: unknown, defName: string, rule: unknown): ValidationFinding[] { + const validate = ajv.getSchema(`${String(schemaId)}#/$defs/${defName}`); + if (!validate) throw new Error(`Schema $defs/${defName} not found — generated schema changed shape`); + if (validate(rule)) return []; + return (validate.errors ?? []).map((e) => ({ + level: 'error' as const, + message: `schema: ${e.instancePath || '(root)'} ${e.message}${e.keyword === 'enum' ? ` (${JSON.stringify((e.params as { allowedValues?: unknown[] }).allowedValues)})` : ''}`, + })); +} + +// ── Guardrail matching (mirrors GatherRuleGuards.cs) ──────────────────────── + +const flatRegistryPrefixes = RULE_GUARDRAILS.registryPrefixes.flatMap((g) => [...g.prefixes]); +const flatCommands = RULE_GUARDRAILS.allowedCommands.flatMap((g) => [...g.commands]); +const flatEventLogChannels = RULE_GUARDRAILS.eventLogChannels.flatMap((g) => [...g.channels]); + +/** Hard blocks enforced in agent CODE (not liftable via guardrails.json). Keep in + * sync with GatherRuleGuards.cs — additions there are fail-safe (agent still blocks), + * missing ones here just cost a pre-flight warning. */ +const HARD_BLOCKED_PATH_PREFIXES = ['C:\\Users', 'C:\\Windows\\System32\\config']; + +/** Segment-bounded prefix match: target equals the prefix or continues with '\'. */ +function pathPrefixAllowed(target: string, prefixes: readonly string[]): boolean { + const t = target.toLowerCase(); + return prefixes.some((p) => { + const pl = p.toLowerCase(); + return t === pl || t.startsWith(pl + '\\'); + }); +} + +function stripRegistryHive(target: string): string { + return target.replace(/^(HKLM|HKEY_LOCAL_MACHINE|HKCU|HKEY_CURRENT_USER)\\/i, ''); +} + +function checkGatherTarget(collectorType: string, target: string): ValidationFinding[] { + const findings: ValidationFinding[] = []; + switch (collectorType) { + case 'registry': { + const subPath = stripRegistryHive(target); + if (!pathPrefixAllowed(subPath, flatRegistryPrefixes)) { + findings.push({ + level: 'error', + message: + `guardrails: registry target "${target}" is not under any allowed prefix. ` + + 'See get_resource(name="rule_guardrails") → registryPrefixes. Note: the last allowed path segment must be followed by "\\" or end-of-string (segment-bounded matching). ' + + 'Widening the allowlist requires a contribution to rules/guardrails.json in the product repository.', + }); + } + break; + } + case 'file': + case 'logparser': { + const blocked = HARD_BLOCKED_PATH_PREFIXES.find((p) => pathPrefixAllowed(target, [p])); + if (blocked) { + findings.push({ level: 'error', message: `guardrails: "${target}" is under the hard-blocked path ${blocked} — the agent always refuses this, it cannot be allow-listed.` }); + break; + } + const allowed = collectorType === 'logparser' + ? [...RULE_GUARDRAILS.filePrefixes, ...RULE_GUARDRAILS.diagnosticsPathPrefixes] + : [...RULE_GUARDRAILS.filePrefixes]; + // logparser targets may carry a glob suffix (*.log) — prefix matching still applies. + if (!pathPrefixAllowed(target, allowed) && !allowed.some((p) => target.toLowerCase().startsWith(p.toLowerCase() + '\\'))) { + findings.push({ + level: 'error', + message: `guardrails: ${collectorType} target "${target}" is not under any allowed file prefix (rule_guardrails → filePrefixes${collectorType === 'logparser' ? ' / diagnosticsPathPrefixes' : ''}).`, + }); + } + break; + } + case 'wmi': { + const t = target.toLowerCase(); + const ok = RULE_GUARDRAILS.wmiQueryPrefixes.some((p) => { + const pl = p.toLowerCase(); + // Whitespace-bounded: the query is the prefix or continues with whitespace + // (mirrors the agent's boundary check, prevents Win32_BIOSX spoofing). + return t === pl || (t.startsWith(pl) && /\s/.test(t.charAt(pl.length))); + }); + if (!ok) { + findings.push({ level: 'error', message: `guardrails: WMI query "${target}" does not start with any allowed query prefix (rule_guardrails → wmiQueryPrefixes).` }); + } + break; + } + case 'command_allowlisted': { + const t = target.trim().toLowerCase(); + if (!flatCommands.some((c) => c.trim().toLowerCase() === t)) { + findings.push({ + level: 'error', + message: `guardrails: command "${target}" is not on the allowlist. Commands match EXACTLY (no extra arguments). See rule_guardrails → allowedCommands.`, + }); + } + break; + } + case 'eventlog': { + const channel = target.split('/')[0]; + const blocked = RULE_GUARDRAILS.blockedEventLogChannels.some((b) => b.toLowerCase() === channel.toLowerCase()); + if (blocked) { + findings.push({ level: 'error', message: `guardrails: event log channel "${channel}" is hard-blocked (security-sensitive) and can never be collected.` }); + break; + } + const ok = flatEventLogChannels.some((c) => { + const cl = c.toLowerCase(); + const tl = target.toLowerCase(); + return tl === cl || tl.startsWith(cl + '/'); + }); + if (!ok) { + findings.push({ level: 'error', message: `guardrails: event log channel "${target}" is not on the allowlist (rule_guardrails → eventLogChannels).` }); + } + break; + } + default: + break; // unknown collectorType is already a schema error + } + return findings; +} + +// ── Analyze-rule semantic lint ────────────────────────────────────────────── + +/** Mirrors the backend evidence auto-capture whitelist (AddDataFieldsToEvidence) and + * the interpolation resolution order (interpolate-rule-template.ts). */ +const AUTO_FIELDS = ['appId', 'appName', 'errorPatternId', 'errorCode', 'exitCode', 'status']; + +/** The only condition strings EvaluateConfidenceFactor parses — anything else is + * silently false in production. Exact spacing matters. */ +const FACTOR_CONDITION_SHAPE = /^(exists|count >= ?\d+|phase_duration > ?\d+)$/; + +interface DraftCondition { + signal?: string; + source?: string; + eventType?: string; + dataField?: string; + operator?: string; + value?: string; + required?: boolean; + correlateEventType?: string; + joinField?: string; +} + +function checkEventTypeKnown(eventType: string | undefined, where: string): ValidationFinding[] { + if (!eventType) return []; + if (isKnownEventType(eventType)) return []; + if (eventType.startsWith('gather_')) { + return [{ + level: 'info', + message: `${where}: eventType "${eventType}" follows the gather_ convention — make sure a gather rule with this outputEventType is deployed, otherwise the condition never matches.`, + }]; + } + return [{ + level: 'warning', + message: `${where}: eventType "${eventType}" is not a known built-in event type (see get_resource(name="event_types")). If it is the outputEventType of a custom gather rule this is fine; otherwise the condition will never match.`, + }]; +} + +function extractTokens(rule: Record): string[] { + const texts: string[] = []; + if (typeof rule.explanation === 'string') texts.push(rule.explanation); + if (Array.isArray(rule.remediation)) { + for (const step of rule.remediation as Array<{ title?: string; steps?: string[] }>) { + if (typeof step?.title === 'string') texts.push(step.title); + for (const s of step?.steps ?? []) if (typeof s === 'string') texts.push(s); + } + } + const tokens = new Set(); + for (const text of texts) { + for (const m of text.matchAll(/\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g)) tokens.add(m[1]); + } + return [...tokens]; +} + +function lintAnalyzeRule(rule: Record): ValidationFinding[] { + const findings: ValidationFinding[] = []; + const conditions = (Array.isArray(rule.conditions) ? rule.conditions : []) as DraftCondition[]; + + const seenSignals = new Set(); + for (let i = 0; i < conditions.length; i++) { + const c = conditions[i]; + const label = `conditions[${i}]`; + + if (c.signal) { + if (seenSignals.has(c.signal)) { + findings.push({ level: 'error', message: `${label}: duplicate signal "${c.signal}" — evidence is keyed by signal, the second entry overwrites the first.` }); + } + seenSignals.add(c.signal); + } + + if (c.source === 'event_correlation') { + if (!c.correlateEventType) findings.push({ level: 'error', message: `${label}: event_correlation requires correlateEventType.` }); + if (!c.joinField) findings.push({ level: 'error', message: `${label}: event_correlation requires joinField.` }); + findings.push(...checkEventTypeKnown(c.correlateEventType, `${label}.correlateEventType`)); + } + + findings.push(...checkEventTypeKnown(c.eventType, label)); + + if ((c.operator === 'regex' || c.operator === 'not_regex') && c.value) { + try { + new RegExp(c.value, 'i'); + } catch (e) { + findings.push({ level: 'error', message: `${label}: value is not a valid regex (${(e as Error).message}). The engine uses .NET regex syntax — stick to the common subset.` }); + } + if (c.operator === 'not_regex' && !c.value.startsWith('^')) { + findings.push({ + level: 'warning', + message: `${label}: not_regex allow-list pattern is not anchored with "^" — unanchored patterns let impostor values (e.g. an allowed name embedded in a longer string) pass. Anchor with ^ and \\b.`, + }); + } + } + + if ((c.source === 'event_data' || c.source === 'event_data_array') && !c.dataField) { + findings.push({ level: 'error', message: `${label}: source ${c.source} requires dataField.` }); + } + } + + // Precondition event types. + const preconditions = (Array.isArray(rule.preconditions) ? rule.preconditions : []) as DraftCondition[]; + for (let i = 0; i < preconditions.length; i++) { + findings.push(...checkEventTypeKnown(preconditions[i].eventType, `preconditions[${i}]`)); + } + + // Confidence factors: exact DSL shapes; "count >= N" counts EVENTS of type factor.signal. + const factors = (Array.isArray(rule.confidenceFactors) ? rule.confidenceFactors : []) as Array<{ signal?: string; condition?: string; weight?: number }>; + let positiveWeights = 0; + for (let i = 0; i < factors.length; i++) { + const f = factors[i]; + if (!f.condition || !FACTOR_CONDITION_SHAPE.test(f.condition)) { + findings.push({ + level: 'error', + message: `confidenceFactors[${i}]: condition "${f.condition}" is not evaluable — supported shapes (exact spacing): "exists", "count >= N", "phase_duration > N". In production this factor would silently never apply.`, + }); + } else if (f.condition === 'exists' && f.signal && !seenSignals.has(f.signal)) { + findings.push({ + level: 'warning', + message: `confidenceFactors[${i}]: "exists" refers to signal "${f.signal}" but no condition declares that signal — the factor can never apply.`, + }); + } else if (f.condition.startsWith('count >=')) { + findings.push(...checkEventTypeKnown(f.signal, `confidenceFactors[${i}].signal (event type for "count >=")`)); + } + if (typeof f.weight === 'number' && f.weight > 0) positiveWeights += f.weight; + } + + // Threshold reachability. + const base = typeof rule.baseConfidence === 'number' ? rule.baseConfidence : 50; + const threshold = typeof rule.confidenceThreshold === 'number' ? rule.confidenceThreshold : 40; + if (threshold > base + positiveWeights) { + findings.push({ + level: 'error', + message: `confidenceThreshold (${threshold}) exceeds baseConfidence + all positive factor weights (${base} + ${positiveWeights}) — the rule can mathematically never fire.`, + }); + } else if (threshold > base) { + findings.push({ + level: 'info', + message: `confidenceThreshold (${threshold}) is above baseConfidence (${base}) — the rule only fires when confidence factors apply. Intentional for corroborated findings; lower the threshold if the required conditions alone should fire it.`, + }); + } + + // {{token}} resolvability (interpolation order: dataField → auto fields → signal). + const resolvable = new Set([...AUTO_FIELDS]); + for (const c of conditions) { + if (c.dataField) resolvable.add(c.dataField); + if (c.signal) resolvable.add(c.signal); + } + for (const token of extractTokens(rule)) { + if (!resolvable.has(token)) { + findings.push({ + level: 'warning', + message: `explanation/remediation references {{${token}}} but no condition dataField/signal or auto-captured field (${AUTO_FIELDS.join(', ')}) resolves it — it will render literally as {{${token}}}.`, + }); + } + } + + if (rule.markSessionAsFailedDefault === true) { + findings.push({ + level: 'info', + message: 'markSessionAsFailedDefault=true escalates every firing of this rule to a terminal FAILED session status (KO criterion). Dry-run against sessions that should NOT fail before enabling.', + }); + } + + return findings; +} + +// ── Gather-rule semantic lint ─────────────────────────────────────────────── + +function lintGatherRule(rule: Record): ValidationFinding[] { + const findings: ValidationFinding[] = []; + const trigger = rule.trigger as string | undefined; + + if (trigger === 'interval' && typeof rule.intervalSeconds !== 'number') { + findings.push({ level: 'error', message: 'trigger "interval" requires intervalSeconds.' }); + } + if (trigger === 'on_event' && !rule.triggerEventType) { + findings.push({ level: 'error', message: 'trigger "on_event" requires triggerEventType.' }); + } + if ((trigger === 'phase_change' || trigger === 'phase_exit') && !rule.triggerPhase) { + findings.push({ level: 'info', message: `trigger "${trigger}" without triggerPhase fires on EVERY phase transition — set triggerPhase to collect once at a specific phase.` }); + } + + const outputEventType = rule.outputEventType as string | undefined; + if (outputEventType) { + if (isKnownEventType(outputEventType)) { + findings.push({ + level: 'warning', + message: `outputEventType "${outputEventType}" collides with a built-in event type — analyze rules and the timeline could no longer tell your collected events from agent events. Use a gather_ prefixed name.`, + }); + } else if (!outputEventType.startsWith('gather_')) { + findings.push({ level: 'warning', message: `outputEventType "${outputEventType}" does not follow the gather_ naming convention (e.g. "gather_${outputEventType}").` }); + } + } + + if (typeof rule.collectorType === 'string' && typeof rule.target === 'string') { + findings.push(...checkGatherTarget(rule.collectorType, rule.target)); + } + + return findings; +} + +// ── Entry point ───────────────────────────────────────────────────────────── + +export function validateRuleDraft(rule: Record): ValidationResult { + const ruleId = typeof rule.ruleId === 'string' ? rule.ruleId : ''; + const ruleType: ValidationResult['ruleType'] = + typeof rule.collectorType === 'string' || ruleId.startsWith('GATHER-') + ? 'gather' + : Array.isArray(rule.conditions) || ruleId.startsWith('ANALYZE-') + ? 'analyze' + : 'unknown'; + + if (ruleType === 'unknown') { + return { + ruleType, + valid: false, + findings: [{ + level: 'error', + message: + 'Cannot tell whether this is a gather rule (collectorType/target, ruleId GATHER-…) or an analyze rule (conditions, ruleId ANALYZE-…). ' + + 'Read get_resource(name="rule_authoring_guide") for the two shapes.', + }], + }; + } + + const findings = ruleType === 'gather' + ? [...schemaValidate(GATHER_RULE_SCHEMA.$id, 'gatherRule', rule), ...lintGatherRule(rule)] + : [...schemaValidate(ANALYZE_RULE_SCHEMA.$id, 'analyzeRule', rule), ...lintAnalyzeRule(rule)]; + + return { + ruleType, + valid: !findings.some((f) => f.level === 'error'), + findings, + }; +} diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools.ts b/src/McpServer/autopilot-monitor-mcp/src/tools.ts index b25f9db8..16f328b4 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools.ts @@ -3,6 +3,7 @@ import type { SearchProvider } from './search-provider.js'; import { registerSessionTools } from './tools/sessions.js'; import { registerSearchTools } from './tools/search.js'; import { registerAdminTools } from './tools/admin.js'; +import { registerRuleTools } from './tools/rules.js'; import type { DocsSearchBundle } from './search-provider.js'; /** @@ -39,6 +40,7 @@ export function registerTools( registerSessionTools(server, ga, delegated); registerSearchTools(server, knowledgeBase, eventTypeIndex, docs, ga, delegated); registerAdminTools(server, ga, strictGa, delegated); + registerRuleTools(server, ga); sortToolCatalog(server); } diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts index 7416d224..5cdada1a 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/admin.ts @@ -1540,9 +1540,14 @@ export function registerAdminTools(server: McpServer, ga: boolean, strictGa: boo 'stateless HTTP MCP servers). Available names:\n' + ' - "event_types": catalog of valid eventType strings for search_sessions_by_event\n' + ' - "device_properties": catalog of dot-notation keys for the deviceProperties filter on search_sessions\n' + - ' - "diag_zip_layout": expected file layout of an agent diagnostics ZIP (what get_session_diagnostics returns you for local analysis)', + ' - "diag_zip_layout": expected file layout of an agent diagnostics ZIP (what get_session_diagnostics returns you for local analysis)\n' + + ' - "rule_authoring_guide": complete guide for authoring gather/analyze rules (read FIRST when asked to create a rule; then validate_rule / test_analyze_rule)\n' + + ' - "rule_schemas": the JSON Schemas for gather and analyze rules (the exact contract)\n' + + ' - "rule_guardrails": on-device collection allowlists a gather rule target must satisfy', inputSchema: { - name: z.enum(['event_types', 'device_properties', 'diag_zip_layout']).describe('Resource name'), + name: z + .enum(['event_types', 'device_properties', 'diag_zip_layout', 'rule_authoring_guide', 'rule_schemas', 'rule_guardrails']) + .describe('Resource name'), }, annotations: READ_ONLY_OPEN, }, diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts new file mode 100644 index 00000000..afedf7f4 --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts @@ -0,0 +1,114 @@ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { apiFetch } from '../client.js'; +import { withToolTelemetry } from '../telemetry.js'; +import { READ_ONLY, MAX_RESULT_SIZE_CHARS, toolResultText, SessionIdSchema } from './shared.js'; +import { toolError } from './error-handler.js'; +import { validateRuleDraft } from '../rule-validation.js'; +import { interpolateRuleTemplate } from '../interpolate-rule-template.js'; + +/** + * Rule-authoring tools: validate a draft gather/analyze rule locally, and dry-run a + * draft analyze rule against a real session via the backend's side-effect-free + * dryrun endpoint. Registered for EVERY role — tenant admins are the primary + * audience (the backend still enforces its own authorization: the dryrun route is + * TenantAdmin / Global Admin / read-only Global Reader). + */ +export function registerRuleTools(server: McpServer, ga: boolean): void { + // Tool: validate_rule (MCP-local, no backend call) + server.registerTool( + 'validate_rule', + { + title: 'Validate Rule Draft', + description: + 'Validates a DRAFT gather or analyze rule without deploying anything: JSON-Schema check ' + + '(the exact contract from get_resource(name="rule_schemas")), guardrail pre-flight for ' + + 'gather targets (registry/file/WMI/command/event-log allowlists with the agent\'s ' + + 'matching semantics), and semantic lint (unreachable confidence threshold, non-evaluable ' + + 'confidenceFactor conditions, unknown event types, unanchored allow-list regex, ' + + 'unresolvable {{token}} placeholders). Returns findings as error/warning/info. ' + + 'Fix all errors, then dry-run analyze rules with test_analyze_rule against a real session. ' + + 'Read get_resource(name="rule_authoring_guide") first when authoring from scratch.', + inputSchema: { + rule: z.record(z.unknown()).describe('The draft rule JSON object (gather or analyze — detected automatically)'), + }, + annotations: READ_ONLY, + }, + async (args) => withToolTelemetry('validate_rule', async () => { + try { + const result = validateRuleDraft(args.rule as Record); + return toolResultText( + { + ruleType: result.ruleType, + valid: result.valid, + errorCount: result.findings.filter((f) => f.level === 'error').length, + findings: result.findings, + nextStep: result.valid + ? (result.ruleType === 'analyze' + ? 'No errors. Dry-run it now: test_analyze_rule(sessionId=, rule=) — once against a session where it SHOULD fire, once where it should not.' + : 'No errors. Gather rules run on-device and cannot be dry-run server-side — deploy to a test device and check the emitted events (the gather debug log setting shows per-rule detail).') + : 'Fix the error-level findings and validate again.', + }, + MAX_RESULT_SIZE_CHARS.small, + ); + } catch (error: unknown) { + return toolError('validate_rule', args, error); + } + }) + ); + + // Tool: test_analyze_rule (backend dry-run, side-effect free) + server.registerTool( + 'test_analyze_rule', + { + title: 'Test Analyze Rule Against Session (Dry-Run)', + description: + 'Dry-runs a DRAFT analyze rule against one real session\'s events and returns the full ' + + 'diagnostic trace: verdict (fired / required_condition_not_met / below_confidence_threshold ' + + '/ skipped_by_precondition / no_conditions_matched / no_events), per-condition ' + + 'matched/evidence with matchingEventCount (how many session events even have that ' + + 'eventType — the first thing to check when a condition unexpectedly misses), the ' + + 'confidence breakdown per factor, and the interpolated explanation preview. ' + + 'NOTHING is persisted — safe to iterate. Use a recent session (older sessions predate ' + + 'newly deployed gather rules). Test both directions: a session where the rule should ' + + 'fire AND one where it should stay silent.' + + (ga ? ' Platform-scope callers can target any tenant\'s session — the tenant is resolved from the session automatically.' : ''), + inputSchema: { + sessionId: SessionIdSchema.describe('Session UUID to evaluate the draft against'), + rule: z.record(z.unknown()).describe('The draft analyze rule JSON object'), + }, + annotations: READ_ONLY, + }, + async (args) => withToolTelemetry('test_analyze_rule', async () => { + try { + const data = await apiFetch('/api/rules/analyze/dryrun', { + method: 'POST', + body: JSON.stringify({ sessionId: args.sessionId, rule: args.rule }), + }) as { result?: { verdict?: string; matchedConditions?: Record } }; + + const result = data.result; + const rule = args.rule as Record; + const mc = result?.matchedConditions; + + // Preview the explanation/remediation exactly as the portal would render them + // for this session's evidence ({{token}} interpolation). + const interpolatedExplanation = typeof rule.explanation === 'string' + ? interpolateRuleTemplate(rule.explanation, mc) + : undefined; + const interpolatedRemediation = Array.isArray(rule.remediation) + ? (rule.remediation as Array<{ title?: string; steps?: string[] }>).map((step) => ({ + title: interpolateRuleTemplate(step?.title, mc), + steps: (step?.steps ?? []).map((s) => interpolateRuleTemplate(s, mc)), + })) + : undefined; + + return toolResultText( + { ...data, interpolatedExplanation, interpolatedRemediation }, + MAX_RESULT_SIZE_CHARS.small, + ); + } catch (error: unknown) { + return toolError('test_analyze_rule', args, error); + } + }) + ); +}