From 75197a4184c8fab5e9f297eb96639b29da209afb Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Wed, 5 Aug 2026 22:48:10 +0200 Subject: [PATCH] Logparser pattern testing with the agent's exact .NET engine (test_log_pattern) (#125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updated commands * feat(backend): logparser pattern test endpoint with agent-exact semantics POST /api/rules/gather/test-pattern evaluates a logparser gather-rule regex against pasted sample lines exactly the way the agent would on the device. Motivated by a field incident: a customer verified a pattern with a PHP regex tester and the .NET engine matched differently. - CmTraceLogParser moves Agent -> AutopilotMonitor.Shared.Logging so agent (IME tracker, logparser collector, stall probe, IME matching mode) and backend run the SAME parsing implementation. Pure mechanical move, agent behavior unchanged (V2.Core suite 2147/2147 green). - TestLogPatternFunction mirrors LogParserCollector verbatim: Regex(Compiled, 1s timeout, NO IgnoreCase -> case-sensitive), first match per line, group "0" excluded / unsuccessful groups omitted, format cmtrace (default, match against parsed message) vs text (raw line), per-line outcomes incl. parse_failed/regex_timeout, and the agent's "set parameter format=text" hint verbatim. Pure compute, no tenant data. Caps: 200 lines, 8KB/line. - Policy: TenantAdminOrGlobalReader (same tier as the analyze dry-run). - Tests pin the parity showcases: case-sensitivity, envelope invisibility in cmtrace mode, optional-group omission, format hint, request validation. Co-Authored-By: Claude Fable 5 * feat(mcp): test_log_pattern tool + logparser lint and guide hardening - test_log_pattern: wraps POST /api/rules/gather/test-pattern — tests a logparser regex against pasted sample lines with the agent-exact .NET semantics (case-sensitive, cmtrace/text mode). Description steers clients away from JS/PHP/Python regex testing. - validate_rule: logparser parameter lint (missing/non-compiling pattern, unknown format value = silently cmtrace on the agent) + info pointing to test_log_pattern; nextStep for valid logparser drafts routes there too. - rule_authoring_guide: logparser section now states the matching semantics (case-SENSITIVE, .NET, first match per line, cmtrace default) and two new pitfalls (engine mismatch, plain-text log under cmtrace default). - Catalog snapshot + handler/validator tests extended (MCP suite 532 green). Co-Authored-By: Claude Fable 5 * docs(okf): test-pattern endpoint + shared CmTraceLogParser in rule authoring surface Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .claude/commands | 2 +- docs/log.md | 4 + docs/rules/rule-authoring-surface.md | 39 ++- .../Monitoring/Ime/CmTraceLogParserTests.cs | 2 +- .../Ime/ImeLogTracker.LogProcessing.cs | 1 + .../SystemSignals/StallProbeCollector.cs | 1 + .../Gather/Collectors/LogParserCollector.cs | 1 + .../Program.ImeMatchingMode.cs | 1 + .../TestLogPatternFunctionTests.cs | 145 +++++++++++ .../Functions/Rules/TestLogPatternFunction.cs | 244 ++++++++++++++++++ .../Security/EndpointAccessPolicyCatalog.cs | 3 + .../src/__tests__/rule-validation.test.ts | 30 +++ .../src/__tests__/tools-catalog-order.test.ts | 8 +- .../src/__tests__/tools-rules.test.ts | 63 +++++ .../src/rule-authoring-guide.ts | 18 +- .../src/rule-validation.ts | 43 +++ .../autopilot-monitor-mcp/src/tools/rules.ts | 42 ++- .../Logging}/CmTraceLogParser.cs | 9 +- 18 files changed, 642 insertions(+), 14 deletions(-) create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/TestLogPatternFunctionTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs rename src/{Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime => Shared/AutopilotMonitor.Shared/Logging}/CmTraceLogParser.cs (91%) diff --git a/.claude/commands b/.claude/commands index 5c66c265..a7e749f2 160000 --- a/.claude/commands +++ b/.claude/commands @@ -1 +1 @@ -Subproject commit 5c66c265a87e1bc235db695e9af218c94fd7983f +Subproject commit a7e749f215f537a58973974e80090387f56108c4 diff --git a/docs/log.md b/docs/log.md index 69c5e9e5..bc8abb51 100644 --- a/docs/log.md +++ b/docs/log.md @@ -1,5 +1,9 @@ # Log +## 2026-08-05 (later) + +* **Update**: `rules/rule-authoring-surface.md` — new section `POST /api/rules/gather/test-pattern`: the logparser pattern tester (agent-exact .NET matching: no IgnoreCase, first-match-per-line, cmtrace vs text mode, verbatim format=text hint), motivated by a field incident where a PHP-verified regex behaved differently under .NET. `CmTraceLogParser` moved Agent → `AutopilotMonitor.Shared.Logging` so agent and backend share the ONE parsing implementation (timestamp caveat: AssumeLocal = server TZ on backend, endpoint reports no timestamps). MCP: new `test_log_pattern` tool, logparser lint in `validate_rule`, guide gains the case-sensitivity/engine-mismatch pitfalls. + ## 2026-08-05 * **Creation**: Added `mcp/tool-telemetry.md` — the three MCP tool-telemetry layers (App Insights `McpToolName`, per-user usage table, `MCP_TOOL_LOGGING` stderr lines) and the new quality signals: soft-error capture (`toolError` returns instead of throwing, so backend telemetry counted those calls as successes), `resultChars`/`overCap` vs. the inline-size hint, bounded args summaries, caller scope, `search_zero_hit` on the three search tools (unmet-demand signal), and `tool_call_rejected` for JSON-RPC-level Zod/unknown-tool rejections sniffed at the `/mcp` route (previously invisible everywhere). Bicep default for `toolLoggingEnabled` flipped to `true` (documented desired state; live flag set via `az containerapp update`). diff --git a/docs/rules/rule-authoring-surface.md b/docs/rules/rule-authoring-surface.md index 4efe54e5..7d264de4 100644 --- a/docs/rules/rule-authoring-surface.md +++ b/docs/rules/rule-authoring-surface.md @@ -100,15 +100,50 @@ Invariants, pinned by `RuleEngineDryRunTests`: explanation/remediation preview via the existing `interpolate-rule-template.ts` against the returned `matchedConditions`. +## POST /api/rules/gather/test-pattern (backend) + +The logparser counterpart to the analyze dry-run, added after a field incident: +a customer verified a logparser regex with a PHP tester and the agent's .NET +engine matched differently. `TestLogPatternFunction` evaluates a pattern against +pasted sample lines with the AGENT's exact matching semantics: + +* Regex construction identical to `LogParserCollector`: `Compiled`, 1 s timeout, + **no IgnoreCase** — logparser matching is case-sensitive (unlike analyze-rule + regex conditions, which run IgnoreCase). +* `format` behaves exactly like the rule parameter: `cmtrace` (default) parses + each line via the shared `CmTraceLogParser` and matches against the parsed + MESSAGE only; `text` matches the raw line. First match per line + (`Regex.Match`), group `"0"` excluded, unsuccessful groups omitted — the + returned `groups` are exactly the event-data fields the device would emit. +* Mirrors the agent's diagnostics: per-line outcomes + (`matched | no_match | parse_failed | regex_timeout`), the all-lines-parse-fail + hint ("set parameter format=text") verbatim, and a case-sensitivity note on + zero matches. +* Pure compute on the request body — no tenant data touched. Same policy tier + as the dry-run (`TenantAdminOrGlobalReader`); caps: 200 lines, 8 KB/line, + 1 MB body. + +Parity is structural, not duplicated: `CmTraceLogParser` moved from the agent +(`Monitoring/Enrollment/Ime`) to **Shared** (`AutopilotMonitor.Shared.Logging`) +so agent and backend run the SAME parsing implementation. Timestamp caveat: +bias-less CMTrace lines parse with `AssumeLocal`, which on the backend is the +server's timezone — the endpoint therefore reports match/parse results, never +timestamps. `test_log_pattern` (MCP) wraps the endpoint; `validate_rule` +additionally lints logparser parameters (missing/non-compiling pattern, unknown +format value) and its `nextStep` routes logparser drafts to `test_log_pattern`. + ## What is deliberately absent * **No gather-rule remote execution** — gather rules run as SYSTEM on devices; - the dry-run surface is analyze-only. Gather authors get guardrail pre-flight - plus the on-device debug log ([gather-rule-debug-log](../agent/gather-rule-debug-log.md)). + session dry-run is analyze-only. Logparser authors get the pattern tester + above; all gather authors get guardrail pre-flight plus the on-device debug + log ([gather-rule-debug-log](../agent/gather-rule-debug-log.md)). * **No rule write/deploy via MCP** — creating rules stays in the portal. # Citations +* `src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs` — logparser pattern tester +* `src/Shared/AutopilotMonitor.Shared/Logging/CmTraceLogParser.cs` — shared CMTrace parser (agent + backend) * `src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.DryRun.cs` — trace twin + DTOs * `src/Backend/AutopilotMonitor.Functions/Functions/Rules/DryRunAnalyzeRuleFunction.cs` — endpoint + draft validation * `src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts` — validator diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/CmTraceLogParserTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/CmTraceLogParserTests.cs index b92ab35c..31f08753 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/CmTraceLogParserTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/CmTraceLogParserTests.cs @@ -1,5 +1,5 @@ using System; -using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime; +using AutopilotMonitor.Shared.Logging; using Xunit; namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.LogProcessing.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.LogProcessing.cs index 477427a6..56860e65 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.LogProcessing.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.LogProcessing.cs @@ -7,6 +7,7 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Shared.Logging; using AutopilotMonitor.Shared.Models; namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/StallProbeCollector.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/StallProbeCollector.cs index eae3ebaf..5fa575ba 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/StallProbeCollector.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/StallProbeCollector.cs @@ -10,6 +10,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime; using AutopilotMonitor.Agent.V2.Core.Orchestration; using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Logging; using AutopilotMonitor.Shared.Models; using Microsoft.Win32; diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Gather/Collectors/LogParserCollector.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Gather/Collectors/LogParserCollector.cs index 3c4d2660..1dbfb40b 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Gather/Collectors/LogParserCollector.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Gather/Collectors/LogParserCollector.cs @@ -10,6 +10,7 @@ using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime; using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment; using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime; using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Shared.Logging; namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather.Collectors { diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Program.ImeMatchingMode.cs b/src/Agent/AutopilotMonitor.Agent.V2/Program.ImeMatchingMode.cs index e94bb3a5..5c514b5b 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Program.ImeMatchingMode.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Program.ImeMatchingMode.cs @@ -6,6 +6,7 @@ using System.Net.Http; using System.Text; using System.Text.RegularExpressions; using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime; +using AutopilotMonitor.Shared.Logging; using AutopilotMonitor.Shared.Models; using Newtonsoft.Json; diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TestLogPatternFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TestLogPatternFunctionTests.cs new file mode 100644 index 00000000..628fe2b6 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TestLogPatternFunctionTests.cs @@ -0,0 +1,145 @@ +using System.Text.RegularExpressions; +using AutopilotMonitor.Functions.Functions.Rules; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Pins the agent-parity semantics of the logparser pattern test endpoint: case-sensitive +/// matching (the field incident that motivated the endpoint: a pattern verified with a PHP +/// regex tester behaved differently under .NET), first-match-per-line, group extraction +/// identical to what LogParserCollector puts into the emitted event data, CMTrace parsing +/// via the SHARED CmTraceLogParser, and the format=text hint. +/// +public class TestLogPatternFunctionTests +{ + private static Regex AgentRegex(string pattern) + => new(pattern, RegexOptions.Compiled, TimeSpan.FromSeconds(1)); + + private const string CmTraceLine = + "" + + ""; + + // ===== text mode ===== + + [Fact] + public void TextMode_NamedGroups_LandInGroups() + { + var result = TestLogPatternFunction.EvaluatePattern( + AgentRegex(@"(?ERROR|FATAL)\s+(?\w+):\s+(?.+)"), + isTextMode: true, + new[] { "ERROR Installer: disk full", "INFO Installer: all good" }); + + Assert.Equal(1, result.MatchCount); + Assert.Equal("matched", result.Lines[0].Outcome); + Assert.Equal("ERROR", result.Lines[0].Groups!["level"]); + Assert.Equal("Installer", result.Lines[0].Groups!["component"]); + Assert.Equal("disk full", result.Lines[0].Groups!["message"]); + Assert.Equal("no_match", result.Lines[1].Outcome); + } + + /// The motivating field case: logparser matching has NO IgnoreCase — a pattern + /// that matched in a case-insensitive tester silently never fires on the device. + [Fact] + public void TextMode_IsCaseSensitive_AndSaysSoInTheNote() + { + var result = TestLogPatternFunction.EvaluatePattern( + AgentRegex("ERROR"), isTextMode: true, new[] { "error: something broke" }); + + Assert.Equal(0, result.MatchCount); + Assert.Contains(result.Notes, n => n.Contains("case-SENSITIVE")); + } + + [Fact] + public void TextMode_UnsuccessfulOptionalGroup_IsOmitted() + { + var result = TestLogPatternFunction.EvaluatePattern( + AgentRegex(@"ERROR(?\s0x[0-9A-F]+)?"), isTextMode: true, new[] { "ERROR without code" }); + + Assert.Equal(1, result.MatchCount); + Assert.False(result.Lines[0].Groups!.ContainsKey("code")); + } + + // ===== cmtrace mode ===== + + [Fact] + public void CmTraceMode_MatchesAgainstParsedMessage_AndReportsComponentAndType() + { + var result = TestLogPatternFunction.EvaluatePattern( + AgentRegex(@"error (?0x[0-9A-Fa-f]+)"), isTextMode: false, new[] { CmTraceLine }); + + Assert.Equal(1, result.MatchCount); + var row = result.Lines[0]; + Assert.Equal("matched", row.Outcome); + Assert.Equal("0x87D1041C", row.Groups!["errorCode"]); + Assert.Equal("AppWorkload", row.Component); + Assert.Equal(3, row.CmTraceType); + Assert.Contains("Installation failed", row.Message); + } + + [Fact] + public void CmTraceMode_RegexDoesNotSeeTheEnvelope() + { + // "component=" only exists in the CMTrace envelope, never in the message the + // agent matches against — a pattern targeting it must NOT match. + var result = TestLogPatternFunction.EvaluatePattern( + AgentRegex("component="), isTextMode: false, new[] { CmTraceLine }); + + Assert.Equal(0, result.MatchCount); + Assert.Equal("no_match", result.Lines[0].Outcome); + } + + [Fact] + public void CmTraceMode_PlainTextLines_ParseFail_WithFormatHint() + { + var result = TestLogPatternFunction.EvaluatePattern( + AgentRegex("ERROR"), isTextMode: false, new[] { "ERROR plain line", "another plain line" }); + + Assert.Equal(2, result.ParseFailureCount); + Assert.All(result.Lines, l => Assert.Equal("parse_failed", l.Outcome)); + Assert.Contains(result.Notes, n => n.Contains("format=text")); + } + + // ===== request validation ===== + + [Fact] + public void Validate_RejectsMissingPatternAndLines() + { + Assert.Contains("pattern", TestLogPatternFunction.ValidateRequest( + new TestLogPatternFunction.TestLogPatternRequest { SampleLines = new List { "x" } })!); + Assert.Contains("sampleLines", TestLogPatternFunction.ValidateRequest( + new TestLogPatternFunction.TestLogPatternRequest { Pattern = "a" })!); + } + + [Fact] + public void Validate_RejectsTooManyLines_AndBadFormat() + { + var tooMany = new TestLogPatternFunction.TestLogPatternRequest + { + Pattern = "a", + SampleLines = Enumerable.Repeat("line", TestLogPatternFunction.MaxSampleLines + 1).ToList(), + }; + Assert.Contains("at most", TestLogPatternFunction.ValidateRequest(tooMany)!); + + var badFormat = new TestLogPatternFunction.TestLogPatternRequest + { + Pattern = "a", + Format = "json", + SampleLines = new List { "x" }, + }; + Assert.Contains("format", TestLogPatternFunction.ValidateRequest(badFormat)!); + } + + [Fact] + public void Validate_AcceptsBothFormatsAndDefault() + { + foreach (var format in new[] { null, "cmtrace", "text", "TEXT" }) + { + Assert.Null(TestLogPatternFunction.ValidateRequest(new TestLogPatternFunction.TestLogPatternRequest + { + Pattern = "a", + Format = format, + SampleLines = new List { "x" }, + })); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs new file mode 100644 index 00000000..b399c391 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Rules/TestLogPatternFunction.cs @@ -0,0 +1,244 @@ +using System.Net; +using System.Text.RegularExpressions; +using AutopilotMonitor.Shared.Logging; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Functions.Functions.Rules +{ + /// + /// Tests a logparser gather-rule regex against pasted sample log lines with the AGENT's + /// exact matching semantics — the ".NET dry-run" for logparser rules, which cannot be + /// dry-run against a session (they execute on devices). + /// + /// Parity is the entire point: authors test regexes in whatever engine their AI client + /// has (JS, PHP, Python) and .NET behaves subtly differently (this endpoint exists + /// because exactly that happened in the field). Mirrored semantics, keep in lock-step + /// with LogParserCollector (agent): + /// - Regex options: Compiled, 1s timeout, NO IgnoreCase — logparser matching is + /// case-SENSITIVE (unlike analyze-rule regex conditions). + /// - First match per line (Regex.Match, not Matches). + /// - format "cmtrace" (default): CmTraceLogParser.TryParseLine first (shared code, the + /// same implementation the agent runs), regex against the parsed MESSAGE only. + /// - format "text": regex against the raw line. + /// - Captured data: every named/numbered group except "0", unsuccessful groups omitted. + /// + /// No tenant data is touched; the endpoint is pure compute on the request body. + /// + public class TestLogPatternFunction + { + private readonly ILogger _logger; + + public TestLogPatternFunction(ILogger logger) + { + _logger = logger; + } + + public sealed class TestLogPatternRequest + { + public string? Pattern { get; set; } + public string? Format { get; set; } + public List? SampleLines { get; set; } + } + + internal const int MaxSampleLines = 200; + internal const int MaxLineLength = 8192; + + [Function("TestLogPattern")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "rules/gather/test-pattern")] HttpRequestData req) + { + // Authentication + TenantAdminOrGlobalReader authorization enforced by PolicyEnforcementMiddleware. + + if (req.Headers.TryGetValues("Content-Length", out var clValues) + && long.TryParse(clValues.FirstOrDefault(), out var contentLength) + && contentLength > 1_048_576) // 1 MB limit + { + return await BadRequestAsync(req, "Request body too large"); + } + + var body = await new StreamReader(req.Body).ReadToEndAsync(); + TestLogPatternRequest? request; + try + { + request = JsonConvert.DeserializeObject(body); + } + catch (JsonException ex) + { + return await BadRequestAsync(req, $"Request body is not valid JSON: {ex.Message}"); + } + + var validationError = ValidateRequest(request); + if (validationError != null) + { + return await BadRequestAsync(req, validationError); + } + + var isTextMode = string.Equals(request!.Format, "text", StringComparison.OrdinalIgnoreCase); + + Regex pattern; + try + { + // EXACT agent construction (LogParserCollector): Compiled + 1s timeout, + // no IgnoreCase — logparser matching is case-sensitive. + pattern = new Regex(request.Pattern!, RegexOptions.Compiled, TimeSpan.FromSeconds(1)); + } + catch (ArgumentException ex) + { + return await BadRequestAsync(req, $"Invalid regex pattern (this is the agent's .NET engine speaking): {ex.Message}"); + } + + var result = EvaluatePattern(pattern, isTextMode, request.SampleLines!); + + _logger.LogInformation( + "Log-pattern test: {LineCount} lines, {MatchCount} matched, {ParseFailures} parse failures, mode={Mode}", + request.SampleLines!.Count, result.MatchCount, result.ParseFailureCount, isTextMode ? "text" : "cmtrace"); + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new { success = true, format = isTextMode ? "text" : "cmtrace", result }); + return response; + } + + internal static string? ValidateRequest(TestLogPatternRequest? request) + { + if (request == null || string.IsNullOrEmpty(request.Pattern)) + return "pattern is required"; + if (request.SampleLines == null || request.SampleLines.Count == 0) + return "sampleLines is required — paste raw lines from the log file"; + if (request.SampleLines.Count > MaxSampleLines) + return $"too many sampleLines ({request.SampleLines.Count}) — send at most {MaxSampleLines} representative lines"; + if (request.SampleLines.Any(l => l != null && l.Length > MaxLineLength)) + return $"a sample line exceeds {MaxLineLength} characters"; + if (!string.IsNullOrEmpty(request.Format) + && !string.Equals(request.Format, "cmtrace", StringComparison.OrdinalIgnoreCase) + && !string.Equals(request.Format, "text", StringComparison.OrdinalIgnoreCase)) + return "format must be \"cmtrace\" (default) or \"text\""; + return null; + } + + /// + /// Per-line evaluation mirroring LogParserCollector's inner loop. Returns one row per + /// sample line so the author sees exactly which lines matched, which failed CMTrace + /// parsing, and which data fields the emitted event would carry. + /// + internal static LogPatternTestResult EvaluatePattern(Regex pattern, bool isTextMode, IReadOnlyList lines) + { + var result = new LogPatternTestResult(); + + for (var i = 0; i < lines.Count; i++) + { + var line = lines[i] ?? string.Empty; + var row = new LogPatternLineResult { LineNumber = i + 1 }; + result.Lines.Add(row); + + string textToMatch; + if (isTextMode) + { + textToMatch = line; + } + else + { + if (!CmTraceLogParser.TryParseLine(line, out var entry)) + { + row.Outcome = "parse_failed"; + result.ParseFailureCount++; + continue; + } + row.Component = entry.Component; + row.CmTraceType = entry.Type; + row.Message = Truncate(entry.Message, 500); + textToMatch = entry.Message; + } + + Match match; + try + { + match = pattern.Match(textToMatch); + } + catch (RegexMatchTimeoutException) + { + row.Outcome = "regex_timeout"; + result.TimeoutCount++; + continue; + } + + if (!match.Success) + { + row.Outcome = "no_match"; + continue; + } + + row.Outcome = "matched"; + result.MatchCount++; + row.MatchedText = Truncate(match.Value, 500); + var groups = new Dictionary(); + foreach (var groupName in pattern.GetGroupNames()) + { + if (groupName == "0") continue; + var group = match.Groups[groupName]; + if (group.Success) + groups[groupName] = Truncate(group.Value, 500); + } + row.Groups = groups; + } + + // Mirror the agent's debug-log hint verbatim — the single most common logparser + // authoring mistake is running a plain-text log through the cmtrace default. + if (!isTextMode && result.Lines.Count > 0 && result.ParseFailureCount == result.Lines.Count) + { + result.Notes.Add("every line failed CMTrace parsing — if this is a plain-text log, set parameter format=text"); + } + if (result.MatchCount == 0 && result.ParseFailureCount < result.Lines.Count) + { + result.Notes.Add("no line matched — remember logparser patterns are case-SENSITIVE (unlike analyze-rule regex conditions)"); + } + + return result; + } + + private static string Truncate(string value, int maxLength) + { + if (string.IsNullOrEmpty(value) || value.Length <= maxLength) + return value; + return value.Substring(0, maxLength) + "..."; + } + + private static async Task BadRequestAsync(HttpRequestData req, string message) + { + var badRequest = req.CreateResponse(HttpStatusCode.BadRequest); + await badRequest.WriteAsJsonAsync(new { success = false, message }); + return badRequest; + } + } + + /// Aggregate outcome of one pattern test. Serialized camelCase to clients. + public sealed class LogPatternTestResult + { + public int MatchCount { get; set; } + public int ParseFailureCount { get; set; } + public int TimeoutCount { get; set; } + public List Lines { get; } = new(); + public List Notes { get; } = new(); + } + + public sealed class LogPatternLineResult + { + public int LineNumber { get; set; } + + /// matched | no_match | parse_failed | regex_timeout + public string Outcome { get; set; } = string.Empty; + + /// The named/numbered capture groups exactly as they would land in the + /// emitted event's data (group "0" excluded, unsuccessful groups omitted). + public Dictionary? Groups { get; set; } + + public string? MatchedText { get; set; } + + /// cmtrace mode only: the parsed component/type/message the regex ran against. + public string? Component { get; set; } + public int? CmTraceType { get; set; } + public string? Message { get; set; } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index 55e2a610..d3db9e79 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -269,6 +269,9 @@ public static class EndpointAccessPolicyCatalog // tier so the read-only Global Reader can help debug rules while tenant Operators/Viewers // stay excluded from the authoring surface. new("POST", "rules/analyze/dryrun", EndpointPolicy.TenantAdminOrGlobalReader), + // Same shape: POST body in, pure compute out (regex vs pasted sample lines with the + // agent's exact .NET matching semantics) — touches no tenant data at all. + new("POST", "rules/gather/test-pattern", EndpointPolicy.TenantAdminOrGlobalReader), // Member tier so Operators/Viewers can see their tenant's configuration read-only in the // Settings UI. Safe because the handler redacts by default: only a Global Admin or the // tenant's OWN admin gets the unredacted secrets — every other admitted caller (Operator, diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts index f88a7cd9..0291e27f 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts @@ -162,6 +162,36 @@ describe('gather guardrails (agent matching semantics)', () => { }); }); +describe('logparser parameter lint', () => { + function logparserRule(parameters: Record): Record { + const r = validGather(); + r.collectorType = 'logparser'; + r.target = 'C:\\Windows\\Logs\\CustomApp\\install.log'; + r.parameters = parameters; + return r; + } + + it('missing pattern is an error', () => { + expect(errors(validateRuleDraft(logparserRule({})).findings).some((m) => m.includes('parameters.pattern'))).toBe(true); + }); + + it('non-compiling pattern is an error', () => { + expect(errors(validateRuleDraft(logparserRule({ pattern: '([unclosed' })).findings) + .some((m) => m.includes('does not compile'))).toBe(true); + }); + + it('valid pattern yields the .NET/case-sensitivity info pointing to test_log_pattern', () => { + const r = validateRuleDraft(logparserRule({ pattern: '(?ERROR)', format: 'text' })); + expect(r.valid).toBe(true); + expect(r.findings.some((f) => f.level === 'info' && f.message.includes('test_log_pattern'))).toBe(true); + }); + + it('unknown format value warns (agent silently treats it as cmtrace)', () => { + expect(warnings(validateRuleDraft(logparserRule({ pattern: 'x', format: 'json' })).findings) + .some((m) => m.includes('CMTrace mode'))).toBe(true); + }); +}); + describe('gather semantic lint', () => { it('interval without intervalSeconds is an error; on_event without triggerEventType is an error', () => { const a = validGather(); diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts index 4d7f37d2..bed3188c 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-catalog-order.test.ts @@ -150,10 +150,12 @@ describe('role catalog snapshot — privilege-leak guard', () => { 'search_sessions', 'search_sessions_by_cve', 'search_sessions_by_event', - // Rule-authoring pair, available to every role: validate_rule is MCP-local - // (no backend call at all) and test_analyze_rule's dryrun endpoint enforces - // TenantAdminOrGlobalReader server-side (a Viewer's call 403s there). + // Rule-authoring trio, available to every role: validate_rule is MCP-local + // (no backend call at all); test_analyze_rule's dryrun endpoint and + // test_log_pattern's test-pattern endpoint enforce TenantAdminOrGlobalReader + // server-side (a Viewer's call 403s there). 'test_analyze_rule', + 'test_log_pattern', 'update_tenant_config', 'validate_rule', ]; diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts index 44e23118..51861fe3 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/tools-rules.test.ts @@ -114,4 +114,67 @@ describe('validate_rule', () => { expect(payload.valid).toBe(true); expect(payload.nextStep).toContain('test_analyze_rule'); }); + + it('valid logparser draft points to test_log_pattern as the next step', async () => { + const handlers = captureToolHandlers(false); + const res = await handlers.validate_rule({ + rule: { + ruleId: 'GATHER-APPS-101', + title: 'CustomApp installer errors', + collectorType: 'logparser', + target: 'C:\\Windows\\Logs\\CustomApp\\install*.log', + parameters: { pattern: '(?ERROR|FATAL)', format: 'text' }, + trigger: 'startup', + outputEventType: 'gather_customapp_errors', + }, + }); + const payload = JSON.parse(res.content[0].text); + expect(payload.valid).toBe(true); + expect(payload.nextStep).toContain('test_log_pattern'); + }); +}); + +describe('test_log_pattern', () => { + it('POSTs pattern + sampleLines + format to the test-pattern endpoint', async () => { + apiFetchMock.mockResolvedValue({ + success: true, + format: 'text', + result: { matchCount: 1, parseFailureCount: 0, timeoutCount: 0, lines: [], notes: [] }, + }); + + const handlers = captureToolHandlers(false); + const res = await handlers.test_log_pattern({ + pattern: '(?ERROR)', + sampleLines: ['ERROR one', 'INFO two'], + format: 'text', + }); + + expect(apiFetchMock).toHaveBeenCalledTimes(1); + const [path, init] = apiFetchMock.mock.calls[0] as [string, RequestInit]; + expect(path).toBe('/api/rules/gather/test-pattern'); + expect(init.method).toBe('POST'); + const body = JSON.parse(String(init.body)); + expect(body.pattern).toBe('(?ERROR)'); + expect(body.sampleLines).toEqual(['ERROR one', 'INFO two']); + expect(body.format).toBe('text'); + + expect(res.isError).toBeFalsy(); + const payload = JSON.parse(res.content[0].text); + expect(payload.result.matchCount).toBe(1); + }); + + it('omits format when not given (backend defaults to cmtrace like the agent)', async () => { + apiFetchMock.mockResolvedValue({ success: true, format: 'cmtrace', result: {} }); + const handlers = captureToolHandlers(false); + await handlers.test_log_pattern({ pattern: 'x', sampleLines: ['line'] }); + const body = JSON.parse(String((apiFetchMock.mock.calls[0] as [string, RequestInit])[1].body)); + expect(body.format).toBeUndefined(); + }); + + it('surfaces backend errors via toolError (no throw)', async () => { + apiFetchMock.mockRejectedValue(new Error('boom')); + const handlers = captureToolHandlers(false); + const res = await handlers.test_log_pattern({ pattern: 'x', sampleLines: ['line'] }); + expect(res.isError).toBe(true); + }); }); diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts index 2bb16112..54777108 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts @@ -34,9 +34,10 @@ export const RULE_AUTHORING_GUIDE = { 'session from your tenant (ideally one where the rule SHOULD fire and one where it should ' + 'NOT). The dry-run returns a per-condition trace with evidence — iterate until the verdict ' + 'matches your expectation. Nothing is persisted by a dry-run.', - '5. Gather rules cannot be dry-run server-side (they execute on devices). Validate them, then ' + - 'deploy to a test device/VM; the agent debug log setting for gather rules shows per-rule ' + - 'execution detail on-device.', + '5. Gather rules cannot be dry-run against a session (they execute on devices). For logparser ' + + 'rules, test the regex with test_log_pattern against sample lines pasted from the real log ' + + 'file — it uses the agent\'s exact .NET matching semantics. Then deploy to a test device/VM; ' + + 'the agent debug log setting for gather rules shows per-rule execution detail on-device.', '6. Create the finished rule in the portal (Settings → Gather Rules / Analyze Rules). Custom ' + 'rules are tenant-scoped. Well-tested rules can be proposed as community rules via GitHub.', ], @@ -70,8 +71,13 @@ export const RULE_AUTHORING_GUIDE = { 'Matching is EXACT (trimmed, case-insensitive) — no arguments can be added or removed.', logparser: 'Parse log files with a regex. target = file/glob under an allow-listed prefix. parameters: ' + - 'pattern (regex, named groups become event data), format ("cmtrace" for IME-style logs), ' + - 'maxLines, trackPosition ("true" = only new lines on re-runs).', + 'pattern (regex, named groups become event data fields on the emitted timeline event), ' + + 'format ("cmtrace" = default: each line is parsed as CMTrace/IME format first and the regex ' + + 'runs against the parsed MESSAGE only; "text" = regex against the raw line), maxLines, ' + + 'trackPosition ("true" = only new lines on re-runs). Matching semantics: .NET regex engine, ' + + 'case-SENSITIVE (unlike analyze-rule regex conditions), first match per line. ALWAYS verify ' + + 'the pattern with test_log_pattern against real lines from the log file — engines like ' + + 'JS/PHP/Python behave subtly differently from the agent\'s .NET engine.', }, triggers: { startup: 'Once when the agent starts.', @@ -248,6 +254,8 @@ export const RULE_AUTHORING_GUIDE = { 'Unanchored allow-list regex with not_regex — anchors (^, \\b) are mandatory or impostor values pass.', 'A confidenceFactor condition like "count > 3" or "Count >= 3" — only the exact shapes "exists", "count >= N", "phase_duration > N" evaluate.', 'Gather targets outside the guardrail allowlists — the agent blocks them at run time; validate_rule checks this up front.', + 'Testing a logparser regex in a JS/PHP/Python tester and assuming it behaves the same on the device — the agent uses the .NET engine and matches case-SENSITIVELY. Use test_log_pattern, which runs the exact agent semantics.', + 'Running a plain-text log through the default cmtrace format — every line fails CMTrace parsing and nothing ever matches. Set parameters.format="text" (test_log_pattern reports this explicitly).', 'Duplicate condition signals — evidence is keyed by signal, the second overwrites the first.', 'Expecting gather events in sessions that ran BEFORE the gather rule was deployed — dry-run against a recent session.', ], diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts index cd1df756..d89cd8f5 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts @@ -361,6 +361,49 @@ function lintGatherRule(rule: Record): ValidationFinding[] { findings.push(...checkGatherTarget(rule.collectorType, rule.target)); } + if (rule.collectorType === 'logparser') { + findings.push(...lintLogparserParameters(rule)); + } + + return findings; +} + +/** logparser-specific parameter lint. Matching runs on the DEVICE with .NET regex — + * this only pre-checks what would make the rule dead; real pattern testing belongs in + * test_log_pattern (agent-exact engine). */ +function lintLogparserParameters(rule: Record): ValidationFinding[] { + const findings: ValidationFinding[] = []; + const parameters = (rule.parameters ?? {}) as Record; + const pattern = parameters.pattern; + + if (typeof pattern !== 'string' || pattern.length === 0) { + findings.push({ level: 'error', message: 'logparser requires parameters.pattern — without it the rule can never match.' }); + return findings; + } + + try { + // JS compile is only an approximation of the agent's .NET engine — good enough to + // catch syntax errors, not equivalence. + new RegExp(pattern); + } catch (e) { + findings.push({ level: 'error', message: `parameters.pattern does not compile (${(e as Error).message}). Note the agent uses .NET regex syntax.` }); + } + + findings.push({ + level: 'info', + message: + 'logparser matching is case-SENSITIVE and runs on the agent\'s .NET regex engine — a pattern verified in a JS/PHP/Python tester can behave differently. ' + + 'Test it with test_log_pattern(pattern, sampleLines) against real lines from the log file before deploying.', + }); + + const format = parameters.format; + if (typeof format === 'string' && format.toLowerCase() !== 'text' && format.toLowerCase() !== 'cmtrace') { + findings.push({ + level: 'warning', + message: `parameters.format "${format}" is not a known format — the agent treats anything other than "text" as CMTrace mode.`, + }); + } + return findings; } diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts index e035493e..30a1f27b 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/rules.ts @@ -46,7 +46,9 @@ export function registerRuleTools(server: McpServer, ga: boolean): void { nextStep: result.valid ? (result.ruleType === 'analyze' ? 'No errors. Dry-run it now: test_analyze_rule(sessionId=, rule=) — once against a session where it SHOULD fire, once where it should not.' - : 'No errors. Gather rules run on-device and cannot be dry-run server-side — deploy to a test device and check the emitted events (the gather debug log setting shows per-rule detail).') + : (args.rule as Record).collectorType === 'logparser' + ? 'No errors. Now test the regex with the agent\'s exact .NET engine: test_log_pattern(pattern=, sampleLines=, format=) — then deploy to a test device.' + : 'No errors. Gather rules run on-device and cannot be dry-run server-side — deploy to a test device and check the emitted events (the gather debug log setting shows per-rule detail).') : 'Fix the error-level findings and validate again.', }, MAX_RESULT_SIZE_CHARS.small, @@ -57,6 +59,44 @@ export function registerRuleTools(server: McpServer, ga: boolean): void { }) ); + // Tool: test_log_pattern (backend .NET-regex test for logparser gather rules) + server.registerTool( + 'test_log_pattern', + { + title: 'Test Logparser Pattern Against Sample Lines', + description: + 'Tests a logparser gather-rule regex against pasted sample log lines using the AGENT\'s ' + + 'exact .NET matching semantics — the dry-run for logparser rules (which run on devices ' + + 'and cannot be tested against a session). Use this INSTEAD of testing the regex in ' + + 'JS/PHP/Python: .NET behaves subtly differently, and logparser matching is ' + + 'case-SENSITIVE (unlike analyze-rule regex conditions). Paste 10-50 representative raw ' + + 'lines from the customer\'s log file (include lines that must match AND lines that must ' + + 'not). format="cmtrace" (default) parses each line as CMTrace/IME format first and runs ' + + 'the regex against the parsed message; format="text" matches the raw line. Returns ' + + 'per-line outcomes with the exact capture groups that would land in the emitted ' + + 'timeline event\'s data. Nothing is stored.', + inputSchema: { + pattern: z.string().min(1).max(2000).describe('The regex (named groups become event data fields), .NET syntax'), + sampleLines: z.array(z.string().max(8192)).min(1).max(200) + .describe('Raw lines pasted from the log file (max 200)'), + format: z.enum(['cmtrace', 'text']).optional() + .describe('Log format, exactly like the rule\'s parameters.format: "cmtrace" (default, IME/SCCM style) or "text" (plain lines)'), + }, + annotations: READ_ONLY, + }, + async (args) => withToolTelemetry('test_log_pattern', args, async () => { + try { + const data = await apiFetch('/api/rules/gather/test-pattern', { + method: 'POST', + body: JSON.stringify({ pattern: args.pattern, format: args.format, sampleLines: args.sampleLines }), + }); + return toolResultText(data, MAX_RESULT_SIZE_CHARS.small); + } catch (error: unknown) { + return toolError('test_log_pattern', args, error); + } + }) + ); + // Tool: test_analyze_rule (backend dry-run, side-effect free) server.registerTool( 'test_analyze_rule', diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/CmTraceLogParser.cs b/src/Shared/AutopilotMonitor.Shared/Logging/CmTraceLogParser.cs similarity index 91% rename from src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/CmTraceLogParser.cs rename to src/Shared/AutopilotMonitor.Shared/Logging/CmTraceLogParser.cs index 1544c211..376ca077 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/CmTraceLogParser.cs +++ b/src/Shared/AutopilotMonitor.Shared/Logging/CmTraceLogParser.cs @@ -2,7 +2,7 @@ using System; using System.Globalization; using System.Text.RegularExpressions; -namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime +namespace AutopilotMonitor.Shared.Logging { /// /// Represents a single parsed line from a CMTrace-format log file @@ -19,6 +19,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime /// /// Parses CMTrace/SCCM log format used by IME and other Microsoft components. /// Format: <![LOG[{message}]LOG]!><time="{time}" date="{date}" component="{comp}" context="" type="{type}" thread="{thread}" file=""> + /// + /// Lives in Shared because it is the SINGLE parsing implementation for both sides: + /// the agent's IME tracker / logparser gather collector (on-device) and the backend's + /// rules/gather/test-pattern endpoint, which must reproduce the agent's matching + /// semantics exactly so authors can test patterns without a device. Timestamp note: + /// bias-less lines parse with DateTimeStyles.AssumeLocal — on the backend that is the + /// SERVER's timezone, so the test endpoint reports match/parse results, not timestamps. /// public static class CmTraceLogParser {