Rule notifications resolve {{token}} placeholders before they are sent

A stored rule result keeps the rule's template; only the web and the MCP
server resolved it. The notification builders sent the explanation as
stored, so a channel card showed {{reason}}, {{appName}},
{{failedSubcategory}} and {{errorCode}} although the session had recorded
all four values.

- RuleTemplateInterpolator: the backend copy of the resolution, applied in
  BuildRuleFiredAlert and AddRuleResultSections before the text is cut.
  In a notification a list item whose value was not recorded is dropped
  and a token in running text becomes an en dash.
- The cut lands on a line, sentence or word boundary instead of a
  character count; the limits are unchanged.
- tests/fixtures/rule-template-interpolation/cases.json: one case file
  run by the backend, the web and the MCP server.
- The engine's evidence auto-field list is one constant. The web, the MCP
  interpolator and the validate_rule token lint gain timeoutHours, which
  the engine already recorded.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-09-30 18:12:02 +02:00
parent f564ee6a06
commit c60e07dfd4
12 changed files with 956 additions and 39 deletions
@@ -0,0 +1,352 @@
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Notifications;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// A stored RuleResult keeps the rule's {{token}} template. The web and the MCP server resolve
/// it when they render; a notification has no renderer, so the backend resolves it before the
/// text leaves. Three things are pinned here: the backend resolution equals the shared cases
/// every renderer runs, no placeholder reaches a notification for any built-in rule, and the
/// cut happens after the resolution and on a text boundary.
/// </summary>
public class RuleTemplateInterpolatorTests
{
private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890";
private const string SessionId = "0f1e2d3c-4b5a-4978-8a6b-5c4d3e2f1a0b";
private const string FailedAppName = "Contoso Productivity Suite";
// ── Shared cases (tests/fixtures/rule-template-interpolation) ─────────
private static readonly Lazy<JObject> Fixture = new(() =>
{
var path = Path.Combine(FindRepoRoot(), "tests", "fixtures", "rule-template-interpolation", "cases.json");
// No date parsing: a timestamp-looking string must stay the string the renderers see.
return JsonConvert.DeserializeObject<JObject>(
File.ReadAllText(path), new JsonSerializerSettings { DateParseHandling = DateParseHandling.None })!;
});
public static IEnumerable<object[]> CaseNames()
=> Fixture.Value["cases"]!.Select(c => new object[] { c.Value<string>("name")! });
[Theory]
[MemberData(nameof(CaseNames))]
public void Interpolate_matches_the_shared_case(string name)
{
var testCase = Fixture.Value["cases"]!.Single(c => c.Value<string>("name") == name);
var template = testCase.Value<string?>("template");
var expected = testCase.Value<string>("expected")!;
var evidence = testCase["matchedConditions"] as JObject;
// Evidence as the rule engine builds it (nested dictionaries) ...
Assert.Equal(expected, RuleTemplateInterpolator.Interpolate(template, AsDictionaries(evidence)));
// ... and as a JSON read without normalization leaves it (JSON nodes as values).
Assert.Equal(expected, RuleTemplateInterpolator.Interpolate(template, AsJsonNodes(evidence)));
}
[Fact]
public void Shared_cases_are_present_and_cover_every_auto_field()
{
Assert.True(Fixture.Value["cases"]!.Count() >= 20, "cases.json lost cases — the parity guard is only as wide as the file.");
Assert.Equal(
Fixture.Value["autoFields"]!.ToObject<string[]>()!,
RuleEngine.EvidenceAutoFields);
}
// ── Notification variant ──────────────────────────────────────────────
[Fact]
public void Notification_drops_a_list_item_whose_value_was_not_recorded()
{
var text = "Facts:\n\n- **Reason:** `{{reason}}`\n- **ESP HRESULT:** `{{errorCode}}`\n* **App:** {{appName}}\n1. step {{exitCode}}\n\nDone.";
var evidence = new Dictionary<string, object>
{
["enrollment_failed_reason"] = new Dictionary<string, object> { ["field"] = "reason", ["value"] = "hello_timeout" },
};
var rendered = RuleTemplateInterpolator.InterpolateForNotification(text, evidence);
Assert.Equal("Facts:\n\n- **Reason:** `hello_timeout`\n\nDone.", rendered);
}
[Fact]
public void Notification_replaces_an_unrecorded_value_in_running_text()
{
var rendered = RuleTemplateInterpolator.InterpolateForNotification(
"App **{{appName}}** failed with HRESULT **{{errorCode}}**.",
new Dictionary<string, object>
{
["failed_app"] = new Dictionary<string, object> { ["field"] = "appName", ["value"] = FailedAppName },
});
Assert.Equal($"App **{FailedAppName}** failed with HRESULT **–**.", rendered);
}
[Fact]
public void Notification_removes_the_sentence_about_literal_placeholders()
{
var rendered = RuleTemplateInterpolator.InterpolateForNotification(
"See ANALYZE-APP-013 for the root cause. Any field still shown as a literal `{{…}}` placeholder was simply not recorded. Check the timeline.",
new Dictionary<string, object>());
Assert.Equal("See ANALYZE-APP-013 for the root cause. Check the timeline.", rendered);
}
[Fact]
public void Notification_without_evidence_or_text_is_safe()
{
Assert.Equal(string.Empty, RuleTemplateInterpolator.InterpolateForNotification(null, null));
Assert.Equal(string.Empty, RuleTemplateInterpolator.InterpolateForNotification("", new Dictionary<string, object>()));
Assert.Equal("Reason: –", RuleTemplateInterpolator.InterpolateForNotification("Reason: {{reason}}", null));
}
[Fact]
public void Notification_collapses_the_gap_a_dropped_paragraph_leaves()
{
var rendered = RuleTemplateInterpolator.InterpolateForNotification(
"First.\n\nOnly a literal `{{…}}` remark.\n\nLast.", new Dictionary<string, object>());
Assert.Equal("First.\n\nLast.", rendered);
}
// ── No placeholder leaves the backend ─────────────────────────────────
[Fact]
public void No_builtin_rule_sends_a_placeholder_whatever_the_evidence()
{
// Worst case for every rule: nothing recorded at all. Both alert builders are the only
// paths that put a rule explanation into a notification.
var offenders = new List<string>();
var templated = 0;
foreach (var rule in BuiltInAnalyzeRules.GetAll())
{
if (rule.Explanation.Contains("{{")) templated++;
var result = new RuleResult
{
RuleId = rule.RuleId,
RuleTitle = rule.Title,
Severity = "critical",
Category = rule.Category,
Explanation = rule.Explanation,
};
var fired = NotificationAlertBuilder.BuildRuleFiredAlert(result, "DESKTOP-TEST01", "SN-0001");
var enrollment = NotificationAlertBuilder.BuildEnrollmentAlert(
"DESKTOP-TEST01", "SN-0001", null, null, success: false, failureReason: "x", duration: null);
NotificationAlertBuilder.AddRuleResultSections(enrollment, new List<RuleResult> { result });
foreach (var section in fired.Sections.Concat(enrollment.Sections))
{
if (section.Text.Contains("{{") || section.Text.Contains("}}"))
offenders.Add($"{rule.RuleId}: {section.Text}");
Assert.True(section.Text.Length <= NotificationAlertBuilder.RuleFiredExplanationMaxLength);
}
}
// Positive control: the catalog really contains templated rules, so an empty offender
// list is a statement about them and not about a catalog without placeholders.
Assert.True(templated >= 10, $"Only {templated} built-in rules carry a placeholder — did the catalog load?");
Assert.True(offenders.Count == 0, "Placeholder reached a notification:\n" + string.Join("\n", offenders));
}
// ── End to end: engine evidence → alert ───────────────────────────────
[Fact]
public async Task Rule_fired_alert_shows_the_values_the_engine_recorded()
{
var result = await RunEnrollmentFailedRuleAsync(withEspFailure: true);
var alert = NotificationAlertBuilder.BuildRuleFiredAlert(result, "DESKTOP-TEST01", "SN-0001");
var text = Assert.Single(alert.Sections).Text;
Assert.Equal(
"The Autopilot enrollment has explicitly failed.\n\n" +
"**Key facts extracted from this session's events:**\n\n" +
"- **Failure reason:** `esp_terminal_failure`\n" +
$"- **First failed app:** `{FailedAppName}`\n" +
"- **ESP failed subcategory:** `Apps`\n" +
"- **ESP HRESULT:** `0x87d1041c` …",
text);
// The stored result is untouched: the web and the MCP server still resolve the template.
Assert.Contains("{{reason}}", result.Explanation);
}
[Fact]
public async Task Rule_fired_alert_reads_evidence_that_went_through_the_table()
{
var result = await RunEnrollmentFailedRuleAsync(withEspFailure: true);
var fresh = NotificationAlertBuilder.BuildRuleFiredAlert(result, null, null).Sections.Single().Text;
// Same write and read as TableStorageService: JSON column, then nested dictionaries.
var json = JsonConvert.SerializeObject(result.MatchedConditions);
result.MatchedConditions = AsDictionaries(JObject.Parse(json))!;
Assert.Equal(fresh, NotificationAlertBuilder.BuildRuleFiredAlert(result, null, null).Sections.Single().Text);
}
[Fact]
public async Task Rule_fired_alert_omits_the_facts_a_non_esp_failure_never_had()
{
var result = await RunEnrollmentFailedRuleAsync(withEspFailure: false);
var text = Assert.Single(NotificationAlertBuilder.BuildRuleFiredAlert(result, null, null).Sections).Text;
Assert.Contains("- **Failure reason:** `hello_timeout`", text);
Assert.DoesNotContain("First failed app", text);
Assert.DoesNotContain("ESP failed subcategory", text);
Assert.DoesNotContain("ESP HRESULT:", text);
}
[Fact]
public async Task Enrollment_alert_rule_section_is_resolved_and_cut_on_a_line()
{
var result = await RunEnrollmentFailedRuleAsync(withEspFailure: true);
var alert = NotificationAlertBuilder.BuildEnrollmentAlert(
"DESKTOP-TEST01", "SN-0001", null, null, success: false, failureReason: "x", duration: null);
NotificationAlertBuilder.AddRuleResultSections(alert, new List<RuleResult> { result });
var text = Assert.Single(alert.Sections).Text;
Assert.True(text.Length <= NotificationAlertBuilder.RuleSectionExplanationMaxLength);
Assert.EndsWith("- **Failure reason:** `esp_terminal_failure` …", text);
}
// ── Cut ───────────────────────────────────────────────────────────────
[Fact]
public void Truncate_keeps_short_text()
=> Assert.Equal("short", NotificationAlertBuilder.TruncateAtBoundary("short", 300));
[Fact]
public void Truncate_prefers_a_line_break_then_a_sentence_then_a_word()
{
var lines = "First paragraph of the text.\n" + new string('a', 40);
Assert.Equal("First paragraph of the text. …", NotificationAlertBuilder.TruncateAtBoundary(lines, 50));
var sentences = "One sentence here. Another sentence that runs on and on without an end";
Assert.Equal("One sentence here. …", NotificationAlertBuilder.TruncateAtBoundary(sentences, 30));
var words = "word word word word word word word word word word";
Assert.Equal("word word word word word …", NotificationAlertBuilder.TruncateAtBoundary(words, 30));
}
[Fact]
public void Truncate_never_exceeds_the_limit_and_survives_text_without_any_boundary()
{
var unbroken = new string('x', 500);
var cut = NotificationAlertBuilder.TruncateAtBoundary(unbroken, 100);
Assert.Equal(100, cut.Length);
Assert.EndsWith(" …", cut);
}
[Fact]
public void Truncate_ignores_a_boundary_that_would_waste_most_of_the_budget()
{
var text = "Hi.\n" + string.Join(" ", Enumerable.Repeat("word", 60));
var cut = NotificationAlertBuilder.TruncateAtBoundary(text, 100);
Assert.True(cut.Length > 50, cut);
Assert.True(cut.Length <= 100);
}
[Fact]
public void Truncate_does_not_leave_a_bold_or_code_span_open()
{
var bold = "See the **App Detection Failure During ESP** rule for the cause and the **remediation steps in detail** below";
Assert.Equal("See the **App Detection Failure During ESP** rule for the cause and the remediation steps …",
NotificationAlertBuilder.TruncateAtBoundary(bold, 94));
var code = "The installer returned `1603 fatal error during installation` and stopped the whole chain";
Assert.Equal("The installer returned 1603 fatal error during …",
NotificationAlertBuilder.TruncateAtBoundary(code, 50));
}
// ── Helpers ───────────────────────────────────────────────────────────
private static async Task<RuleResult> RunEnrollmentFailedRuleAsync(bool withEspFailure)
{
var rule = BuiltInAnalyzeRules.GetAll().Single(r => r.RuleId == "ANALYZE-ENRL-001");
var events = new List<EnrollmentEvent>
{
Event("enrollment_failed", 163, new Dictionary<string, object>
{
["reason"] = withEspFailure ? "esp_terminal_failure" : "hello_timeout",
}),
};
if (withEspFailure)
{
events.Add(Event("esp_failure_settle_started", 161, new Dictionary<string, object>
{
["failedSubcategory"] = "Apps",
["errorCode"] = "0x87d1041c",
}));
events.Add(Event("app_install_failed", 158, new Dictionary<string, object>
{
["appId"] = "00000000-0000-4000-8000-000000000001",
["appName"] = FailedAppName,
["errorCode"] = "0x87d1041c",
}));
}
var ruleRepo = new Mock<IRuleRepository>();
ruleRepo.Setup(r => r.GetAnalyzeRulesAsync("global")).ReturnsAsync(new List<AnalyzeRule> { rule });
ruleRepo.Setup(r => r.GetAnalyzeRulesAsync(TenantId)).ReturnsAsync(new List<AnalyzeRule>());
ruleRepo.Setup(r => r.GetRuleStatesAsync(It.IsAny<string>())).ReturnsAsync(new Dictionary<string, RuleState>());
ruleRepo.Setup(r => r.GetRuleResultsAsync(It.IsAny<string>(), It.IsAny<string>())).ReturnsAsync(new List<RuleResult>());
var sessionRepo = new Mock<ISessionRepository>();
sessionRepo.Setup(s => s.GetSessionEventsStrictAsync(TenantId, SessionId, It.IsAny<int>())).ReturnsAsync(events);
var ruleService = new AnalyzeRuleService(ruleRepo.Object, NullLogger<AnalyzeRuleService>.Instance);
var engine = new RuleEngine(ruleService, ruleRepo.Object, sessionRepo.Object, NullLogger<RuleTemplateInterpolatorTests>.Instance);
var outcome = await engine.AnalyzeSessionAsync(TenantId, SessionId);
return Assert.Single(outcome.Results);
}
private static EnrollmentEvent Event(string eventType, long sequence, Dictionary<string, object> data) => new()
{
EventId = Guid.NewGuid().ToString(),
TenantId = TenantId,
SessionId = SessionId,
EventType = eventType,
Timestamp = DateTime.UtcNow,
Sequence = sequence,
Data = data,
};
private static Dictionary<string, object>? AsJsonNodes(JObject? evidence)
=> evidence?.Properties().ToDictionary(p => p.Name, p => (object)p.Value);
private static Dictionary<string, object>? AsDictionaries(JObject? evidence)
=> evidence?.Properties().ToDictionary(p => p.Name, p => Plain(p.Value)!);
private static object? Plain(JToken token) => token switch
{
JObject o => o.Properties().ToDictionary(p => p.Name, p => Plain(p.Value)!),
JArray a => a.Select(Plain).ToList(),
JValue v => v.Value,
_ => null,
};
private static string FindRepoRoot()
{
var dir = new DirectoryInfo(AppContext.BaseDirectory);
while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AutopilotMonitor.sln")))
{
dir = dir.Parent;
}
Assert.NotNull(dir);
return dir!.FullName;
}
}
@@ -233,13 +233,9 @@ namespace AutopilotMonitor.Functions.Services.Notifications
},
};
if (!string.IsNullOrEmpty(result.Explanation))
{
var explanation = result.Explanation.Length > 300
? result.Explanation[..300] + "..."
: result.Explanation;
var explanation = RenderExplanation(result, RuleFiredExplanationMaxLength);
if (explanation.Length > 0)
alert.Sections.Add(new NotificationSection { Title = "Explanation", Text = explanation });
}
if (!string.IsNullOrEmpty(sessionUrl))
alert.Actions.Add(new NotificationAction { Type = "openUrl", Title = "Open session", Url = sessionUrl });
@@ -273,18 +269,77 @@ namespace AutopilotMonitor.Functions.Services.Notifications
_ => "\ud83d\udfe1"
};
var explanation = r.Explanation?.Length > 200
? r.Explanation[..200] + "..."
: r.Explanation ?? "";
alert.Sections.Add(new NotificationSection
{
Title = $"{emoji} {r.RuleTitle}",
Text = explanation
Text = RenderExplanation(r, RuleSectionExplanationMaxLength)
});
}
}
internal const int RuleFiredExplanationMaxLength = 300;
internal const int RuleSectionExplanationMaxLength = 200;
/// <summary>
/// A stored <see cref="RuleResult.Explanation"/> is the rule's template. Tokens are
/// resolved BEFORE the cut: cutting first would split a placeholder or spend the
/// budget on placeholder names instead of the values.
/// </summary>
private static string RenderExplanation(RuleResult result, int maxLength)
=> TruncateAtBoundary(
RuleTemplateInterpolator.InterpolateForNotification(result.Explanation, result.MatchedConditions),
maxLength);
/// <summary>
/// Shortens to at most <paramref name="maxLength"/> characters at the last line break
/// inside the limit, else the last sentence end, else the last word. A boundary in the
/// first half of the limit is not taken, so the cut never throws away most of the budget.
/// A bold or code span the cut left open is closed by dropping its opening marker.
/// </summary>
internal static string TruncateAtBoundary(string text, int maxLength)
{
const string marker = " …";
if (text.Length <= maxLength)
return text;
var window = text[..(maxLength - marker.Length)];
var floor = window.Length / 2;
var cut = window.LastIndexOf('\n');
if (cut < floor)
{
cut = -1;
for (var i = window.Length - 2; i >= floor; i--)
{
if (window[i] is '.' or '!' or '?' && char.IsWhiteSpace(window[i + 1]))
{
cut = i + 1;
break;
}
}
}
if (cut < floor)
cut = window.LastIndexOf(' ');
if (cut < floor)
cut = window.Length;
var kept = DropUnpairedMarker(DropUnpairedMarker(window[..cut], "**"), "`");
return kept.TrimEnd() + marker;
}
private static string DropUnpairedMarker(string text, string markdownMarker)
{
var count = 0;
var last = -1;
for (var i = text.IndexOf(markdownMarker, StringComparison.Ordinal); i >= 0;
i = text.IndexOf(markdownMarker, i + markdownMarker.Length, StringComparison.Ordinal))
{
count++;
last = i;
}
return count % 2 == 0 ? text : text.Remove(last, markdownMarker.Length);
}
/// <summary>
/// Builds a notification alert for an SLA breach. Dispatches on
/// <paramref name="breachType"/>: <c>SuccessRate</c>, <c>Duration</c>, or <c>AppInstall</c>.
@@ -1307,6 +1307,17 @@ namespace AutopilotMonitor.Functions.Services
return (true, primary);
}
/// <summary>
/// Same-event fields copied into every evidence entry. Whitelist: short identifiers only;
/// errorDetail/message are excluded (can be multi-KB). Every renderer resolves these as
/// <c>{{token}}</c> — the web, the MCP server and <see cref="RuleTemplateInterpolator"/>
/// carry the same list, pinned by tests/fixtures/rule-template-interpolation/cases.json.
/// </summary>
internal static readonly IReadOnlyList<string> EvidenceAutoFields = new[]
{
"appId", "appName", "errorPatternId", "errorCode", "exitCode", "status", "timeoutHours"
};
/// <summary>
/// Adds slim identifying data fields from an event to the evidence dictionary with a prefix.
/// Free-text fields like <c>errorDetail</c> (often a stack trace) and <c>message</c> are
@@ -1317,10 +1328,7 @@ namespace AutopilotMonitor.Functions.Services
{
if (evt.Data == null) return;
// Whitelist: short identifiers only. errorDetail/message removed (can be multi-KB).
var knownFields = new[] { "appId", "appName", "errorPatternId", "errorCode", "exitCode", "status", "timeoutHours" };
foreach (var field in knownFields)
foreach (var field in EvidenceAutoFields)
{
var val = GetDataFieldValue(evt, field);
if (val != null)
@@ -0,0 +1,199 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace AutopilotMonitor.Functions.Services
{
/// <summary>
/// Resolves the <c>{{token}}</c> placeholders of a rule's explanation/remediation text from a
/// result's <see cref="RuleResult.MatchedConditions"/>. A stored <see cref="RuleResult"/> keeps
/// the rule's raw template; the web (<c>lib/interpolateRuleTemplate.ts</c>) and the MCP server
/// (<c>src/interpolate-rule-template.ts</c>) resolve it when they render. This is the backend
/// copy for text that leaves without passing either of them (notifications).
/// <para>
/// Resolution order, identical in all three and pinned by
/// <c>tests/fixtures/rule-template-interpolation/cases.json</c>:
/// </para>
/// <list type="number">
/// <item>an evidence entry whose <c>field</c> equals the token (the last such entry wins);</item>
/// <item>a same-event auto-field named like the token (<see cref="RuleEngine.EvidenceAutoFields"/>;
/// the first non-empty one wins, which pins it to the rule's earliest matched condition);</item>
/// <item>an evidence entry whose signal name equals the token;</item>
/// <item>otherwise the token stays as written.</item>
/// </list>
/// </summary>
public static class RuleTemplateInterpolator
{
/// <summary>Shown in a notification where running text names a value that was not recorded.</summary>
internal const string NotRecorded = "–";
private static readonly Regex TokenRegex = new Regex(
@"\{\{\s*([a-zA-Z0-9_]+)\s*\}\}", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex ListItemRegex = new Regex(
@"^\s*(?:[-*+]|\d+[.)])\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex SentenceBreakRegex = new Regex(
@"(?<=[.!?])\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private static readonly Regex BlankRunRegex = new Regex(
@"\n{3,}", RegexOptions.Compiled | RegexOptions.CultureInvariant);
/// <summary>
/// Replaces every resolvable token; an unresolved one stays literal, exactly as the web
/// and the MCP server render it.
/// </summary>
public static string Interpolate(string? text, IReadOnlyDictionary<string, object>? matchedConditions)
{
if (string.IsNullOrEmpty(text))
return string.Empty;
var lookup = Lookup.From(matchedConditions);
return TokenRegex.Replace(text, m => lookup.Resolve(m.Groups[1].Value) ?? m.Value);
}
/// <summary>
/// Interpolation for text that leaves the backend as a notification. The recipient has no
/// renderer and no footnote explaining a literal placeholder, so none may survive: a list
/// item whose token was not recorded is dropped, a token in running text becomes
/// <see cref="NotRecorded"/>, and a sentence that talks about literal placeholders is removed.
/// </summary>
public static string InterpolateForNotification(string? text, IReadOnlyDictionary<string, object>? matchedConditions)
{
if (string.IsNullOrEmpty(text))
return string.Empty;
var lookup = Lookup.From(matchedConditions);
var lines = new List<string>();
foreach (var line in text.Replace("\r\n", "\n").Split('\n'))
{
var unresolved = false;
var rendered = TokenRegex.Replace(line, m =>
{
var value = lookup.Resolve(m.Groups[1].Value);
if (value != null)
return value;
unresolved = true;
return NotRecorded;
});
if (unresolved && ListItemRegex.IsMatch(line))
continue;
if (rendered.Contains("{{", StringComparison.Ordinal))
{
rendered = string.Join(" ", SentenceBreakRegex.Split(rendered)
.Where(sentence => !sentence.Contains("{{", StringComparison.Ordinal)));
if (rendered.Length == 0)
continue;
}
lines.Add(rendered);
}
return BlankRunRegex.Replace(string.Join("\n", lines), "\n\n").Trim();
}
private sealed class Lookup
{
private readonly Dictionary<string, string> _byField = new Dictionary<string, string>(StringComparer.Ordinal);
private readonly Dictionary<string, string> _byAutoField = new Dictionary<string, string>(StringComparer.Ordinal);
private readonly Dictionary<string, string> _bySignal = new Dictionary<string, string>(StringComparer.Ordinal);
public string? Resolve(string token)
{
if (_byField.TryGetValue(token, out var value)) return value;
if (_byAutoField.TryGetValue(token, out value)) return value;
if (_bySignal.TryGetValue(token, out value)) return value;
return null;
}
public static Lookup From(IReadOnlyDictionary<string, object>? matchedConditions)
{
var lookup = new Lookup();
if (matchedConditions == null || matchedConditions.Count == 0)
return lookup;
// Evidence arrives in two shapes: nested dictionaries straight from the rule engine,
// or whatever the table read produced. Going through JSON gives both the form the
// web and the MCP server see. JsonSerializer.Create() ignores the process-wide
// default settings, so a naming strategy configured there cannot rename the keys.
JObject root;
try
{
root = JObject.FromObject(matchedConditions, JsonSerializer.Create());
}
catch (Exception)
{
// Evidence that cannot be serialized resolves nothing; the text is still sent.
return lookup;
}
foreach (var property in root.Properties())
{
if (property.Value is not JObject evidence)
{
if (property.Value is JValue scalar && scalar.Type != JTokenType.Null && scalar.Type != JTokenType.Undefined)
lookup._bySignal[property.Name] = Format(scalar);
continue;
}
var hasValue = evidence.TryGetValue("value", out var valueToken);
if (hasValue)
{
if (evidence["field"] is JValue { Type: JTokenType.String } field)
lookup._byField[(string)field.Value!] = Format(valueToken);
lookup._bySignal[property.Name] = Format(valueToken);
}
foreach (var autoField in RuleEngine.EvidenceAutoFields)
{
if (lookup._byAutoField.ContainsKey(autoField))
continue;
var formatted = Format(evidence[autoField]);
if (formatted.Length > 0)
lookup._byAutoField[autoField] = formatted;
}
}
return lookup;
}
// Mirrors the TypeScript formatValue: strings as they are, numbers and booleans the
// way JavaScript prints them, anything structured as compact JSON.
private static string Format(JToken? token)
{
if (token == null)
return string.Empty;
switch (token.Type)
{
case JTokenType.Null:
case JTokenType.Undefined:
return string.Empty;
case JTokenType.String:
return (string)((JValue)token).Value!;
case JTokenType.Boolean:
return (bool)token ? "true" : "false";
case JTokenType.Integer:
return Convert.ToString(((JValue)token).Value, CultureInfo.InvariantCulture) ?? string.Empty;
case JTokenType.Float:
return Convert.ToDouble(((JValue)token).Value, CultureInfo.InvariantCulture)
.ToString(CultureInfo.InvariantCulture);
case JTokenType.Object:
case JTokenType.Array:
return token.ToString(Formatting.None);
default:
// Date, Guid, TimeSpan, Uri: their JSON string form without the quotes.
return token.ToString(Formatting.None).Trim('"');
}
}
}
}
}
@@ -1,5 +1,45 @@
import { describe, it, expect } from 'vitest';
import { interpolateRuleTemplate, interpolateAnalysisResults } from '../interpolate-rule-template.js';
import { existsSync, readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
interpolateRuleTemplate,
interpolateAnalysisResults,
RULE_TEMPLATE_AUTO_FIELDS,
} from '../interpolate-rule-template.js';
interface SharedCase {
name: string;
template: string | null;
matchedConditions: Record<string, unknown> | null;
expected: string;
}
/** tests/fixtures/rule-template-interpolation at the repo root — the cases every renderer runs. */
function loadSharedCases(): { autoFields: string[]; cases: SharedCase[] } {
let dir = dirname(fileURLToPath(import.meta.url));
while (!existsSync(join(dir, 'AutopilotMonitor.sln'))) {
const parent = dirname(dir);
if (parent === dir) throw new Error('repository root (AutopilotMonitor.sln) not found');
dir = parent;
}
return JSON.parse(
readFileSync(join(dir, 'tests', 'fixtures', 'rule-template-interpolation', 'cases.json'), 'utf8'),
);
}
describe('interpolateRuleTemplate — shared cases (web = MCP = backend)', () => {
const shared = loadSharedCases();
it('has cases and the same auto-field list as the rule engine', () => {
expect(shared.cases.length).toBeGreaterThanOrEqual(20);
expect([...RULE_TEMPLATE_AUTO_FIELDS]).toEqual(shared.autoFields);
});
it.each(shared.cases.map((c) => [c.name, c] as const))('%s', (_name, c) => {
expect(interpolateRuleTemplate(c.template, c.matchedConditions)).toBe(c.expected);
});
});
describe('interpolateRuleTemplate', () => {
// Shape of a real ANALYZE-ENRL-001 matchedConditions map (the case from the
@@ -8,15 +8,33 @@
* Keep the resolution order in lock-step with the web copy:
* 1. matchedConditions entry whose `field` equals `token`.
* 2. matchedConditions entry carrying a whitelisted same-event auto-field
* named `token` (backend `AddDataFieldsToEvidence`: appId, appName,
* errorPatternId, errorCode, exitCode, status). First non-empty wins so the
* named `token` (`RULE_TEMPLATE_AUTO_FIELDS`). First non-empty wins so the
* value pins to the rule's required-condition event.
* 3. matchedConditions entry whose key (signal name) equals `token`.
* 4. Unresolved tokens are left untouched so authors notice the typo (and the
* reworded ANALYZE-ENRL-001 footnote explains a genuinely-absent field).
*
* Token chars: [a-zA-Z0-9_]. Whitespace inside the braces is tolerated.
*
* Web, MCP and the backend (notifications) all run the cases in
* `tests/fixtures/rule-template-interpolation/cases.json` at the repo root.
*/
/**
* Mirror of the backend evidence auto-capture whitelist (`RuleEngine.EvidenceAutoFields`).
* Explicit list keeps a rogue evidence key (e.g. accidental `description`) from shadowing
* a token. Also the list validate_rule lints tokens against.
*/
export const RULE_TEMPLATE_AUTO_FIELDS = [
'appId',
'appName',
'errorPatternId',
'errorCode',
'exitCode',
'status',
'timeoutHours',
] as const;
export function interpolateRuleTemplate(
text: string | null | undefined,
matchedConditions: Record<string, unknown> | null | undefined
@@ -28,10 +46,6 @@ export function interpolateRuleTemplate(
const byAutoField = new Map<string, string>();
const bySignal = new Map<string, string>();
// Mirror of the backend AddDataFieldsToEvidence whitelist. Explicit list keeps a
// rogue evidence key (e.g. accidental `description`) from shadowing a token.
const AUTO_FIELDS = ['appId', 'appName', 'errorPatternId', 'errorCode', 'exitCode', 'status'];
for (const [signal, evidence] of Object.entries(matchedConditions)) {
if (!evidence || typeof evidence !== 'object') {
if (evidence != null) bySignal.set(signal, formatValue(evidence));
@@ -44,7 +58,7 @@ export function interpolateRuleTemplate(
if ('value' in e) {
bySignal.set(signal, formatValue(e.value));
}
for (const af of AUTO_FIELDS) {
for (const af of RULE_TEMPLATE_AUTO_FIELDS) {
if (byAutoField.has(af)) continue;
const v = e[af];
if (v != null && v !== '') byAutoField.set(af, formatValue(v));
@@ -250,7 +250,7 @@ export const RULE_AUTHORING_GUIDE = {
'explanation and remediation text may contain {{token}} placeholders, resolved from the ' +
'matched evidence at display time: first a matched condition whose dataField equals the ' +
'token, then the auto-captured fields appId / appName / errorPatternId / errorCode / ' +
'exitCode / status from the matched events, then a condition SIGNAL name. Unresolved tokens ' +
'exitCode / status / timeoutHours from the matched events, then a condition SIGNAL name. Unresolved tokens ' +
'stay visible as {{token}} so typos are noticeable. test_analyze_rule returns the ' +
'interpolated preview for a real session.',
markSessionAsFailed:
@@ -27,6 +27,7 @@ import addFormatsModule from 'ajv-formats';
const addFormats = addFormatsModule as unknown as (ajv: Ajv2020) => void;
import { GATHER_RULE_SCHEMA, ANALYZE_RULE_SCHEMA, RULE_GUARDRAILS } from './rule-authoring.generated.js';
import { isKnownEventType } from './resource-catalog.js';
import { RULE_TEMPLATE_AUTO_FIELDS } from './interpolate-rule-template.js';
export type FindingLevel = 'error' | 'warning' | 'info';
@@ -274,9 +275,8 @@ function checkGatherTarget(collectorType: string, target: string): ValidationFin
// ── Analyze-rule semantic lint ──────────────────────────────────────────────
/** Mirrors the backend evidence auto-capture whitelist (AddDataFieldsToEvidence) and
* the interpolation resolution order (interpolate-rule-template.ts). */
const AUTO_FIELDS = ['appId', 'appName', 'errorPatternId', 'errorCode', 'exitCode', 'status'];
/** The auto-captured fields a token may resolve to — the interpolator's own list. */
const AUTO_FIELDS = RULE_TEMPLATE_AUTO_FIELDS;
/** The only condition strings EvaluateConfidenceFactor parses — anything else is
* silently false in production. Exact spacing matters. */
@@ -1,5 +1,40 @@
import { describe, it, expect } from "vitest";
import { interpolateRuleTemplate } from "../interpolateRuleTemplate";
import fs from "node:fs";
import path from "node:path";
import { interpolateRuleTemplate, RULE_TEMPLATE_AUTO_FIELDS } from "../interpolateRuleTemplate";
interface SharedCase {
name: string;
template: string | null;
matchedConditions: Record<string, unknown> | null;
expected: string;
}
/** tests/fixtures/rule-template-interpolation at the repo root — the cases every renderer runs. */
function loadSharedCases(): { autoFields: string[]; cases: SharedCase[] } {
let dir = __dirname;
while (!fs.existsSync(path.join(dir, "AutopilotMonitor.sln"))) {
const parent = path.dirname(dir);
if (parent === dir) throw new Error("repository root (AutopilotMonitor.sln) not found");
dir = parent;
}
return JSON.parse(
fs.readFileSync(path.join(dir, "tests", "fixtures", "rule-template-interpolation", "cases.json"), "utf8")
);
}
describe("interpolateRuleTemplate — shared cases (web = MCP = backend)", () => {
const shared = loadSharedCases();
it("has cases and the same auto-field list as the rule engine", () => {
expect(shared.cases.length).toBeGreaterThanOrEqual(20);
expect([...RULE_TEMPLATE_AUTO_FIELDS]).toEqual(shared.autoFields);
});
it.each(shared.cases.map((c) => [c.name, c] as const))("%s", (_name, c) => {
expect(interpolateRuleTemplate(c.template, c.matchedConditions)).toBe(c.expected);
});
});
describe("interpolateRuleTemplate", () => {
const matched = {
@@ -3,13 +3,13 @@
* with values pulled from the rule's `matchedConditions` evidence map.
*
* Resolution order for `{{token}}`:
* 1. The first matchedConditions entry whose `field` equals `token`
* (e.g. {{uefiCA2023Status}} -> entry where field === "uefiCA2023Status").
* 1. A matchedConditions entry whose `field` equals `token`
* (e.g. {{uefiCA2023Status}} -> entry where field === "uefiCA2023Status");
* when several entries name the same field, the last one wins.
* 2. The first matchedConditions entry that carries a same-event auto-field
* named `token` (added by the backend's
* `RuleEngine.ConditionEvaluators.AddDataFieldsToEvidence` whitelist —
* currently `appId`, `appName`, `errorPatternId`, `errorCode`, `exitCode`,
* `status`). Using "first" pins these to the same source event as the
* `RULE_TEMPLATE_AUTO_FIELDS`). Using "first" pins these to the same source event as the
* rule's required condition, avoiding cross-event interpolation drift
* when a session has multiple `app_install_failed` events.
* 3. A matchedConditions entry whose key (signal name) equals `token`
@@ -17,7 +17,27 @@
* 4. Unresolved tokens are left untouched so authors notice the typo.
*
* Token chars: [a-zA-Z0-9_]. Whitespace inside the braces is tolerated.
*
* The MCP server and the backend (notifications) carry the same resolution; all three run
* the cases in `tests/fixtures/rule-template-interpolation/cases.json` at the repo root.
*/
/**
* Whitelist of auto-injected event fields the backend rule engine adds to every
* matched-condition evidence dict (`RuleEngine.EvidenceAutoFields`). Kept explicit so a
* rogue evidence key (e.g. accidental `description`) can't shadow a template token from
* another resolution path.
*/
export const RULE_TEMPLATE_AUTO_FIELDS = [
"appId",
"appName",
"errorPatternId",
"errorCode",
"exitCode",
"status",
"timeoutHours",
] as const;
export function interpolateRuleTemplate(
text: string | null | undefined,
matchedConditions: Record<string, unknown> | null | undefined
@@ -29,12 +49,6 @@ export function interpolateRuleTemplate(
const byAutoField = new Map<string, string>();
const bySignal = new Map<string, string>();
// Whitelist of auto-injected event fields the backend rule engine adds to every
// matched-condition evidence dict via AddDataFieldsToEvidence. Mirror keeps the
// mapping explicit so a rogue evidence key (e.g. accidental `description`) can't
// shadow a template token from another resolution path.
const AUTO_FIELDS = ["appId", "appName", "errorPatternId", "errorCode", "exitCode", "status"];
for (const [signal, evidence] of Object.entries(matchedConditions)) {
if (!evidence || typeof evidence !== "object") {
if (evidence != null) bySignal.set(signal, formatValue(evidence));
@@ -50,7 +64,7 @@ export function interpolateRuleTemplate(
// Auto-field fallback: first non-empty wins (insertion order of
// matchedConditions mirrors rule.conditions order, so the required
// condition's event is checked first).
for (const af of AUTO_FIELDS) {
for (const af of RULE_TEMPLATE_AUTO_FIELDS) {
if (byAutoField.has(af)) continue;
const v = e[af];
if (v != null && v !== "") byAutoField.set(af, formatValue(v));
+19
View File
@@ -0,0 +1,19 @@
# Rule template interpolation — shared cases
A rule result stores the rule's text with its `{{token}}` placeholders; each renderer resolves
them from the result's `matchedConditions`. `cases.json` is the one definition of that
resolution, and every implementation runs all of it:
- Web — `interpolateRuleTemplate` in `src/Web/autopilot-monitor-web/lib/interpolateRuleTemplate.ts`
(test: `lib/__tests__/interpolateRuleTemplate.test.ts`)
- MCP server — `interpolateRuleTemplate` in `src/McpServer/autopilot-monitor-mcp/src/interpolate-rule-template.ts`
(test: `src/__tests__/interpolate-rule-template.test.ts`)
- Backend — `RuleTemplateInterpolator.Interpolate` in
`src/Backend/AutopilotMonitor.Functions/Services/RuleTemplateInterpolator.cs`
(test: `RuleTemplateInterpolatorTests.cs`)
`autoFields` is the list of same-event fields the rule engine copies into every evidence
entry (`RuleEngine.EvidenceAutoFields`). Each implementation's list is compared against it.
A change to the resolution goes into `cases.json` first; the three tests then say which
implementation is behind.
+181
View File
@@ -0,0 +1,181 @@
{
"autoFields": ["appId", "appName", "errorPatternId", "errorCode", "exitCode", "status", "timeoutHours"],
"cases": [
{
"name": "token resolves from the entry whose field equals it",
"template": "Status: `{{uefiCA2023Status}}`",
"matchedConditions": {
"ca2023_not_updated": { "eventId": "evt-1", "sequence": 23, "timestamp": "2026-04-28T18:24:16Z", "eventType": "secureboot_status", "field": "uefiCA2023Status", "value": "NotStarted" }
},
"expected": "Status: `NotStarted`"
},
{
"name": "several tokens in one text, engine evidence shape",
"template": "- **Failure reason:** `{{reason}}`\n- **First failed app:** `{{appName}}`\n- **ESP failed subcategory:** `{{failedSubcategory}}`\n- **ESP HRESULT:** `{{errorCode}}`",
"matchedConditions": {
"enrollment_failed": { "eventId": "evt-9", "sequence": 431, "timestamp": "2026-05-28T10:41:16Z", "eventType": "enrollment_failed", "count": 1, "errorCode": "0x87d1041c" },
"enrollment_failed_reason": { "eventId": "evt-9", "sequence": 431, "timestamp": "2026-05-28T10:41:16Z", "eventType": "enrollment_failed", "field": "reason", "value": "esp_terminal_failure", "errorCode": "0x87d1041c" },
"esp_failure_subcategory": { "eventId": "evt-8", "sequence": 429, "timestamp": "2026-05-28T10:41:12Z", "eventType": "esp_failure_settle_started", "field": "failedSubcategory", "value": "Apps", "errorCode": "0x87d1041c" },
"esp_failure_errorcode": { "eventId": "evt-8", "sequence": 429, "timestamp": "2026-05-28T10:41:12Z", "eventType": "esp_failure_settle_started", "field": "errorCode", "value": "0x87d1041c", "errorCode": "0x87d1041c" },
"failed_app": { "eventId": "evt-7", "sequence": 420, "timestamp": "2026-05-28T10:41:06Z", "eventType": "app_install_failed", "field": "appName", "value": "Contoso Productivity Suite", "appId": "00000000-0000-4000-8000-000000000001", "appName": "Contoso Productivity Suite", "errorCode": "0x87d1041c" },
"factor_failed_app": true
},
"expected": "- **Failure reason:** `esp_terminal_failure`\n- **First failed app:** `Contoso Productivity Suite`\n- **ESP failed subcategory:** `Apps`\n- **ESP HRESULT:** `0x87d1041c`"
},
{
"name": "token without evidence stays literal",
"template": "sub: `{{failedSubcategory}}` reason: {{reason}}",
"matchedConditions": {
"enrollment_failed_reason": { "field": "reason", "value": "hello_timeout" }
},
"expected": "sub: `{{failedSubcategory}}` reason: hello_timeout"
},
{
"name": "whitespace inside the braces is tolerated",
"template": "[{{ appName }}]",
"matchedConditions": { "failed_app": { "field": "appName", "value": "Contoso VPN" } },
"expected": "[Contoso VPN]"
},
{
"name": "no evidence map leaves the text unchanged",
"template": "hello {{reason}}",
"matchedConditions": null,
"expected": "hello {{reason}}"
},
{
"name": "empty text",
"template": "",
"matchedConditions": { "sig": { "field": "reason", "value": "x" } },
"expected": ""
},
{
"name": "missing text yields an empty string",
"template": null,
"matchedConditions": { "sig": { "field": "reason", "value": "x" } },
"expected": ""
},
{
"name": "text without placeholders",
"template": "plain text with {single} braces",
"matchedConditions": { "sig": { "field": "reason", "value": "x" } },
"expected": "plain text with {single} braces"
},
{
"name": "numbers and booleans render without type decoration",
"template": "seq={{n}} ok={{b}} ratio={{r}} whole={{w}} on={{t}}",
"matchedConditions": {
"a": { "field": "n", "value": 42 },
"b": { "field": "b", "value": false },
"c": { "field": "r", "value": 1.5 },
"d": { "field": "w", "value": 120.0 },
"e": { "field": "t", "value": true }
},
"expected": "seq=42 ok=false ratio=1.5 whole=120 on=true"
},
{
"name": "signal name resolves when no field matches",
"template": "{{ca2023_not_updated}}",
"matchedConditions": {
"ca2023_not_updated": { "field": "uefiCA2023Status", "value": "NotStarted" }
},
"expected": "NotStarted"
},
{
"name": "scalar evidence resolves by signal name",
"template": "{{factor_failed_app}} {{note}} {{hits}}",
"matchedConditions": { "factor_failed_app": true, "note": "plain", "hits": 3 },
"expected": "true plain 3"
},
{
"name": "auto-field comes from the first entry that carries it",
"template": "App={{appName}} HR={{errorCode}}",
"matchedConditions": {
"esp_apps_detection_failure": { "field": "failureType", "value": "esp_apps_detection_failure", "appName": "Contoso Productivity Suite", "errorCode": "0x87d1041c" },
"another_app_failure": { "field": "isError", "value": "true", "appName": "Some Other App", "errorCode": "0x80070005" }
},
"expected": "App=Contoso Productivity Suite HR=0x87d1041c"
},
{
"name": "empty auto-field is skipped for a later entry",
"template": "{{exitCode}}/{{status}}",
"matchedConditions": {
"first": { "field": "x", "value": "1", "exitCode": "", "status": null },
"second": { "field": "y", "value": "2", "exitCode": "1603", "status": "Failed" }
},
"expected": "1603/Failed"
},
{
"name": "every auto-field is resolvable",
"template": "{{appId}}|{{appName}}|{{errorPatternId}}|{{errorCode}}|{{exitCode}}|{{status}}|{{timeoutHours}}",
"matchedConditions": {
"sig": { "field": "other", "value": "v", "appId": "id-1", "appName": "Contoso VPN", "errorPatternId": "pattern-1", "errorCode": "0x1", "exitCode": "2", "status": "Failed", "timeoutHours": "5" }
},
"expected": "id-1|Contoso VPN|pattern-1|0x1|2|Failed|5"
},
{
"name": "explicit field beats an auto-field, whatever the order",
"template": "{{appName}} / {{errorCode}}",
"matchedConditions": {
"with_auto": { "field": "failureType", "value": "x", "appName": "Auto-field value", "errorCode": "auto" },
"explicit_appname": { "field": "appName", "value": "Explicit value" },
"explicit_errorcode": { "field": "errorCode", "value": "explicit" }
},
"expected": "Explicit value / explicit"
},
{
"name": "two entries with the same field: the later one wins",
"template": "{{appName}}",
"matchedConditions": {
"first_app": { "field": "appName", "value": "First App" },
"second_app": { "field": "appName", "value": "Second App" }
},
"expected": "Second App"
},
{
"name": "field beats a signal of the same name",
"template": "{{reason}}",
"matchedConditions": {
"reason": { "field": "other", "value": "from signal" },
"enrollment_failed_reason": { "field": "reason", "value": "from field" }
},
"expected": "from field"
},
{
"name": "internal evidence keys are not tokens",
"template": "{{eventId}} {{sequence}} {{timestamp}} {{eventType}} {{field}} {{value}} {{count}}",
"matchedConditions": {
"sig": { "eventId": "evt-secret", "sequence": 42, "timestamp": "2026-05-28T10:00:00Z", "eventType": "app_install_failed", "field": "failureType", "value": "esp_apps_detection_failure", "count": 2 }
},
"expected": "{{eventId}} {{sequence}} {{timestamp}} {{eventType}} {{field}} {{value}} {{count}}"
},
{
"name": "prefixed correlation fields are not auto-fields",
"template": "{{appName}} {{eventA_appName}}",
"matchedConditions": {
"pair": { "eventA_appName": "Contoso VPN", "eventB_appName": "Contoso VPN", "totalMatches": 1 }
},
"expected": "{{appName}} {{eventA_appName}}"
},
{
"name": "null value renders empty",
"template": "[{{reason}}]",
"matchedConditions": { "sig": { "field": "reason", "value": null } },
"expected": "[]"
},
{
"name": "object and array values render as compact JSON",
"template": "{{detail}} {{names}}",
"matchedConditions": {
"a": { "field": "detail", "value": { "a": 1, "b": "x" } },
"b": { "field": "names", "value": ["one", "two"] }
},
"expected": "{\"a\":1,\"b\":\"x\"} [\"one\",\"two\"]"
},
{
"name": "the meta placeholder of the rule footnote is not a token",
"template": "Any field still shown as a literal `{{…}}` placeholder was not recorded: {{reason}}",
"matchedConditions": { "sig": { "field": "reason", "value": "max_lifetime" } },
"expected": "Any field still shown as a literal `{{…}}` placeholder was not recorded: max_lifetime"
}
]
}