diff --git a/.claude/commands b/.claude/commands index 171813b2..9b48783b 160000 --- a/.claude/commands +++ b/.claude/commands @@ -1 +1 @@ -Subproject commit 171813b2cba00c7db4e7ac70fe2d3a7d484bb190 +Subproject commit 9b48783b262ab414d50959696c250fe47bc1c891 diff --git a/internal b/internal index b6c8e277..3b57dea3 160000 --- a/internal +++ b/internal @@ -1 +1 @@ -Subproject commit b6c8e277eb65ea14f97e42f7d4393c48c70b44fa +Subproject commit 3b57dea3196117b7844329ba1be1a75dd24f3b2d diff --git a/rules/guardrails.json b/rules/guardrails.json index 0072a443..f85b2500 100644 --- a/rules/guardrails.json +++ b/rules/guardrails.json @@ -282,5 +282,25 @@ "log_entry", "agent_trace", "stall_probe_check" + ], + "imeLogDerivedEventTypes": [ + "app_install_started", + "app_install_completed", + "app_install_failed", + "app_install_skipped", + "app_download_started", + "app_tracking_summary", + "download_progress", + "do_telemetry", + "script_started", + "script_completed", + "script_failed", + "script_timeout_suspected", + "esp_phase_changed", + "ime_agent_version", + "ime_token_failure", + "ime_user_session_completed", + "ime_process_exited", + "ime_user_token_acquired" ] } diff --git a/rules/schema/guardrails.schema.json b/rules/schema/guardrails.schema.json index 0c173f10..886b80c3 100644 --- a/rules/schema/guardrails.schema.json +++ b/rules/schema/guardrails.schema.json @@ -16,6 +16,7 @@ "blockedFilePrefixes": { "$ref": "#/$defs/stringList" }, "blockedCommandPatterns": { "$ref": "#/$defs/stringList" }, "blockedInterimTriggerEventTypes": { "$ref": "#/$defs/stringList" }, + "imeLogDerivedEventTypes": { "$ref": "#/$defs/stringList" }, "maxCommandLength": { "type": "integer", "minimum": 1, "maximum": 100000 } }, "required": [ diff --git a/rules/scripts/combine.js b/rules/scripts/combine.js index f0701dea..bfac3d8b 100644 --- a/rules/scripts/combine.js +++ b/rules/scripts/combine.js @@ -142,6 +142,7 @@ if (fs.existsSync(guardrailsPath)) { assertStringList(guardrails[key], key); } assertStringList(guardrails.blockedInterimTriggerEventTypes ?? [], 'blockedInterimTriggerEventTypes'); + assertStringList(guardrails.imeLogDerivedEventTypes ?? [], 'imeLogDerivedEventTypes'); if (!Number.isInteger(guardrails.maxCommandLength) || guardrails.maxCommandLength < 1) { fail(`maxCommandLength: expected a positive integer, got ${JSON.stringify(guardrails.maxCommandLength)}`); } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Runtime/DiagnosticsPackageServiceTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Runtime/DiagnosticsPackageServiceTests.cs index 943aac4f..482eeba4 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Runtime/DiagnosticsPackageServiceTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Runtime/DiagnosticsPackageServiceTests.cs @@ -903,5 +903,87 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime NtfsLinks.RemoveLink(link); } } + + // ============================================================= packaging counters + + [Fact] + public void BuildArchiveBytes_records_packaging_counters_by_reason_and_kind_without_paths() + { + using var rig = new Rig(); + File.WriteAllText(Path.Combine(rig.StateFolder, "small.json"), "{}"); + File.WriteAllText(Path.Combine(rig.StateFolder, "big-1.json"), new string('x', 64)); + File.WriteAllText(Path.Combine(rig.StateFolder, "big-2.json"), new string('y', 64)); + var blockedPath = @"C:\Windows\System32\config\SYSTEM"; + + var service = rig.Build(cfg => cfg.DiagnosticsLogPaths.Add( + new AutopilotMonitor.Shared.Models.DiagnosticsLogPath { Path = blockedPath })); + service.Budget = new DiagnosticsBudget { MaxSingleFileBytes = 16, MaxTotalUncompressedBytes = 1L << 30, MaxFileCount = 5000 }; + + Assert.Null(service.LastPackaging); + var bytes = service.BuildArchiveBytes(enrollmentSucceeded: true); + + var stats = service.LastPackaging; + Assert.NotNull(stats); + Assert.True(stats!.Truncated); + Assert.Equal(2, stats.SkippedFiles); + Assert.Equal(2, stats.SkippedByReason["size"]); + Assert.Single(stats.SkippedByReason); + Assert.Equal(1, stats.ProblemsByKind["path-guard"]); + Assert.Single(stats.ProblemsByKind); + Assert.True(stats.IncludedFiles >= 1, "the small file must have been included"); + Assert.True(stats.IncludedBytes >= 2); + + // The ZIP still carries the per-file list; the counters never do. + Assert.NotNull(ReadZipEntryText(bytes, "_TRUNCATED.txt")); + foreach (var key in stats.SkippedByReason.Keys.Concat(stats.ProblemsByKind.Keys)) + { + Assert.DoesNotContain("\\", key); + Assert.DoesNotContain(":", key); + Assert.DoesNotContain("/", key); + } + } + + [Fact] + public void BuildArchiveBytes_reports_a_clean_build_as_not_truncated_with_empty_maps() + { + using var rig = new Rig(); + File.WriteAllText(Path.Combine(rig.StateFolder, "snapshot.json"), "{}"); + + var service = rig.Build(); + service.BuildArchiveBytes(enrollmentSucceeded: true); + + var stats = service.LastPackaging; + Assert.NotNull(stats); + Assert.False(stats!.Truncated); + Assert.Equal(0, stats.SkippedFiles); + Assert.Empty(stats.SkippedByReason); + Assert.Empty(stats.ProblemsByKind); + Assert.True(stats.IncludedFiles >= 1); + } + + [Fact] + public void BuildArchiveBytes_counts_a_rejected_source_folder_as_a_problem() + { + using var rig = new Rig(); + using var outside = new TempDirectory(); + File.WriteAllText(Path.Combine(outside.Path, "snapshot.json"), "{}"); + var link = Path.Combine(rig.Tmp.Path, "StateLink"); + NtfsLinks.CreateJunction(link, outside.Path); + + try + { + var service = rig.Build(sectionFolderOverrides: new Dictionary { ["AgentState"] = link }); + service.BuildArchiveBytes(enrollmentSucceeded: true); + + // The section is walked once per search pattern (11 patterns) but the folder + // failed once — the counter says so. + Assert.Equal(1, service.LastPackaging!.ProblemsByKind["folder-rejected"]); + Assert.False(service.LastPackaging.Truncated); + } + finally + { + NtfsLinks.RemoveLink(link); + } + } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs index 9a410fe4..9eb3f3d8 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs @@ -1701,5 +1701,86 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination rig.PromotionCalls[0].failureType); Assert.Null(rig.PromotionCalls[0].errorCode); } + + // ============================================================= packaging counters on the diag events + + [Fact] + public void Handle_diagnostics_uploaded_carries_packaging_counters_when_the_packager_reported_them() + { + using var rig = new Rig(); + rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build(); + rig.DiagnosticsResult = new DiagnosticsUploadResult + { + BlobName = "diag-blob.zip", + Packaging = new DiagnosticsPackagingStats + { + IncludedFiles = 412, + IncludedBytes = 123456, + SkippedFiles = 3, + SkippedByReason = new Dictionary { ["size"] = 2, ["total"] = 1 }, + ProblemsByKind = new Dictionary { ["copy"] = 1 }, + }, + }; + var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: false); + + rig.Build(cfg).Handle(sender: null!, + Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed)); + + var uploaded = rig.DataOf("diagnostics_uploaded"); + Assert.NotNull(uploaded); + Assert.Equal(true, uploaded!["truncated"]); + Assert.Equal(412, uploaded["includedFiles"]); + Assert.Equal(123456L, uploaded["includedBytes"]); + Assert.Equal(3, uploaded["skippedFiles"]); + var byReason = Assert.IsType>(uploaded["skippedByReason"]); + Assert.Equal(2, byReason["size"]); + Assert.Equal(1, byReason["total"]); + var problems = Assert.IsType>(uploaded["problemsByKind"]); + Assert.Equal(1, problems["copy"]); + // Counters only — no key or value carries a file path. + foreach (var kv in uploaded) + Assert.False(kv.Value is string s && (s.Contains("\\") || s.Contains("/")) && kv.Key != "sasUrlPrefix", + $"payload key {kv.Key} looks like a path: {kv.Value}"); + } + + [Fact] + public void Handle_diagnostics_upload_failed_after_a_build_carries_the_counters_too() + { + using var rig = new Rig(); + rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build(); + rig.DiagnosticsResult = new DiagnosticsUploadResult + { + ErrorCode = "upload_5xx", + Packaging = new DiagnosticsPackagingStats { IncludedFiles = 40, IncludedBytes = 1000 }, + }; + var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: false); + + rig.Build(cfg).Handle(sender: null!, + Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Failed, SessionStage.Failed)); + + var failed = rig.DataOf("diagnostics_upload_failed"); + Assert.NotNull(failed); + Assert.Equal("upload_5xx", (string)failed!["errorCode"]); + Assert.Equal(false, failed["truncated"]); + Assert.Equal(40, failed["includedFiles"]); + Assert.Empty(Assert.IsType>(failed["skippedByReason"])); + } + + [Fact] + public void Handle_diagnostics_events_omit_the_counters_when_no_package_was_built() + { + using var rig = new Rig(); + rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Completed }.Build(); + rig.DiagnosticsResult = new DiagnosticsUploadResult { BlobName = "diag-blob.zip" }; + var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: false); + + rig.Build(cfg).Handle(sender: null!, + Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed)); + + var uploaded = rig.DataOf("diagnostics_uploaded"); + Assert.NotNull(uploaded); + Assert.False(uploaded!.ContainsKey("truncated")); + Assert.False(uploaded.ContainsKey("skippedByReason")); + } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/DiagnosticsPackageService.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/DiagnosticsPackageService.cs index 31ac1d76..379ac5b6 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/DiagnosticsPackageService.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/DiagnosticsPackageService.cs @@ -65,6 +65,44 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime public static DiagnosticsUploadResult SkippedBy(string gate) => new DiagnosticsUploadResult { Skipped = true, ErrorCode = gate }; + + /// + /// Counters of the archive build that produced (or preceded) this result; null when no + /// package was built (configuration gate, build exception). Surfaced into the + /// diagnostics_uploaded / diagnostics_upload_failed events so a truncated or + /// partially failed package is visible without opening the ZIP. + /// + public DiagnosticsPackagingStats Packaging { get; set; } + } + + /// + /// What the packager included, skipped and failed on — as bounded counters only. File paths + /// stay inside the ZIP (package-manifest.txt, _TRUNCATED.txt): they can carry + /// user names, and a skip list has no upper bound while this payload must stay small no + /// matter how many files a cap rejected. + /// + public sealed class DiagnosticsPackagingStats + { + private static readonly IReadOnlyDictionary Empty = new Dictionary(); + + public int IncludedFiles { get; set; } + public long IncludedBytes { get; set; } + + /// Files rejected by a cap or a reparse-point check (the _TRUNCATED.txt list). + public int SkippedFiles { get; set; } + + /// True when at least one file was skipped — the same condition that writes _TRUNCATED.txt. + public bool Truncated => SkippedFiles > 0; + + /// Skip count per reason: size, count, total, reparse, resolved-outside-folder. + public IReadOnlyDictionary SkippedByReason { get; set; } = Empty; + + /// + /// Packaging problems that are not cap skips, per kind: path-guard (configured path + /// refused), no-user-session (user-profile token without a signed-in user), + /// folder-rejected, enumerate, open, copy. + /// + public IReadOnlyDictionary ProblemsByKind { get; set; } = Empty; } /// @@ -134,6 +172,41 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime Skipped.Add(new SkipRecord(path, reason, size)); public bool HasSkips => Skipped.Count > 0; + + // Non-cap problems (guard refusals, enumeration/open/copy failures) by kind — the + // manifest names the files, the event carries only these counts. + public Dictionary Problems { get; } = new Dictionary(StringComparer.Ordinal); + private readonly HashSet _problemKeys = new HashSet(StringComparer.OrdinalIgnoreCase); + + /// + /// Counts one problem of . A section is walked once per search + /// pattern, so a folder-level failure would otherwise be counted once per pattern — + /// pass the folder as to count it once. + /// + public void RecordProblem(string kind, string dedupeKey = null) + { + if (dedupeKey != null && !_problemKeys.Add(kind + "|" + dedupeKey)) return; + Problems.TryGetValue(kind, out var n); + Problems[kind] = n + 1; + } + + public DiagnosticsPackagingStats ToStats() + { + var byReason = new Dictionary(StringComparer.Ordinal); + foreach (var s in Skipped) + { + byReason.TryGetValue(s.Reason, out var n); + byReason[s.Reason] = n + 1; + } + return new DiagnosticsPackagingStats + { + IncludedFiles = FileCount, + IncludedBytes = TotalBytes, + SkippedFiles = Skipped.Count, + SkippedByReason = byReason, + ProblemsByKind = new Dictionary(Problems, StringComparer.Ordinal), + }; + } } private readonly struct SkipRecord @@ -262,6 +335,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime var zipFileName = $"AgentDiagnostics-{_configuration.SessionId}-{timestamp}{suffix}.zip"; var zipBytes = BuildArchiveBytes(enrollmentSucceeded); + var packaging = LastPackaging; _logger.Info($"Diagnostics package created: {zipFileName} ({zipBytes.Length / 1024} KB)"); @@ -276,7 +350,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { var errorCode = uploadUrlResponse?.Message ?? "Failed to get diagnostics upload URL from backend"; _logger.Warning($"Failed to get diagnostics upload URL from backend — skipping upload: {errorCode}"); - return new DiagnosticsUploadResult { ErrorCode = errorCode }; + return new DiagnosticsUploadResult { ErrorCode = errorCode, Packaging = packaging }; } var sasUrlPrefix = BuildSasUrlPrefix(uploadUrlResponse.UploadUrl); @@ -305,6 +379,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime BlobName = persistedBlobName, Destination = uploadUrlResponse.Destination, SasUrlPrefix = sasUrlPrefix, + Packaging = packaging, }; } @@ -313,6 +388,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime Destination = uploadUrlResponse.Destination, SasUrlPrefix = sasUrlPrefix, ErrorCode = uploadErrorCode, + Packaging = packaging, }; } catch (Exception ex) @@ -351,6 +427,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime // INSIDE the ZIP (field case: sessions a11102f4 / 3ae7528b, missing evtx undiagnosable). private StringBuilder _manifest; + /// Counters of the most recent ; null before the first build. + internal DiagnosticsPackagingStats LastPackaging { get; private set; } + private void ManifestLine(string text) { _manifest?.AppendLine($"[{DateTime.UtcNow:HH:mm:ss.fff}Z] {text}"); @@ -391,6 +470,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { _logger.Warning($"Diagnostics path blocked by guard: {entry.Path}"); ManifestLine($"BLOCKED (path guard): {entry.Path}"); + tracker.RecordProblem("path-guard"); continue; } var expandedPath = UserProfileResolver.ExpandCustomTokens(entry.Path); @@ -398,6 +478,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { _logger.Warning($"Diagnostics path skipped (no user session for token): {entry.Path}"); ManifestLine($"SKIPPED (no user session for token): {entry.Path}"); + tracker.RecordProblem("no-user-session"); continue; } var folder = Path.GetDirectoryName(expandedPath); @@ -427,6 +508,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime WriteTruncatedMarker(archive, tracker); } + LastPackaging = tracker.ToStats(); return ms.ToArray(); } } @@ -498,6 +580,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { _logger.Warning($"Built-in diagnostics section skipped (no user session for token): {section.Id}"); ManifestLine($"BUILT-IN SKIPPED (no user session for token): {section.Id} '{section.SourceFolder}'"); + tracker.RecordProblem("no-user-session"); return; } @@ -584,6 +667,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { _logger.Warning($"Diagnostics source folder rejected ({folderRejection}): {sourceFolder}"); ManifestLine($"FOLDER REJECTED ({folderRejection}): {sourceFolder} (pattern '{searchPattern}')"); + tracker.RecordProblem("folder-rejected", sourceFolder); } return; } @@ -597,6 +681,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { _logger.Warning($"Failed to enumerate log files in {sourceFolder}: {ex.Message}"); ManifestLine($"ENUMERATION FAILED: {sourceFolder} (pattern '{searchPattern}'): {ex.Message}"); + tracker.RecordProblem("enumerate", sourceFolder); return; } @@ -681,6 +766,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { _logger.Warning($"Failed to add log file to diagnostics package: {file} - {ex.Message}"); ManifestLine($"FAILED (copy): {file}: {ex.Message}"); + tracker.RecordProblem("copy"); } finally { @@ -720,6 +806,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime default: _logger.Warning($"Failed to open {file}: {rejection}"); ManifestLine($"FAILED (open): {file}: {rejection}"); + tracker.RecordProblem("open"); break; } return null; diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs index c969fcb5..538c4af6 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs @@ -1007,6 +1007,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination if (result != null && result.Success) { _logger.Info($"EnrollmentTerminationHandler: diagnostics uploaded (blob={result.BlobName})."); + var data = new Dictionary + { + { "blobName", result.BlobName ?? string.Empty }, + // Tells the backend which storage the blob landed in so it can + // stamp Session.DiagnosticsBlobDestination and route downloads + // even after a future tenant destination switch. Empty when the + // backend predates the field (agent falls through harmlessly). + { "destination", result.Destination ?? string.Empty }, + { "sasUrlPrefix", result.SasUrlPrefix ?? string.Empty }, + }; + AppendPackagingStats(data, result.Packaging); EmitEventSafe(new EnrollmentEvent { SessionId = _configuration.SessionId, @@ -1015,17 +1026,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination Severity = EventSeverity.Info, Source = "EnrollmentTerminationHandler", Phase = EnrollmentPhase.Unknown, - Message = $"Diagnostics package uploaded ({result.BlobName}).", - Data = new Dictionary - { - { "blobName", result.BlobName ?? string.Empty }, - // Tells the backend which storage the blob landed in so it can - // stamp Session.DiagnosticsBlobDestination and route downloads - // even after a future tenant destination switch. Empty when the - // backend predates the field (agent falls through harmlessly). - { "destination", result.Destination ?? string.Empty }, - { "sasUrlPrefix", result.SasUrlPrefix ?? string.Empty }, - }, + Message = $"Diagnostics package uploaded ({result.BlobName})." + + (result.Packaging != null && result.Packaging.Truncated + ? $" Truncated: {result.Packaging.SkippedFiles} file(s) skipped by the packaging caps." + : string.Empty), + Data = data, ImmediateUpload = true, }); } @@ -1033,6 +1038,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination { var errorCode = result?.ErrorCode ?? "null-result"; _logger.Warning($"EnrollmentTerminationHandler: diagnostics upload failed: {errorCode}."); + var data = new Dictionary + { + { "errorCode", errorCode }, + { "blobName", result?.BlobName ?? string.Empty }, + { "destination", result?.Destination ?? string.Empty }, + }; + AppendPackagingStats(data, result?.Packaging); EmitEventSafe(new EnrollmentEvent { SessionId = _configuration.SessionId, @@ -1042,17 +1054,30 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination Source = "EnrollmentTerminationHandler", Phase = EnrollmentPhase.Unknown, Message = $"Diagnostics upload failed: {errorCode}.", - Data = new Dictionary - { - { "errorCode", errorCode }, - { "blobName", result?.BlobName ?? string.Empty }, - { "destination", result?.Destination ?? string.Empty }, - }, + Data = data, ImmediateUpload = true, }); } } + /// + /// Adds the packager's counters to a diagnostics event payload — counts only, never a + /// file path (paths live in the ZIP's manifest and may carry user names). Absent when no + /// package was built, so consumers treat the keys as optional. + /// + private static void AppendPackagingStats(Dictionary data, DiagnosticsPackagingStats packaging) + { + if (packaging == null) return; + data["truncated"] = packaging.Truncated; + data["includedFiles"] = packaging.IncludedFiles; + data["includedBytes"] = packaging.IncludedBytes; + data["skippedFiles"] = packaging.SkippedFiles; + data["skippedByReason"] = new Dictionary( + (IDictionary)packaging.SkippedByReason ?? new Dictionary(), StringComparer.Ordinal); + data["problemsByKind"] = new Dictionary( + (IDictionary)packaging.ProblemsByKind ?? new Dictionary(), StringComparer.Ordinal); + } + private void WriteEnrollmentCompleteMarker(EnrollmentTerminatedEventArgs args) { // On WhiteGlove Part 1 exit we keep the session alive for Part 2 — DO NOT write the diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/BuiltInRuleCatalogPolicyTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/BuiltInRuleCatalogPolicyTests.cs index 90ea36a3..464d9f9c 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/BuiltInRuleCatalogPolicyTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/BuiltInRuleCatalogPolicyTests.cs @@ -53,4 +53,94 @@ public class BuiltInRuleCatalogPolicyTests Assert.True(duplicates.Count == 0, $"Duplicate built-in gather rule IDs: {string.Join(", ", duplicates)}"); } + + /// + /// Built-in rules that assert an ABSENCE (required not_exists) of an event the agent derives + /// from the IME logs, and deliberately do NOT carry the ime_tracker_degraded coverage gate. + /// Every entry is a decision with a reason; adding one here is the way to ship such a rule. + /// + private static readonly IReadOnlyDictionary AbsenceWithoutCoverageGate = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["ANALYZE-ID-004"] = + "A hybrid session whose user never obtained an Entra token must not lose its finding because " + + "the tracker skipped the remaining patterns on one line; the tracker state is shown next to " + + "the finding in get_session_summary (coverage) instead (plan session-coverage-and-absence-audit).", + }; + + [Fact] + public void Required_not_exists_on_ime_derived_events_is_coverage_gated_or_documented() + { + // "Absence proves nothing": a required not_exists on an IME-log-derived event evaluates + // to true both when the event never happened and when the IME log tracker skipped the + // line that carried it (ime_tracker_degraded). A built-in rule that asserts such an + // absence either suppresses itself on degraded sessions (precondition + // ime_tracker_degraded not_exists) or is listed above with the reason it does not. + // The IME-derived list is the guardrails.json mirror the MCP validate_rule lint reads, + // so catalog and pre-flight judge by one list. + var guardrailsPath = Path.Combine(FindRepoRoot(), "rules", "guardrails.json"); + var json = Newtonsoft.Json.Linq.JObject.Parse(File.ReadAllText(guardrailsPath)); + var imeDerived = new HashSet( + json["imeLogDerivedEventTypes"]!.ToObject()!, StringComparer.OrdinalIgnoreCase); + Assert.Contains("ime_user_token_acquired", imeDerived); + + var offenders = new List(); + foreach (var rule in BuiltInAnalyzeRules.GetAll()) + { + var assertsImeAbsence = rule.Conditions.Any(c => + c.Required + && string.Equals(c.Operator, "not_exists", StringComparison.OrdinalIgnoreCase) + && string.IsNullOrEmpty(c.DataField) + && !string.IsNullOrEmpty(c.EventType) + && imeDerived.Contains(c.EventType)); + if (!assertsImeAbsence) continue; + + var gated = rule.Preconditions.Any(p => + string.Equals(p.EventType, "ime_tracker_degraded", StringComparison.OrdinalIgnoreCase) + && string.Equals(p.Operator, "not_exists", StringComparison.OrdinalIgnoreCase) + && string.IsNullOrEmpty(p.DataField)); + if (gated || AbsenceWithoutCoverageGate.ContainsKey(rule.RuleId)) continue; + offenders.Add(rule.RuleId); + } + + Assert.True(offenders.Count == 0, + "Built-in rules assert the absence of an IME-log-derived event without the " + + "ime_tracker_degraded coverage precondition and without a documented exception: " + + string.Join(", ", offenders)); + + // The exception list must not outlive the rules it excuses. + var catalogIds = new HashSet(BuiltInAnalyzeRules.GetAll().Select(r => r.RuleId), StringComparer.OrdinalIgnoreCase); + foreach (var excused in AbsenceWithoutCoverageGate.Keys) + Assert.True(catalogIds.Contains(excused), $"Exception list names a rule that is not in the catalog: {excused}"); + } + + [Fact] + public void Ime_derived_list_names_only_real_event_types_and_the_tracker_event_exists() + { + var guardrailsPath = Path.Combine(FindRepoRoot(), "rules", "guardrails.json"); + var json = Newtonsoft.Json.Linq.JObject.Parse(File.ReadAllText(guardrailsPath)); + var imeDerived = json["imeLogDerivedEventTypes"]!.ToObject()!; + + var known = new HashSet( + typeof(AutopilotMonitor.Shared.Constants.EventTypes) + .GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Static) + .Where(f => f.IsLiteral && f.FieldType == typeof(string)) + .Select(f => (string)f.GetRawConstantValue()!), + StringComparer.Ordinal); + + var phantom = imeDerived.Where(t => !known.Contains(t)).ToList(); + Assert.True(phantom.Count == 0, $"imeLogDerivedEventTypes lists unknown event types: {string.Join(", ", phantom)}"); + Assert.Contains(AutopilotMonitor.Shared.Constants.EventTypes.ImeTrackerDegraded, known); + } + + private static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AutopilotMonitor.sln"))) + { + dir = dir.Parent; + } + Assert.NotNull(dir); + return dir!.FullName; + } } diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/event-types-drift.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/event-types-drift.test.ts index fcedc1ea..1686a5e0 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/event-types-drift.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/event-types-drift.test.ts @@ -14,6 +14,7 @@ import { dirname, join } from 'node:path'; import { existsSync, readFileSync } from 'node:fs'; import { EVENT_TYPES_CATALOG, INTERNAL_EVENT_TYPES, ALL_EVENT_TYPES } from '../resource-catalog.js'; import { EXCLUDED_EVENT_TYPES, KEY_EVENT_TYPES, PHASE_EVENT_TYPES } from '../tools/sessions.js'; +import { COVERAGE_EVENT_TYPES } from '../session-coverage.js'; /** Walk up from this test file to the repo root (marked by AutopilotMonitor.sln). */ function findConstantsCs(): string | null { @@ -95,10 +96,11 @@ describe('get_session_summary event-type sets ⊆ canonical catalog', () => { // key-ranks. They bypass the C#-drift test above (which only checks the // public catalog), so guard them directly against ALL_EVENT_TYPES. const known = new Set(ALL_EVENT_TYPES); - const cases: Array<[string, Set]> = [ + const cases: Array<[string, ReadonlySet]> = [ ['EXCLUDED_EVENT_TYPES', EXCLUDED_EVENT_TYPES], ['KEY_EVENT_TYPES', KEY_EVENT_TYPES], ['PHASE_EVENT_TYPES', PHASE_EVENT_TYPES], + ['COVERAGE_EVENT_TYPES', COVERAGE_EVENT_TYPES], ]; for (const [name, set] of cases) { it(`${name} contains only real event types`, () => { diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/lean-defaults.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/lean-defaults.test.ts index b8dbd79a..0a46dec5 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/lean-defaults.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/lean-defaults.test.ts @@ -166,6 +166,31 @@ describe('first-page defaults', () => { expect(SUMMARY_EVENT_FIELDS).toContain('data.rejectedSourceTimestamp'); // The error-code keys and their enriched siblings travel as slices too (errorCode/errorText). expect(SUMMARY_EVENT_FIELDS.split(',')).toEqual(expect.arrayContaining(['data.errorCode', 'data.errorCodeInfo', 'data.hresultSymbol', 'data.exitCodeInfo'])); + // The coverage block's payload keys travel as slices too (session-coverage.ts). + expect(SUMMARY_EVENT_FIELDS.split(',')).toEqual(expect.arrayContaining(['data.firstSkippedPattern', 'data.bootToAgentStartSeconds', 'data.collector', 'data.skippedByReason'])); + }); + + it('get_session_summary places coverage right after overview and reports gaps', async () => { + const handler = handlerFor('get_session_summary'); + stubFetchCapture({ + success: true, + session: { sessionId: SESSION, tenantId: 't', status: 'Succeeded', startedAt: '2026-09-01T00:00:00Z' }, + events: [ + { eventType: 'agent_started', severity: 'Info', timestamp: '2026-09-01T00:00:05Z', data: {} }, + { eventType: 'ime_tracker_degraded', severity: 'Warning', timestamp: '2026-09-01T00:10:00Z', data: { file: 'AppWorkload.log', lineBudgetBreaks: 1, firstSkippedPattern: 'IME-NO-ACTION' } }, + ], + results: [], + annotations: [], + count: 0, + }); + + const result = await runWithCaller(GA, () => handler({ sessionId: SESSION }, extra)); + const body = resultJson(result); + expect(Object.keys(body).slice(0, 3)).toEqual(['overview', 'coverage', 'keyEvents']); + const coverage = body.coverage as { imeTracker: { degraded: boolean }; gaps: string[]; window: { observedFrom: string } }; + expect(coverage.window.observedFrom).toBe('2026-09-01T00:00:05Z'); + expect(coverage.imeTracker.degraded).toBe(true); + expect(coverage.gaps).toEqual([expect.stringContaining('IME-NO-ACTION')]); }); }); diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts index 9e1c7268..8e70bafc 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/rule-validation.test.ts @@ -4,7 +4,7 @@ * schema-valid-but-silently-dead rules. */ import { describe, it, expect } from 'vitest'; -import { validateRuleDraft, type ValidationFinding } from '../rule-validation.js'; +import { validateRuleDraft, IME_LOG_DERIVED_EVENT_TYPES, type ValidationFinding } from '../rule-validation.js'; const errors = (findings: ValidationFinding[]) => findings.filter((f) => f.level === 'error').map((f) => f.message); const warnings = (findings: ValidationFinding[]) => findings.filter((f) => f.level === 'warning').map((f) => f.message); @@ -523,3 +523,52 @@ describe('reserved built-in namespace', () => { expect(warnings(g.findings).some((m) => m.includes('reserved built-in namespace'))).toBe(false); }); }); + +describe('absence lint (IME-log-derived not_exists)', () => { + const absenceRule = (extra: Record = {}): Record => ({ + ...validAnalyze(), + ruleId: 'ANALYZE-CUSTOM-004', + conditions: [ + { signal: 'desktop', source: 'event_type', eventType: 'desktop_arrived', operator: 'exists', value: '', required: true }, + { signal: 'no_token', source: 'event_type', eventType: 'ime_user_token_acquired', operator: 'not_exists', value: '', required: true }, + ], + ...extra, + }); + + it('warns on a required not_exists of an IME-log-derived event without the tracker precondition', () => { + const r = validateRuleDraft(absenceRule()); + expect(r.valid).toBe(true); + const w = warnings(r.findings); + expect(w.some((m) => m.includes('conditions[1]') && m.includes('ime_user_token_acquired') && m.includes('ime_tracker_degraded'))).toBe(true); + }); + + it('is silent once the ime_tracker_degraded not_exists precondition is present', () => { + const r = validateRuleDraft(absenceRule({ + preconditions: [{ source: 'event_data', eventType: 'ime_tracker_degraded', operator: 'not_exists' }], + })); + expect(warnings(r.findings).some((m) => m.includes('ime_tracker_degraded'))).toBe(false); + }); + + it('does not fire for optional conditions, field-level absence, or non-IME event types', () => { + const optional = absenceRule(); + (optional.conditions as Array>)[1].required = false; + expect(warnings(validateRuleDraft(optional).findings).some((m) => m.includes('IME-log-derived'))).toBe(false); + + const fieldLevel = absenceRule(); + (fieldLevel.conditions as Array>)[1] = { + signal: 'no_field', source: 'event_data', eventType: 'app_install_failed', dataField: 'appId', operator: 'not_exists', value: '', required: true, + }; + expect(warnings(validateRuleDraft(fieldLevel).findings).some((m) => m.includes('IME-log-derived'))).toBe(false); + + const terminal = absenceRule(); + (terminal.conditions as Array>)[1].eventType = 'enrollment_complete'; + expect(warnings(validateRuleDraft(terminal).findings).some((m) => m.includes('IME-log-derived'))).toBe(false); + }); + + it('reads the IME-derived list from the baked guardrails mirror', () => { + expect(IME_LOG_DERIVED_EVENT_TYPES.has('ime_user_token_acquired')).toBe(true); + expect(IME_LOG_DERIVED_EVENT_TYPES.has('app_install_failed')).toBe(true); + expect(IME_LOG_DERIVED_EVENT_TYPES.has('enrollment_complete')).toBe(false); + expect(IME_LOG_DERIVED_EVENT_TYPES.size).toBeGreaterThan(10); + }); +}); diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts new file mode 100644 index 00000000..bc286d3e --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts @@ -0,0 +1,277 @@ +/** + * Observation-coverage block of get_session_summary: a pure fold over the health/lifecycle + * events the agent already emits. Every branch here mirrors one agent emit site — the payload + * keys are the ones the agent writes (see session-coverage.ts), typed loosely because half of + * them arrive as strings (Dictionary posts) and half as numbers. + */ +import { describe, it, expect } from 'vitest'; +import { buildSessionCoverage, COVERAGE_EVENT_FIELDS, COVERAGE_EVENT_TYPES } from '../session-coverage.js'; +import { SUMMARY_EVENT_FIELDS } from '../tools/shared.js'; + +const T0 = Date.UTC(2026, 8, 3, 15, 0, 0); +const at = (offsetSec: number): string => new Date(T0 + offsetSec * 1000).toISOString(); + +const ev = (eventType: string, offsetSec: number, data: Record = {}, extra: Record = {}) => ({ + eventType, + timestamp: at(offsetSec), + severity: 'Info', + data, + ...extra, +}); + +const SESSION = { startedAt: at(0) }; + +describe('buildSessionCoverage — clean session', () => { + it('reports full coverage with no gaps', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 5, { previousExitType: '' }), + ev('esp_phase_changed', 60), + ev('diagnostics_uploaded', 900, { blobName: 'x.zip', destination: 'Hosted' }), + ]); + expect(c.gaps).toEqual([]); + expect(c.window.observedFrom).toBe(at(5)); + expect(c.window.startedAtBackdatedSeconds).toBe(5); + expect(c.window.agentStarts).toBe(1); + expect(c.window.previousExitTypes).toEqual([]); + expect(c.window.lateStart).toBeNull(); + expect(c.window.historicReplaySuppressed).toBeNull(); + expect(c.imeTracker).toEqual({ degraded: false }); + expect(c.collectors).toEqual([]); + expect(c.upload).toEqual({ spoolPressure: false, poisonedItems: 0, blocked: false, backpressureEpisodes: 0, diskSpaceLow: false }); + // Older agents report no packaging counters — status only, no truncated flag. + expect(c.diagnostics).toEqual({ status: 'uploaded' }); + }); + + it('ignores events outside the coverage set and tolerates missing data', () => { + const c = buildSessionCoverage({ startedAt: null }, [ + { eventType: 'app_install_failed', timestamp: at(1) }, + { eventType: 'agent_started', timestamp: at(2) }, + ]); + expect(c.window.sessionStartedAt).toBeNull(); + expect(c.window.startedAtBackdatedSeconds).toBeNull(); + expect(c.gaps).toEqual([]); + }); +}); + +describe('buildSessionCoverage — observation window', () => { + it('flags a missing agent_started', () => { + const c = buildSessionCoverage(SESSION, [ev('esp_phase_changed', 10)]); + expect(c.window.observedFrom).toBeNull(); + expect(c.diagnostics.status).toBe('not_collected'); + expect(c.gaps).toHaveLength(1); + expect(c.gaps[0]).toContain('No agent_started'); + }); + + it('flags a back-dated session start (IME replay reconstruction) beyond a minute', () => { + const c = buildSessionCoverage({ startedAt: at(-3600) }, [ev('agent_started', 0)]); + expect(c.window.startedAtBackdatedSeconds).toBe(3600); + expect(c.gaps.some((g) => g.includes('3600 s before the first agent_started'))).toBe(true); + }); + + it('never reports a negative back-date when startedAt is after the first start', () => { + const c = buildSessionCoverage({ startedAt: at(100) }, [ev('agent_started', 0)]); + expect(c.window.startedAtBackdatedSeconds).toBe(0); + expect(c.gaps).toEqual([]); + }); + + it('counts restarts and their previous exit types, earliest start first regardless of input order', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 700, { previousExitType: 'Crash' }), + ev('agent_started', 0), + ev('agent_started', 300, { previousExitType: 'Reboot' }), + ev('agent_started', 900, { previousExitType: 'Crash' }), + ]); + expect(c.window.observedFrom).toBe(at(0)); + expect(c.window.agentStarts).toBe(4); + expect(c.window.previousExitTypes).toEqual(['Reboot', 'Crash']); + expect(c.gaps).toEqual([expect.stringContaining('started 4 times (previous exit: Reboot, Crash)')]); + }); + + it('surfaces agent_late_start even at Info severity (successful session) with the agent note', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('agent_late_start', 200, { bootToAgentStartSeconds: '1800', agentUptimeSeconds: 120, outcome: 'Succeeded', note: 'Agent arrived after the work was done.' }), + ]); + expect(c.window.lateStart).toEqual({ bootToAgentStartSeconds: 1800, agentUptimeSeconds: 120, outcome: 'Succeeded', note: 'Agent arrived after the work was done.' }); + expect(c.gaps).toEqual([expect.stringMatching(/^Low observation coverage \(agent started 1800 s after boot, ran 120 s\): Agent arrived after the work was done\.$/)]); + }); + + it('surfaces suppressed historic IME replay with the earliest rejected timestamp', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('historic_ime_replay_detected', 1, { decision: 'historic_ime_replay_suppressed', earliestRejectedSourceTimestamp: '2026-08-01T00:00:00.0000000Z' }), + ]); + expect(c.window.historicReplaySuppressed).toEqual({ earliestRejectedSourceTimestamp: '2026-08-01T00:00:00.0000000Z' }); + expect(c.gaps).toEqual([expect.stringContaining('earliest 2026-08-01T00:00:00.0000000Z')]); + }); +}); + +describe('buildSessionCoverage — IME log tracker', () => { + it('projects ime_tracker_degraded with a calibrated budget-break line', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('ime_tracker_degraded', 50, { file: 'AgentExecutor.log', oversizedLines: 0, regexTimeouts: 0, lineBudgetBreaks: 1, unanchoredPatterns: 0, firstSkippedPattern: 'PS-SCRIPT-GENERATED' }, { severity: 'Warning' }), + ]); + expect(c.imeTracker).toEqual({ + degraded: true, + file: 'AgentExecutor.log', + firstSkippedPattern: 'PS-SCRIPT-GENERATED', + lineBudgetBreaks: 1, + regexTimeouts: 0, + oversizedLines: 0, + unanchoredPatterns: 0, + }); + expect(c.gaps).toEqual([ + 'IME log tracker skipped the remaining patterns on 1 line(s) (first skipped: PS-SCRIPT-GENERATED) in AgentExecutor.log: the absence of an IME-derived event is not fully provable for this session.', + ]); + }); + + it('prefers whole-session totals from ime_pattern_hits over the first-pass counters', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('ime_tracker_degraded', 50, { file: 'AppWorkload.log', lineBudgetBreaks: 1, regexTimeouts: 0, oversizedLines: 0 }), + ev('ime_pattern_hits', 900, { linesRead: 12000, lineBudgetBreaks: 7, regexTimeouts: 2, oversizedLines: 1 }, { severity: 'Debug' }), + ]); + expect(c.imeTracker.sessionTotals).toEqual({ linesRead: 12000, lineBudgetBreaks: 7, regexTimeouts: 2, oversizedLines: 1 }); + expect(c.gaps[0]).toContain('skipped the remaining patterns on 7 line(s)'); + expect(c.gaps[0]).toContain('gave up on 2 line(s) after a regex timeout'); + expect(c.gaps[0]).toContain('dropped 1 oversized line(s)'); + }); + + it('keeps session totals on a healthy tracker without declaring a gap', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('ime_pattern_hits', 900, { linesRead: 800, lineBudgetBreaks: 0, regexTimeouts: 0, oversizedLines: 0 }), + ]); + expect(c.imeTracker).toEqual({ degraded: false, sessionTotals: { linesRead: 800, lineBudgetBreaks: 0, regexTimeouts: 0, oversizedLines: 0 } }); + expect(c.gaps).toEqual([]); + }); +}); + +describe('buildSessionCoverage — collectors, upload, disk', () => { + it('deduplicates collector_degraded by (collector, reason) and keeps the first occurrence', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('collector_degraded', 20, { collector: 'HelloTracker', reason: 'watcher_arm_failed', errorType: 'UnauthorizedAccessException' }, { source: 'HelloTracker' }), + ev('collector_degraded', 25, { collector: 'HelloTracker', reason: 'watcher_arm_failed' }), + ev('collector_degraded', 30, { collector: 'WindowsUpdateTracker', reason: 'subscription_failed' }), + ]); + expect(c.collectors).toEqual([ + { collector: 'HelloTracker', reason: 'watcher_arm_failed', errorType: 'UnauthorizedAccessException', at: at(20) }, + { collector: 'WindowsUpdateTracker', reason: 'subscription_failed', errorType: null, at: at(30) }, + ]); + expect(c.gaps).toEqual([ + expect.stringContaining('Collector HelloTracker degraded (watcher_arm_failed, UnauthorizedAccessException)'), + expect.stringContaining('Collector WindowsUpdateTracker degraded (subscription_failed)'), + ]); + }); + + it('folds spool pressure, poison (summed itemCount), block, backpressure and low disk', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('spool_pressure_detected', 100, { pendingItemCount: 2400, pendingBytes: 6000000 }), + ev('telemetry_upload_poisoned', 110, { kind: 'Oversize', itemCount: 1 }), + ev('telemetry_upload_poisoned', 120, { kind: 'BackendDeclared', itemCount: '3' }), + ev('telemetry_upload_blocked', 130, { reason: 'permanent 4xx' }), + ev('ingress_backpressure', 140, { origin: 'ime' }), + ev('ingress_backpressure', 141, { origin: 'ime' }), + ev('disk_space_low', 150, { disk_free_gb: 1.4, disk_total_gb: 256, threshold_gb: 2 }), + ]); + expect(c.upload).toEqual({ spoolPressure: true, poisonedItems: 4, blocked: true, backpressureEpisodes: 2, diskSpaceLow: true }); + expect(c.gaps).toEqual([ + expect.stringContaining('Telemetry spool pressure (2400 items pending)'), + expect.stringContaining('4 telemetry item(s) were quarantined as poison and never uploaded (Oversize, BackendDeclared)'), + expect.stringContaining('Telemetry upload was blocked'), + expect.stringContaining('System drive ran low on space (1.4 GB free)'), + ]); + }); + + it('counts a poison event without itemCount as one item', () => { + const c = buildSessionCoverage(SESSION, [ev('agent_started', 0), ev('telemetry_upload_poisoned', 1, {})]); + expect(c.upload.poisonedItems).toBe(1); + }); +}); + +describe('buildSessionCoverage — diagnostics package', () => { + it('reports packaging counters and a truncation gap (no paths)', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('diagnostics_uploaded', 900, { + blobName: 'x.zip', + truncated: 'true', + includedFiles: 412, + includedBytes: 123456789, + skippedFiles: 3, + skippedByReason: { size: 2, total: 1, reparse: 0 }, + problemsByKind: { copy: 1 }, + }), + ]); + expect(c.diagnostics).toEqual({ + status: 'uploaded', + truncated: true, + includedFiles: 412, + skippedFiles: 3, + skippedByReason: { size: 2, total: 1 }, + problemsByKind: { copy: 1 }, + }); + expect(c.gaps).toEqual([ + 'Diagnostics ZIP is truncated: 3 file(s) skipped by the packaging caps (size: 2, total: 1); the full list is in _TRUNCATED.txt inside the ZIP.', + 'Diagnostics packaging hit problems (copy: 1): those files are missing from the ZIP; package-manifest.txt names them.', + ]); + }); + + it('reports a complete package with counters as no gap', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('diagnostics_uploaded', 900, { truncated: false, includedFiles: 40, includedBytes: 1000, skippedFiles: 0, skippedByReason: {}, problemsByKind: {} }), + ]); + expect(c.diagnostics).toEqual({ status: 'uploaded', truncated: false, includedFiles: 40, skippedFiles: 0 }); + expect(c.gaps).toEqual([]); + }); + + it('distinguishes a configuration skip from an upload failure', () => { + const skipped = buildSessionCoverage(SESSION, [ev('agent_started', 0), ev('diagnostics_upload_failed', 900, { errorCode: 'diagnostics_skipped_mode_off' })]); + expect(skipped.diagnostics).toEqual({ status: 'skipped', errorCode: 'diagnostics_skipped_mode_off' }); + expect(skipped.gaps).toEqual([expect.stringContaining('skipped by configuration (diagnostics_skipped_mode_off)')]); + + const failed = buildSessionCoverage(SESSION, [ev('agent_started', 0), ev('diagnostics_upload_failed', 900, { errorCode: 'upload_5xx' })]); + expect(failed.diagnostics).toEqual({ status: 'failed', errorCode: 'upload_5xx' }); + expect(failed.gaps).toEqual([expect.stringContaining('Diagnostics upload failed (upload_5xx)')]); + }); + + it('lets a later upload win over an earlier failure (retry) and flags a collection that never finished', () => { + const retried = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('diagnostics_upload_failed', 800, { errorCode: 'upload_5xx' }), + ev('diagnostics_uploaded', 900, { blobName: 'x.zip' }), + ]); + expect(retried.diagnostics.status).toBe('uploaded'); + expect(retried.gaps).toEqual([]); + + const died = buildSessionCoverage(SESSION, [ev('agent_started', 0), ev('diagnostics_collecting', 800, { mode: 'Always' })]); + expect(died.diagnostics.status).toBe('not_collected'); + expect(died.gaps).toEqual([expect.stringContaining('neither an upload nor a failure was reported')]); + }); +}); + +describe('coverage wiring', () => { + it('SUMMARY_EVENT_FIELDS carries every coverage slice, as data. entries only', () => { + const fields = SUMMARY_EVENT_FIELDS.split(','); + for (const f of COVERAGE_EVENT_FIELDS.split(',')) { + expect(f.startsWith('data.')).toBe(true); + expect(fields).toContain(f); + } + expect(fields).not.toContain('data'); + }); + + it('every coverage event type is read by the builder (no dead member)', () => { + // A type in the set that the fold never consults would be a maintenance trap: keep the + // set equal to what the builder actually switches on. + const consulted = [ + 'agent_started', 'agent_late_start', 'historic_ime_replay_detected', 'ime_tracker_degraded', 'ime_pattern_hits', + 'collector_degraded', 'spool_pressure_detected', 'telemetry_upload_poisoned', 'telemetry_upload_blocked', + 'ingress_backpressure', 'disk_space_low', 'diagnostics_collecting', 'diagnostics_uploaded', 'diagnostics_upload_failed', + ]; + expect([...COVERAGE_EVENT_TYPES].sort()).toEqual([...consulted].sort()); + }); +}); diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index 35f4a3cd..1e0a4b03 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -138,7 +138,7 @@ export interface AdminConfiguration { maxDiagnosticsDownloadSizeMB: number; /** Timeout in seconds for the entire diagnostics download+stream operation. 0 = no timeout. Default: 120 seconds. */ diagnosticsDownloadTimeoutSeconds: number; - /** Controls who can access the remote MCP server. "Disabled" = MCP off, "WhitelistOnly" = GlobalAdmins + McpUsers table (default), "AllMembers" = any authenticated user. */ + /** Controls who can access the remote MCP server (evaluated by McpUserService.IsAllowedAsync). "Disabled" = MCP off for everyone, platform roles included. "WhitelistOnly" = platform roles, delegated (MSP) admins and enabled McpUsers rows (default). "AllMembers" = the same grants plus every account holding an effective member role (Admin / Operator / Viewer) in the tenant its token was issued for — never "any authenticated token": an employee without a role, or a tenant that never onboarded, is denied. An explicitly disabled McpUsers row denies under every policy. */ mcpAccessPolicy: string; /** NVD API key for higher rate limits (50 req/30s vs 5 req/30s without key). Free registration at https://nvd.nist.gov/developers/request-an-api-key null = operate without API key (slower, still functional). */ nvdApiKey: string; diff --git a/src/McpServer/autopilot-monitor-mcp/src/mcp-server-factory.ts b/src/McpServer/autopilot-monitor-mcp/src/mcp-server-factory.ts index bf22aaad..541347a0 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/mcp-server-factory.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/mcp-server-factory.ts @@ -60,7 +60,7 @@ export function buildInstructions(deps: ServerDeps, ga: boolean, strictGa: boole return [ headline, '', - 'Investigating one session: call get_session_summary FIRST (status, filtered timeline, stats, rule analysis in one call), then drill in.', + 'Investigating one session: call get_session_summary FIRST (status, observation coverage, filtered timeline, stats, rule analysis in one call), then drill in. Read coverage.gaps before treating a missing event as evidence — the agent watches only from its own start, and a degraded tracker or collector makes an absence unprovable.', ...(deps.docs ? ['Product questions ("how do I…", "what does X mean", "where is my data stored"): use search_docs — the ' + 'published customer documentation. search_knowledge is a DIFFERENT corpus (analysis rules and IME log ' + diff --git a/src/McpServer/autopilot-monitor-mcp/src/prompts.ts b/src/McpServer/autopilot-monitor-mcp/src/prompts.ts index 354c8b8f..d5a95f59 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/prompts.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/prompts.ts @@ -34,7 +34,9 @@ export function registerPrompts(server: McpServer, ga: boolean, strictGa: boolea `Investigate enrollment session ${sessionId} and explain why it ended the way it did.\n\n` + 'Follow this order:\n' + `1. Call get_session_summary(sessionId="${sessionId}") first — it gives status, the ` + - 'noise-filtered key-event timeline, aggregate stats, and any rule analysis in one shot.\n' + + 'observation coverage (coverage.gaps: what the agent could not see), the noise-filtered ' + + 'key-event timeline, aggregate stats, and any rule analysis in one shot. Never treat a ' + + 'missing event as proof while coverage.gaps names a relevant blind spot.\n' + '2. If the summary shows errors or a failure, escalate: use search_events ' + '(hybrid; depth="deep" if the fast pass is thin) for the failing area, then get_session_events for the full ' + 'chronological stream around the first error. Follow nextLink until the relevant window is covered.\n' + @@ -78,7 +80,7 @@ export function registerPrompts(server: McpServer, ga: boolean, strictGa: boolea '.\n\n' + 'Do a correlated client↔backend analysis:\n' + `1. Call get_session_summary(sessionId="${sessionId}"${tenantId ? `, tenantId="${tenantId}"` : ''}) ` + - 'first — status, noise-filtered timeline, stats, rule analysis in one shot.\n' + + 'first — status, observation coverage (coverage.gaps), noise-filtered timeline, stats, rule analysis in one shot.\n' + `2. Call get_session_diagnostics(sessionId="${sessionId}"${tenantId ? `, tenantId="${tenantId}"` : ''}). ` + 'If available=true, DOWNLOAD the ZIP from downloadUrl using your local tools (no auth header — ' + 'it is a short-lived signed ticket), unzip it locally, and read files in the priority order of the diag_zip_layout resource (get_resource(name="diag_zip_layout")): ' + diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts index 7b7d9091..d0893562 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts @@ -205,6 +205,18 @@ export const RULE_AUTHORING_GUIDE = { 'Optional gate BEFORE conditions: source must be "event_data"; all preconditions are ' + 'AND-combined and a failing one silently skips the rule (no finding, no UI card). Use for ' + 'applicability ("skip on virtual machines", "only when marker event absent via not_exists").', + absence: + 'ABSENCE PROVES NOTHING. The agent observes an enrollment only from its own start (IME bootstrap) ' + + 'and only as well as its collectors work: a missing event can mean "never happened" or "not seen". ' + + 'Prefer positive agent signals (app_install_starved, entra_user_affinity_pending) over asserting that ' + + 'something did not occur. Use not_exists as a SUPPRESSION (precondition: "skip when enrollment_complete ' + + 'exists"), rarely as a required condition. A required not_exists on an IME-log-derived event ' + + '(guardrails imeLogDerivedEventTypes: app_install_*, script_*, ime_*, …) is only as good as the log ' + + 'tracker: add a precondition { eventType: "ime_tracker_degraded", operator: "not_exists" } so the rule ' + + 'stays silent when the tracker skipped work, or accept the risk knowingly — validate_rule warns. ' + + 'collector_degraded is the same gate for kernel/event-log watchers (precondition on ' + + 'collector_degraded not_exists, or dataField collector not_equals ). When reading a session, ' + + 'get_session_summary → coverage.gaps lists the blind spots before the findings.', suppressByEvent: 'On a condition, suppressByEvent {eventType, joinField} drops matches that a resolving event ' + 'shares the joinField value with — e.g. app_install_failed suppressed by a later ' + diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts index 34e43b5c..1282cee3 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts @@ -1053,5 +1053,25 @@ export const RULE_GUARDRAILS = { "log_entry", "agent_trace", "stall_probe_check" + ], + "imeLogDerivedEventTypes": [ + "app_install_started", + "app_install_completed", + "app_install_failed", + "app_install_skipped", + "app_download_started", + "app_tracking_summary", + "download_progress", + "do_telemetry", + "script_started", + "script_completed", + "script_failed", + "script_timeout_suspected", + "esp_phase_changed", + "ime_agent_version", + "ime_token_failure", + "ime_user_session_completed", + "ime_process_exited", + "ime_user_token_acquired" ] } as const; diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts index cee5ee08..c8d886e9 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-validation.ts @@ -339,6 +339,32 @@ function lintAnalyzeRule(rule: Record): ValidationFinding[] { findings.push(...checkEventTypeKnown(preconditions[i].eventType, `preconditions[${i}]`)); } + // Absence proves nothing: a required not_exists on an event the agent derives from the IME + // logs is only as good as the tracker that would have matched it. When the tracker reports + // ime_tracker_degraded (a line whose remaining patterns were skipped, a regex timeout, an + // oversized line) the engine still evaluates the absence as true. The precondition below + // suppresses the rule on those sessions; without it the author has to accept that risk + // knowingly — hence a warning, never an error (production does not reject the rule). + const hasTrackerCoverageGate = preconditions.some( + (p) => p.eventType === IME_TRACKER_DEGRADED_EVENT_TYPE && p.operator === 'not_exists' && !p.dataField, + ); + if (!hasTrackerCoverageGate) { + for (let i = 0; i < conditions.length; i++) { + const c = conditions[i]; + if (c.operator !== 'not_exists' || c.required !== true || !c.eventType || c.dataField) continue; + if (!IME_LOG_DERIVED_EVENT_TYPES.has(c.eventType)) continue; + findings.push({ + level: 'warning', + message: + `conditions[${i}]: required not_exists on "${c.eventType}" asserts an ABSENCE of an IME-log-derived event — ` + + 'it is only as good as the log tracker that would have matched it. Either add a precondition ' + + `{ eventType: "${IME_TRACKER_DEGRADED_EVENT_TYPE}", operator: "not_exists" } so the rule stays silent when the tracker ` + + 'skipped work, or build the rule on a positive agent signal instead (pattern: app_install_starved). ' + + 'get_session_summary → coverage.gaps shows the tracker state per session.', + }); + } + } + // Confidence factors: exact DSL shapes; "count >= N" counts EVENTS of type factor.signal. const factors = (Array.isArray(rule.confidenceFactors) ? rule.confidenceFactors : []) as Array<{ signal?: string; condition?: string; weight?: number }>; let positiveWeights = 0; @@ -465,6 +491,20 @@ const BLOCKED_INTERIM_TRIGGER_EVENT_TYPES = new Set( ((RULE_GUARDRAILS as Record).blockedInterimTriggerEventTypes as readonly string[] | undefined) ?? [], ); +/** + * Event types the agent derives from the IME log files (ImeLogTracker + its adapter). Their + * absence in a session can mean "never happened" OR "the tracker skipped the line" — the + * absence lint above tells the two apart via the ime_tracker_degraded precondition. Mirrored + * from rules/guardrails.json (imeLogDerivedEventTypes); the backend catalog-policy test reads + * the same file, so the built-in catalog and this lint judge by one list. + */ +export const IME_LOG_DERIVED_EVENT_TYPES: ReadonlySet = new Set( + ((RULE_GUARDRAILS as Record).imeLogDerivedEventTypes as readonly string[] | undefined) ?? [], +); + +/** The tracker's one-shot self-report of skipped work (Constants.EventTypes.ImeTrackerDegraded). */ +export const IME_TRACKER_DEGRADED_EVENT_TYPE = 'ime_tracker_degraded'; + // ── Gather-rule semantic lint ─────────────────────────────────────────────── function lintGatherRule(rule: Record): ValidationFinding[] { diff --git a/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts b/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts new file mode 100644 index 00000000..303e23de --- /dev/null +++ b/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts @@ -0,0 +1,385 @@ +/** + * Observation coverage of one session — a pure projection of events the agent already emits. + * + * The agent only sees an enrollment from the moment the IME bootstrapped it, and even inside + * that window only as well as its IME log tracker, collectors, spool and upload path worked. + * Every one of those degradations is an event today (`agent_late_start`, `ime_tracker_degraded`, + * `collector_degraded`, `telemetry_upload_poisoned`, …), but the triage timeline ranks them as + * ordinary warnings — first to fall off the 50-entry cap on a noisy session, and the Info-severity + * ones (`agent_late_start` on a successful session, `diagnostics_uploaded`) never make it at all. + * + * `buildSessionCoverage` folds them into one block that `get_session_summary` places BEFORE the + * key events, so a reader knows what was and was not observable before interpreting findings — + * in particular before trusting the absence of an event. No backend field, no new event type: + * the block is computed from the same event list the summary already fetches. + */ +import type { EnrollmentEvent } from './generated/wire-types.generated.js'; + +/** Event types the coverage block reads. Guarded against the catalog by event-types-drift.test.ts. */ +export const COVERAGE_EVENT_TYPES: ReadonlySet = new Set([ + 'agent_started', + 'agent_late_start', + 'historic_ime_replay_detected', + 'ime_tracker_degraded', + 'ime_pattern_hits', + 'collector_degraded', + 'spool_pressure_detected', + 'telemetry_upload_poisoned', + 'telemetry_upload_blocked', + 'ingress_backpressure', + 'disk_space_low', + 'diagnostics_collecting', + 'diagnostics_uploaded', + 'diagnostics_upload_failed', +]); + +/** + * `data.` slices the coverage block needs from the events endpoint, appended to + * SUMMARY_EVENT_FIELDS so the backend never ships whole payloads for a summary. + */ +export const COVERAGE_EVENT_FIELDS = + 'data.previousExitType,data.oobeStateAtAgentStart,' + + 'data.bootToAgentStartSeconds,data.agentUptimeSeconds,data.outcome,data.note,' + + 'data.earliestRejectedSourceTimestamp,' + + 'data.file,data.firstSkippedPattern,data.lineBudgetBreaks,data.regexTimeouts,data.oversizedLines,data.unanchoredPatterns,data.linesRead,' + + 'data.collector,data.reason,data.errorType,' + + 'data.pendingItemCount,data.pendingBytes,data.kind,data.itemCount,data.disk_free_gb,' + + 'data.truncated,data.includedFiles,data.includedBytes,data.skippedFiles,data.skippedByReason,data.problemsByKind'; + +export interface CoverageWindow { + /** Timestamp of the first `agent_started` — the earliest moment anything was observed live. */ + observedFrom: string | null; + /** `session.startedAt` — back-dated to the earliest replayed IME log line when the agent arrived late. */ + sessionStartedAt: string | null; + /** Seconds between `sessionStartedAt` and `observedFrom`; > 0 means the start was reconstructed from IME logs, not observed. */ + startedAtBackdatedSeconds: number | null; + /** Number of `agent_started` events — every restart is a gap in observation. */ + agentStarts: number; + /** Distinct `previousExitType` values reported by restarts (the first start has none). */ + previousExitTypes: string[]; + lateStart: { + bootToAgentStartSeconds: number | null; + agentUptimeSeconds: number | null; + outcome: string | null; + note: string | null; + } | null; + historicReplaySuppressed: { earliestRejectedSourceTimestamp: string | null } | null; +} + +export interface ImeTrackerCoverage { + degraded: boolean; + file?: string | null; + firstSkippedPattern?: string | null; + lineBudgetBreaks?: number | null; + regexTimeouts?: number | null; + oversizedLines?: number | null; + unanchoredPatterns?: number | null; + /** Whole-session counters from `ime_pattern_hits` (emitted at termination), when present. */ + sessionTotals?: { + linesRead: number | null; + lineBudgetBreaks: number | null; + regexTimeouts: number | null; + oversizedLines: number | null; + }; +} + +export interface CollectorDegradation { + collector: string; + reason: string; + errorType: string | null; + at: string | null; +} + +export interface UploadCoverage { + spoolPressure: boolean; + poisonedItems: number; + blocked: boolean; + backpressureEpisodes: number; + diskSpaceLow: boolean; +} + +export type DiagnosticsStatus = 'uploaded' | 'failed' | 'skipped' | 'not_collected'; + +export interface DiagnosticsCoverage { + status: DiagnosticsStatus; + errorCode?: string; + /** Present once the agent reports packaging counters (never file paths — those live in the ZIP). */ + truncated?: boolean; + includedFiles?: number; + skippedFiles?: number; + skippedByReason?: Record; + problemsByKind?: Record; +} + +export interface SessionCoverage { + window: CoverageWindow; + imeTracker: ImeTrackerCoverage; + collectors: CollectorDegradation[]; + upload: UploadCoverage; + diagnostics: DiagnosticsCoverage; + /** One calibrated line per observation gap; empty means nothing reported a gap. */ + gaps: string[]; +} + +type Ev = Partial; + +function str(v: unknown): string | null { + if (v === undefined || v === null) return null; + const s = String(v).trim(); + return s.length > 0 ? s : null; +} + +function num(v: unknown): number | null { + if (v === undefined || v === null || v === '') return null; + const n = typeof v === 'number' ? v : Number(v); + return Number.isFinite(n) ? n : null; +} + +function bool(v: unknown): boolean { + if (typeof v === 'boolean') return v; + return String(v ?? '').toLowerCase() === 'true'; +} + +function countMap(v: unknown): Record | undefined { + if (!v || typeof v !== 'object' || Array.isArray(v)) return undefined; + const out: Record = {}; + for (const [k, raw] of Object.entries(v as Record)) { + const n = num(raw); + if (n !== null && n > 0) out[k] = n; + } + return Object.keys(out).length > 0 ? out : undefined; +} + +function ms(ts: unknown): number | null { + const s = str(ts); + if (!s) return null; + const t = Date.parse(s); + return Number.isFinite(t) ? t : null; +} + +function byTime(a: Ev, b: Ev): number { + return (ms(a.timestamp) ?? 0) - (ms(b.timestamp) ?? 0) || ((a.sequence ?? 0) - (b.sequence ?? 0)); +} + +function fmtCounts(m: Record): string { + return Object.entries(m).map(([k, n]) => `${k}: ${n}`).join(', '); +} + +/** + * Folds the session's events (already stripped of historic-replay rows by the caller) into the + * coverage block. Pure: no fetch, no clock. + */ +export function buildSessionCoverage( + session: { startedAt?: string | null }, + events: readonly Ev[], +): SessionCoverage { + const byType = new Map(); + for (const e of events) { + const et = String(e.eventType ?? ''); + if (!COVERAGE_EVENT_TYPES.has(et)) continue; + const list = byType.get(et); + if (list) list.push(e); + else byType.set(et, [e]); + } + for (const list of byType.values()) list.sort(byTime); + const of = (type: string): Ev[] => byType.get(type) ?? []; + const first = (type: string): Ev | undefined => of(type)[0]; + const last = (type: string): Ev | undefined => of(type)[of(type).length - 1]; + + const gaps: string[] = []; + + // ── Observation window ────────────────────────────────────────────────────── + const starts = of('agent_started'); + const observedFrom = str(starts[0]?.timestamp); + const sessionStartedAt = str(session.startedAt); + const observedMs = ms(observedFrom); + const startedMs = ms(sessionStartedAt); + const startedAtBackdatedSeconds = + observedMs !== null && startedMs !== null ? Math.max(0, Math.round((observedMs - startedMs) / 1000)) : null; + const previousExitTypes = [...new Set(starts.slice(1).map((e) => str(e.data?.previousExitType)).filter((x): x is string => !!x))]; + + const late = last('agent_late_start'); + const lateStart = late + ? { + bootToAgentStartSeconds: num(late.data?.bootToAgentStartSeconds), + agentUptimeSeconds: num(late.data?.agentUptimeSeconds), + outcome: str(late.data?.outcome), + note: str(late.data?.note), + } + : null; + + const replay = first('historic_ime_replay_detected'); + const historicReplaySuppressed = replay + ? { earliestRejectedSourceTimestamp: str(replay.data?.earliestRejectedSourceTimestamp) } + : null; + + if (!observedFrom) { + gaps.push('No agent_started event in this session: the live observation window cannot be established (events may stem from a partial or replayed upload).'); + } else if (startedAtBackdatedSeconds !== null && startedAtBackdatedSeconds > 60) { + gaps.push( + `Session start is ${startedAtBackdatedSeconds} s before the first agent_started (${observedFrom}): the earlier part was reconstructed from IME log replay, not observed live.`, + ); + } + if (lateStart) { + const detail = [ + lateStart.bootToAgentStartSeconds !== null ? `agent started ${lateStart.bootToAgentStartSeconds} s after boot` : null, + lateStart.agentUptimeSeconds !== null ? `ran ${lateStart.agentUptimeSeconds} s` : null, + ].filter(Boolean).join(', '); + gaps.push(`Low observation coverage${detail ? ` (${detail})` : ''}: ${lateStart.note ?? 'the agent observed only the end state of this enrollment.'}`); + } + if (historicReplaySuppressed) { + gaps.push( + `IME log content from a previous enrollment (earliest ${historicReplaySuppressed.earliestRejectedSourceTimestamp ?? 'unknown'}) was suppressed; app and script events from before this enrollment are intentionally absent.`, + ); + } + if (starts.length > 1) { + const exits = previousExitTypes.length > 0 ? ` (previous exit: ${previousExitTypes.join(', ')})` : ''; + gaps.push(`The agent started ${starts.length} times${exits}: signals during the restart gaps were not observed.`); + } + + // ── IME log tracker ───────────────────────────────────────────────────────── + const degradedEv = first('ime_tracker_degraded'); + const hits = last('ime_pattern_hits'); + const sessionTotals = hits + ? { + linesRead: num(hits.data?.linesRead), + lineBudgetBreaks: num(hits.data?.lineBudgetBreaks), + regexTimeouts: num(hits.data?.regexTimeouts), + oversizedLines: num(hits.data?.oversizedLines), + } + : undefined; + let imeTracker: ImeTrackerCoverage; + if (degradedEv) { + imeTracker = { + degraded: true, + file: str(degradedEv.data?.file), + firstSkippedPattern: str(degradedEv.data?.firstSkippedPattern), + lineBudgetBreaks: num(degradedEv.data?.lineBudgetBreaks), + regexTimeouts: num(degradedEv.data?.regexTimeouts), + oversizedLines: num(degradedEv.data?.oversizedLines), + unanchoredPatterns: num(degradedEv.data?.unanchoredPatterns), + ...(sessionTotals ? { sessionTotals } : {}), + }; + const parts: string[] = []; + const totals = sessionTotals ?? { + linesRead: null, + lineBudgetBreaks: imeTracker.lineBudgetBreaks, + regexTimeouts: imeTracker.regexTimeouts, + oversizedLines: imeTracker.oversizedLines, + }; + if ((totals.lineBudgetBreaks ?? 0) > 0) { + parts.push(`skipped the remaining patterns on ${totals.lineBudgetBreaks} line(s)${imeTracker.firstSkippedPattern ? ` (first skipped: ${imeTracker.firstSkippedPattern})` : ''}`); + } + if ((totals.regexTimeouts ?? 0) > 0) parts.push(`gave up on ${totals.regexTimeouts} line(s) after a regex timeout`); + if ((totals.oversizedLines ?? 0) > 0) parts.push(`dropped ${totals.oversizedLines} oversized line(s)`); + if (parts.length === 0) parts.push('skipped work on at least one line'); + gaps.push( + `IME log tracker ${parts.join(', ')} in ${imeTracker.file ?? 'an IME log'}: the absence of an IME-derived event is not fully provable for this session.`, + ); + } else { + imeTracker = { degraded: false, ...(sessionTotals ? { sessionTotals } : {}) }; + } + + // ── Collectors ────────────────────────────────────────────────────────────── + const collectors: CollectorDegradation[] = []; + const seen = new Set(); + for (const e of of('collector_degraded')) { + const collector = str(e.data?.collector) ?? str(e.source) ?? 'unknown'; + const reason = str(e.data?.reason) ?? 'unknown'; + const key = `${collector}|${reason}`; + if (seen.has(key)) continue; + seen.add(key); + collectors.push({ collector, reason, errorType: str(e.data?.errorType), at: str(e.timestamp) }); + } + for (const c of collectors) { + gaps.push(`Collector ${c.collector} degraded (${c.reason}${c.errorType ? `, ${c.errorType}` : ''}): its signals are missing from this session, not proven negative.`); + } + + // ── Spool / upload ────────────────────────────────────────────────────────── + const spool = last('spool_pressure_detected'); + const poisoned = of('telemetry_upload_poisoned'); + const poisonedItems = poisoned.reduce((n, e) => n + (num(e.data?.itemCount) ?? 1), 0); + const blocked = of('telemetry_upload_blocked').length > 0; + const backpressureEpisodes = of('ingress_backpressure').length; + const disk = last('disk_space_low'); + const upload: UploadCoverage = { + spoolPressure: !!spool, + poisonedItems, + blocked, + backpressureEpisodes, + diskSpaceLow: !!disk, + }; + if (spool) { + const pending = num(spool.data?.pendingItemCount); + gaps.push(`Telemetry spool pressure${pending !== null ? ` (${pending} items pending)` : ''}: events reached the backend late; anything after the last upload may still be on the device.`); + } + if (poisonedItems > 0) { + const kinds = [...new Set(poisoned.map((e) => str(e.data?.kind)).filter((x): x is string => !!x))]; + gaps.push(`${poisonedItems} telemetry item(s) were quarantined as poison and never uploaded${kinds.length > 0 ? ` (${kinds.join(', ')})` : ''}; they remain in the spool inside the diagnostics ZIP.`); + } + if (blocked) { + gaps.push('Telemetry upload was blocked by a permanent backend rejection at some point; later events reached the backend only once the block cleared.'); + } + if (disk) { + const free = num(disk.data?.disk_free_gb); + gaps.push(`System drive ran low on space${free !== null ? ` (${free} GB free)` : ''}: spool and diagnostics writes may have failed.`); + } + + // ── Diagnostics package ───────────────────────────────────────────────────── + const uploaded = last('diagnostics_uploaded'); + const failed = last('diagnostics_upload_failed'); + const collecting = last('diagnostics_collecting'); + let diagnostics: DiagnosticsCoverage; + if (uploaded) { + diagnostics = { status: 'uploaded' }; + const d = uploaded.data ?? {}; + if (d.truncated !== undefined || d.includedFiles !== undefined) { + diagnostics.truncated = bool(d.truncated); + const included = num(d.includedFiles); + const skipped = num(d.skippedFiles); + if (included !== null) diagnostics.includedFiles = included; + if (skipped !== null) diagnostics.skippedFiles = skipped; + const byReason = countMap(d.skippedByReason); + const problems = countMap(d.problemsByKind); + if (byReason) diagnostics.skippedByReason = byReason; + if (problems) diagnostics.problemsByKind = problems; + if (diagnostics.truncated) { + gaps.push(`Diagnostics ZIP is truncated: ${skipped ?? 'some'} file(s) skipped by the packaging caps${byReason ? ` (${fmtCounts(byReason)})` : ''}; the full list is in _TRUNCATED.txt inside the ZIP.`); + } + if (problems) { + gaps.push(`Diagnostics packaging hit problems (${fmtCounts(problems)}): those files are missing from the ZIP; package-manifest.txt names them.`); + } + } + } else if (failed) { + const errorCode = str(failed.data?.errorCode) ?? 'unknown'; + if (errorCode.startsWith('diagnostics_skipped_')) { + diagnostics = { status: 'skipped', errorCode }; + gaps.push(`No diagnostics ZIP for this session: collection was skipped by configuration (${errorCode}).`); + } else { + diagnostics = { status: 'failed', errorCode }; + gaps.push(`Diagnostics upload failed (${errorCode}): no ZIP is available for this session.`); + } + } else { + diagnostics = { status: 'not_collected' }; + if (collecting) { + gaps.push('Diagnostics collection started but neither an upload nor a failure was reported: the agent may have died while packaging.'); + } + } + + return { + window: { + observedFrom, + sessionStartedAt, + startedAtBackdatedSeconds, + agentStarts: starts.length, + previousExitTypes, + lateStart, + historicReplaySuppressed, + }, + imeTracker, + collectors, + upload, + diagnostics, + gaps, + }; +} diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts index 03269170..ba24434d 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/sessions.ts @@ -7,6 +7,7 @@ import { toolError } from './error-handler.js'; import { lookupErrorCode } from '../error-code-catalog.js'; import { assertKnownEventType, assertKnownDevicePropertyKeys } from '../resource-catalog.js'; import { interpolateAnalysisResults } from '../interpolate-rule-template.js'; +import { buildSessionCoverage } from '../session-coverage.js'; import { API_BASE_URL } from '../config.js'; import type { AppMetricsResponse, @@ -483,6 +484,9 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated: description: 'Get a concise, structured summary of an enrollment session optimized for analysis. ' + 'Returns: session overview (status, duration, device, enrollment config), ' + + 'observation coverage (coverage: from when the agent actually watched, IME log tracker / collector / ' + + 'upload / diagnostics-package health, and coverage.gaps — one line per known blind spot; read it before ' + + 'treating a missing event as proof that something did not happen), ' + 'key events timeline (errors, warnings, phase transitions, app installs — noise filtered out, ' + 'capped at 50 most-relevant entries; stats.keyEventsTruncated indicates if more were dropped), ' + 'rule analysis results (probable cause, remediation), and aggregate stats. A key event that carries an ' + @@ -564,6 +568,10 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated: // cleans all three. stats.totalEvents therefore counts non-replayed events. const allEvents = (eventsData.events ?? []).filter((e) => !isHistoricImeReplay(e)); + // Observation coverage: what the agent could and could not see, folded from the + // health/lifecycle events that the triage timeline below ranks low or drops. + const coverage = buildSessionCoverage(s, allEvents); + let errorCount = 0; let warningCount = 0; let appTotal = 0; @@ -664,6 +672,7 @@ export function registerSessionTools(server: McpServer, ga: boolean, delegated: const result = { overview, + coverage, keyEvents: mappedEvents, analysis, annotations, diff --git a/src/McpServer/autopilot-monitor-mcp/src/tools/shared.ts b/src/McpServer/autopilot-monitor-mcp/src/tools/shared.ts index 1375711b..8ee54bdc 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/tools/shared.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/tools/shared.ts @@ -2,6 +2,7 @@ import type { ToolAnnotations } from '@modelcontextprotocol/server'; import { z } from 'zod'; import { PRETTY_JSON } from '../config.js'; import { isPrettyJsonRequested } from '../client.js'; +import { COVERAGE_EVENT_FIELDS } from '../session-coverage.js'; /** * Projection sent by get_session_events when the caller omits `fields`: everything an event @@ -76,7 +77,10 @@ export const SUMMARY_EVENT_FIELDS = 'data.scriptType,data.script_type,data.scriptPart,data.script_part,data.result,' + 'data.rejectedSourceTimestamp,data.rejected_source_timestamp,' + 'data.errorCode,data.exitCode,data.hresult,data.hresultFromWin32,data.hresultSymbol,' + - 'data.errorCodeInfo,data.exitCodeInfo,data.hresultInfo,data.hresultFromWin32Info'; + 'data.errorCodeInfo,data.exitCodeInfo,data.hresultInfo,data.hresultFromWin32Info,' + + // Observation-coverage slices (session-coverage.ts): the health/lifecycle payload keys the + // `coverage` block folds — still per-key, never the whole payload. + COVERAGE_EVENT_FIELDS; /** * Zod validator for session IDs. Sessions are UUIDs and the value is diff --git a/src/Web/autopilot-monitor-web/utils/shared-manifests.json b/src/Web/autopilot-monitor-web/utils/shared-manifests.json index 6c0d9974..c2db8f4c 100644 --- a/src/Web/autopilot-monitor-web/utils/shared-manifests.json +++ b/src/Web/autopilot-monitor-web/utils/shared-manifests.json @@ -1096,7 +1096,7 @@ "kind": "primitive", "name": "string" }, - "doc": "Controls who can access the remote MCP server. \"Disabled\" = MCP off, \"WhitelistOnly\" = GlobalAdmins + McpUsers table (default), \"AllMembers\" = any authenticated user." + "doc": "Controls who can access the remote MCP server (evaluated by McpUserService.IsAllowedAsync). \"Disabled\" = MCP off for everyone, platform roles included. \"WhitelistOnly\" = platform roles, delegated (MSP) admins and enabled McpUsers rows (default). \"AllMembers\" = the same grants plus every account holding an effective member role (Admin / Operator / Viewer) in the tenant its token was issued for — never \"any authenticated token\": an employee without a role, or a tenant that never onboarded, is denied. An explicitly disabled McpUsers row denies under every policy." }, { "name": "nvdApiKey", diff --git a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts index 50af5153..0389fdd9 100644 --- a/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/wire-types.generated.ts @@ -138,7 +138,7 @@ export interface AdminConfiguration { maxDiagnosticsDownloadSizeMB: number; /** Timeout in seconds for the entire diagnostics download+stream operation. 0 = no timeout. Default: 120 seconds. */ diagnosticsDownloadTimeoutSeconds: number; - /** Controls who can access the remote MCP server. "Disabled" = MCP off, "WhitelistOnly" = GlobalAdmins + McpUsers table (default), "AllMembers" = any authenticated user. */ + /** Controls who can access the remote MCP server (evaluated by McpUserService.IsAllowedAsync). "Disabled" = MCP off for everyone, platform roles included. "WhitelistOnly" = platform roles, delegated (MSP) admins and enabled McpUsers rows (default). "AllMembers" = the same grants plus every account holding an effective member role (Admin / Operator / Viewer) in the tenant its token was issued for — never "any authenticated token": an employee without a role, or a tenant that never onboarded, is denied. An explicitly disabled McpUsers row denies under every policy. */ mcpAccessPolicy: string; /** NVD API key for higher rate limits (50 req/30s vs 5 req/30s without key). Free registration at https://nvd.nist.gov/developers/request-an-api-key null = operate without API key (slower, still functional). */ nvdApiKey: string;