diff --git a/rules/analyze/ANALYZE-DEV-006.json b/rules/analyze/ANALYZE-DEV-006.json new file mode 100644 index 00000000..8c8d67e0 --- /dev/null +++ b/rules/analyze/ANALYZE-DEV-006.json @@ -0,0 +1,65 @@ +{ + "$schema": "../schema/analyze-rule.schema.json", + "ruleId": "ANALYZE-DEV-006", + "title": "OS Build Changed During Enrollment", + "description": "Informational: the OS build (CurrentBuild.UBR) changed across a reboot during enrollment — an update was installed, regardless of which servicing path delivered it. Deterministic corroboration that works even when the Windows Update event channel shows nothing (registry comparison across restart cannot be lost to timing or logging-path differences).", + "severity": "info", + "category": "device", + "version": "1.0.0", + "enabled": true, + "trigger": "single", + "baseConfidence": 70, + "confidenceThreshold": 50, + "conditions": [ + { + "signal": "build_changed", + "source": "event_type", + "eventType": "os_build_changed", + "operator": "exists", + "value": "", + "required": true + }, + { + "signal": "previous_build", + "source": "event_data", + "eventType": "os_build_changed", + "dataField": "previousBuild", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "current_build", + "source": "event_data", + "eventType": "os_build_changed", + "dataField": "currentBuild", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "wu_channel_blind", + "source": "event_type", + "eventType": "windows_update_channel_census", + "operator": "exists", + "value": "", + "required": false + } + ], + "confidenceFactors": [ + { "signal": "wu_channel_blind", "condition": "exists", "weight": 10 } + ], + "explanation": "The OS build changed from **{{previousBuild}}** to **{{currentBuild}}** across a reboot during enrollment — a Windows update was installed mid-provisioning. This is normal when quality updates run during OOBE, but it adds a reboot and extends enrollment time. If a windows_update_channel_census event is also present, the update installed via a servicing path the Windows Update watcher does not capture yet — its EventID histogram tells which IDs to enable.", + "remediation": [ + { + "title": "Optional review", + "steps": [ + "No action required — this is informational context for enrollment duration and reboot analysis.", + "Correlate the build jump with reboots and any behavior change after enrollment.", + "If quality updates during OOBE are not desired for this Autopilot profile, review the ESP 'Install quality updates during OOBE' setting." + ] + } + ], + "markSessionAsFailedDefault": false, + "tags": ["device", "windows-update", "oobe", "reboot"] +} diff --git a/rules/dist/analyze-rules.json b/rules/dist/analyze-rules.json index 271a5f66..b9171e6f 100644 --- a/rules/dist/analyze-rules.json +++ b/rules/dist/analyze-rules.json @@ -1042,6 +1042,79 @@ "oobe" ] }, + { + "ruleId": "ANALYZE-DEV-006", + "title": "OS Build Changed During Enrollment", + "description": "Informational: the OS build (CurrentBuild.UBR) changed across a reboot during enrollment — an update was installed, regardless of which servicing path delivered it. Deterministic corroboration that works even when the Windows Update event channel shows nothing (registry comparison across restart cannot be lost to timing or logging-path differences).", + "severity": "info", + "category": "device", + "version": "1.0.0", + "enabled": true, + "trigger": "single", + "baseConfidence": 70, + "confidenceThreshold": 50, + "conditions": [ + { + "signal": "build_changed", + "source": "event_type", + "eventType": "os_build_changed", + "operator": "exists", + "value": "", + "required": true + }, + { + "signal": "previous_build", + "source": "event_data", + "eventType": "os_build_changed", + "dataField": "previousBuild", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "current_build", + "source": "event_data", + "eventType": "os_build_changed", + "dataField": "currentBuild", + "operator": "exists", + "value": "", + "required": false + }, + { + "signal": "wu_channel_blind", + "source": "event_type", + "eventType": "windows_update_channel_census", + "operator": "exists", + "value": "", + "required": false + } + ], + "confidenceFactors": [ + { + "signal": "wu_channel_blind", + "condition": "exists", + "weight": 10 + } + ], + "explanation": "The OS build changed from **{{previousBuild}}** to **{{currentBuild}}** across a reboot during enrollment — a Windows update was installed mid-provisioning. This is normal when quality updates run during OOBE, but it adds a reboot and extends enrollment time. If a windows_update_channel_census event is also present, the update installed via a servicing path the Windows Update watcher does not capture yet — its EventID histogram tells which IDs to enable.", + "remediation": [ + { + "title": "Optional review", + "steps": [ + "No action required — this is informational context for enrollment duration and reboot analysis.", + "Correlate the build jump with reboots and any behavior change after enrollment.", + "If quality updates during OOBE are not desired for this Autopilot profile, review the ESP 'Install quality updates during OOBE' setting." + ] + } + ], + "markSessionAsFailedDefault": false, + "tags": [ + "device", + "windows-update", + "oobe", + "reboot" + ] + }, { "ruleId": "ANALYZE-ENRL-001", "title": "Enrollment Failed", diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/WindowsUpdateTrackerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/WindowsUpdateTrackerTests.cs index 4689fd08..8bbfd8b0 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/WindowsUpdateTrackerTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/WindowsUpdateTrackerTests.cs @@ -250,5 +250,117 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals Assert.Equal("S_OK", WindowsUpdateTracker.DecodeHResult(0x00000000)); Assert.Equal("WU_E_UNKNOWN", WindowsUpdateTracker.DecodeHResult(0x8888DEAD)); } + + // --------------------------------------------------------------------------- + // Blind-spot channel census (session 7443317c) + // --------------------------------------------------------------------------- + + private WindowsUpdateTracker BuildCensusTracker( + bool censusEnabled, + bool osBuildChanged, + Func? scanner = null) + { + var post = new InformationalEventPost(_sink, new VirtualClock(At)); + var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info); + var tracker = new WindowsUpdateTracker( + sessionId: "sess-wu", + tenantId: "tenant-wu", + post: post, + logger: logger, + backfillEnabled: false, + stateDirectory: null, + channelCensusEnabled: censusEnabled, + osBuildChangedProvider: () => osBuildChanged); + tracker.CensusScannerOverride = scanner ?? ((channel, lookback) => + new WindowsUpdateTracker.ChannelCensusScan( + channel == WindowsUpdateTracker.Channel + ? new Dictionary { { 25, 3 }, { 21, 1 } } + : new Dictionary { { 200, 2 } }, + truncated: false)); + return tracker; + } + + [Fact] + public void Census_BuildChanged_ZeroTargetedEmitted_EmitsHistogramEvent() + { + var tracker = BuildCensusTracker(censusEnabled: true, osBuildChanged: true); + + tracker.EmitChannelCensusIfBlind(); + + var s = Assert.Single(ByType(Constants.EventTypes.WindowsUpdateChannelCensus)); + Assert.Equal("Debug", s.Payload![SignalPayloadKeys.Severity]); + var data = Data(s); + Assert.Equal("21=1,25=3", data["wuClientCensus"]); + Assert.Equal("200=2", data["updateOrchestratorCensus"]); + Assert.Equal("19,20,43,44", data["targetedEventIds"]); + Assert.False(data.ContainsKey("censusTruncated")); + } + + [Fact] + public void Census_BuildUnchanged_EmitsNothing() + { + var tracker = BuildCensusTracker(censusEnabled: true, osBuildChanged: false); + + tracker.EmitChannelCensusIfBlind(); + + Assert.Empty(ByType(Constants.EventTypes.WindowsUpdateChannelCensus)); + } + + [Fact] + public void Census_TargetedEventsWereEmittedThisRun_EmitsNothing() + { + // The watcher captured the update through its normal path — no blind spot to report. + var tracker = BuildCensusTracker(censusEnabled: true, osBuildChanged: true); + tracker.ProcessEvent(WindowsUpdateTracker.EventId_InstallSuccess, 4, recordId: 9, timeCreatedUtc: At, + updateTitle: "KB1", updateGuid: null, updateRevisionNumber: null, errorCode: null, + formattedDescription: null, isBackfill: false); + + tracker.EmitChannelCensusIfBlind(); + + Assert.Empty(ByType(Constants.EventTypes.WindowsUpdateChannelCensus)); + } + + [Fact] + public void Census_DisabledByConfig_EmitsNothing() + { + var tracker = BuildCensusTracker(censusEnabled: false, osBuildChanged: true); + + tracker.EmitChannelCensusIfBlind(); + + Assert.Empty(ByType(Constants.EventTypes.WindowsUpdateChannelCensus)); + } + + [Fact] + public void Census_TruncatedScan_SaysSoInPayload() + { + // No silent caps: a capped scan must be visible in the evidence. + var tracker = BuildCensusTracker(censusEnabled: true, osBuildChanged: true, + scanner: (channel, lookback) => new WindowsUpdateTracker.ChannelCensusScan( + new Dictionary { { 19, WindowsUpdateTracker.CensusRecordCap } }, truncated: true)); + + tracker.EmitChannelCensusIfBlind(); + + var s = Assert.Single(ByType(Constants.EventTypes.WindowsUpdateChannelCensus)); + Assert.Equal(true, Data(s)["censusTruncated"]); + } + + [Fact] + public void Census_ScannerThrows_IsSwallowed_NeverBreaksTheWatcher() + { + var tracker = BuildCensusTracker(censusEnabled: true, osBuildChanged: true, + scanner: (channel, lookback) => throw new InvalidOperationException("boom")); + + tracker.EmitChannelCensusIfBlind(); + + Assert.Empty(ByType(Constants.EventTypes.WindowsUpdateChannelCensus)); + } + + [Fact] + public void FormatHistogram_OrdersByEventId() + { + Assert.Equal("19=2,25=1,43=7", + WindowsUpdateTracker.FormatHistogram(new Dictionary { { 43, 7 }, { 19, 2 }, { 25, 1 } })); + Assert.Equal("", WindowsUpdateTracker.FormatHistogram(new Dictionary())); + } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/OsBuildChangeHostTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/OsBuildChangeHostTests.cs new file mode 100644 index 00000000..01d34399 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/OsBuildChangeHostTests.cs @@ -0,0 +1,153 @@ +#nullable enable +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; +using AutopilotMonitor.Shared; +using Newtonsoft.Json; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration +{ + /// + /// OsBuildChangeHost — deterministic Windows-Update corroboration (session 7443317c). + /// Persists CurrentBuild.UBR across restarts, emits os_build_changed on a diff and + /// feeds BuildChanged to the WU channel census. All system access is seamed: the + /// build comes from an injected reader, persistence goes to a temp state directory. + /// + public sealed class OsBuildChangeHostTests : IDisposable + { + private static readonly DateTime At = new DateTime(2026, 7, 10, 14, 0, 0, DateTimeKind.Utc); + + private readonly TempDirectory _tmp = new TempDirectory(); + private readonly FakeSignalIngressSink _sink = new FakeSignalIngressSink(); + private readonly AgentLogger _logger; + + public OsBuildChangeHostTests() + { + _logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info); + } + + public void Dispose() => _tmp.Dispose(); + + private OsBuildChangeHost Build(string? build, string? stateDirectory = null) => + new OsBuildChangeHost( + sessionId: "sess-osb", + tenantId: "tenant-osb", + logger: _logger, + ingress: _sink, + clock: new VirtualClock(At), + stateDirectory: stateDirectory ?? _tmp.Path, + buildReader: () => build); + + private IReadOnlyList ChangedEvents() => + _sink.Posted.Where(p => + p.Kind == DecisionSignalKind.InformationalEvent + && p.Payload != null + && p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et) + && et == Constants.EventTypes.OsBuildChanged).ToList(); + + private string StateFilePath => Path.Combine(_tmp.Path, OsBuildChangeHost.StateFileName); + + [Fact] + public void FirstRun_SeedsStateFile_EmitsNothing() + { + var host = Build("26200.8037"); + + host.Start(); + + Assert.False(host.BuildChanged); + Assert.Empty(ChangedEvents()); + var state = JsonConvert.DeserializeObject(File.ReadAllText(StateFilePath))!; + Assert.Equal("26200.8037", state.OsBuild); + } + + [Fact] + public void SameBuildAcrossRestart_EmitsNothing() + { + Build("26200.8037").Start(); + + var second = Build("26200.8037"); + second.Start(); + + Assert.False(second.BuildChanged); + Assert.Empty(ChangedEvents()); + } + + [Fact] + public void ChangedBuildAcrossRestart_EmitsOsBuildChanged_AndSetsFlag() + { + // Session 7443317c: 26200.8037 → 26200.8655 across the mid-OOBE WU reboot. + Build("26200.8037").Start(); + + var second = Build("26200.8655"); + second.Start(); + + Assert.True(second.BuildChanged); + var s = Assert.Single(ChangedEvents()); + Assert.Equal("Info", s.Payload![SignalPayloadKeys.Severity]); + Assert.Equal("true", s.Payload![SignalPayloadKeys.ImmediateUpload]); + Assert.Contains("26200.8037 -> 26200.8655", s.Payload![SignalPayloadKeys.Message]); + + var data = (IReadOnlyDictionary)s.TypedPayload!; + Assert.Equal("26200.8037", data["previousBuild"]); + Assert.Equal("26200.8655", data["currentBuild"]); + Assert.True(data.ContainsKey("previousCapturedUtc")); + + // The new build is persisted — a third run on the same build stays silent. + var third = Build("26200.8655"); + third.Start(); + Assert.False(third.BuildChanged); + Assert.Single(ChangedEvents()); + } + + [Fact] + public void UnreadableBuild_SkipsWithoutSeeding() + { + var host = Build(null); + + host.Start(); + + Assert.False(host.BuildChanged); + Assert.Empty(ChangedEvents()); + Assert.False(File.Exists(StateFilePath)); + } + + [Fact] + public void CorruptStateFile_ReseedsSilently() + { + File.WriteAllText(StateFilePath, "{ not json !!"); + + var host = Build("26200.8655"); + host.Start(); + + Assert.False(host.BuildChanged); + Assert.Empty(ChangedEvents()); + var state = JsonConvert.DeserializeObject(File.ReadAllText(StateFilePath))!; + Assert.Equal("26200.8655", state.OsBuild); + } + + [Fact] + public void NoStateDirectory_IsToleratedWithoutEmission() + { + var host = new OsBuildChangeHost( + sessionId: "s", + tenantId: "t", + logger: _logger, + ingress: _sink, + clock: new VirtualClock(At), + stateDirectory: null, + buildReader: () => "26200.8655"); + + host.Start(); + + Assert.False(host.BuildChanged); + Assert.Empty(ChangedEvents()); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/WindowsUpdateTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/WindowsUpdateTracker.cs index 345fb5e9..06ecc025 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/WindowsUpdateTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/WindowsUpdateTracker.cs @@ -46,6 +46,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals internal sealed class WindowsUpdateTracker : IDisposable { internal const string Channel = "Microsoft-Windows-WindowsUpdateClient/Operational"; + internal const string OrchestratorChannel = "Microsoft-Windows-UpdateOrchestrator/Operational"; internal const int EventId_InstallSuccess = 19; internal const int EventId_InstallFailure = 20; @@ -54,6 +55,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals internal const string WatermarkStateFileName = "windows-update-watermark.json"; + // Hard bound for the unfiltered census scan — during OOBE both channels carry at most a + // few hundred records in the lookback window; the cap only guards against a pathological + // log. A truncated census says so in its payload (no silent caps). + internal const int CensusRecordCap = 5000; + private readonly AgentLogger _logger; private readonly string _sessionId; private readonly string _tenantId; @@ -62,6 +68,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals private readonly bool _backfillEnabled; private readonly int _backfillLookbackMinutes; private readonly string _stateDirectory; + private readonly bool _channelCensusEnabled; + private readonly Func _osBuildChangedProvider; + + // Count of targeted WU events emitted THIS run (live + backfill). Zero after the backfill + // while the OS build provably changed = the watcher is blind to the update's channel/IDs. + private int _emittedThisRun; private EventLogWatcher _watcher; @@ -91,7 +103,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals int[] targetedEventIds = null, bool backfillEnabled = true, int backfillLookbackMinutes = 60, - string stateDirectory = null) + string stateDirectory = null, + bool channelCensusEnabled = true, + Func osBuildChangedProvider = null) { _sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId)); _tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId)); @@ -103,6 +117,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _backfillEnabled = backfillEnabled; _backfillLookbackMinutes = backfillLookbackMinutes; _stateDirectory = stateDirectory != null ? Environment.ExpandEnvironmentVariables(stateDirectory) : null; + _channelCensusEnabled = channelCensusEnabled; + _osBuildChangedProvider = osBuildChangedProvider; } public void Start() @@ -118,6 +134,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals { _logger.Info("WindowsUpdate backfill disabled by config"); } + + EmitChannelCensusIfBlind(); } public void Stop() @@ -261,6 +279,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals if (!MarkProcessed(recordId)) return; // already emitted (cross-restart or duplicate delivery) + System.Threading.Interlocked.Increment(ref _emittedThisRun); + string eventType; EventSeverity severity; bool immediateUpload; @@ -528,6 +548,135 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals } } + // ----------------------------------------------------------------------- + // Blind-spot channel census (session 7443317c) + // ----------------------------------------------------------------------- + + /// + /// Self-evidencing blind spot diagnosis. Session 7443317c: an OOBE quality update + /// provably installed (OS build changed across the mid-enrollment reboot) yet neither + /// the live watcher nor two 60-min backfills captured a single targeted event — the + /// update logged via a channel/EventID set the watcher is blind to. With diagnostics + /// upload tenant-disabled there is no client-log path to find out which one; so when + /// the blind spot provably occurred (build changed AND zero targeted events emitted + /// this run), emit a one-shot unfiltered EventID histogram of both update channels + /// over the lookback window. The histogram arrives via normal event upload and tells + /// us which IDs to add to WindowsUpdateTargetedEventIds via remote config — + /// no agent redeploy, no diagnostics access needed. + /// + internal void EmitChannelCensusIfBlind() + { + if (!_channelCensusEnabled) return; + + try + { + if (_osBuildChangedProvider == null || !_osBuildChangedProvider()) + return; + if (System.Threading.Volatile.Read(ref _emittedThisRun) > 0) + return; + + var lookback = _backfillLookbackMinutes > 0 ? _backfillLookbackMinutes : 60; + var scanner = CensusScannerOverride ?? ScanChannel; + var wuClient = scanner(Channel, lookback); + var orchestrator = scanner(OrchestratorChannel, lookback); + + var wuClientHistogram = FormatHistogram(wuClient.Histogram); + var orchestratorHistogram = FormatHistogram(orchestrator.Histogram); + _logger.Info( + $"WindowsUpdate channel census (blind spot): wuClient=[{wuClientHistogram}] " + + $"updateOrchestrator=[{orchestratorHistogram}] lookback={lookback}min"); + + var data = new Dictionary + { + { "wuClientCensus", wuClientHistogram }, + { "updateOrchestratorCensus", orchestratorHistogram }, + { "lookbackMinutes", lookback }, + { "restartWatermark", _restartWatermark }, + { "targetedEventIds", string.Join(",", _targetedEventIds.OrderBy(id => id)) }, + }; + if (wuClient.Truncated || orchestrator.Truncated) + data["censusTruncated"] = true; + + _post.Emit(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.WindowsUpdateChannelCensus, + Severity = EventSeverity.Debug, + Source = "WindowsUpdateWatcher", + Phase = EnrollmentPhase.Unknown, + Message = "Windows Update channel census: OS build changed across restart but no targeted WU events were captured — EventID histogram attached", + Data = data, + ImmediateUpload = false, + }); + } + catch (Exception ex) + { + // Diagnosis must never break the watcher. + _logger.Warning($"WindowsUpdate channel census failed: {ex.Message}"); + } + } + + /// Test seam — replaces the real per-channel scan (Windows event log access). + internal Func CensusScannerOverride { get; set; } + + internal sealed class ChannelCensusScan + { + public Dictionary Histogram { get; } + public bool Truncated { get; } + + public ChannelCensusScan(Dictionary histogram, bool truncated) + { + Histogram = histogram ?? new Dictionary(); + Truncated = truncated; + } + } + + /// + /// Unfiltered EventID→count histogram of over the lookback + /// window. Fail-soft per channel (a missing UpdateOrchestrator channel returns an empty + /// histogram). Bounded by . + /// + internal static ChannelCensusScan ScanChannel(string channel, int lookbackMinutes) + { + var truncated = false; + var histogram = new Dictionary(); + try + { + var lookbackMs = (long)lookbackMinutes * 60 * 1000; + var xpath = $"*[System[TimeCreated[timediff(@SystemTime) <= {lookbackMs}]]]"; + var query = new EventLogQuery(channel, PathType.LogName, xpath); + + var scanned = 0; + using (var reader = new EventLogReader(query)) + { + EventRecord record; + while ((record = reader.ReadEvent()) != null) + { + using (record) + { + histogram.TryGetValue(record.Id, out var count); + histogram[record.Id] = count + 1; + if (++scanned >= CensusRecordCap) + { + truncated = true; + break; + } + } + } + } + } + catch (EventLogNotFoundException) + { + // Channel absent (older builds / test environments) — empty histogram says so. + } + return new ChannelCensusScan(histogram, truncated); + } + + /// Compact "id=count" form ordered by EventID, e.g. "19=2,25=1,43=2". + internal static string FormatHistogram(Dictionary histogram) => + string.Join(",", histogram.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key}={kv.Value}")); + // ----------------------------------------------------------------------- // Watermark dedup (cross-restart) // ----------------------------------------------------------------------- diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs index e6b92735..e23069f4 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs @@ -303,6 +303,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration harmlessModernDeploymentEventIds: collectors.ModernDeploymentHarmlessEventIds)); } + // Deterministic update corroboration — compares the persisted OS build (CurrentBuild.UBR) + // across agent restarts and emits os_build_changed when it differs (session 7443317c: + // build jump proved an OOBE quality update the WU channel never showed). Must be added + // BEFORE the WindowsUpdateWatcherHost: hosts start in list order and the watcher's + // channel census reads BuildChanged at its own start. + var osBuildChangeHost = new OsBuildChangeHost( + sessionId: sessionId, + tenantId: tenantId, + logger: logger, + ingress: ingress, + clock: clock, + stateDirectory: _stateDirectory); + hosts.Add(osBuildChangeHost); + // Windows Update during OOBE watcher — subscribes to WindowsUpdateClient/Operational and // backfills recent events (OOBE quality updates run before the agent starts). Surfaces // quality/cumulative updates installing/failing DURING enrollment — otherwise invisible. @@ -316,7 +330,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration clock: clock, targetedEventIds: collectors.WindowsUpdateTargetedEventIds, backfillLookbackMinutes: collectors.WindowsUpdateBackfillLookbackMinutes, - stateDirectory: _stateDirectory)); + stateDirectory: _stateDirectory, + channelCensusEnabled: collectors.WindowsUpdateChannelCensusEnabled, + osBuildChangedProvider: () => osBuildChangeHost.BuildChanged)); } // ----- Peripheral hosts (event-only; driven by remote-config toggles) -------------- diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/OsBuildChangeHost.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/OsBuildChangeHost.cs new file mode 100644 index 00000000..46da7571 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/OsBuildChangeHost.cs @@ -0,0 +1,210 @@ +#nullable enable +using System; +using System.IO; +using System.Threading; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; +using Microsoft.Win32; +using Newtonsoft.Json; + +namespace AutopilotMonitor.Agent.V2.Core.Orchestration +{ + /// + /// Deterministic Windows-Update corroboration (session 7443317c, 2026-07-10): persists the + /// OS build (CurrentBuild.UBR) across agent restarts and emits a one-shot + /// os_build_changed event when it differs — an update provably installed during the + /// enrollment, no matter which servicing path it took. This closes the WindowsUpdateTracker + /// blind spot deterministically: in 7443317c the build jumped 26200.8037→26200.8655 across a + /// mid-OOBE reboot while the WU channel carried none of the targeted EventIDs. The comparison + /// happens AFTER the reboot by construction, so unlike event-channel capture it cannot be + /// lost to timing, watermarks, or an unexpected logging path. + /// + /// feeds the channel census + /// (start-order: this host runs first) so a captured-nothing run can self-report the EventID + /// histogram of the update channels. + /// + /// + /// First run seeds the state file silently. Reads use the 64-bit registry view — the agent is + /// a 32-bit process on x64, and the default view would redirect to WOW6432Node. + /// + /// + internal sealed class OsBuildChangeHost : ICollectorHost + { + internal const string StateFileName = "os-build-state.json"; + private const string RegKeyCurrentVersion = @"SOFTWARE\Microsoft\Windows NT\CurrentVersion"; + + public string Name => "OsBuildChangeDetector"; + + private readonly string _sessionId; + private readonly string _tenantId; + private readonly AgentLogger _logger; + private readonly InformationalEventPost _post; + private readonly string? _stateDirectory; + private readonly Func _buildReader; + private int _disposed; + + /// True once found a build different from the persisted one. + public bool BuildChanged { get; private set; } + + public OsBuildChangeHost( + string sessionId, + string tenantId, + AgentLogger logger, + ISignalIngressSink ingress, + IClock clock, + string? stateDirectory, + Func? buildReader = null) + { + if (ingress == null) throw new ArgumentNullException(nameof(ingress)); + if (clock == null) throw new ArgumentNullException(nameof(clock)); + _sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId)); + _tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _post = new InformationalEventPost(ingress, clock, logger); + _stateDirectory = stateDirectory != null ? Environment.ExpandEnvironmentVariables(stateDirectory) : null; + _buildReader = buildReader ?? ReadOsBuildFromRegistry; + } + + public void Start() + { + try + { + DetectAndEmit(); + } + catch (Exception ex) + { + // One-shot observability — never let it break the host pipeline. + _logger.Warning($"OsBuildChangeDetector: detection failed: {ex.Message}"); + } + } + + public void Stop() { } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) return; + Stop(); + } + + internal void DetectAndEmit() + { + var currentBuild = _buildReader(); + if (string.IsNullOrEmpty(currentBuild)) + { + _logger.Warning("OsBuildChangeDetector: could not read the current OS build — skipping"); + return; + } + + if (string.IsNullOrEmpty(_stateDirectory)) + { + _logger.Debug("OsBuildChangeDetector: no state directory configured — skipping"); + return; + } + + var state = LoadState(); + var previousBuild = state?.OsBuild; + if (state == null || string.IsNullOrEmpty(previousBuild)) + { + // First run of this enrollment: seed silently — there is no "previous" to compare. + PersistState(new OsBuildState { OsBuild = currentBuild, CapturedUtc = DateTime.UtcNow }); + _logger.Info($"OsBuildChangeDetector: seeded OS build {currentBuild}"); + return; + } + + if (string.Equals(previousBuild, currentBuild, StringComparison.Ordinal)) + { + _logger.Debug($"OsBuildChangeDetector: OS build unchanged ({currentBuild})"); + return; + } + + BuildChanged = true; + _logger.Info( + $"OsBuildChangeDetector: OS build changed across restart {previousBuild} -> {currentBuild} " + + $"(previous captured {state.CapturedUtc:o})"); + + _post.Emit(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.OsBuildChanged, + Severity = EventSeverity.Info, + Source = Name, + Phase = EnrollmentPhase.Unknown, + Message = $"OS build changed across restart: {previousBuild} -> {currentBuild} — a Windows update was likely installed during enrollment", + Data = new System.Collections.Generic.Dictionary + { + { "previousBuild", previousBuild! }, + { "currentBuild", currentBuild! }, + { "previousCapturedUtc", state.CapturedUtc.ToString("o") }, + }, + ImmediateUpload = true, + }); + + PersistState(new OsBuildState { OsBuild = currentBuild, CapturedUtc = DateTime.UtcNow }); + } + + /// + /// CurrentBuild.UBR (e.g. "26200.8655") from the 64-bit HKLM view, or null when + /// either value is unreadable. + /// + internal static string? ReadOsBuildFromRegistry() + { + try + { + using (var baseKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry64)) + using (var key = baseKey.OpenSubKey(RegKeyCurrentVersion)) + { + if (key == null) return null; + var currentBuild = key.GetValue("CurrentBuild")?.ToString(); + var ubr = key.GetValue("UBR")?.ToString(); + if (string.IsNullOrEmpty(currentBuild) || string.IsNullOrEmpty(ubr)) return null; + return $"{currentBuild}.{ubr}"; + } + } + catch + { + return null; + } + } + + private OsBuildState? LoadState() + { + try + { + var path = Path.Combine(_stateDirectory!, StateFileName); + if (!File.Exists(path)) return null; + return JsonConvert.DeserializeObject(File.ReadAllText(path)); + } + catch (Exception ex) + { + // Corrupt state → treat as first run (re-seed); losing one comparison beats crashing. + _logger.Warning($"OsBuildChangeDetector: state file unreadable, re-seeding: {ex.Message}"); + return null; + } + } + + private void PersistState(OsBuildState state) + { + try + { + Directory.CreateDirectory(_stateDirectory!); + File.WriteAllText( + Path.Combine(_stateDirectory!, StateFileName), + JsonConvert.SerializeObject(state)); + } + catch (Exception ex) + { + _logger.Warning($"OsBuildChangeDetector: failed to persist state: {ex.Message}"); + } + } + + internal sealed class OsBuildState + { + public string? OsBuild { get; set; } + public DateTime CapturedUtc { get; set; } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/WindowsUpdateWatcherHost.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/WindowsUpdateWatcherHost.cs index 4a621223..8022b65d 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/WindowsUpdateWatcherHost.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/WindowsUpdateWatcherHost.cs @@ -31,7 +31,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration IClock clock, int[]? targetedEventIds, int backfillLookbackMinutes, - string stateDirectory) + string stateDirectory, + bool channelCensusEnabled = true, + Func? osBuildChangedProvider = null) { if (ingress == null) throw new ArgumentNullException(nameof(ingress)); if (clock == null) throw new ArgumentNullException(nameof(clock)); @@ -46,7 +48,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration targetedEventIds: targetedEventIds, backfillEnabled: backfillLookbackMinutes > 0, backfillLookbackMinutes: backfillLookbackMinutes, - stateDirectory: stateDirectory); + stateDirectory: stateDirectory, + channelCensusEnabled: channelCensusEnabled, + osBuildChangedProvider: osBuildChangedProvider); } public void Start() diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/WindowsUpdateRuleTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/WindowsUpdateRuleTests.cs index 43897060..6a38edf6 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/WindowsUpdateRuleTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/WindowsUpdateRuleTests.cs @@ -7,12 +7,15 @@ using Moq; namespace AutopilotMonitor.Functions.Tests; /// -/// Windows Update during OOBE — pins the firing behaviour of the two built-in analyze rules +/// Windows Update during OOBE — pins the firing behaviour of the built-in analyze rules /// that surface update activity captured by the agent's WindowsUpdateTracker: /// - ANALYZE-DEV-004 (high): a quality/cumulative update FAILED during enrollment. /// - ANALYZE-DEV-005 (info): a quality/cumulative update INSTALLED during enrollment. -/// Also verifies the matched-condition evidence carries updateTitle / hresult / hresultSymbol so -/// the web interpolator can render {{updateTitle}} and {{hresultSymbol}} in the explanation. +/// - ANALYZE-DEV-006 (info): the OS build changed across a mid-enrollment reboot — +/// deterministic corroboration that works even when the WU channel showed nothing +/// (session 7443317c blind spot). +/// Also verifies the matched-condition evidence carries updateTitle / hresult / hresultSymbol / +/// previousBuild / currentBuild so the web interpolator can render the explanation tokens. /// public class WindowsUpdateRuleTests { @@ -122,8 +125,102 @@ public class WindowsUpdateRuleTests "exists=true must satisfy the reboot_pending condition."); } + [Fact] + public async Task ANALYZE_DEV_006_fires_info_on_os_build_change_with_interpolation_material() + { + var rule = BuiltInAnalyzeRules.GetAll().First(r => r.RuleId == "ANALYZE-DEV-006"); + Assert.True(rule.Enabled); + Assert.Equal("info", rule.Severity); + Assert.False(rule.MarkSessionAsFailedDefault); + + var events = new List + { + OsBuildChanged("26200.8037", "26200.8655"), + }; + + var outcome = await RunAsync(rule, events); + + var result = Assert.Single(outcome.Results); + Assert.Equal("ANALYZE-DEV-006", result.RuleId); + Assert.Equal("info", result.Severity); + + // Interpolation material for {{previousBuild}} / {{currentBuild}} + var previous = AsDict(result.MatchedConditions["previous_build"]); + Assert.Equal("26200.8037", AsString(previous["value"])); + var current = AsDict(result.MatchedConditions["current_build"]); + Assert.Equal("26200.8655", AsString(current["value"])); + } + + [Fact] + public async Task ANALYZE_DEV_006_counts_channel_census_as_corroboration() + { + // Session 7443317c shape: build change + census = the update came through a path the + // WU watcher is blind to; the census signal should be part of the matched evidence. + var rule = BuiltInAnalyzeRules.GetAll().First(r => r.RuleId == "ANALYZE-DEV-006"); + + var events = new List + { + OsBuildChanged("26200.8037", "26200.8655"), + WindowsUpdateChannelCensus(), + }; + + var outcome = await RunAsync(rule, events); + + var result = Assert.Single(outcome.Results); + Assert.True(result.MatchedConditions.ContainsKey("wu_channel_blind"), + "the census event must be counted as corroboration evidence."); + } + + [Fact] + public async Task ANALYZE_DEV_006_does_not_fire_without_a_build_change() + { + var rule = BuiltInAnalyzeRules.GetAll().First(r => r.RuleId == "ANALYZE-DEV-006"); + + var events = new List + { + WindowsUpdateSucceeded("2026-07 Cumulative Update (KB5099999)"), + WindowsUpdateChannelCensus(), + }; + + var outcome = await RunAsync(rule, events); + Assert.Empty(outcome.Results); + } + // ===== Event builders — mirror the WindowsUpdateTracker emit shape ===== + private static EnrollmentEvent OsBuildChanged(string previousBuild, string currentBuild) => new() + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "os_build_changed", + Timestamp = DateTime.UtcNow, + Sequence = 45, + Data = new Dictionary + { + ["previousBuild"] = previousBuild, + ["currentBuild"] = currentBuild, + ["previousCapturedUtc"] = DateTime.UtcNow.AddMinutes(-30).ToString("o"), + } + }; + + private static EnrollmentEvent WindowsUpdateChannelCensus() => new() + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "windows_update_channel_census", + Timestamp = DateTime.UtcNow, + Sequence = 46, + Data = new Dictionary + { + ["wuClientCensus"] = "21=1,25=3", + ["updateOrchestratorCensus"] = "200=2", + ["lookbackMinutes"] = 60, + ["targetedEventIds"] = "19,20,43,44", + } + }; + private static EnrollmentEvent WindowsUpdateFailed(string title, string hresult, string hresultSymbol) => new() { EventId = Guid.NewGuid().ToString(), diff --git a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts index 28ee65d7..c1fb26d8 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts @@ -128,6 +128,8 @@ export const EVENT_TYPES_CATALOG = { 'windows_update_started', 'windows_update_reboot_pending', 'windows_update_history', + 'os_build_changed', + 'windows_update_channel_census', ], lifecycle_events: [ 'agent_started', diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index abf4d2b3..40578d18 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -386,6 +386,11 @@ namespace AutopilotMonitor.Shared // update landed during enrollment even when the watcher's backfill window missed it. public const string WindowsUpdateRebootPending = "windows_update_reboot_pending"; // CBS / WU Auto-Update pending-reboot regkey snapshot public const string WindowsUpdateHistory = "windows_update_history"; // Get-HotFix installed-updates snapshot (QFE-registered only) + // Deterministic update corroboration + blind-spot self-diagnosis (session 7443317c: + // OS build jumped 26200.8037→.8655 across a mid-OOBE reboot, yet the WU channel carried + // none of the targeted EventIDs — the update installed via a path the watcher is blind to). + public const string OsBuildChanged = "os_build_changed"; // OS build (CurrentBuild.UBR) differs across an agent restart — an update was installed during enrollment + public const string WindowsUpdateChannelCensus = "windows_update_channel_census"; // One-shot EventID histogram of WU Client + UpdateOrchestrator channels when os_build_changed but zero targeted WU events captured // TEMPORARY: shadow SM rollout verification — remove when CompletionStateMachine is promoted to primary public const string ShadowDiscrepancy = "shadow_discrepancy"; diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs b/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs index 690ef11a..d75eb923 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs @@ -457,6 +457,19 @@ namespace AutopilotMonitor.Shared.Models /// public int WindowsUpdateBackfillLookbackMinutes { get; set; } = 60; + /// + /// Self-evidencing blind-spot census (session 7443317c: an OOBE quality update installed — + /// OS build changed across the reboot — yet the WindowsUpdateClient channel carried NONE of + /// the targeted EventIDs). When enabled and the OS build changed across an agent restart + /// while zero targeted WU events were captured, the tracker emits a one-shot + /// windows_update_channel_census event with an unfiltered EventID histogram of the + /// WindowsUpdateClient AND UpdateOrchestrator channels over the backfill lookback window — + /// field evidence for which EventIDs/channel to add to + /// via remote config, without diagnostics + /// upload access. Default: true. + /// + public bool WindowsUpdateChannelCensusEnabled { get; set; } = true; + /// /// Creates default collector configuration /// diff --git a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts index 513c859b..05a0f6c3 100644 --- a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts +++ b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts @@ -78,6 +78,10 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [ description: "Startup snapshot of the CBS RebootPending registry key. Present at agent start = a high-confidence corroborator that a (often pre-agent, OOBE-time) update landed and needs a reboot, which can delay/block completion. Gather rule GATHER-DEVICE-005 (on by default)." }, { value: "windows_update_history", label: "windows_update_history", category: "esp", description: "Get-HotFix snapshot of recently installed updates (secondary corroboration). Gather rule GATHER-DEVICE-004 — opt-in (cmdlet cost; QFE-registered updates only)." }, + { value: "os_build_changed", label: "os_build_changed", category: "esp", + description: "The OS build (CurrentBuild.UBR) differs across an agent restart — an update was installed during enrollment, no matter which servicing path it took. Deterministic corroboration for the Windows Update watcher (cannot be lost to timing, watermarks or unexpected logging channels). Carries previousBuild and currentBuild." }, + { value: "windows_update_channel_census", label: "windows_update_channel_census", category: "esp", + description: "Blind-spot self-diagnosis: the OS build changed but zero targeted Windows Update events were captured. Carries an unfiltered EventID histogram of the WindowsUpdateClient and UpdateOrchestrator channels (wuClientCensus / updateOrchestratorCensus, 'id=count' format) — field evidence for which EventIDs to add to WindowsUpdateTargetedEventIds via remote config." }, // -------- ESP signals -------- { value: "esp_phase_changed", label: "esp_phase_changed", category: "esp",