From 9e24b92fbfa514cd223aeee55d5d6ab9122e0ce0 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 23 Jul 2026 15:20:47 +0200 Subject: [PATCH] feat(agent): ESP policy-provider stall detector + Autopilot registry guardrail Customer field case: pre-provisioned devices with the ConfigMgr client as a blocking app occasionally park the user ESP at "Apps (Identifying)" for days. Root cause (verified against the EnrollmentStatusTracking CSP doc): only a "ConfigMgr" provider is registered under Device\Setup\Apps\PolicyProviders (no "Sidecar" = IME) and TrackingPoliciesCreated never becomes 1 - a wait the CSP defines WITHOUT any timeout. Renaming the key to Sidecar unblocks immediately. Microsoft documents that ESP providers are not aware of each other and that pre-provisioning is unsupported with co-management. New always-on esp_policy_provider_stalled tripwire, generalized to any provider in both kinds (Setup/Apps TrackingPoliciesCreated != 1, DevicePreparation InstallationState not in {2,3}): - EspPolicyProviderProbe: fail-soft registry read of the EnrollmentStatusTracking mirror (device + user-SID scopes), defensive value parsing (missing/unparseable = incomplete), ScopedOverride test seam. Research and CSP contract quotes captured in the XML doc header. - EspPolicyProviderStallDetector: pure EvaluateStallTransition on a 15-min continuous-incompleteness dwell (resets on completion/disappearance; absent key = normal early enrollment), one-shot per provider, StartupEventGate fingerprint over the sorted stalled key set for cross-restart dedup. Warning + ImmediateUpload via InformationalEventPost; purely observational, no decision-engine coupling, no config toggle, no ConfigVersion bump (disk_space_low precedent). - EspPolicyProviderStallHost: 60-s wall-clock tick, deliberately outside the StallProbe idle gate (idle clock resets on session activity) and the idle-stopped periodic collectors. - Registered in Constants.EventTypes, MCP resource-catalog (health_events), web gather-rule autocomplete (stall category). Guardrail companion: SOFTWARE\Microsoft\Windows\Autopilot joins the gather-rule registry allowlist ("Autopilot / OOBE / Setup") so tenants can build custom rules on the subtree (e.g. not_exists on the Sidecar provider key); generated web mirror regenerated via combine.js; positive + sibling- spoof guard tests added. Docs: OKF concept doc agent/esp-policy-provider-stall.md (+ index/log). Tests: 22 new detector tests (pure transition matrix + emission surface incl. the ConfigMgr-only customer scenario and gate-across-restart dedup); agent suite 2013/2013, web vitest 589/589 + tsc clean, MCP 449 passed incl. event-types-drift. Both the detector and the embedded allowlist take effect with the next agent release. Co-Authored-By: Claude Fable 5 --- docs/agent/esp-policy-provider-stall.md | 125 ++++++ docs/index.md | 1 + docs/log.md | 2 + rules/guardrails.json | 3 +- .../Gather/GatherRuleCollectorGuardTests.cs | 11 + .../EspPolicyProviderStallDetectorTests.cs | 359 ++++++++++++++++++ .../SystemSignals/EspPolicyProviderProbe.cs | 301 +++++++++++++++ .../EspPolicyProviderStallDetector.cs | 226 +++++++++++ .../Orchestration/DefaultComponentFactory.cs | 17 + .../Hosts/EspPolicyProviderStallHost.cs | 94 +++++ .../src/resource-catalog.ts | 1 + .../AutopilotMonitor.Shared/Constants.cs | 1 + .../app/gather-rules/eventTypes.ts | 2 + .../utils/guardrails.generated.ts | 2 + 14 files changed, 1144 insertions(+), 1 deletion(-) create mode 100644 docs/agent/esp-policy-provider-stall.md create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/EspPolicyProviderStallDetectorTests.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderProbe.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderStallDetector.cs create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/EspPolicyProviderStallHost.cs diff --git a/docs/agent/esp-policy-provider-stall.md b/docs/agent/esp-policy-provider-stall.md new file mode 100644 index 00000000..b1cd78d5 --- /dev/null +++ b/docs/agent/esp-policy-provider-stall.md @@ -0,0 +1,125 @@ +--- +type: Concept +title: ESP Policy-Provider Stall Detection — the no-timeout Setup/Apps wait and the co-management Sidecar gap +description: Why a registered EnrollmentStatusTracking CSP policy provider that never sets TrackingPoliciesCreated parks the user ESP at "Apps (Identifying)" indefinitely, the pre-provisioning + ConfigMgr field case, and how the always-on esp_policy_provider_stalled tripwire (15-min wall-clock dwell, StartupEventGate cross-restart dedup) surfaces it. +resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderProbe.cs +tags: + - agent + - esp + - enrollment-status-tracking + - co-management + - stall-detection +timestamp: 2026-07-23T00:00:00+02:00 +--- + +# ESP Policy-Provider Stall Detection + +## Schema + +The Enrollment Status Page decides when its "Apps" category may progress from +"Identifying" using the **EnrollmentStatusTracking CSP** +(), +mirrored in the registry under +`HKLM\SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking\{Device|}\…`. +Two contracts matter: + +1. **`Setup/Apps/PolicyProviders/`** — per the CSP doc: *"Existence of this node + indicates to the ESP that it shouldn't show the tracking status message until the + TrackingPoliciesCreated node has been set to true."* This wait has **no timeout**. A + registered provider that never sets `TrackingPoliciesCreated=1` parks the ESP at + "Apps (Identifying)" indefinitely — observed for days in the field. Exists in device + scope (`Device`) and user scope (`S-`, appears at sign-in). +2. **`DevicePreparation/PolicyProviders//InstallationState`** — must reach 2 + (NotRequired) or 3 (Completed); 1=NotInstalled, 4=Error. Device scope only. The ESP + applies a default 15-minute timeout here, so these stalls self-surface as ESP errors — + the detector still records them for the diagnostic trail. + +Known provider names: **`Sidecar` is the Intune Management Extension** (the CSP doc names +"Intune's agents, such as SideCar"); **`ConfigMgr`** is the Configuration Manager client, +registered during Autopilot-into-co-management — the troubleshooting section of + queries +exactly these registry paths. The same document states that ESP policy providers **are not +aware of each other** and that **pre-provisioning is unsupported** with Autopilot into +co-management. + +## The field case + +A tenant pre-provisions devices and installs the ConfigMgr client as a blocking app. +Occasionally the user-phase ESP reaches "Apps (Identifying)" and never leaves it — for +hours to days. On those devices `Device\Setup\Apps\PolicyProviders` contains **only a +`ConfigMgr` key (no `Sidecar`)** and `TrackingPoliciesCreated` is never set. Manually +renaming the key to `Sidecar` unblocks the ESP immediately (the IME then finds its +provider node and reports tracking-policies-created). The mechanism is exactly the +documented no-timeout wait; the unsupported pre-provisioning + co-management combination +explains why only some devices are affected. Why the Sidecar registration is missing or +displaced on those devices is not publicly documented — irrelevant for detection, since +the signature is deterministic. + +## Detection + +`EspPolicyProviderProbe` (fail-soft, static, `ScopedOverride` test seam — structural clone +of `EspTrackingInfoProbe`) enumerates all providers in both kinds and scopes. +`EspPolicyProviderStallDetector` evaluates a pure transition function on a 60-second tick +from the always-on `EspPolicyProviderStallHost`: + +* A provider **continuously incomplete for ≥ 15 minutes** (wall-clock) is stalled. The + dwell clock starts at first incomplete observation and resets on completion or + disappearance. Missing values parse as incomplete (fail-toward-observation). +* Missing root key / empty provider list = normal early enrollment: clears all dwell + state, never fires. +* One `esp_policy_provider_stalled` **Warning** per newly stalled provider set + (`ImmediateUpload`), one-shot per provider in-process, cross-restart deduped via a + `StartupEventGate` fingerprint over the sorted currently-stalled key set — a restart + into the same stall stays silent, a new provider joining the stall re-reports. +* Payload: `providers[]` (full table: name, scope, kind, raw values, complete flag), + `stalledProviders[]` with `stalledForMinutes`, `sidecarRegistered`, `dwellMinutes`. +* **Purely observational**: emitted via `InformationalEventPost` (dispatch-guard exempt + pass-through) — never touches decision state. No config toggle, no ConfigVersion bump + (precedent: `disk_space_low`). + +Deliberately **not** built on `StallProbeCollector` (its idle clock resets on any session +activity — a provider stall coexists with a chatty session) and not on the periodic +collectors (idle-stopped after 15 minutes — exactly the dormant window the dwell must +observe). + +## Guardrail companion + +`rules/guardrails.json` allow-lists `SOFTWARE\Microsoft\Windows\Autopilot` (category +"Autopilot / OOBE / Setup") so tenants can build their own gather rules on the subtree — +e.g. a `not_exists` registry rule on the `Sidecar` provider key. The subtree carries +deployment state only (provider registrations, tracking lists, app IDs, per-user SIDs); +sensitivity is at or below existing entries. Enforcement is agent-side (embedded +resource), so both the detector and the guardrail change ship with the agent release. + +## Examples + +Customer-case event (abbreviated): + +```json +{ + "eventType": "esp_policy_provider_stalled", + "severity": "Warning", + "data": { + "dwellMinutes": 15, + "sidecarRegistered": false, + "providers": [ + { "name": "ConfigMgr", "scope": "device", "kind": "setupApps", + "trackingPoliciesCreated": null, "installationState": null, "complete": false } + ], + "stalledProviders": [ + { "name": "ConfigMgr", "scope": "device", "kind": "setupApps", "stalledForMinutes": 15.0 } + ] + } +} +``` + +`sidecarRegistered=false` while `setupApps` providers exist is the co-management +signature; `sidecarRegistered=true` with a stalled third-party provider generalizes the +same contract violation. + +# Citations + +* EnrollmentStatusTracking CSP — +* Windows Autopilot into co-management (requirements, provider-conflict warning, troubleshooting registry paths) — +* Agent code: `EspPolicyProviderProbe.cs`, `EspPolicyProviderStallDetector.cs`, `EspPolicyProviderStallHost.cs` (SystemSignals / Orchestration.Hosts) +* Related: [MDM Reboot Coalescing](mdm-reboot-coalescing.md), [Gather-Rule Guardrails](../rules/gather-rule-guardrails.md) diff --git a/docs/index.md b/docs/index.md index 61023c4e..9f2841a5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,6 +22,7 @@ https://docs.autopilotmonitor.com (separate repository). * [Agent Endpoint Migration](agent/endpoint-migration.md) - How the backend re-homes agents to a new API base URL via the config channel (backend move or per-tenant region move): live-fetch-only, allowlist-validated on both sides, one hop, kill wins. * [MDM Reboot Coalescing](agent/mdm-reboot-coalescing.md) - Attributing the mid-ESP coalesced reboot ("second sign-in") to the device-assigned policy URIs that forced it: aggregated DM-Enterprise 2800 observations (neutral, flush-time watermark) + the ANALYZE-ESP-005 advisory gated on an actually-observed reboot. * [Secure Boot CA-2023 Detection](agent/secureboot-ca2023-detection.md) - Why ANALYZE-SEC-001 v3 reads the UEFI db/KEK variables directly instead of trusting the SecureBoot\Servicing registry, and how the one-sided uefiCa2023FirmwareConfirmed marker plus a not_exists precondition give absence-tolerant suppression across old and new agents. +* [ESP Policy-Provider Stall Detection](agent/esp-policy-provider-stall.md) - The EnrollmentStatusTracking CSP's no-timeout Setup/Apps wait: a registered provider that never sets TrackingPoliciesCreated (co-management "ConfigMgr" without the IME's "Sidecar") parks the user ESP at "Apps (Identifying)" for days; the always-on esp_policy_provider_stalled tripwire and the SOFTWARE\Microsoft\Windows\Autopilot gather-rule allowlist entry that surface it. # Rules diff --git a/docs/log.md b/docs/log.md index 21830727..83ceaaeb 100644 --- a/docs/log.md +++ b/docs/log.md @@ -2,6 +2,8 @@ ## 2026-07-23 +* **Creation**: Added `agent/esp-policy-provider-stall.md` — a customer diagnosed that pre-provisioned devices with the ConfigMgr client as a blocking app occasionally park the user ESP at "Apps (Identifying)" for days: only a `ConfigMgr` provider is registered under `EnrollmentStatusTracking\Device\Setup\Apps\PolicyProviders` (no `Sidecar` = IME) and `TrackingPoliciesCreated` never becomes 1 — a wait the CSP defines without any timeout; renaming the key to `Sidecar` unblocks immediately. The agent now ships the always-on `esp_policy_provider_stalled` tripwire (60-s tick, 15-min continuous-incompleteness dwell, one-shot per provider, `StartupEventGate` fingerprint across restarts, purely observational via the InformationalEvent rail — no toggle, no ConfigVersion bump), generalized to any provider in both `Setup/Apps` and `DevicePreparation` kinds rather than hardcoding the ConfigMgr case. Companion change: `SOFTWARE\Microsoft\Windows\Autopilot` joins the gather-rule registry allowlist so tenants can target the subtree with custom rules (e.g. `not_exists` on the Sidecar key). + * **Creation**: Added `agent/secureboot-ca2023-detection.md` — ANALYZE-SEC-001 warned on devices whose firmware already carried the 2023 certificate, because the agent only read the `SecureBoot\Servicing\UEFICA2023Status` registry value, which mirrors the Windows Update rollout rather than the firmware (customer report). The agent now reads the UEFI `db`/`KEK` variables natively (`GetFirmwareEnvironmentVariableExW` with `SeSystemEnvironmentPrivilege` enabled on the process token — first privilege-adjustment code in the repo) and emits per-certificate booleans on `secureboot_status`. Suppression rides on a one-sided marker, `uefiCa2023FirmwareConfirmed`, emitted only on positive confirmation: the engine's `not_exists` precondition fails on any present value, so a two-sided boolean could not distinguish "confirmed" from "checked and missing", while the marker's absence keeps old agents and firmware-missing devices on the v2 path. Rule v3.0.0 is therefore safe to seed before the agent release. * **Update**: `url-registry.md` — the first release after the alias migration surfaced Front Door caching as a correctness hazard: the alias served the previous `version.json` (`X-Cache: TCP_REMOTE_HIT`) after the origin had rotated, and since manifest and ZIP are independent cache entries, a stale manifest with a fresh ZIP fails the bootstrap SHA-256 check. Route caching on `/agent/*` is now disabled and every blob upload stamps `Cache-Control: no-cache` (CI `Send-Blob`, local `build.ps1`, `versions/*.json` manifests) — all agent-container blobs rotate in place, so nothing there is safely cacheable. diff --git a/rules/guardrails.json b/rules/guardrails.json index 4365b5d6..5bf774af 100644 --- a/rules/guardrails.json +++ b/rules/guardrails.json @@ -43,7 +43,8 @@ "prefixes": [ "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Setup", "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\OOBE", - "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon" + "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon", + "SOFTWARE\\Microsoft\\Windows\\Autopilot" ] }, { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Gather/GatherRuleCollectorGuardTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Gather/GatherRuleCollectorGuardTests.cs index ebc7873f..c4065c07 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Gather/GatherRuleCollectorGuardTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Gather/GatherRuleCollectorGuardTests.cs @@ -290,9 +290,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather [InlineData(@"SAM\SAM\Domains\Account\Users")] [InlineData(@"SECURITY\Policy\Secrets")] [InlineData(@"HKLM\SOFTWARE\Microsoft\EnrollmentsSomethingElse")] // prefix spoofing + [InlineData(@"HKLM\SOFTWARE\Microsoft\Windows\AutopilotSomethingElse")] // sibling spoof of the Autopilot prefix public void RegistryCollector_OffAllowlistPath_IsBlockedBeforeTheHiveIsOpened(string target) => AssertBlocked(new RegistryCollector().Execute(Rule("registry", target), Context()), target); + // Session 5d735290: the EnrollmentStatusTracking subtree (ESP policy providers / tracking + // state) is allow-listed so tenants can build gather rules on the co-management stall + // signature the esp_policy_provider_stalled detector reports (e.g. not_exists on the + // Sidecar provider key). + [Theory] + [InlineData(@"SOFTWARE\Microsoft\Windows\Autopilot")] + [InlineData(@"SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking\Device\Setup\Apps\PolicyProviders\Sidecar")] + public void RegistryGuard_AutopilotEnrollmentTrackingSubtree_IsAllowed(string subPath) + => Assert.True(GatherRuleGuards.IsRegistryPathAllowed(subPath)); + [Theory] [InlineData("SELECT * FROM Win32_UserAccount")] [InlineData("SELECT * FROM Win32_ShadowCopy")] diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/EspPolicyProviderStallDetectorTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/EspPolicyProviderStallDetectorTests.cs new file mode 100644 index 00000000..f59a3e7b --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/EspPolicyProviderStallDetectorTests.cs @@ -0,0 +1,359 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Persistence; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals +{ + /// + /// Session 5d735290 — pure detection logic of : + /// the CSP completion rule and the dwell/one-shot transition. No registry, no timers. + /// + public sealed class EspPolicyProviderStallLogicTests + { + private static readonly DateTime T0 = new DateTime(2026, 7, 23, 10, 0, 0, DateTimeKind.Utc); + private static readonly TimeSpan Dwell = TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes); + + private static PolicyProviderState SetupApps(string name, int? tracking, string scope = "device") => + new PolicyProviderState(name, scope, EspPolicyProviderProbe.KindSetupApps, tracking, null); + + private static PolicyProviderState DevicePrep(string name, int? installState) => + new PolicyProviderState(name, "device", EspPolicyProviderProbe.KindDevicePreparation, null, installState); + + private static EspPolicyProviderSnapshot Snapshot(params PolicyProviderState[] providers) => + new EspPolicyProviderSnapshot(providers); + + // ------------------------------------------------- completion rule (CSP contract) + + [Theory] + [InlineData(1, true)] // TrackingPoliciesCreated=true + [InlineData(0, false)] // explicit false (the CSP default) + [InlineData(null, false)] // value missing — the field case + [InlineData(2, false)] // anything else is not "true" + public void SetupApps_provider_complete_only_at_tracking_1(int? tracking, bool expected) + => Assert.Equal(expected, EspPolicyProviderProbe.IsProviderComplete( + EspPolicyProviderProbe.KindSetupApps, tracking, null)); + + [Theory] + [InlineData(2, true)] // NotRequired + [InlineData(3, true)] // Completed + [InlineData(1, false)] // NotInstalled + [InlineData(4, false)] // Error + [InlineData(null, false)] + public void DevicePreparation_provider_complete_at_state_2_or_3(int? state, bool expected) + => Assert.Equal(expected, EspPolicyProviderProbe.IsProviderComplete( + EspPolicyProviderProbe.KindDevicePreparation, null, state)); + + [Fact] + public void TryReadInt_tolerates_dword_string_and_bool_forms() + { + Assert.Equal(1, EspPolicyProviderProbe.TryReadInt(1)); + Assert.Equal(3, EspPolicyProviderProbe.TryReadInt(3L)); + Assert.Equal(1, EspPolicyProviderProbe.TryReadInt(" 1 ")); + Assert.Equal(1, EspPolicyProviderProbe.TryReadInt(true)); + Assert.Equal(0, EspPolicyProviderProbe.TryReadInt(false)); + Assert.Null(EspPolicyProviderProbe.TryReadInt(null)); + Assert.Null(EspPolicyProviderProbe.TryReadInt("not-a-number")); + Assert.Null(EspPolicyProviderProbe.TryReadInt(new byte[] { 1 })); + } + + // ------------------------------------------------- dwell transition + + [Fact] + public void Empty_or_missing_snapshot_clears_dwell_state_and_never_stalls() + { + var previous = new Dictionary { ["setupApps|device|ConfigMgr"] = T0 }; + + var missing = EspPolicyProviderStallDetector.EvaluateStallTransition( + EspPolicyProviderSnapshot.Empty, previous, new HashSet(), T0.AddHours(2), Dwell); + Assert.Empty(missing.UpdatedFirstSeenUtc); + Assert.Empty(missing.NewlyStalled); + + var registeredButEmpty = EspPolicyProviderStallDetector.EvaluateStallTransition( + Snapshot(), previous, new HashSet(), T0.AddHours(2), Dwell); + Assert.Empty(registeredButEmpty.UpdatedFirstSeenUtc); + Assert.Empty(registeredButEmpty.NewlyStalled); + } + + [Fact] + public void Incomplete_provider_accrues_dwell_but_does_not_stall_before_threshold() + { + var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: null)); + + var round1 = EspPolicyProviderStallDetector.EvaluateStallTransition( + snapshot, new Dictionary(), new HashSet(), T0, Dwell); + Assert.Equal(T0, round1.UpdatedFirstSeenUtc["setupApps|device|ConfigMgr"]); + Assert.Empty(round1.NewlyStalled); + + var round2 = EspPolicyProviderStallDetector.EvaluateStallTransition( + snapshot, round1.UpdatedFirstSeenUtc, new HashSet(), T0.AddMinutes(14), Dwell); + Assert.Equal(T0, round2.UpdatedFirstSeenUtc["setupApps|device|ConfigMgr"]); // first-seen preserved + Assert.Empty(round2.NewlyStalled); + } + + [Fact] + public void Provider_stalls_at_dwell_threshold_with_correct_minutes() + { + var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: 0)); + var seen = new Dictionary { ["setupApps|device|ConfigMgr"] = T0 }; + + var eval = EspPolicyProviderStallDetector.EvaluateStallTransition( + snapshot, seen, new HashSet(), T0.AddMinutes(15), Dwell); + + var stalled = Assert.Single(eval.NewlyStalled); + Assert.Equal("ConfigMgr", stalled.Provider.Name); + Assert.Equal(15.0, stalled.StalledForMinutes, precision: 3); + } + + [Fact] + public void Completion_or_disappearance_resets_the_dwell_clock() + { + var key = "setupApps|device|ConfigMgr"; + var seen = new Dictionary { [key] = T0 }; + + // Completes mid-dwell → dropped from the map. + var completed = EspPolicyProviderStallDetector.EvaluateStallTransition( + Snapshot(SetupApps("ConfigMgr", tracking: 1)), seen, new HashSet(), T0.AddMinutes(10), Dwell); + Assert.Empty(completed.UpdatedFirstSeenUtc); + + // Re-appears incomplete much later → dwell restarts from the new now, no stall yet. + var reappeared = EspPolicyProviderStallDetector.EvaluateStallTransition( + Snapshot(SetupApps("ConfigMgr", tracking: 0)), completed.UpdatedFirstSeenUtc, + new HashSet(), T0.AddHours(3), Dwell); + Assert.Equal(T0.AddHours(3), reappeared.UpdatedFirstSeenUtc[key]); + Assert.Empty(reappeared.NewlyStalled); + + // Disappearance behaves the same as completion. + var disappeared = EspPolicyProviderStallDetector.EvaluateStallTransition( + Snapshot(SetupApps("Sidecar", tracking: 1)), new Dictionary { [key] = T0 }, + new HashSet(), T0.AddMinutes(20), Dwell); + Assert.False(disappeared.UpdatedFirstSeenUtc.ContainsKey(key)); + } + + [Fact] + public void Already_fired_provider_is_never_reported_again() + { + var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: null)); + var seen = new Dictionary { ["setupApps|device|ConfigMgr"] = T0 }; + var fired = new HashSet(StringComparer.OrdinalIgnoreCase) { "setupApps|device|ConfigMgr" }; + + var eval = EspPolicyProviderStallDetector.EvaluateStallTransition( + snapshot, seen, fired, T0.AddHours(5), Dwell); + + Assert.Empty(eval.NewlyStalled); + Assert.True(eval.UpdatedFirstSeenUtc.ContainsKey("setupApps|device|ConfigMgr")); // still tracked + } + + [Fact] + public void Mixed_kinds_and_scopes_are_tracked_under_distinct_keys() + { + var snapshot = Snapshot( + SetupApps("ConfigMgr", tracking: null), + SetupApps("ConfigMgr", tracking: null, scope: "user:S-1-5-21-1111"), + DevicePrep("ConfigMgr", installState: 1)); + + var eval = EspPolicyProviderStallDetector.EvaluateStallTransition( + snapshot, new Dictionary(), new HashSet(), T0, Dwell); + + Assert.Equal(3, eval.UpdatedFirstSeenUtc.Count); + Assert.Contains("setupApps|device|ConfigMgr", eval.UpdatedFirstSeenUtc.Keys); + Assert.Contains("setupApps|user:S-1-5-21-1111|ConfigMgr", eval.UpdatedFirstSeenUtc.Keys); + Assert.Contains("devicePreparation|device|ConfigMgr", eval.UpdatedFirstSeenUtc.Keys); + } + + [Fact] + public void SidecarRegistered_matches_setupApps_sidecar_only() + { + Assert.True(Snapshot(SetupApps("Sidecar", tracking: 0)).SidecarRegistered); + Assert.True(Snapshot(SetupApps("sidecar", tracking: 1, scope: "user:S-1-5-21-1")).SidecarRegistered); + Assert.False(Snapshot(SetupApps("ConfigMgr", tracking: 1)).SidecarRegistered); + Assert.False(Snapshot(DevicePrep("Sidecar", installState: 3)).SidecarRegistered); // wrong kind + Assert.False(EspPolicyProviderSnapshot.Empty.SidecarRegistered); + } + } + + /// + /// Emission surface of the detector via the real rail. + /// Static probe override → serial collection (same contract as + /// EspConfigDetectedTrackingFieldsTests). + /// + [Collection("SerialThreading")] + public sealed class EspPolicyProviderStallDetectorEmissionTests : IDisposable + { + private static readonly DateTime At = new DateTime(2026, 7, 23, 12, 0, 0, DateTimeKind.Utc); + + private readonly TempDirectory _tmp = new TempDirectory(); + + public void Dispose() => _tmp.Dispose(); + + private (EspPolicyProviderStallDetector sut, FakeSignalIngressSink sink, VirtualClock clock) + BuildDetector(StartupEventGate? gate = null) + { + var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info); + var sink = new FakeSignalIngressSink(); + var clock = new VirtualClock(At); + var post = new InformationalEventPost(sink, clock); + var sut = new EspPolicyProviderStallDetector("S1", "T1", post, logger, clock, gate); + return (sut, sink, clock); + } + + private static EspPolicyProviderSnapshot ConfigMgrOnly() => new EspPolicyProviderSnapshot(new[] + { + new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindSetupApps, + trackingPoliciesCreated: null, installationState: null), + }); + + private static EspPolicyProviderSnapshot SidecarCompleteConfigMgrIncomplete() => new EspPolicyProviderSnapshot(new[] + { + new PolicyProviderState("Sidecar", "device", EspPolicyProviderProbe.KindSetupApps, 1, null), + new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindSetupApps, 0, null), + }); + + private static IReadOnlyList StalledEvents(FakeSignalIngressSink sink) => + sink.Posted.Where(p => + p.Payload != null + && p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et) + && et == "esp_policy_provider_stalled").ToList(); + + [Fact] + public void CustomerScenario_configmgr_without_sidecar_emits_one_warning_after_dwell() + { + using var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => ConfigMgrOnly()); + var (sut, sink, clock) = BuildDetector(); + + sut.Tick(); // first sighting — dwell starts + Assert.Empty(StalledEvents(sink)); + + clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes)); + sut.Tick(); // threshold crossed → emit + + var evt = Assert.Single(StalledEvents(sink)); + Assert.Equal(DecisionSignalKind.InformationalEvent, evt.Kind); + Assert.Equal("Warning", evt.Payload![SignalPayloadKeys.Severity]); + Assert.Equal("true", evt.Payload[SignalPayloadKeys.ImmediateUpload]); + Assert.Equal(EspPolicyProviderStallDetector.SourceLabel, evt.Payload[SignalPayloadKeys.Source]); + + var data = Assert.IsAssignableFrom>(evt.TypedPayload); + Assert.Equal(false, data["sidecarRegistered"]); + Assert.Equal(EspPolicyProviderStallDetector.DwellMinutes, data["dwellMinutes"]); + var stalled = Assert.IsAssignableFrom>>(data["stalledProviders"]); + var entry = Assert.Single(stalled); + Assert.Equal("ConfigMgr", entry["name"]); + Assert.Equal(EspPolicyProviderProbe.KindSetupApps, entry["kind"]); + Assert.Equal(15.0, entry["stalledForMinutes"]); + } + + [Fact] + public void Sidecar_complete_configmgr_incomplete_reports_only_configmgr_with_sidecar_flag() + { + using var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => SidecarCompleteConfigMgrIncomplete()); + var (sut, sink, clock) = BuildDetector(); + + sut.Tick(); + clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes)); + sut.Tick(); + + var evt = Assert.Single(StalledEvents(sink)); + var data = Assert.IsAssignableFrom>(evt.TypedPayload); + Assert.Equal(true, data["sidecarRegistered"]); + var stalled = Assert.IsAssignableFrom>>(data["stalledProviders"]); + Assert.Equal("ConfigMgr", Assert.Single(stalled)["name"]); + var providers = Assert.IsAssignableFrom>>(data["providers"]); + Assert.Equal(2, providers.Count); // full table always attached + } + + [Fact] + public void Latch_prevents_second_emission_for_the_same_provider() + { + using var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => ConfigMgrOnly()); + var (sut, sink, clock) = BuildDetector(); + + sut.Tick(); + clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes)); + sut.Tick(); + clock.Advance(TimeSpan.FromMinutes(30)); + sut.Tick(); + sut.Tick(); + + Assert.Single(StalledEvents(sink)); + } + + [Fact] + public void Startup_gate_suppresses_same_stall_across_restart_but_reports_new_provider_set() + { + var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info); + + // Run 1: fires and persists the fingerprint. + using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => ConfigMgrOnly())) + { + var (sut, sink, clock) = BuildDetector(new StartupEventGate(_tmp.Path, logger)); + sut.Tick(); + clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes)); + sut.Tick(); + Assert.Single(StalledEvents(sink)); + } + + // Run 2 (simulated restart — fresh detector, gate reloaded from the same state dir): + // identical stall re-accrues dwell but lands on the same fingerprint → suppressed. + using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => ConfigMgrOnly())) + { + var (sut, sink, clock) = BuildDetector(new StartupEventGate(_tmp.Path, logger)); + sut.Tick(); + clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes)); + sut.Tick(); + Assert.Empty(StalledEvents(sink)); + } + + // Run 3: a SECOND provider joins the stall → fingerprint changes → reported. + var widened = new EspPolicyProviderSnapshot(new[] + { + new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindSetupApps, null, null), + new PolicyProviderState("Contoso", "device", EspPolicyProviderProbe.KindSetupApps, 0, null), + }); + using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => widened)) + { + var (sut, sink, clock) = BuildDetector(new StartupEventGate(_tmp.Path, logger)); + sut.Tick(); + clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes)); + sut.Tick(); + Assert.Single(StalledEvents(sink)); + } + } + + [Fact] + public void Healthy_or_absent_tracking_never_emits() + { + var healthy = new EspPolicyProviderSnapshot(new[] + { + new PolicyProviderState("Sidecar", "device", EspPolicyProviderProbe.KindSetupApps, 1, null), + new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindDevicePreparation, null, 3), + }); + using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => healthy)) + { + var (sut, sink, clock) = BuildDetector(); + sut.Tick(); + clock.Advance(TimeSpan.FromHours(4)); + sut.Tick(); + Assert.Empty(StalledEvents(sink)); + } + + using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => EspPolicyProviderSnapshot.Empty)) + { + var (sut, sink, clock) = BuildDetector(); + sut.Tick(); + clock.Advance(TimeSpan.FromHours(4)); + sut.Tick(); + Assert.Empty(StalledEvents(sink)); + } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderProbe.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderProbe.cs new file mode 100644 index 00000000..c0809a05 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderProbe.cs @@ -0,0 +1,301 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using Microsoft.Win32; + +namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals +{ + /// + /// Session 5d735290 (2026-07-23) — reads the ESP's registered policy providers from + /// HKLM\SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking, the registry + /// mirror of the EnrollmentStatusTracking CSP + /// (https://learn.microsoft.com/en-us/windows/client-management/mdm/enrollmentstatustracking-csp). + /// + /// CSP contract the ESP enforces (quotes from the doc above): + /// + /// Setup/Apps/PolicyProviders/<Name> — "Existence of this node indicates + /// to the ESP that it shouldn't show the tracking status message until the + /// TrackingPoliciesCreated node has been set to true." This wait has NO timeout: a + /// registered provider that never sets TrackingPoliciesCreated=1 parks the ESP at + /// "Apps (Identifying)" indefinitely (observed for days in the field). + /// DevicePreparation/PolicyProviders/<Name>/InstallationState — must reach + /// 2 (NotRequired) or 3 (Completed); 1=NotInstalled, 4=Error. ESP applies a default + /// 15-minute timeout here, so stalls in this stage are self-surfacing — still reported + /// for the diagnostic trail. + /// + /// Registry layout: Device subkey for device scope, S-<SID> subkeys for user + /// scope (appear at sign-in), each containing Setup\Apps\PolicyProviders\<Name>; + /// DevicePreparation\PolicyProviders exists in device scope only. + /// + /// + /// Known provider names: Sidecar is the Intune Management Extension ("Intune's agents, + /// such as SideCar" per the CSP doc); ConfigMgr is the Configuration Manager client + /// (registered during Autopilot-into-co-management, see the troubleshooting registry paths in + /// https://learn.microsoft.com/en-us/intune/configmgr/comanage/autopilot-enrollment). Microsoft + /// warns that ESP policy providers are not aware of each other and that pre-provisioning is + /// unsupported with co-management. Field case that motivated this probe: a pre-provisioned + /// device with the ConfigMgr client installed as a blocking app ended up with ONLY + /// ConfigMgr registered under Device\Setup\Apps\PolicyProviders (no + /// Sidecar) and TrackingPoliciesCreated never set — the user ESP hung at + /// "Apps (Identifying)" for hours to days; manually renaming the key to Sidecar + /// unblocked it immediately. turns this contract + /// into the esp_policy_provider_stalled tripwire. + /// + /// + /// Fail-soft like : any failure is Debug-logged and yields + /// ; a missing root key (early enrollment, + /// non-Autopilot device) is not an error. Values are parsed defensively — a missing or + /// unparseable value counts as incomplete (fail-toward-observation). + /// + /// + internal static class EspPolicyProviderProbe + { + internal const string TrackingRootKeyPath = + @"SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking"; + + /// Provider kind: Setup\Apps\PolicyProviders (TrackingPoliciesCreated wait, no timeout). + internal const string KindSetupApps = "setupApps"; + + /// Provider kind: DevicePreparation\PolicyProviders (InstallationState wait, 15-min OS timeout). + internal const string KindDevicePreparation = "devicePreparation"; + + /// The IME's provider name — its absence while other providers are registered is the field-case signature. + internal const string SidecarProviderName = "Sidecar"; + + internal const string DeviceScope = "device"; + + private const string TrackingPoliciesCreatedValueName = "TrackingPoliciesCreated"; + private const string InstallationStateValueName = "InstallationState"; + + /// + /// Test seam — when non-null, delegates to this func instead of + /// touching the live registry. Use to guarantee cleanup. + /// + internal static Func TestOverride; + + /// + /// Reads all registered ESP policy providers. + /// is true when the EnrollmentStatusTracking root key existed and was readable + /// (even with no providers registered); otherwise. + /// + /// Optional logger — Debug-level trace only; no warn/error. + public static EspPolicyProviderSnapshot Read(AgentLogger logger = null) + { + var probe = TestOverride; + if (probe != null) return probe(logger); + + try + { + using (var rootKey = Registry.LocalMachine.OpenSubKey(TrackingRootKeyPath)) + { + if (rootKey == null) + { + logger?.Debug("EspPolicyProviderProbe: EnrollmentStatusTracking registry key not found"); + return EspPolicyProviderSnapshot.Empty; + } + + var providers = new List(); + + foreach (var scopeName in rootKey.GetSubKeyNames()) + { + var isDevice = string.Equals(scopeName, "Device", StringComparison.OrdinalIgnoreCase); + // User scope repeats the Setup structure under the signed-in user's SID. + var isUserSid = scopeName.StartsWith("S-", StringComparison.OrdinalIgnoreCase); + if (!isDevice && !isUserSid) continue; + + var scopeLabel = isDevice ? DeviceScope : "user:" + scopeName; + using (var scopeKey = rootKey.OpenSubKey(scopeName)) + { + if (scopeKey == null) continue; + + CollectProviders( + scopeKey, @"Setup\Apps\PolicyProviders", scopeLabel, KindSetupApps, + TrackingPoliciesCreatedValueName, providers, logger); + + if (isDevice) + { + CollectProviders( + scopeKey, @"DevicePreparation\PolicyProviders", scopeLabel, KindDevicePreparation, + InstallationStateValueName, providers, logger); + } + } + } + + return new EspPolicyProviderSnapshot(providers); + } + } + catch (Exception ex) + { + logger?.Debug($"EspPolicyProviderProbe: read threw: {ex.Message}"); + return EspPolicyProviderSnapshot.Empty; + } + } + + private static void CollectProviders( + RegistryKey scopeKey, + string providersSubPath, + string scopeLabel, + string kind, + string stateValueName, + List providers, + AgentLogger logger) + { + try + { + using (var providersKey = scopeKey.OpenSubKey(providersSubPath)) + { + if (providersKey == null) return; + + foreach (var providerName in providersKey.GetSubKeyNames()) + { + using (var providerKey = providersKey.OpenSubKey(providerName)) + { + if (providerKey == null) continue; + var value = TryReadInt(providerKey.GetValue(stateValueName)); + providers.Add(new PolicyProviderState( + name: providerName, + scope: scopeLabel, + kind: kind, + trackingPoliciesCreated: kind == KindSetupApps ? value : null, + installationState: kind == KindDevicePreparation ? value : null)); + } + } + } + } + catch (Exception ex) + { + logger?.Debug($"EspPolicyProviderProbe: '{providersSubPath}' ({scopeLabel}) read threw: {ex.Message}"); + } + } + + /// + /// Defensive value parse: REG_DWORD expected per CSP (booleans stored as 0/1), but string + /// forms are tolerated. Anything missing or unparseable is null — which the + /// completion rule treats as incomplete. + /// + internal static int? TryReadInt(object rawValue) + { + switch (rawValue) + { + case null: return null; + case int i: return i; + case long l when l >= int.MinValue && l <= int.MaxValue: return (int)l; + case string s when int.TryParse(s.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed): + return parsed; + case bool b: return b ? 1 : 0; + default: return null; + } + } + + /// + /// Pure completion rule per the CSP contract: setupApps providers are complete when + /// TrackingPoliciesCreated == 1; devicePreparation providers when + /// InstallationState is 2 (NotRequired) or 3 (Completed). + /// + internal static bool IsProviderComplete(string kind, int? trackingPoliciesCreated, int? installationState) + => kind == KindSetupApps + ? trackingPoliciesCreated == 1 + : installationState == 2 || installationState == 3; + + /// + /// Disposable scope that sets for the lifetime of the scope + /// and restores the previous value on Dispose. Nestable. Internal test-only helper. + /// + internal sealed class ScopedOverride : IDisposable + { + private readonly Func _previous; + private int _disposed; + + public ScopedOverride(Func probe) + { + _previous = TestOverride; + TestOverride = probe; + } + + public void Dispose() + { + if (System.Threading.Interlocked.Exchange(ref _disposed, 1) == 1) return; + TestOverride = _previous; + } + } + } + + /// + /// One registered ESP policy provider with its completion-relevant state. + /// is the stable identity used for dwell tracking and one-shot latching. + /// + internal readonly struct PolicyProviderState + { + public PolicyProviderState(string name, string scope, string kind, int? trackingPoliciesCreated, int? installationState) + { + Name = name ?? string.Empty; + Scope = scope ?? string.Empty; + Kind = kind ?? string.Empty; + TrackingPoliciesCreated = trackingPoliciesCreated; + InstallationState = installationState; + IsComplete = EspPolicyProviderProbe.IsProviderComplete(kind, trackingPoliciesCreated, installationState); + } + + /// Provider key name, e.g. "Sidecar" or "ConfigMgr". + public string Name { get; } + + /// "device" or "user:S-1-5-21-…". + public string Scope { get; } + + /// or . + public string Kind { get; } + + /// Raw TrackingPoliciesCreated value (setupApps only); null when missing/unparseable. + public int? TrackingPoliciesCreated { get; } + + /// Raw InstallationState value (devicePreparation only); null when missing/unparseable. + public int? InstallationState { get; } + + /// Completion per . + public bool IsComplete { get; } + + /// Stable dwell/latch identity: kind|scope|name. + public string Key => Kind + "|" + Scope + "|" + Name; + } + + /// + /// Snapshot of all registered ESP policy providers. is false only + /// when the EnrollmentStatusTracking root key was missing or unreadable — an empty provider + /// list with = true is valid data (nothing registered yet). + /// + internal readonly struct EspPolicyProviderSnapshot + { + public static readonly EspPolicyProviderSnapshot Empty = default; + + public EspPolicyProviderSnapshot(IReadOnlyList providers) + { + HasData = true; + Providers = providers ?? Array.Empty(); + } + + /// true when the EnrollmentStatusTracking root key existed and was readable. + public bool HasData { get; } + + /// All registered providers across scopes and kinds; empty when none registered. + public IReadOnlyList Providers { get; } + + /// + /// true when a Setup\Apps provider named "Sidecar" (the IME) is registered in + /// any scope — its absence while other providers are registered is the co-management + /// field-case signature. + /// + public bool SidecarRegistered + { + get + { + var providers = Providers; + if (providers == null) return false; + return providers.Any(p => + p.Kind == EspPolicyProviderProbe.KindSetupApps && + string.Equals(p.Name, EspPolicyProviderProbe.SidecarProviderName, StringComparison.OrdinalIgnoreCase)); + } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderStallDetector.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderStallDetector.cs new file mode 100644 index 00000000..84bf58d3 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderStallDetector.cs @@ -0,0 +1,226 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Persistence; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.Shared; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals +{ + /// + /// Turns the ESP policy-provider contract read by into + /// the one-shot esp_policy_provider_stalled Warning: a registered provider that stays + /// continuously incomplete for minutes is reported once, with the + /// full provider table in the payload. See the probe's header for the CSP contract, the + /// official sources, and the co-management field case (ConfigMgr registered without Sidecar + /// → user ESP parked at "Apps (Identifying)" with no OS timeout) this tripwire exists for. + /// + /// Dwell semantics: the dwell clock per provider starts when it is first observed incomplete + /// and RESETS when the provider completes or disappears — only continuously incomplete + /// providers fire. A missing root key / empty provider list is normal early enrollment and + /// clears all dwell state (probe D4). Purely observational: emission goes through + /// (InformationalEvent pass-through, dispatch-guard + /// exempt), never into decision state. + /// + /// + /// Duplicate suppression: in-process one-shot per provider key, plus a + /// fingerprint over the sorted set of currently-stalled keys so + /// an agent restart into the same stall does not re-report it — while a NEW provider joining + /// the stall changes the fingerprint and is reported (the event carries the full list). + /// + /// + internal sealed class EspPolicyProviderStallDetector + { + /// + /// Wall-clock minutes a provider must stay continuously incomplete before it is reported. + /// Mirrors the CSP's own DevicePreparation default timeout; the Setup/Apps wait this + /// tripwire mainly targets has no OS timeout at all. + /// + internal const int DwellMinutes = 15; + + internal const string SourceLabel = "EspPolicyProviderStallDetector"; + + private readonly string _sessionId; + private readonly string _tenantId; + private readonly InformationalEventPost _post; + private readonly AgentLogger _logger; + private readonly IClock _clock; + private readonly StartupEventGate? _startupGate; + private readonly TimeSpan _dwell; + + private readonly Dictionary _firstSeenIncompleteUtc = + new Dictionary(StringComparer.OrdinalIgnoreCase); + private readonly HashSet _firedKeys = + new HashSet(StringComparer.OrdinalIgnoreCase); + + public EspPolicyProviderStallDetector( + string sessionId, + string tenantId, + InformationalEventPost post, + AgentLogger logger, + IClock clock, + StartupEventGate? startupGate = null, + TimeSpan? dwell = null) + { + _sessionId = sessionId; + _tenantId = tenantId; + _post = post ?? throw new ArgumentNullException(nameof(post)); + _logger = logger; + _clock = clock ?? throw new ArgumentNullException(nameof(clock)); + _startupGate = startupGate; + _dwell = dwell ?? TimeSpan.FromMinutes(DwellMinutes); + } + + /// Host timer callback and test seam — one probe + evaluation round. + internal void Tick() + { + var snapshot = EspPolicyProviderProbe.Read(_logger); + var eval = EvaluateStallTransition(snapshot, _firstSeenIncompleteUtc, _firedKeys, _clock.UtcNow, _dwell); + + _firstSeenIncompleteUtc.Clear(); + foreach (var kv in eval.UpdatedFirstSeenUtc) + _firstSeenIncompleteUtc[kv.Key] = kv.Value; + + if (eval.NewlyStalled.Count > 0) + EmitStalled(snapshot, eval); + } + + /// + /// Pure evaluation of one probe round — the entire detection decision, testable without + /// registry, timer, or ingress (pattern: PerformanceCollector.EvaluateDiskLowTransition). + /// + /// Current provider snapshot. + /// Dwell state from the previous round (provider key → first-seen-incomplete). + /// Provider keys already reported this run (never re-report). + /// Evaluation timestamp. + /// Continuous-incompleteness threshold. + internal static StallEvaluation EvaluateStallTransition( + EspPolicyProviderSnapshot snapshot, + IReadOnlyDictionary firstSeenIncompleteUtc, + ISet alreadyFired, + DateTime nowUtc, + TimeSpan dwell) + { + var result = new StallEvaluation(); + + // Missing root key or no registered providers = normal early enrollment: clear all + // dwell state (a provider must be continuously OBSERVED incomplete to accrue dwell). + if (!snapshot.HasData || snapshot.Providers.Count == 0) + return result; + + foreach (var provider in snapshot.Providers) + { + if (provider.IsComplete) + continue; // completed → falls out of the dwell map (reset) + + var firstSeen = firstSeenIncompleteUtc.TryGetValue(provider.Key, out var seen) + ? seen + : nowUtc; + result.UpdatedFirstSeenUtc[provider.Key] = firstSeen; + + var stalledFor = nowUtc - firstSeen; + if (stalledFor >= dwell && !alreadyFired.Contains(provider.Key)) + result.NewlyStalled.Add(new StalledProvider(provider, stalledFor.TotalMinutes)); + } + + return result; + } + + private void EmitStalled(EspPolicyProviderSnapshot snapshot, StallEvaluation eval) + { + foreach (var stalled in eval.NewlyStalled) + _firedKeys.Add(stalled.Provider.Key); + + // Cross-restart dedup: fingerprint = sorted set of all fired keys that are still + // incomplete. A restart into the same stall re-accrues dwell and lands on the same + // fingerprint → suppressed; a new provider joining the stall changes it → re-emitted. + var currentlyStalledKeys = snapshot.Providers + .Where(p => !p.IsComplete && _firedKeys.Contains(p.Key)) + .Select(p => p.Key) + .OrderBy(k => k, StringComparer.OrdinalIgnoreCase) + .ToList(); + var fingerprint = StartupEventGate.HashString(string.Join(";", currentlyStalledKeys)); + if (_startupGate != null && + !_startupGate.ShouldEmit(Constants.EventTypes.EspPolicyProviderStalled, fingerprint)) + { + _logger?.Debug( + "EspPolicyProviderStallDetector: stalled provider set already reported by a " + + "previous agent run (fingerprint match) — suppressed."); + return; + } + + var stalledNames = string.Join(", ", + eval.NewlyStalled.Select(s => $"{s.Provider.Name} ({s.Provider.Scope}/{s.Provider.Kind})")); + + var data = new Dictionary(StringComparer.Ordinal) + { + ["dwellMinutes"] = DwellMinutes, + ["sidecarRegistered"] = snapshot.SidecarRegistered, + ["providers"] = snapshot.Providers.Select(p => new Dictionary(StringComparer.Ordinal) + { + ["name"] = p.Name, + ["scope"] = p.Scope, + ["kind"] = p.Kind, + ["trackingPoliciesCreated"] = p.TrackingPoliciesCreated, + ["installationState"] = p.InstallationState, + ["complete"] = p.IsComplete, + }).ToList(), + ["stalledProviders"] = eval.NewlyStalled.Select(s => new Dictionary(StringComparer.Ordinal) + { + ["name"] = s.Provider.Name, + ["scope"] = s.Provider.Scope, + ["kind"] = s.Provider.Kind, + ["stalledForMinutes"] = Math.Round(s.StalledForMinutes, 1), + }).ToList(), + }; + + _post.Emit(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + Timestamp = _clock.UtcNow, + EventType = Constants.EventTypes.EspPolicyProviderStalled, + Severity = EventSeverity.Warning, + Source = SourceLabel, + Phase = EnrollmentPhase.Unknown, + ImmediateUpload = true, + Message = $"ESP policy provider(s) incomplete for >= {DwellMinutes} min: {stalledNames} " + + $"(sidecarRegistered={snapshot.SidecarRegistered}) — the ESP waits on " + + "Setup/Apps providers without any timeout", + Data = data + }); + _startupGate?.MarkEmitted(Constants.EventTypes.EspPolicyProviderStalled); + + _logger?.Warning( + $"EspPolicyProviderStallDetector: {stalledNames} incomplete for >= {DwellMinutes} min " + + $"(sidecarRegistered={snapshot.SidecarRegistered})."); + } + + /// Result of one round. + internal sealed class StallEvaluation + { + /// Next round's dwell state — only currently-incomplete providers survive. + public Dictionary UpdatedFirstSeenUtc { get; } = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + /// Providers that crossed the dwell threshold this round and were not yet reported. + public List NewlyStalled { get; } = new List(); + } + + internal readonly struct StalledProvider + { + public StalledProvider(PolicyProviderState provider, double stalledForMinutes) + { + Provider = provider; + StalledForMinutes = stalledForMinutes; + } + + public PolicyProviderState Provider { get; } + public double StalledForMinutes { get; } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs index 6ebd12ce..ef4bd862 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs @@ -36,6 +36,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration /// StallProbeHost — + /// . Owns its 60-s idle-check timer (the /// collector itself has no timer — it's a pure probe invoked from outside). + /// EspPolicyProviderStallHost — always-on 60-s wall-clock tick for the + /// esp_policy_provider_stalled tripwire (registered EnrollmentStatusTracking CSP + /// provider continuously incomplete ≥ 15 min — the no-timeout ESP wait). /// /// Optional peripheral hosts (driven by toggles): /// @@ -303,6 +306,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration harmlessModernDeploymentEventIds: collectors.ModernDeploymentHarmlessEventIds)); } + // ESP policy-provider stall tripwire — always-on kernel host (no config gate, like + // disk_space_low): a registered EnrollmentStatusTracking CSP provider that never + // completes (e.g. co-management "ConfigMgr" without "Sidecar" on a pre-provisioned + // device) parks the user ESP at "Apps (Identifying)" WITHOUT any OS timeout. Wall-clock + // dwell, deliberately outside the StallProbe gate: the idle clock there resets on any + // session activity, and the stall must accrue regardless of it. + hosts.Add(new EspPolicyProviderStallHost( + sessionId: sessionId, + tenantId: tenantId, + logger: logger, + ingress: ingress, + clock: clock, + startupGate: _startupEventGate)); + // Deterministic update corroboration — compares the persisted OS build (CurrentBuild.UBR) // across agent restarts and emits os_build_changed when it differs (session 7443317c: // build jump proved an OOBE quality update the WU channel never showed). Must be added diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/EspPolicyProviderStallHost.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/EspPolicyProviderStallHost.cs new file mode 100644 index 00000000..c50642e3 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/EspPolicyProviderStallHost.cs @@ -0,0 +1,94 @@ +#nullable enable +using System; +using System.Threading; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Persistence; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; + +namespace AutopilotMonitor.Agent.V2.Core.Orchestration +{ + /// + /// Drives the from a 60-s wall-clock tick. + /// Deliberately NOT idle-coupled (a policy-provider stall coexists with a chatty session — + /// 's idle clock would keep resetting) and NOT idle-stopped + /// ( would sleep through exactly the dormant + /// window the dwell must observe). Always-on kernel host, no config gate — precedent: + /// the disk_space_low tripwire. + /// + /// First tick fires after 60 s, not at Start: the EnrollmentStatusTracking key legitimately + /// doesn't exist at agent start, and nothing can be 15 min stale at t=0. + /// + /// + internal sealed class EspPolicyProviderStallHost : ICollectorHost + { + public string Name => "EspPolicyProviderStallDetector"; + + private static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(60); + + private readonly EspPolicyProviderStallDetector _detector; + private readonly AgentLogger _logger; + private Timer? _tickTimer; + private int _disposed; + + public EspPolicyProviderStallHost( + string sessionId, + string tenantId, + AgentLogger logger, + ISignalIngressSink ingress, + IClock clock, + StartupEventGate? startupGate) + { + if (ingress == null) throw new ArgumentNullException(nameof(ingress)); + if (clock == null) throw new ArgumentNullException(nameof(clock)); + _logger = logger; + var post = new InformationalEventPost(ingress, clock); + _detector = new EspPolicyProviderStallDetector( + sessionId: sessionId, + tenantId: tenantId, + post: post, + logger: logger, + clock: clock, + startupGate: startupGate); + } + + public void Start() + { + _tickTimer = new Timer( + _ => SafeTick(), + state: null, + dueTime: PollInterval, + period: PollInterval); + _logger.Info($"EspPolicyProviderStallHost: started (tick every {PollInterval.TotalSeconds}s)."); + } + + public void Stop() + { + try + { + _tickTimer?.Dispose(); + _tickTimer = null; + } + catch (Exception ex) { _logger.Warning($"EspPolicyProviderStallHost: timer dispose failed: {ex.Message}"); } + } + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 1) return; + Stop(); + } + + private void SafeTick() + { + try + { + _detector.Tick(); + } + catch (Exception ex) + { + _logger.Error("EspPolicyProviderStallHost: tick failed.", ex); + } + } + } +} diff --git a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts index 615742ba..16b72483 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts @@ -167,6 +167,7 @@ export const EVENT_TYPES_CATALOG = { 'telemetry_upload_blocked', 'session_parked_without_deadline', 'disk_space_low', + 'esp_policy_provider_stalled', 'agent_late_start', ], metrics_events: ['performance_snapshot', 'agent_metrics_snapshot', 'ingress_backpressure'], diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index b96d37bd..48d47949 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -368,6 +368,7 @@ namespace AutopilotMonitor.Shared public const string TelemetryUploadBlocked = "telemetry_upload_blocked"; // Warning (one-shot) when the drain is blocked by a retained unknown permanent 4xx (400 contract bug / 404 route mismatch / tenant-validation). Data is NOT discarded. LOCAL diagnostic (agent log + diag ZIP): this marker is queued behind the blocking batch and only reaches the backend after the block clears — no out-of-band bypass by design. public const string SessionParkedWithoutDeadline = "session_parked_without_deadline"; // One-shot Warning tripwire (liveness plan PR1): a non-terminal session entered the post-AccountSetup dead-end zone (ESP final exit or advisory failure) with NO resolution-capable deadline armed. After the 2026-06-12 dead-end fixes this must never fire in production — every occurrence is a bug report for an unknown parking variant. public const string DiskSpaceLow = "disk_space_low"; // One-shot Warning (state-change-only, 1 GB hysteresis — NOT periodic) when the system drive drops below 2 GB free. Review MON-B10: disk-full was previously only visible inside Debug performance_snapshot payloads (filtered out of timelines), so a disk-starved enrollment carried no actionable threshold signal. + public const string EspPolicyProviderStalled = "esp_policy_provider_stalled"; // One-shot-per-provider Warning: a registered EnrollmentStatusTracking CSP policy provider stayed incomplete >= 15 min (Setup/Apps TrackingPoliciesCreated != 1, or DevicePreparation InstallationState not in {2,3}). The ESP waits on Setup/Apps providers WITHOUT any OS timeout — e.g. pre-provisioning + co-management leaves only "ConfigMgr" registered (no "Sidecar" = IME) and the user ESP sits at "Apps (Identifying)" for days. Observational only; payload carries the full provider list. See EspPolicyProviderProbe. public const string KeepAwakeEngaged = "keep_awake_engaged"; // Info (state-change, NOT periodic) emitted by the opt-in UserEspKeepAwake host when it holds the device awake (system + display) on entering the User-ESP/AccountSetup phase, so idle standby cannot stall app installs / account provisioning. Reboots are unaffected. Per-tenant toggle (AnalyzerConfiguration.KeepAwakeDuringUserEsp, default off). public const string KeepAwakeReleased = "keep_awake_released"; // Info (Warning when reason=safety_cap) emitted when the keep-awake hold is released — reason = account_setup_complete | host_stop | safety_cap. Brackets KeepAwakeEngaged; the OS also auto-clears the hold on process exit/reboot. diff --git a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts index f698db6d..5f20b209 100644 --- a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts +++ b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts @@ -58,6 +58,8 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [ description: "Stall probe found an anomaly (ModernDeployment error, DeploymentErrorCode≠0, IME EnforcementState Failed)." }, { value: "stall_probe_check", label: "stall_probe_check", category: "stall", description: "Trace-level heartbeat from a stall probe — proves the logic ran (default: only the 15 min probe)." }, + { value: "esp_policy_provider_stalled", label: "esp_policy_provider_stalled", category: "stall", + description: "A registered ESP policy provider (EnrollmentStatusTracking CSP) stayed incomplete for 15+ minutes — e.g. co-management leaves only ConfigMgr registered without the Intune Sidecar provider, which parks the user ESP at 'Apps (Identifying)' with no OS timeout. One-shot Warning; payload carries the full provider table, per-provider state, and sidecarRegistered." }, { value: "agent_late_start", label: "agent_late_start", category: "stall", description: "Low observation coverage: the agent started long after device boot (≥10 min) AND lived only briefly (≤5 min) before a terminal outcome — it arrived after the enrollment had already decided, so its diagnosis is a post-mortem of the end-state, not a live observation. Carries bootToAgentStartSeconds, agentUptimeSeconds and outcome. Often paired with script_timeout_suspected (a script that hung ahead of the agent bootstrap)." }, diff --git a/src/Web/autopilot-monitor-web/utils/guardrails.generated.ts b/src/Web/autopilot-monitor-web/utils/guardrails.generated.ts index 4f886a7b..5bd65daa 100644 --- a/src/Web/autopilot-monitor-web/utils/guardrails.generated.ts +++ b/src/Web/autopilot-monitor-web/utils/guardrails.generated.ts @@ -57,6 +57,7 @@ export const REGISTRY_PREFIX_CATEGORIES: readonly GuardrailCategory[] = [ "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Setup", "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\OOBE", "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon", + "SOFTWARE\\Microsoft\\Windows\\Autopilot", ], }, { @@ -242,6 +243,7 @@ export const ALLOWED_REGISTRY_PREFIXES: readonly string[] = [ "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Setup", "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\OOBE", "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon", + "SOFTWARE\\Microsoft\\Windows\\Autopilot", "SYSTEM\\CurrentControlSet\\Services\\TPM", "SOFTWARE\\Microsoft\\Tpm", "SYSTEM\\CurrentControlSet\\Control\\SecureBoot",