diff --git a/internal b/internal index 0ab6ced5..31957733 160000 --- a/internal +++ b/internal @@ -1 +1 @@ -Subproject commit 0ab6ced5232a5cf5edd6cab26b1bfdea795185fd +Subproject commit 31957733a275a4161d2ae2e2250b6640f26db4d0 diff --git a/rules/analyze/ANALYZE-ESP-004.json b/rules/analyze/ANALYZE-ESP-004.json index 3a5cccfc..a963aa46 100644 --- a/rules/analyze/ANALYZE-ESP-004.json +++ b/rules/analyze/ANALYZE-ESP-004.json @@ -1,11 +1,11 @@ { "$schema": "../schema/analyze-rule.schema.json", "ruleId": "ANALYZE-ESP-004", - "title": "ESP Timeout with 'Continue Anyway' (Soft Failure)", - "description": "Detects an ESP terminal failure on a profile that allows 'Continue anyway'. The enrollment is reported as failed, but the user most likely dismissed the error screen and reached the desktop — a soft failure rather than an unusable device.", + "title": "ESP Gave Up on a Blocking App with 'Continue Anyway' (Soft Failure)", + "description": "Detects an ESP terminal failure where the ESP gave up while a blocking app was still installing (the agent's own esp_apps_timeout verdict — no per-app HRESULT was reported) on a profile that allows 'Continue anyway'. The enrollment is reported as failed, but the user most likely dismissed the error screen and reached the desktop — a soft failure rather than an unusable device. An ESP failure that carries a specific error code (a failed installer, a detection mismatch) is a different case and stays with the rule that explains that code.", "severity": "warning", "category": "esp", - "version": "1.0.1", + "version": "1.1.0", "enabled": true, "baseConfidence": 80, "conditions": [ @@ -26,6 +26,27 @@ "value": "true", "required": true }, + { + "signal": "esp_gave_up_on_app", + "source": "event_type", + "eventType": "app_install_failed", + "dataField": "failureType", + "operator": "equals", + "value": "esp_apps_timeout", + "required": true + }, + { + "signal": "appName", + "source": "event_data", + "eventType": "app_install_failed", + "dataField": "appName", + "operator": "exists", + "value": "", + "filterField": "failureType", + "filterOperator": "equals", + "filterValue": "esp_apps_timeout", + "required": false + }, { "signal": "espSyncFailureTimeoutMinutes", "source": "event_type", @@ -38,34 +59,25 @@ { "signal": "failedSubcategory", "source": "event_type", - "eventType": "enrollment_failed", + "eventType": "esp_failure_settle_started", "dataField": "failedSubcategory", "operator": "exists", "value": "", "required": false - }, - { - "signal": "appName", - "source": "event_type", - "eventType": "app_install_failed", - "dataField": "appName", - "operator": "exists", - "value": "", - "required": false } ], "confidenceFactors": [ - { "signal": "appName", "condition": "exists", "weight": 10 } + { "signal": "failedSubcategory", "condition": "exists", "weight": 10 } ], - "explanation": "This enrollment was reported as **failed** because the Enrollment Status Page reached its configured terminal timeout (`{{espSyncFailureTimeoutMinutes}} min`) before all blocking items finished. The ESP profile, however, allows **'Continue anyway'** — so the user most likely dismissed the error screen and reached the desktop with a working, though not fully provisioned, device.\n\nThe agent only observes the ESP terminal failure on the device side; it cannot see what the user did after the failure screen. Treat this as a **soft failure**: the device may well be in productive use, while at least one blocking item (`{{failedSubcategory}}` subcategory / app `{{appName}}`) did not complete inside the ESP window.\n\nA cluster of these across a tenant almost always points at one slow blocking app holding the whole fleet at the ESP timeout wall, not at broken devices. Any field still shown as a literal `{{…}}` placeholder was simply not recorded for this session (e.g. a certificate-only failure has no failed app).", + "explanation": "This enrollment was reported as **failed** because the Enrollment Status Page gave up while the blocking app **{{appName}}** was still installing — the agent recorded no per-app error code, only that the app never finished inside the ESP window (the profile's configured limit is `{{espSyncFailureTimeoutMinutes}} min`; the ESP may have stopped earlier than that). The ESP profile, however, allows **'Continue anyway'** — so the user most likely dismissed the error screen and reached the desktop with a working, though not fully provisioned, device.\n\nThe agent only observes the ESP terminal failure on the device side; it cannot see what the user did after the failure screen. Treat this as a **soft failure**: the device may well be in productive use, while the `{{failedSubcategory}}` subcategory did not complete inside the ESP window.\n\nA cluster of these across a tenant almost always points at one slow blocking app holding the whole fleet at the ESP limit, not at broken devices. This rule deliberately stays silent when the failed app reported a specific error code — a failed installer or a detection mismatch is explained by the rule for that code, and 'Continue anyway' does not make it a timing problem. Any field still shown as a literal `{{…}}` placeholder was simply not recorded for this session.", "remediation": [ { "title": "Relieve ESP pressure so enrollments finish inside the window", "steps": [ - "Identify the slow blocking app (`{{appName}}`) and either remove it from the ESP 'block device use until these required apps are installed' list, or raise the ESP installation timeout (currently {{espSyncFailureTimeoutMinutes}} min).", - "If the app legitimately needs longer than the ESP allows, make it non-blocking and let it install after the desktop arrives.", - "Verify the app's detection rule matches after install — an app that installs but fails detection times out the ESP on every device (see ANALYZE-APP-013).", - "If a subcategory such as Certificates is reported under '{{failedSubcategory}}', confirm the certificate/connector actually delivers — 'Continue anyway' hides a missing certificate, it does not fix it." + "Look at how long `{{appName}}` takes on a device that completes: if it legitimately needs longer than the ESP allows, make it non-blocking (remove it from 'block device use until these required apps are installed') and let it install after the desktop arrives.", + "If the app must stay blocking, raise the ESP setting 'Show error when installation takes longer than' (configured at {{espSyncFailureTimeoutMinutes}} min) — a limit, not a target; it only helps when the app reliably finishes within the new value.", + "Check the app's dependencies and install order: a blocking app waiting on a large dependency download (Delivery Optimization, proxy throughput) is the usual reason it never finishes in time.", + "If the ESP reported a non-Apps subcategory under '{{failedSubcategory}}', confirm the certificate/connector actually delivers — 'Continue anyway' hides a missing certificate, it does not fix it." ] } ], diff --git a/rules/dist/analyze-rules.json b/rules/dist/analyze-rules.json index 544ad601..409c4642 100644 --- a/rules/dist/analyze-rules.json +++ b/rules/dist/analyze-rules.json @@ -2496,11 +2496,11 @@ }, { "ruleId": "ANALYZE-ESP-004", - "title": "ESP Timeout with 'Continue Anyway' (Soft Failure)", - "description": "Detects an ESP terminal failure on a profile that allows 'Continue anyway'. The enrollment is reported as failed, but the user most likely dismissed the error screen and reached the desktop — a soft failure rather than an unusable device.", + "title": "ESP Gave Up on a Blocking App with 'Continue Anyway' (Soft Failure)", + "description": "Detects an ESP terminal failure where the ESP gave up while a blocking app was still installing (the agent's own esp_apps_timeout verdict — no per-app HRESULT was reported) on a profile that allows 'Continue anyway'. The enrollment is reported as failed, but the user most likely dismissed the error screen and reached the desktop — a soft failure rather than an unusable device. An ESP failure that carries a specific error code (a failed installer, a detection mismatch) is a different case and stays with the rule that explains that code.", "severity": "warning", "category": "esp", - "version": "1.0.1", + "version": "1.1.0", "enabled": true, "baseConfidence": 80, "conditions": [ @@ -2521,6 +2521,27 @@ "value": "true", "required": true }, + { + "signal": "esp_gave_up_on_app", + "source": "event_type", + "eventType": "app_install_failed", + "dataField": "failureType", + "operator": "equals", + "value": "esp_apps_timeout", + "required": true + }, + { + "signal": "appName", + "source": "event_data", + "eventType": "app_install_failed", + "dataField": "appName", + "operator": "exists", + "value": "", + "filterField": "failureType", + "filterOperator": "equals", + "filterValue": "esp_apps_timeout", + "required": false + }, { "signal": "espSyncFailureTimeoutMinutes", "source": "event_type", @@ -2533,38 +2554,29 @@ { "signal": "failedSubcategory", "source": "event_type", - "eventType": "enrollment_failed", + "eventType": "esp_failure_settle_started", "dataField": "failedSubcategory", "operator": "exists", "value": "", "required": false - }, - { - "signal": "appName", - "source": "event_type", - "eventType": "app_install_failed", - "dataField": "appName", - "operator": "exists", - "value": "", - "required": false } ], "confidenceFactors": [ { - "signal": "appName", + "signal": "failedSubcategory", "condition": "exists", "weight": 10 } ], - "explanation": "This enrollment was reported as **failed** because the Enrollment Status Page reached its configured terminal timeout (`{{espSyncFailureTimeoutMinutes}} min`) before all blocking items finished. The ESP profile, however, allows **'Continue anyway'** — so the user most likely dismissed the error screen and reached the desktop with a working, though not fully provisioned, device.\n\nThe agent only observes the ESP terminal failure on the device side; it cannot see what the user did after the failure screen. Treat this as a **soft failure**: the device may well be in productive use, while at least one blocking item (`{{failedSubcategory}}` subcategory / app `{{appName}}`) did not complete inside the ESP window.\n\nA cluster of these across a tenant almost always points at one slow blocking app holding the whole fleet at the ESP timeout wall, not at broken devices. Any field still shown as a literal `{{…}}` placeholder was simply not recorded for this session (e.g. a certificate-only failure has no failed app).", + "explanation": "This enrollment was reported as **failed** because the Enrollment Status Page gave up while the blocking app **{{appName}}** was still installing — the agent recorded no per-app error code, only that the app never finished inside the ESP window (the profile's configured limit is `{{espSyncFailureTimeoutMinutes}} min`; the ESP may have stopped earlier than that). The ESP profile, however, allows **'Continue anyway'** — so the user most likely dismissed the error screen and reached the desktop with a working, though not fully provisioned, device.\n\nThe agent only observes the ESP terminal failure on the device side; it cannot see what the user did after the failure screen. Treat this as a **soft failure**: the device may well be in productive use, while the `{{failedSubcategory}}` subcategory did not complete inside the ESP window.\n\nA cluster of these across a tenant almost always points at one slow blocking app holding the whole fleet at the ESP limit, not at broken devices. This rule deliberately stays silent when the failed app reported a specific error code — a failed installer or a detection mismatch is explained by the rule for that code, and 'Continue anyway' does not make it a timing problem. Any field still shown as a literal `{{…}}` placeholder was simply not recorded for this session.", "remediation": [ { "title": "Relieve ESP pressure so enrollments finish inside the window", "steps": [ - "Identify the slow blocking app (`{{appName}}`) and either remove it from the ESP 'block device use until these required apps are installed' list, or raise the ESP installation timeout (currently {{espSyncFailureTimeoutMinutes}} min).", - "If the app legitimately needs longer than the ESP allows, make it non-blocking and let it install after the desktop arrives.", - "Verify the app's detection rule matches after install — an app that installs but fails detection times out the ESP on every device (see ANALYZE-APP-013).", - "If a subcategory such as Certificates is reported under '{{failedSubcategory}}', confirm the certificate/connector actually delivers — 'Continue anyway' hides a missing certificate, it does not fix it." + "Look at how long `{{appName}}` takes on a device that completes: if it legitimately needs longer than the ESP allows, make it non-blocking (remove it from 'block device use until these required apps are installed') and let it install after the desktop arrives.", + "If the app must stay blocking, raise the ESP setting 'Show error when installation takes longer than' (configured at {{espSyncFailureTimeoutMinutes}} min) — a limit, not a target; it only helps when the app reliably finishes within the new value.", + "Check the app's dependencies and install order: a blocking app waiting on a large dependency download (Delivery Optimization, proxy throughput) is the usual reason it never finishes in time.", + "If the ESP reported a non-Apps subcategory under '{{failedSubcategory}}', confirm the certificate/connector actually delivers — 'Continue anyway' hides a missing certificate, it does not fix it." ] } ], diff --git a/rules/schema/analyze-rule.schema.json b/rules/schema/analyze-rule.schema.json index 02852d59..31f21c37 100644 --- a/rules/schema/analyze-rule.schema.json +++ b/rules/schema/analyze-rule.schema.json @@ -190,7 +190,7 @@ }, "operator": { "type": "string", - "description": "Comparison operator.", + "description": "Comparison operator. exists/not_exists with a dataField test whether ANY event of the type carries a non-empty value at that field: not_exists matches when no event does — including when the event type is absent altogether — so a rule can gate on a missing field (e.g. an ESP failure without an HRESULT). Without a dataField they test only the presence of the event type.", "enum": ["equals", "not_equals", "contains", "not_contains", "regex", "not_regex", "gt", "lt", "gte", "lte", "exists", "not_exists", "count_gte", "count_per_group_gte", "in", "not_in"] }, "value": { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/EspAndHelloTrackerShellCoreFailureInheritanceTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/EspAndHelloTrackerShellCoreFailureInheritanceTests.cs new file mode 100644 index 00000000..2bc2f5d5 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/EspAndHelloTrackerShellCoreFailureInheritanceTests.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.SignalAdapters; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.DecisionCore.Signals; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring +{ + /// + /// Session 683f1eff — a Shell-Core 62407 failure (WhiteGlove_Failed) arrived four + /// seconds into the provisioning tracker's 30-s settle window and terminalised the session + /// with nothing but its failure type: the HRESULT (0x80070652) and the failed subcategory + /// (Apps) the registry had already named never reached the terminal event, the backend + /// FailureReason or the termination-time app classification. The coordinator now inherits + /// the pending settle detail onto the Shell-Core args and records the terminal-failure + /// snapshot from it. + /// + public sealed class EspAndHelloTrackerShellCoreFailureInheritanceTests + { + private static readonly DateTime Fixed = new DateTime(2026, 9, 8, 18, 11, 6, DateTimeKind.Utc); + + private sealed class Fixture : IDisposable + { + public TempDirectory Tmp { get; } = new TempDirectory(); + public AgentLogger Logger { get; } + public FakeSignalIngressSink Ingress { get; } = new FakeSignalIngressSink(); + public FakeSignalIngressSink TrackerPostSink { get; } = new FakeSignalIngressSink(); + public VirtualClock Clock { get; } = new VirtualClock(Fixed); + public List Forwarded { get; } = new List(); + + public Fixture() { Logger = new AgentLogger(Tmp.Path, AgentLogLevel.Debug); } + + public EspAndHelloTracker BuildCoordinator(Func pendingProbe) + { + var coordinator = new EspAndHelloTracker( + sessionId: "S1", + tenantId: "T1", + post: new InformationalEventPost(TrackerPostSink, Clock), + logger: Logger, + skipConfigProbe: () => ((bool?)false, (bool?)false), + accountSetupActivityProbe: () => false, + pendingEspFailureProbe: pendingProbe); + coordinator.EspFailureDetected += (_, args) => Forwarded.Add(args); + return coordinator; + } + + public FakeSignalIngressSink.PostedSignal TerminalSignal() => + Assert.Single(Ingress.Posted, p => p.Kind == DecisionSignalKind.EspTerminalFailure); + + public void Dispose() { Tmp.Dispose(); } + } + + private static EspFailureDetectedEventArgs PendingAppsFailure() => new EspFailureDetectedEventArgs( + failureType: "Provisioning_DeviceSetup_Apps_Failed", + errorCode: "0x80070652", + failedSubcategory: "Apps", + category: "DeviceSetup", + likelyCulpritApps: new[] { "ESP Finalizer US" }); + + [Fact] + public void ShellCore_failure_inside_the_settle_window_inherits_the_registry_detail() + { + using var f = new Fixture(); + using var coordinator = f.BuildCoordinator(PendingAppsFailure); + using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock); + + coordinator.TriggerShellCoreEspFailureForTest("WhiteGlove_Failed"); + + var args = Assert.Single(f.Forwarded); + Assert.Equal("WhiteGlove_Failed", args.FailureType); // the signal that terminalised + Assert.Equal("0x80070652", args.ErrorCode); + Assert.Equal("Apps", args.FailedSubcategory); + Assert.Equal("DeviceSetup", args.Category); + Assert.Equal(new[] { "ESP Finalizer US" }, args.LikelyCulpritApps); + + // Wire-through: the EspTerminalFailure signal payload carries the detail the + // DecisionEngine copies onto enrollment_failed. + var signal = f.TerminalSignal(); + Assert.Equal("WhiteGlove_Failed", signal.Payload!["failureType"]); + Assert.Equal("0x80070652", signal.Payload["errorCode"]); + Assert.Equal("Apps", signal.Payload["failedSubcategory"]); + Assert.Equal("DeviceSetup", signal.Payload["category"]); + Assert.Equal("ESP Finalizer US", signal.Payload["likelyCulpritApps"]); + + // Termination-time classification reads the same context. + Assert.NotNull(coordinator.LastEspTerminalFailure); + Assert.Equal("0x80070652", coordinator.LastEspTerminalFailure.ErrorCode); + Assert.True(coordinator.LastEspTerminalFailure.IsAppsSubcategory); + } + + [Fact] + public void ShellCore_failure_without_a_pending_window_stays_bare() + { + using var f = new Fixture(); + using var coordinator = f.BuildCoordinator(() => null); + using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock); + + coordinator.TriggerShellCoreEspFailureForTest("ESPProgress_Timeout"); + + var args = Assert.Single(f.Forwarded); + Assert.Equal("ESPProgress_Timeout", args.FailureType); + Assert.Null(args.ErrorCode); + Assert.Null(args.FailedSubcategory); + Assert.Null(args.Category); + Assert.Empty(args.LikelyCulpritApps); + Assert.Null(coordinator.LastEspTerminalFailure); + + var signal = f.TerminalSignal(); + Assert.False(signal.Payload!.ContainsKey("errorCode")); + Assert.False(signal.Payload.ContainsKey("failedSubcategory")); + } + + [Fact] + public void A_throwing_probe_degrades_to_the_bare_failure() + { + using var f = new Fixture(); + using var coordinator = f.BuildCoordinator(() => throw new InvalidOperationException("probe")); + + coordinator.TriggerShellCoreEspFailureForTest("WhiteGlove_Failed"); + + var args = Assert.Single(f.Forwarded); + Assert.Equal("WhiteGlove_Failed", args.FailureType); + Assert.Null(args.ErrorCode); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningStatusTrackerPendingFailureTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningStatusTrackerPendingFailureTests.cs new file mode 100644 index 00000000..efaf3c4e --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/SystemSignals/ProvisioningStatusTrackerPendingFailureTests.cs @@ -0,0 +1,92 @@ +using System; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals +{ + /// + /// exposes the registry + /// failure detail while its settle window is open and nothing once the window has fired — + /// the coordinator reads it when a Shell-Core failure terminalises first (session 683f1eff). + /// + public sealed class ProvisioningStatusTrackerPendingFailureTests + { + private static readonly DateTime Fixed = new DateTime(2026, 9, 8, 18, 11, 2, DateTimeKind.Utc); + + private const string InProgressJson = @"{ + ""categorySucceeded"": null, + ""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working)""} + }"; + private const string AppsFailedJson = @"{ + ""categorySucceeded"": null, + ""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (0x80070652)""} + }"; + + private sealed class Fixture : IDisposable + { + public TempDirectory Tmp { get; } = new TempDirectory(); + public ProvisioningStatusTracker Tracker { get; } + + public Fixture() + { + var post = new InformationalEventPost(new FakeSignalIngressSink(), new VirtualClock(Fixed)); + var logger = new AgentLogger(Tmp.Path, AgentLogLevel.Info); + Tracker = new ProvisioningStatusTracker( + sessionId: "S1", + tenantId: "T1", + post: post, + logger: logger, + appxScanner: new FakeAppxDeploymentFailureScanner(), + backgroundDispatcher: action => action()); + } + + public void Dispose() + { + Tracker.Dispose(); + Tmp.Dispose(); + } + } + + [Fact] + public void Nothing_pending_before_any_failure() + { + using var f = new Fixture(); + Assert.Null(f.Tracker.TryGetPendingFailureArgs()); + + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", InProgressJson); + Assert.Null(f.Tracker.TryGetPendingFailureArgs()); + } + + [Fact] + public void Armed_settle_window_exposes_the_registry_detail() + { + using var f = new Fixture(); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", InProgressJson); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", AppsFailedJson); + + var pending = f.Tracker.TryGetPendingFailureArgs(); + + Assert.NotNull(pending); + Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", pending!.FailureType); + Assert.Equal("0x80070652", pending.ErrorCode); + Assert.Equal("Apps", pending.FailedSubcategory); + Assert.Equal("DeviceSetup", pending.Category); + } + + [Fact] + public void Expired_settle_window_leaves_nothing_pending() + { + using var f = new Fixture(); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", InProgressJson); + f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", AppsFailedJson); + + f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status"); + + Assert.Null(f.Tracker.TryGetPendingFailureArgs()); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs index 0a1537ff..e82c00af 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs @@ -44,6 +44,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals private readonly Func _userEspAppsSettledProbe; private readonly Func> _starvedUserEspAppsProbe; private readonly Func> _packageStatesProbe; + // Session 683f1eff: registry failure detail still sitting in an open settle window, + // consulted when a Shell-Core failure terminalises first (see OnShellCoreEspFailureDetected). + private readonly Func _pendingEspFailureProbe; // Session caa6cf50 gate-starvation fix (2026-06-11) — fire-once guard for the // user-ESP-apps-settled AccountSetup synthesis (see MaybeSynthesizeAccountSetupComplete). @@ -174,8 +177,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals /// late "Installing-→-Error" promotion can classify the failure correctly /// AND refuse to promote when the failure originated outside the Apps /// subcategory (a non-Apps HRESULT does not describe per-app outcome). - /// Null when no HRESULT-carrying ESP failure has fired (or all observed - /// failures came from ShellCoreTracker, which has no HRESULT surface). + /// Null when no registry-derived failure detail has been observed — neither from + /// a settle window that expired nor from one a Shell-Core failure inherited. /// public Termination.EspTerminalFailureSnapshot LastEspTerminalFailure { get; private set; } @@ -243,7 +246,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals Func accountSetupActivityProbe = null, Func userEspAppsSettledProbe = null, Func> starvedUserEspAppsProbe = null, - Func> packageStatesProbe = null) + Func> packageStatesProbe = null, + Func pendingEspFailureProbe = null) { _sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId)); _tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId)); @@ -277,6 +281,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals // provisioning tracker name not-completed apps on an Apps-subcategory failure. _packageStatesProbe = packageStatesProbe ?? (() => Array.Empty()); + // Test seam: defaults to the provisioning tracker's open settle window (null when + // none is armed or the tracker is not running, e.g. Device Preparation). + _pendingEspFailureProbe = pendingEspFailureProbe + ?? (() => _provisioningTracker?.TryGetPendingFailureArgs()); } /// @@ -658,6 +666,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals internal void TriggerEspExitedForTest(DateTime occurredAtUtc) => OnEspExited(this, new EspExitedEventArgs(occurredAtUtc)); + // Test seam for a Shell-Core 62407 failure — drives the coordinator forwarder including + // the pending-settle inheritance, without a real ShellCoreTracker. + internal void TriggerShellCoreEspFailureForTest(string failureType) => + OnShellCoreEspFailureDetected(this, failureType); + // Test seam for HelloWizardStarted — same contract as TriggerEspExitedForTest. internal void TriggerHelloWizardStartedForTest(DateTime occurredAtUtc) => OnHelloWizardStarted(this, new HelloWizardStartedEventArgs(occurredAtUtc)); @@ -673,33 +686,56 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals // Forwarder for ShellCoreTracker (event-log-derived ESP failures, e.g. Shell-Core 62407 // failure descriptions). Source-event timestamp is mirrored so the adapter can stamp the // DecisionSignal with the historical instant on backfill. ShellCoreTracker has no HRESULT - // surface, so only FailureType is set on the args. + // surface; the registry detail is inherited from an open settle window (see below). private void OnShellCoreEspFailureDetected(object sender, string failureType) { LastEventOccurredAtUtc = (sender as ShellCoreTracker)?.LastEventOccurredAtUtc; try { - EspFailureDetected?.Invoke(this, new EspFailureDetectedEventArgs(failureType)); + EspFailureDetected?.Invoke(this, BuildShellCoreFailureArgs(failureType)); } catch (Exception ex) { _logger.Error($"Error forwarding ShellCore EspFailureDetected for '{failureType}'", ex); } finally { LastEventOccurredAtUtc = null; } } - // Forwarder for ProvisioningStatusTracker (registry-derived ESP failures). Args carry the - // full failure detail (FailureType, ErrorCode, FailedSubcategory, Category) extracted - // from the failed subcategory's statusText. Provisioning has no source-event timestamp - // surfaced today, so the adapter falls back to clock. - private void OnProvisioningEspFailureDetected(object sender, EspFailureDetectedEventArgs args) + // Session 683f1eff: the Shell-Core 62407 failure usually lands INSIDE the provisioning + // tracker's 30-s settle window — after the registry named the failed subcategory and its + // HRESULT, before the settle timer would have forwarded them. The engine terminalises on + // the first EspTerminalFailure and the agent stops seconds later, so whatever this signal + // carries is all the terminal event, the backend FailureReason and the termination-time + // app classification ever see (a third of all ESP terminal failures took this path). The + // pending detail is inherited here; FailureType stays the Shell-Core value because that + // is the signal that actually terminalised. + private EspFailureDetectedEventArgs BuildShellCoreFailureArgs(string failureType) + { + EspFailureDetectedEventArgs pending = null; + try { pending = _pendingEspFailureProbe(); } + catch (Exception ex) { _logger.Warning($"EspAndHelloTracker: pending ESP failure probe threw: {ex.Message}"); } + + if (pending == null) + return new EspFailureDetectedEventArgs(failureType); + + RecordTerminalFailureSnapshot(pending); + _logger.Info( + $"EspAndHelloTracker: Shell-Core failure '{failureType}' inherits the pending provisioning failure detail " + + $"(category={pending.Category ?? "n/a"}, failedSubcategory={pending.FailedSubcategory ?? "n/a"}, " + + $"errorCode={pending.ErrorCode ?? "n/a"})"); + return new EspFailureDetectedEventArgs( + failureType, + errorCode: pending.ErrorCode, + failedSubcategory: pending.FailedSubcategory, + category: pending.Category, + likelyCulpritApps: pending.LikelyCulpritApps); + } + + // Session 080edee9 follow-up + Codex review (P2/P3) — snapshot the full failure context + // (HRESULT + failedSubcategory + category) so EnrollmentTerminationHandler can read it on + // the terminal-failure pathway. Only update when the args carry at least one of the three + // fields, so a bare Shell-Core failure cannot wipe out the registry snapshot. Deliberately + // NOT gated on ErrorCode alone — a non-Apps subcategory failure without HRESULT is still + // useful for downstream "should we promote?" gating. + private void RecordTerminalFailureSnapshot(EspFailureDetectedEventArgs args) { - LastEventOccurredAtUtc = null; - // Session 080edee9 follow-up + Codex review (P2/P3) — snapshot the full - // failure context (HRESULT + failedSubcategory + category) before - // forwarding so EnrollmentTerminationHandler can read it on the terminal- - // failure pathway. Only update when the args carry at least one of the - // three fields, so a later ShellCore-derived failure (which has none of - // them) cannot wipe out the registry snapshot. We deliberately do NOT - // gate on ErrorCode alone — a non-Apps subcategory failure without - // HRESULT is still useful for downstream "should we promote?" gating. if (!string.IsNullOrEmpty(args?.ErrorCode) || !string.IsNullOrEmpty(args?.FailedSubcategory) || !string.IsNullOrEmpty(args?.Category)) @@ -709,6 +745,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals failedSubcategory: args.FailedSubcategory, category: args.Category); } + } + + // Forwarder for ProvisioningStatusTracker (registry-derived ESP failures). Args carry the + // full failure detail (FailureType, ErrorCode, FailedSubcategory, Category) extracted + // from the failed subcategory's statusText. Provisioning has no source-event timestamp + // surfaced today, so the adapter falls back to clock. + private void OnProvisioningEspFailureDetected(object sender, EspFailureDetectedEventArgs args) + { + LastEventOccurredAtUtc = null; + RecordTerminalFailureSnapshot(args); try { EspFailureDetected?.Invoke(this, args); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs index f5a6dec8..ba83f719 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ProvisioningStatusTracker.cs @@ -80,6 +80,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals "AccountSetupCategory.Status" }; + // Order in which TryGetPendingFailureArgs resolves concurrently armed settle windows. + private static readonly string[] PendingFailurePreference = + { + "DeviceSetupCategory.Status", + "AccountSetupCategory.Status", + "DevicePreparationCategory.Status" + }; + private readonly AgentLogger _logger; private readonly string _sessionId; private readonly string _tenantId; @@ -1711,6 +1719,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals internal void TriggerSettleTimerForTest(string categoryName) => OnProvisioningFailureSettleExpired(categoryName); + /// + /// The registry-derived failure detail of a settle window that is still open (armed, + /// not yet expired or retracted) — null when none is pending. Read by the coordinator + /// when a Shell-Core 62407 failure arrives inside the window, so the terminal signal + /// inherits HRESULT / failed subcategory / category instead of racing them away + /// (session 683f1eff). Windows fail one category at a time, but should two be armed + /// the device-phase category wins: it is the one the ESP page reports. + /// + internal EspFailureDetectedEventArgs TryGetPendingFailureArgs() + { + lock (_stateLock) + { + if (_provisioningFailureSettleArgs == null || _provisioningFailureSettleArgs.Count == 0) + return null; + foreach (var category in PendingFailurePreference) + { + if (_provisioningFailureSettleArgs.TryGetValue(category, out var args) && args != null) + return args; + } + return null; + } + } + private bool HasCategorySucceededChanged(string categoryName, bool? newValue) { if (!_lastCategorySucceeded.TryGetValue(categoryName, out var oldValue)) diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEspSoftFailureTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEspSoftFailureTests.cs index 60318b65..5843eca3 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEspSoftFailureTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEspSoftFailureTests.cs @@ -7,19 +7,21 @@ using Moq; namespace AutopilotMonitor.Functions.Tests; /// -/// Pins ANALYZE-ESP-004 ("ESP Timeout with 'Continue Anyway' — Soft Failure"). +/// Pins ANALYZE-ESP-004 ("ESP Gave Up on a Blocking App with 'Continue Anyway' — Soft Failure"). /// /// Background (tenant c9787ba2, session cb7036a6): a slow blocking app (Encompass /// Hybrid Installer) does not finish inside the 30-min Device-ESP window, so the ESP /// fails terminally in DeviceSetup. The profile allows "Continue anyway", so the user /// most likely dismissed the failure screen and reached the desktop — but the agent -/// only ever sees the terminal failure and the session is recorded as Failed. The -/// DecisionEngine already stamps the enrollment_failed event with -/// mayHaveContinuedAnyway=true; this rule turns that into a customer-visible -/// "soft failure" advisory with the root-cause remediation. +/// only ever sees the terminal failure and the session is recorded as Failed. /// -/// The rule keys ONLY on mayHaveContinuedAnyway==true (set exclusively on the -/// ESP terminal-failure path), so an ordinary hard failure must not trip it. +/// The rule needs THREE facts: the terminal enrollment_failed, the DecisionEngine's +/// mayHaveContinuedAnyway=true stamp (profile allows the button), and the agent's own +/// timeout verdict — an app_install_failed promoted with failureType=esp_apps_timeout +/// (ESP gave up while the app was still installing, no per-app HRESULT). Session 683f1eff: +/// an ESP failure with a real HRESULT (0x80070652, installer collision) on a Continue-anyway +/// profile fired the rule and rendered the configured 90-min limit as if it had elapsed — +/// the continue-anyway flag alone proves nothing about timing. /// public class RuleEngineEspSoftFailureTests { @@ -37,8 +39,9 @@ public class RuleEngineEspSoftFailureTests var events = new List { + SettleStartedEvent(errorCode: null), EnrollmentFailedEvent(mayHaveContinuedAnyway: "true"), - AppInstallFailed(), + AppInstallFailed(failureType: "esp_apps_timeout"), }; var outcome = await RunAsync(rule, events); @@ -47,13 +50,15 @@ public class RuleEngineEspSoftFailureTests Assert.Equal("ANALYZE-ESP-004", result.RuleId); Assert.Equal("warning", result.Severity); - // Required gate matched on the continue-anyway flag. + // Required gates: the continue-anyway flag AND the agent's timeout verdict. var gate = AsDict(result.MatchedConditions["may_have_continued"]); Assert.Equal("true", AsString(gate["value"])); + var verdict = AsDict(result.MatchedConditions["esp_gave_up_on_app"]); + Assert.Equal("esp_apps_timeout", AsString(verdict["value"])); - // Interpolation material for the explanation / remediation templates is present: - // {{espSyncFailureTimeoutMinutes}}, {{failedSubcategory}} (from enrollment_failed) - // and {{appName}} (from app_install_failed). + // Interpolation material: {{espSyncFailureTimeoutMinutes}} (enrollment_failed), + // {{failedSubcategory}} (esp_failure_settle_started — the terminal event drops it when + // the Shell-Core failure wins the race) and {{appName}} (the promoted app). var timeout = AsDict(result.MatchedConditions["espSyncFailureTimeoutMinutes"]); Assert.Equal("30", AsString(timeout["value"])); @@ -65,6 +70,48 @@ public class RuleEngineEspSoftFailureTests Assert.Equal(FailedAppName, AsString(failedApp["value"])); } + [Fact] + public async Task ANALYZE_ESP_004_stays_silent_on_a_hard_failure_with_continue_anyway() + { + // Session 683f1eff: continue-anyway profile, but the ESP failed on a real installer + // error (0x80070652 / MSI 1618) after 19 minutes. The failed app carries no + // esp_apps_timeout verdict — the rule must not turn an installer collision into + // "the ESP reached its 90-min timeout". + var rule = BuiltInAnalyzeRules.GetAll().First(r => r.RuleId == "ANALYZE-ESP-004"); + + var events = new List + { + SettleStartedEvent(errorCode: "0x80070652"), + EnrollmentFailedEvent(mayHaveContinuedAnyway: "true"), + AppInstallFailed(failureType: null, exitCodeClass: "Retry"), + }; + + var outcome = await RunAsync(rule, events); + Assert.Empty(outcome.Results); + } + + [Fact] + public async Task ANALYZE_ESP_004_names_the_timed_out_app_not_another_failed_one() + { + // Two failed apps: one hard installer failure earlier in the run, one promoted + // esp_apps_timeout at the terminal. {{appName}} must be the timed-out app. + var rule = BuiltInAnalyzeRules.GetAll().First(r => r.RuleId == "ANALYZE-ESP-004"); + + var events = new List + { + AppInstallFailed(failureType: null, exitCodeClass: "Failed", appName: "Contoso Agent", sequence: 100), + SettleStartedEvent(errorCode: null), + EnrollmentFailedEvent(mayHaveContinuedAnyway: "true"), + AppInstallFailed(failureType: "esp_apps_timeout"), + }; + + var outcome = await RunAsync(rule, events); + + var result = Assert.Single(outcome.Results); + var failedApp = AsDict(result.MatchedConditions["appName"]); + Assert.Equal(FailedAppName, AsString(failedApp["value"])); + } + [Fact] public async Task ANALYZE_ESP_004_does_not_fire_when_continue_anyway_is_false() { @@ -75,7 +122,7 @@ public class RuleEngineEspSoftFailureTests var events = new List { EnrollmentFailedEvent(mayHaveContinuedAnyway: "false"), - AppInstallFailed(), + AppInstallFailed(failureType: "esp_apps_timeout"), }; var outcome = await RunAsync(rule, events); @@ -92,7 +139,7 @@ public class RuleEngineEspSoftFailureTests var events = new List { EnrollmentFailedEvent(mayHaveContinuedAnyway: null), - AppInstallFailed(), + AppInstallFailed(failureType: "esp_apps_timeout"), }; var outcome = await RunAsync(rule, events); @@ -132,23 +179,61 @@ public class RuleEngineEspSoftFailureTests return data; } - private static EnrollmentEvent AppInstallFailed() => new() + private static EnrollmentEvent SettleStartedEvent(string? errorCode) { - EventId = Guid.NewGuid().ToString(), - TenantId = TenantId, - SessionId = SessionId, - EventType = "app_install_failed", - Timestamp = DateTime.UtcNow, - Sequence = 220, - Data = new Dictionary + var data = new Dictionary { - ["appName"] = FailedAppName, + ["category"] = "DeviceSetup", + ["failedSubcategory"] = "Certificates", + ["failureType"] = "Provisioning_DeviceSetup_Certificates_Failed", + ["settleSeconds"] = 30, + ["reason"] = "wait_for_late_ime_signals", + }; + if (errorCode != null) + data["errorCode"] = errorCode; + return new EnrollmentEvent + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "esp_failure_settle_started", + Timestamp = DateTime.UtcNow, + Sequence = 212, + Data = data, + }; + } + + private static EnrollmentEvent AppInstallFailed( + string? failureType, + string? exitCodeClass = null, + string appName = FailedAppName, + int sequence = 220) + { + var data = new Dictionary + { + ["appName"] = appName, ["state"] = "Error", ["targeted"] = "Device", - ["failureType"] = "esp_apps_timeout", - ["errorDetail"] = "Install status unconfirmed — ESP gave up while this app was still installing.", + }; + if (failureType != null) + { + data["failureType"] = failureType; + data["confidence"] = "presumed"; + data["errorDetail"] = "Install status unconfirmed — ESP gave up while this app was still installing."; } - }; + if (exitCodeClass != null) + data["exitCodeClass"] = exitCodeClass; + return new EnrollmentEvent + { + EventId = Guid.NewGuid().ToString(), + TenantId = TenantId, + SessionId = SessionId, + EventType = "app_install_failed", + Timestamp = DateTime.UtcNow, + Sequence = sequence, + Data = data, + }; + } private static Dictionary AsDict(object o) { diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEventTypeNotExistsTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEventTypeNotExistsTests.cs index 4c56b9b9..55f3772f 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEventTypeNotExistsTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/RuleEngineEventTypeNotExistsTests.cs @@ -7,12 +7,17 @@ using Moq; namespace AutopilotMonitor.Functions.Tests; /// -/// Pins the absence gate on event_type conditions: not_exists without a -/// dataField matches when no event of that type occurs in the session and is disproved by -/// a single such event. Before this branch an absent type always evaluated false, so a required -/// "must not have happened" condition could never be expressed (ANALYZE-ID-004 v2 relies on it). -/// The dataField variant keeps its historical meaning (some event of the type has an empty field) -/// and is not touched here. +/// Pins the absence gates on conditions. +/// +/// Without a dataField, not_exists matches when no event of that type occurs in +/// the session and is disproved by a single such event (ANALYZE-ID-004 v2 relies on it). +/// +/// With a dataField, not_exists matches when no event of the type carries a +/// non-empty value at that field — including when the type is absent — and is disproved by a +/// single non-empty value; on event_data the same-event filter narrows the inspected set +/// first. Before this the per-event loop's null short-circuit rejected a missing field before +/// the operator ever ran, so "the ESP failure carried no HRESULT" (session 683f1eff) could not be +/// expressed at all, while the precondition gate already had the absence semantics. /// public class RuleEngineEventTypeNotExistsTests { @@ -24,7 +29,7 @@ public class RuleEngineEventTypeNotExistsTests { var events = new List { Event("marker_seen", 1) }; - var outcome = await RunAsync(Rule(), events); + var outcome = await RunAsync(TypeAbsenceRule(), events); var result = Assert.Single(outcome.Results); var evidence = Assert.IsType>(result.MatchedConditions["disproof_absent"]); @@ -37,12 +42,111 @@ public class RuleEngineEventTypeNotExistsTests { var events = new List { Event("marker_seen", 1), Event("disproof", 2) }; - var outcome = await RunAsync(Rule(), events); + var outcome = await RunAsync(TypeAbsenceRule(), events); Assert.Empty(outcome.Results); } - private static AnalyzeRule Rule() => new() + [Fact] + public async Task Field_not_exists_matches_when_no_event_carries_the_field() + { + // The event type is present, the field is not — the shape of an ESP failure without an + // HRESULT (esp_failure_settle_started without errorCode). + var events = new List { Event("marker_seen", 1), Event("disproof", 2) }; + + var outcome = await RunAsync(FieldAbsenceRule("event_type"), events); + + var result = Assert.Single(outcome.Results); + var evidence = Assert.IsType>(result.MatchedConditions["code_absent"]); + Assert.Equal("disproof", evidence["eventType"]); + Assert.Equal("errorCode", evidence["field"]); + Assert.Equal(1, evidence["count"]); + } + + [Fact] + public async Task Field_not_exists_matches_when_the_event_type_is_absent() + { + var events = new List { Event("marker_seen", 1) }; + + var outcome = await RunAsync(FieldAbsenceRule("event_type"), events); + + var result = Assert.Single(outcome.Results); + var evidence = Assert.IsType>(result.MatchedConditions["code_absent"]); + Assert.Equal(0, evidence["count"]); + } + + [Theory] + [InlineData("event_type")] + [InlineData("event_data")] + public async Task A_non_empty_value_on_any_event_vetoes_a_required_field_not_exists(string source) + { + var events = new List + { + Event("marker_seen", 1), + Event("disproof", 2), + Event("disproof", 3, ("errorCode", "0x80070652")), + }; + + var outcome = await RunAsync(FieldAbsenceRule(source), events); + + Assert.Empty(outcome.Results); + } + + [Fact] + public async Task An_empty_string_does_not_count_as_a_carried_value() + { + var events = new List { Event("marker_seen", 1), Event("disproof", 2, ("errorCode", "")) }; + + var outcome = await RunAsync(FieldAbsenceRule("event_type"), events); + + Assert.Single(outcome.Results); + } + + [Fact] + public async Task Event_data_field_not_exists_inspects_only_the_filtered_events() + { + // The carrier of the field is filtered out (kind=a); among kind=b events nobody carries it. + var events = new List + { + Event("marker_seen", 1), + Event("disproof", 2, ("kind", "a"), ("errorCode", "0x80070652")), + Event("disproof", 3, ("kind", "b")), + }; + + var matched = await RunAsync(FieldAbsenceRule("event_data", filterKind: "b"), events); + var result = Assert.Single(matched.Results); + var evidence = Assert.IsType>(result.MatchedConditions["code_absent"]); + Assert.Equal(1, evidence["count"]); + + var vetoed = await RunAsync(FieldAbsenceRule("event_data", filterKind: "a"), events); + Assert.Empty(vetoed.Results); + } + + private static AnalyzeRule TypeAbsenceRule() => Rule( + new RuleCondition { Signal = "disproof_absent", Source = "event_type", EventType = "disproof", Operator = "not_exists", Value = "", Required = true }); + + private static AnalyzeRule FieldAbsenceRule(string source, string? filterKind = null) + { + var absence = new RuleCondition + { + Signal = "code_absent", + Source = source, + EventType = "disproof", + DataField = "errorCode", + Operator = "not_exists", + Value = "", + Required = true, + }; + if (filterKind != null) + { + absence.FilterField = "kind"; + absence.FilterOperator = "equals"; + absence.FilterValue = filterKind; + } + return Rule(absence); + } + + private static AnalyzeRule Rule(RuleCondition absence) => new() { // Custom-namespace rule: a built-in ID outside the live catalog is hidden by the // sunset filter in AnalyzeRuleService (IsBuiltIn defaults to true). @@ -58,11 +162,11 @@ public class RuleEngineEventTypeNotExistsTests Conditions = new List { new() { Signal = "marker", Source = "event_type", EventType = "marker_seen", Operator = "exists", Value = "", Required = true }, - new() { Signal = "disproof_absent", Source = "event_type", EventType = "disproof", Operator = "not_exists", Value = "", Required = true }, + absence, }, }; - private static EnrollmentEvent Event(string eventType, int sequence) => new() + private static EnrollmentEvent Event(string eventType, int sequence, params (string key, string value)[] data) => new() { EventId = Guid.NewGuid().ToString(), TenantId = TenantId, @@ -70,7 +174,7 @@ public class RuleEngineEventTypeNotExistsTests EventType = eventType, Timestamp = DateTime.UtcNow.AddMinutes(sequence), Sequence = sequence, - Data = new Dictionary(), + Data = data.ToDictionary(d => d.key, d => (object)d.value), }; private static async Task RunAsync(AnalyzeRule rule, List events) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs index 5c7e515a..dcdb0fa5 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/RuleEngine.ConditionEvaluators.cs @@ -127,6 +127,18 @@ namespace AutopilotMonitor.Functions.Services }); } + // Field-absence gate: not_exists WITH a dataField matches when no event of the type + // carries a non-empty value at that field — including when the type is absent — the + // same semantics as the precondition gate. Must run before the "no matching events" + // veto and the per-event loop below, whose null short-circuit could never express + // "the field is missing" (session 683f1eff: "esp_failure_settle_started.errorCode + // not_exists" was silently dead). + if (!string.IsNullOrEmpty(condition.DataField) + && string.Equals(condition.Operator, "not_exists", StringComparison.OrdinalIgnoreCase)) + { + return EvaluateFieldAbsence(condition, matchingEvents); + } + if (!matchingEvents.Any()) return (false, "no matching events"); @@ -206,6 +218,11 @@ namespace AutopilotMonitor.Functions.Services { var matchingEvents = ApplyValueFilter(condition, events.Where(e => MatchesEventType(e, condition.EventType)).ToList()); + // Field-absence gate — see EvaluateEventTypeCondition; here it applies to the + // filtered event set, so "no event with filterField=x carries dataField" is expressible. + if (string.Equals(condition.Operator, "not_exists", StringComparison.OrdinalIgnoreCase)) + return EvaluateFieldAbsence(condition, matchingEvents); + foreach (var evt in matchingEvents) { var fieldValue = GetDataFieldValue(evt, condition.DataField); @@ -227,6 +244,26 @@ namespace AutopilotMonitor.Functions.Services return (false, "no matching data"); } + /// + /// not_exists with a dataField: true when none of + /// carries a non-empty value at the field (an empty set counts as absent). A single + /// non-empty value disproves it. Evidence names the type, the field and how many events + /// of the type were inspected, so a finding can say "0 esp_failure_settle_started events + /// carried errorCode" rather than pointing at one event. + /// + private static (bool matched, object evidence) EvaluateFieldAbsence(RuleCondition condition, List matchingEvents) + { + if (matchingEvents.Any(e => !string.IsNullOrEmpty(GetDataFieldValue(e, condition.DataField)))) + return (false, "data field present"); + + return (true, new Dictionary + { + ["eventType"] = condition.EventType, + ["field"] = condition.DataField, + ["count"] = matchingEvents.Count + }); + } + /// /// Evidence values feed the finding UI and {{token}} interpolation, both of which /// render via ToString() — a raw List would surface as its .NET type name diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts index c99c71e9..d2394ccd 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring-guide.ts @@ -201,8 +201,11 @@ export const RULE_AUTHORING_GUIDE = { ], operatorNotes: 'String comparisons are case-insensitive; regex runs case-insensitive with a 1s timeout ' + - '(.NET regex syntax); in/not_in take a comma-separated value list. An operator the engine ' + - 'does not know silently evaluates to FALSE in production — validate_rule catches this.', + '(.NET regex syntax); in/not_in take a comma-separated value list. exists/not_exists with a ' + + 'dataField test whether ANY event of the type carries a non-empty value there — not_exists ' + + 'matches when none does, also when the event type is absent (e.g. "the ESP failure carried no ' + + 'errorCode"); without a dataField they test only the presence of the event type. An operator ' + + 'the engine does not know silently evaluates to FALSE in production — validate_rule catches this.', required: 'required=true conditions are the firing core: ALL of them must match or the rule produces ' + 'nothing. required=false conditions are optional reinforcers whose evidence feeds ' + diff --git a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts index acaf7d6c..0572b1ed 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts @@ -470,7 +470,7 @@ export const ANALYZE_RULE_SCHEMA: Record = { }, "operator": { "type": "string", - "description": "Comparison operator.", + "description": "Comparison operator. exists/not_exists with a dataField test whether ANY event of the type carries a non-empty value at that field: not_exists matches when no event does — including when the event type is absent altogether — so a rule can gate on a missing field (e.g. an ESP failure without an HRESULT). Without a dataField they test only the presence of the event type.", "enum": [ "equals", "not_equals",