From 5055db2de1b3689028bc93669bf45ad0caf01bf9 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Thu, 27 Aug 2026 19:40:17 +0200 Subject: [PATCH] feat(rules): detect installer-mutex app failures (0x80070652) and SWG-blocked diagnostics uploads Two analyze rules born from the pilatus-aircraft diagnosis (2026-08-27): - ANALYZE-APP-017 "App Install Blocked by Another Installer (0x80070652)": fires on app_install_failed with hresultFromWin32 -2147023278 (ERROR_INSTALL_ALREADY_RUNNING); confidence boosts for MSI exit code 1618 and for an ESP failure settling with the same code. The 08-26 burst failed 16 sessions deterministically at ~21 min (IME's fixed msiexec-busy retry budget) right after a preceding installer left the Windows Installer mutex held - previously only generic "Enrollment Failed" fired, naming no cause. - ANALYZE-NET-001 "Diagnostics Upload Blocked by Web Security Gateway": fires on diagnostics_upload_failed answered by an HTML block page carrying a recognizable security-product marker (McAfee/Skyhigh, Zscaler, Netskope, ...). OnFailure diagnostics are lost exactly on the sessions that need them; plain HTML errors without a vendor marker (stale/stopped endpoint) stay excluded on purpose. Both validated via MCP validate_rule (0 errors) and dry-run with test_analyze_rule: fire on the burst session (confidence 100 / 90), silent on a timeout-shaped failure and on a succeeded session that carried unrelated app_install_failed events. Co-Authored-By: Claude Fable 5 --- rules/analyze/ANALYZE-APP-017.json | 92 ++++++++++++++++ rules/analyze/ANALYZE-NET-001.json | 81 ++++++++++++++ rules/dist/analyze-rules.json | 171 +++++++++++++++++++++++++++++ 3 files changed, 344 insertions(+) create mode 100644 rules/analyze/ANALYZE-APP-017.json create mode 100644 rules/analyze/ANALYZE-NET-001.json diff --git a/rules/analyze/ANALYZE-APP-017.json b/rules/analyze/ANALYZE-APP-017.json new file mode 100644 index 00000000..eaaa07aa --- /dev/null +++ b/rules/analyze/ANALYZE-APP-017.json @@ -0,0 +1,92 @@ +{ + "$schema": "../schema/analyze-rule.schema.json", + "ruleId": "ANALYZE-APP-017", + "title": "App Install Blocked by Another Installer (0x80070652)", + "description": "Detects Win32 app installs that failed because the Windows Installer was busy with another installation (HRESULT 0x80070652 / MSI exit code 1618, ERROR_INSTALL_ALREADY_RUNNING). The IME retries on a fixed budget (~15-17 minutes across several attempts) and then reports the app as failed; when the app is ESP-blocking, the Device Setup Apps subcategory terminates with it.", + "severity": "high", + "category": "apps", + "version": "1.0.0", + "author": "Autopilot Monitor", + "enabled": true, + "baseConfidence": 70, + "conditions": [ + { + "signal": "mutex_blocked_install", + "source": "event_type", + "eventType": "app_install_failed", + "dataField": "hresultFromWin32", + "operator": "equals", + "value": "-2147023278", + "required": true + }, + { + "signal": "installer_exit_1618", + "source": "event_data", + "eventType": "app_install_failed", + "dataField": "exitCode", + "operator": "equals", + "value": "1618", + "required": false + }, + { + "signal": "esp_failed_with_mutex_code", + "source": "event_data", + "eventType": "esp_failure_settle_started", + "dataField": "errorCode", + "operator": "equals", + "value": "0x80070652", + "required": false + } + ], + "confidenceFactors": [ + { + "signal": "installer_exit_1618", + "condition": "exists", + "weight": 10 + }, + { + "signal": "esp_failed_with_mutex_code", + "condition": "exists", + "weight": 20 + } + ], + "confidenceThreshold": 65, + "explanation": "App **{{appName}}** failed to install because the Windows Installer service was already busy with another installation — HRESULT **0x80070652** / MSI exit code **1618** (`ERROR_INSTALL_ALREADY_RUNNING`). Only one Windows Installer transaction can run at a time; the Intune Management Extension retried on its fixed budget (typically ~15-17 minutes across several attempts) and then reported the app as failed. If this app is required in the Enrollment Status Page, the Device Setup Apps subcategory fails with it and the enrollment terminates.\n\nThis failure is **deterministic per attempt**: as long as the competing installer keeps winning the race, a plain re-enrollment fails again at roughly the same minute mark — retrying without fixing the collision only churns devices.\n\nCommon sources of the competing installation:\n\n- The **preceding Win32 app in the install order**: its installer exits 0 (so IME moves on) but leaves a child `msiexec` transaction or post-install MSI work running. Security agents and proxy clients that chain multiple MSIs are frequent offenders.\n- **OEM factory software** installing on first boot (factory-image provisioning packages, vendor updaters) — typical on brand-new hardware batches.\n- **Microsoft 365 Apps (C2R), driver updaters or Windows servicing** running installations in parallel outside IME's control.", + "remediation": [ + { + "title": "Identify the competing installer", + "steps": [ + "In this session's timeline, note which app finished installing immediately before {{appName}} started — its installer is the prime suspect for holding the Windows Installer mutex", + "On a reproduction device, when the failure window is active, check for running installers: Get-Process msiexec | Select-Object Id,StartTime and the in-progress marker key HKLM\\SOFTWARE\\Microsoft\\Windows\\Installer\\InProgress", + "Check the Application event log for MsiInstaller events around the failure timestamp to see which product held the transaction", + "On factory-fresh hardware, also look for OEM first-boot installers (vendor updater services, factory provisioning packages) that run outside Intune" + ] + }, + { + "title": "Serialize the install chain", + "steps": [ + "Add an Intune Win32 dependency so {{appName}} depends on the app whose installer causes the collision — IME then orders them explicitly instead of racing", + "Alternatively, wrap the offending predecessor so its install script waits until msiexec has fully finished (poll the InProgress registry key or the _MSIExecute mutex with a bounded timeout) before exiting", + "If the app does not have to gate the ESP, remove it from the blocking-app list so a transient 1618 no longer terminates the enrollment; Intune retries it after the desktop" + ] + } + ], + "relatedDocs": [ + { + "title": "Microsoft Learn: MsiExec.exe error messages (1618 = another installation is in progress)", + "url": "https://learn.microsoft.com/en-us/windows/win32/msi/error-codes" + }, + { + "title": "Microsoft Learn: Troubleshoot Win32 app installations in Intune", + "url": "https://learn.microsoft.com/en-us/troubleshoot/mem/intune/app-management/troubleshoot-win32-app-install" + } + ], + "tags": [ + "apps", + "msiexec", + "mutex", + "0x80070652", + "1618", + "esp" + ] +} diff --git a/rules/analyze/ANALYZE-NET-001.json b/rules/analyze/ANALYZE-NET-001.json new file mode 100644 index 00000000..f9ad40c9 --- /dev/null +++ b/rules/analyze/ANALYZE-NET-001.json @@ -0,0 +1,81 @@ +{ + "$schema": "../schema/analyze-rule.schema.json", + "ruleId": "ANALYZE-NET-001", + "title": "Diagnostics Upload Blocked by Web Security Gateway", + "description": "Detects diagnostics package uploads that were answered by a security appliance's HTML block page instead of the Autopilot Monitor backend. A forward proxy / secure web gateway (SWG) with SSL inspection is intercepting the upload endpoint, so exactly the sessions where diagnostics matter most (OnFailure uploads) are the ones losing them.", + "severity": "warning", + "category": "network", + "version": "1.0.0", + "author": "Autopilot Monitor", + "enabled": true, + "baseConfidence": 75, + "conditions": [ + { + "signal": "foreign_html_response", + "source": "event_data", + "eventType": "diagnostics_upload_failed", + "dataField": "errorCode", + "operator": "contains", + "value": "HTML platform error page", + "required": true + }, + { + "signal": "security_gateway_marker", + "source": "event_data", + "eventType": "diagnostics_upload_failed", + "dataField": "errorCode", + "operator": "regex", + "value": "mcafee|skyhigh|zscaler|netskope|blue ?coat|proxysg|forcepoint|websense|palo ?alto|fortigate|fortiguard|sophos|watchguard|barracuda|iboss|umbrella|lightspeed|smoothwall|contentkeeper|web ?gateway|web ?filter|content ?filter|secure ?gateway", + "required": true + }, + { + "signal": "http_403_block", + "source": "event_data", + "eventType": "diagnostics_upload_failed", + "dataField": "errorCode", + "operator": "contains", + "value": "403", + "required": false + } + ], + "confidenceFactors": [ + { + "signal": "http_403_block", + "condition": "exists", + "weight": 15 + } + ], + "confidenceThreshold": 70, + "explanation": "The agent collected a diagnostics package but the upload was answered with an **HTML block page from a web security gateway** instead of a response from the Autopilot Monitor backend — the event message names the product that served the page. A forward proxy / SWG with SSL inspection is intercepting HTTPS traffic from the enrollment network and blocking the upload endpoint.\n\nThe consequence is quietly painful: diagnostics upload is typically configured **OnFailure**, so the sessions that fail — the ones where the IME logs and diagnostics ZIP would explain the root cause — are exactly the sessions whose evidence never arrives. Enrollment telemetry itself may still flow (the agent often starts before the gateway's client enforcement does, or other endpoints are treated differently), which makes the gap easy to miss until a failure needs investigating.\n\nNote this rule only fires when the blocking page carries a recognizable security-product marker. A plain HTML error page without one usually means a stale or stopped endpoint instead — that is a different problem.", + "remediation": [ + { + "title": "Exempt the upload endpoint in the web gateway", + "steps": [ + "Identify the product from the block-page title quoted in this event's message (e.g. a Skyhigh/McAfee Web Gateway notification page)", + "In that gateway, add the Autopilot Monitor API host shown in the event message to the SSL-inspection bypass AND the URL allowlist for the enrollment network segment", + "Enrollment traffic often originates from a dedicated provisioning VLAN or from devices without the proxy client yet — make sure the exemption applies to that segment, not only to managed clients", + "If the gateway enforces per-user policy, remember ESP-phase traffic runs as SYSTEM/device before any user signs in — the exemption must cover unauthenticated device traffic" + ] + }, + { + "title": "Verify the fix", + "steps": [ + "Re-run an enrollment (or wait for the next failed one) and confirm a diagnostics package appears on the session in the portal", + "Optionally set diagnostics upload to Always temporarily to validate the path without needing a failure" + ] + } + ], + "relatedDocs": [ + { + "title": "Microsoft Learn: Windows Autopilot networking requirements", + "url": "https://learn.microsoft.com/en-us/autopilot/networking-requirements" + } + ], + "tags": [ + "network", + "proxy", + "ssl-inspection", + "swg", + "diagnostics" + ] +} diff --git a/rules/dist/analyze-rules.json b/rules/dist/analyze-rules.json index e3a5dacd..ff2cbd04 100644 --- a/rules/dist/analyze-rules.json +++ b/rules/dist/analyze-rules.json @@ -943,6 +943,97 @@ "delivery-gap" ] }, + { + "ruleId": "ANALYZE-APP-017", + "title": "App Install Blocked by Another Installer (0x80070652)", + "description": "Detects Win32 app installs that failed because the Windows Installer was busy with another installation (HRESULT 0x80070652 / MSI exit code 1618, ERROR_INSTALL_ALREADY_RUNNING). The IME retries on a fixed budget (~15-17 minutes across several attempts) and then reports the app as failed; when the app is ESP-blocking, the Device Setup Apps subcategory terminates with it.", + "severity": "high", + "category": "apps", + "version": "1.0.0", + "author": "Autopilot Monitor", + "enabled": true, + "baseConfidence": 70, + "conditions": [ + { + "signal": "mutex_blocked_install", + "source": "event_type", + "eventType": "app_install_failed", + "dataField": "hresultFromWin32", + "operator": "equals", + "value": "-2147023278", + "required": true + }, + { + "signal": "installer_exit_1618", + "source": "event_data", + "eventType": "app_install_failed", + "dataField": "exitCode", + "operator": "equals", + "value": "1618", + "required": false + }, + { + "signal": "esp_failed_with_mutex_code", + "source": "event_data", + "eventType": "esp_failure_settle_started", + "dataField": "errorCode", + "operator": "equals", + "value": "0x80070652", + "required": false + } + ], + "confidenceFactors": [ + { + "signal": "installer_exit_1618", + "condition": "exists", + "weight": 10 + }, + { + "signal": "esp_failed_with_mutex_code", + "condition": "exists", + "weight": 20 + } + ], + "confidenceThreshold": 65, + "explanation": "App **{{appName}}** failed to install because the Windows Installer service was already busy with another installation — HRESULT **0x80070652** / MSI exit code **1618** (`ERROR_INSTALL_ALREADY_RUNNING`). Only one Windows Installer transaction can run at a time; the Intune Management Extension retried on its fixed budget (typically ~15-17 minutes across several attempts) and then reported the app as failed. If this app is required in the Enrollment Status Page, the Device Setup Apps subcategory fails with it and the enrollment terminates.\n\nThis failure is **deterministic per attempt**: as long as the competing installer keeps winning the race, a plain re-enrollment fails again at roughly the same minute mark — retrying without fixing the collision only churns devices.\n\nCommon sources of the competing installation:\n\n- The **preceding Win32 app in the install order**: its installer exits 0 (so IME moves on) but leaves a child `msiexec` transaction or post-install MSI work running. Security agents and proxy clients that chain multiple MSIs are frequent offenders.\n- **OEM factory software** installing on first boot (factory-image provisioning packages, vendor updaters) — typical on brand-new hardware batches.\n- **Microsoft 365 Apps (C2R), driver updaters or Windows servicing** running installations in parallel outside IME's control.", + "remediation": [ + { + "title": "Identify the competing installer", + "steps": [ + "In this session's timeline, note which app finished installing immediately before {{appName}} started — its installer is the prime suspect for holding the Windows Installer mutex", + "On a reproduction device, when the failure window is active, check for running installers: Get-Process msiexec | Select-Object Id,StartTime and the in-progress marker key HKLM\\SOFTWARE\\Microsoft\\Windows\\Installer\\InProgress", + "Check the Application event log for MsiInstaller events around the failure timestamp to see which product held the transaction", + "On factory-fresh hardware, also look for OEM first-boot installers (vendor updater services, factory provisioning packages) that run outside Intune" + ] + }, + { + "title": "Serialize the install chain", + "steps": [ + "Add an Intune Win32 dependency so {{appName}} depends on the app whose installer causes the collision — IME then orders them explicitly instead of racing", + "Alternatively, wrap the offending predecessor so its install script waits until msiexec has fully finished (poll the InProgress registry key or the _MSIExecute mutex with a bounded timeout) before exiting", + "If the app does not have to gate the ESP, remove it from the blocking-app list so a transient 1618 no longer terminates the enrollment; Intune retries it after the desktop" + ] + } + ], + "relatedDocs": [ + { + "title": "Microsoft Learn: MsiExec.exe error messages (1618 = another installation is in progress)", + "url": "https://learn.microsoft.com/en-us/windows/win32/msi/error-codes" + }, + { + "title": "Microsoft Learn: Troubleshoot Win32 app installations in Intune", + "url": "https://learn.microsoft.com/en-us/troubleshoot/mem/intune/app-management/troubleshoot-win32-app-install" + } + ], + "tags": [ + "apps", + "msiexec", + "mutex", + "0x80070652", + "1618", + "esp" + ] + }, { "ruleId": "ANALYZE-CORR-003", "title": "Proxy Configuration Causing Download Failure", @@ -2964,6 +3055,86 @@ "esp" ] }, + { + "ruleId": "ANALYZE-NET-001", + "title": "Diagnostics Upload Blocked by Web Security Gateway", + "description": "Detects diagnostics package uploads that were answered by a security appliance's HTML block page instead of the Autopilot Monitor backend. A forward proxy / secure web gateway (SWG) with SSL inspection is intercepting the upload endpoint, so exactly the sessions where diagnostics matter most (OnFailure uploads) are the ones losing them.", + "severity": "warning", + "category": "network", + "version": "1.0.0", + "author": "Autopilot Monitor", + "enabled": true, + "baseConfidence": 75, + "conditions": [ + { + "signal": "foreign_html_response", + "source": "event_data", + "eventType": "diagnostics_upload_failed", + "dataField": "errorCode", + "operator": "contains", + "value": "HTML platform error page", + "required": true + }, + { + "signal": "security_gateway_marker", + "source": "event_data", + "eventType": "diagnostics_upload_failed", + "dataField": "errorCode", + "operator": "regex", + "value": "mcafee|skyhigh|zscaler|netskope|blue ?coat|proxysg|forcepoint|websense|palo ?alto|fortigate|fortiguard|sophos|watchguard|barracuda|iboss|umbrella|lightspeed|smoothwall|contentkeeper|web ?gateway|web ?filter|content ?filter|secure ?gateway", + "required": true + }, + { + "signal": "http_403_block", + "source": "event_data", + "eventType": "diagnostics_upload_failed", + "dataField": "errorCode", + "operator": "contains", + "value": "403", + "required": false + } + ], + "confidenceFactors": [ + { + "signal": "http_403_block", + "condition": "exists", + "weight": 15 + } + ], + "confidenceThreshold": 70, + "explanation": "The agent collected a diagnostics package but the upload was answered with an **HTML block page from a web security gateway** instead of a response from the Autopilot Monitor backend — the event message names the product that served the page. A forward proxy / SWG with SSL inspection is intercepting HTTPS traffic from the enrollment network and blocking the upload endpoint.\n\nThe consequence is quietly painful: diagnostics upload is typically configured **OnFailure**, so the sessions that fail — the ones where the IME logs and diagnostics ZIP would explain the root cause — are exactly the sessions whose evidence never arrives. Enrollment telemetry itself may still flow (the agent often starts before the gateway's client enforcement does, or other endpoints are treated differently), which makes the gap easy to miss until a failure needs investigating.\n\nNote this rule only fires when the blocking page carries a recognizable security-product marker. A plain HTML error page without one usually means a stale or stopped endpoint instead — that is a different problem.", + "remediation": [ + { + "title": "Exempt the upload endpoint in the web gateway", + "steps": [ + "Identify the product from the block-page title quoted in this event's message (e.g. a Skyhigh/McAfee Web Gateway notification page)", + "In that gateway, add the Autopilot Monitor API host shown in the event message to the SSL-inspection bypass AND the URL allowlist for the enrollment network segment", + "Enrollment traffic often originates from a dedicated provisioning VLAN or from devices without the proxy client yet — make sure the exemption applies to that segment, not only to managed clients", + "If the gateway enforces per-user policy, remember ESP-phase traffic runs as SYSTEM/device before any user signs in — the exemption must cover unauthenticated device traffic" + ] + }, + { + "title": "Verify the fix", + "steps": [ + "Re-run an enrollment (or wait for the next failed one) and confirm a diagnostics package appears on the session in the portal", + "Optionally set diagnostics upload to Always temporarily to validate the path without needing a failure" + ] + } + ], + "relatedDocs": [ + { + "title": "Microsoft Learn: Windows Autopilot networking requirements", + "url": "https://learn.microsoft.com/en-us/autopilot/networking-requirements" + } + ], + "tags": [ + "network", + "proxy", + "ssl-inspection", + "swg", + "diagnostics" + ] + }, { "ruleId": "ANALYZE-OFFICE-001", "title": "Microsoft 365 Apps Install Failed",