mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 13:10:13 +00:00
Compare commits
52 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fd7d3973e7 | |||
| fd2b5c2da2 | |||
| a44e37a7c2 | |||
| d8a252562d | |||
| a5d12ba1d0 | |||
| 530c978bf8 | |||
| 09898d7ba0 | |||
| 71eda93e8b | |||
| 1cfc32c9aa | |||
| 11339c9868 | |||
| 540f543834 | |||
| 088c3b0479 | |||
| 3224774752 | |||
| e7ea431ccf | |||
| 9e24b92fbf | |||
| 45d7c03ccf | |||
| 90012d9a66 | |||
| e8ef4f53ae | |||
| a0d10b8471 | |||
| ad3632e9fb | |||
| 05e1569503 | |||
| ba3581e253 | |||
| 829ec4c969 | |||
| b2d2d37c6f | |||
| e0ad51b965 | |||
| bbb55034e3 | |||
| 58c8cf87a4 | |||
| 71d2c0e47f | |||
| 3b2f5ed46d | |||
| 7329f12217 | |||
| c9a9df99f1 | |||
| 757427ffea | |||
| 71832b9deb | |||
| ce4d884abc | |||
| b56c774103 | |||
| 68767f436f | |||
| c7b06be8b7 | |||
| 479247cdae | |||
| b6ba87f072 | |||
| fa640ebe61 | |||
| dbc20a9f66 | |||
| cbda62d131 | |||
| 3951000b40 | |||
| c59842262e | |||
| c3e9621a78 | |||
| 3ed21c37d1 | |||
| 4a1194569e | |||
| d0794fc7c5 | |||
| f34c9d2322 | |||
| 380a8417c8 | |||
| 63826a4869 | |||
| 7df208fac9 |
+1
-1
Submodule .claude/commands updated: 3ed910ab50...af1b8468c7
@@ -0,0 +1,31 @@
|
||||
# CodeQL configuration.
|
||||
#
|
||||
# The default query suites stay active — this file only filters out two C# queries
|
||||
# that fire structurally on this codebase, where the flagged behavior is the feature,
|
||||
# not a defect:
|
||||
#
|
||||
# cs/exposure-of-sensitive-information
|
||||
# Audit and operational logs record WHO performed an action (actor UPN / email)
|
||||
# and notification services deliver mail to a recipient address. Writing that
|
||||
# identity to the log / notification IS the purpose of the field. Every such site
|
||||
# is a legitimate, reviewed use — the query cannot distinguish it from a leak.
|
||||
#
|
||||
# cs/cleartext-storage-of-sensitive-information
|
||||
# The rate limiters and distress guard use the trusted client IP as their bucket
|
||||
# key. The IP is an identifier for throttling, not a secret at rest; storing it in
|
||||
# the in-memory / table rate-limit key is by design.
|
||||
#
|
||||
# Genuine one-off false positives from OTHER queries (e.g. an open-redirect the taint
|
||||
# model can't see is allow-listed, a rate-limited route it doesn't recognize) are
|
||||
# dismissed individually in the Security tab so those queries stay sharp — they are
|
||||
# NOT suppressed here.
|
||||
#
|
||||
# Log-injection (cs/log-forging) is deliberately NOT filtered: user-controlled values
|
||||
# are sanitized at the sink via Helpers/LogSanitizer, so the query stays useful for
|
||||
# catching a future unsanitized sink.
|
||||
|
||||
query-filters:
|
||||
- exclude:
|
||||
id: cs/exposure-of-sensitive-information
|
||||
- exclude:
|
||||
id: cs/cleartext-storage-of-sensitive-information
|
||||
@@ -20,6 +20,11 @@ updates:
|
||||
web:
|
||||
patterns: ['*']
|
||||
update-types: [minor, patch]
|
||||
# Major bumps are breaking and reviewed deliberately, not auto-PR'd.
|
||||
# Security updates ignore this filter, so the Security tab stays clean.
|
||||
ignore:
|
||||
- dependency-name: '*'
|
||||
update-types: ['version-update:semver-major']
|
||||
|
||||
- package-ecosystem: npm
|
||||
directory: /src/McpServer/autopilot-monitor-mcp
|
||||
@@ -30,6 +35,11 @@ updates:
|
||||
mcp:
|
||||
patterns: ['*']
|
||||
update-types: [minor, patch]
|
||||
# Major bumps are breaking and reviewed deliberately, not auto-PR'd.
|
||||
# Security updates ignore this filter, so the Security tab stays clean.
|
||||
ignore:
|
||||
- dependency-name: '*'
|
||||
update-types: ['version-update:semver-major']
|
||||
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Reserves a monotone build number from a counter blob (ETag compare-and-swap).
|
||||
|
||||
.DESCRIPTION
|
||||
Single source of build numbers for every component that ships from this repo
|
||||
(agent, backend, MCP). Each component owns one counter blob holding a plain
|
||||
integer; this script reads it, writes value+1 back under If-Match, and returns
|
||||
the reserved number.
|
||||
|
||||
Reserve-BEFORE-build: two concurrent builds can never mint the same number.
|
||||
A failed build burns a number, which is harmless -- uniqueness matters,
|
||||
density does not.
|
||||
|
||||
The blob is public-read; only the write needs the container SAS.
|
||||
|
||||
Why a counter and not the published version manifest: manifests are written
|
||||
on publish, not on build (the agent writes version.json only for stable
|
||||
releases), so two dev builds in a row would derive the same number and
|
||||
overwrite each other's versioned artifact.
|
||||
|
||||
.PARAMETER CounterUrl
|
||||
Full URL of the counter blob, without SAS.
|
||||
|
||||
.PARAMETER SasToken
|
||||
Container SAS with create+write permission. A leading '?' is tolerated.
|
||||
|
||||
.PARAMETER Override
|
||||
Explicit build number instead of counter+1. If it is not greater than the
|
||||
current counter the blob is left untouched (re-build of a number that was
|
||||
already reserved). 0 = no override.
|
||||
|
||||
.PARAMETER MaxAttempts
|
||||
CAS retries before giving up.
|
||||
|
||||
.OUTPUTS
|
||||
[int] the reserved build number. Progress goes to the host, not the pipeline.
|
||||
|
||||
.EXAMPLE
|
||||
$n = ./Request-BuildNumber.ps1 -CounterUrl $url -SasToken $sas
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$CounterUrl,
|
||||
[Parameter(Mandatory = $true)][string]$SasToken,
|
||||
[int]$Override = 0,
|
||||
[int]$MaxAttempts = 5
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# Tolerate both secret formats ("sp=cw&..." and "?sp=cw&...").
|
||||
$writeUrl = $CounterUrl + '?' + $SasToken.TrimStart('?')
|
||||
|
||||
function Get-HttpStatus {
|
||||
param($ErrorRecord)
|
||||
if ($ErrorRecord.Exception.PSObject.Properties['Response'] -and $ErrorRecord.Exception.Response) {
|
||||
return [int]$ErrorRecord.Exception.Response.StatusCode
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function Read-Counter {
|
||||
try {
|
||||
$resp = Invoke-WebRequest -Uri $CounterUrl -Method Get -UseBasicParsing
|
||||
} catch {
|
||||
if ((Get-HttpStatus $_) -eq 404) {
|
||||
throw "Counter blob not found at $CounterUrl. Seed it once with the last used build number: PUT the plain integer with 'x-ms-blob-type: BlockBlob' and 'If-None-Match: *'. See docs/versioning.md."
|
||||
}
|
||||
throw
|
||||
}
|
||||
$raw = if ($resp.Content -is [byte[]]) { [System.Text.Encoding]::UTF8.GetString($resp.Content) } else { [string]$resp.Content }
|
||||
# If-Match requires the quoted ETag form; PowerShell strips the quotes on read.
|
||||
$etag = '"' + ([string]($resp.Headers['ETag'] | Select-Object -First 1)).Trim('"') + '"'
|
||||
return [pscustomobject]@{ Value = [int]$raw.Trim(); ETag = $etag }
|
||||
}
|
||||
|
||||
for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) {
|
||||
$counter = Read-Counter
|
||||
|
||||
if ($Override -gt 0 -and $Override -le $counter.Value) {
|
||||
Write-Host " Build number $Override <= counter $($counter.Value) -- already reserved, counter blob unchanged"
|
||||
return $Override
|
||||
}
|
||||
$target = if ($Override -gt 0) { $Override } else { $counter.Value + 1 }
|
||||
|
||||
try {
|
||||
$headers = @{ 'x-ms-blob-type' = 'BlockBlob'; 'If-Match' = $counter.ETag }
|
||||
Invoke-RestMethod -Uri $writeUrl -Method Put -Headers $headers -Body "$target" -ContentType 'text/plain' | Out-Null
|
||||
Write-Host " Reserved build number $target (CAS write ok, attempt $attempt)"
|
||||
return $target
|
||||
} catch {
|
||||
if ((Get-HttpStatus $_) -eq 412) {
|
||||
Write-Host " CAS conflict on attempt $attempt (another build reserved concurrently) -- retrying"
|
||||
Start-Sleep -Seconds ($attempt * 2)
|
||||
} else {
|
||||
throw
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw "Could not reserve a build number after $MaxAttempts CAS attempts"
|
||||
@@ -56,78 +56,27 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v7
|
||||
|
||||
# SDK 10 = single build toolchain (parity with local build.ps1 / global.json).
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@v5
|
||||
uses: actions/setup-dotnet@v6
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
# Build numbers come from the shared counter blob (single source of truth for CI
|
||||
# AND local build.ps1). Reserve-before-build with ETag-CAS: concurrent builds can
|
||||
# never mint the same number; a failed build burns a number, which is harmless.
|
||||
# The blob is public-read (like the rest of the container); writes need the SAS.
|
||||
# Build numbers come from the shared counter blob -- one counter and one
|
||||
# implementation for CI and the local build (.github/scripts/Request-BuildNumber.ps1).
|
||||
# Reserve-before-build with ETag-CAS: concurrent builds can never mint the same
|
||||
# number; a failed build burns a number, which is harmless.
|
||||
- name: Reserve build number (counter blob CAS)
|
||||
id: version
|
||||
shell: pwsh
|
||||
run: |
|
||||
$counterBlob = "buildcounter-v$env:LINE_MAJOR.txt"
|
||||
$counterUrl = "https://autopilotmonitoreu.blob.core.windows.net/agent/$counterBlob"
|
||||
# Tolerate both secret formats ("sp=cw&..." and "?sp=cw&...") -- the URL below adds the '?'.
|
||||
$sasToken = "${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}".TrimStart('?')
|
||||
$override = "${{ inputs.build_number }}"
|
||||
$override = 0
|
||||
if ('${{ inputs.build_number }}' -ne '') { $override = [int]'${{ inputs.build_number }}' }
|
||||
|
||||
function Get-Counter {
|
||||
try {
|
||||
$resp = Invoke-WebRequest -Uri $counterUrl -Method Get
|
||||
$raw = if ($resp.Content -is [byte[]]) { [System.Text.Encoding]::UTF8.GetString($resp.Content) } else { [string]$resp.Content }
|
||||
# If-Match requires the quoted ETag form; pwsh strips the quotes on read.
|
||||
$etag = '"' + ([string]($resp.Headers['ETag'] | Select-Object -First 1)).Trim('"') + '"'
|
||||
return @{ Value = [int]$raw.Trim(); ETag = $etag }
|
||||
} catch {
|
||||
$status = if ($_.Exception.Response) { [int]$_.Exception.Response.StatusCode } else { 0 }
|
||||
if ($status -eq 404) {
|
||||
Write-Error "Counter blob '$counterBlob' not found. Seed it once with the last used build number (see docs/agent/build-counter-blob.md) -- do NOT seed from the repo's buildcounter.txt, it is a stale offline fallback."
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
$buildNumber = $null
|
||||
for ($attempt = 1; $attempt -le 5; $attempt++) {
|
||||
$counter = Get-Counter
|
||||
if ($override -ne '') {
|
||||
$target = [int]$override
|
||||
if ($target -le $counter.Value) {
|
||||
# Re-build of an already-reserved number: counter stays monotonic, no write.
|
||||
Write-Host "Manual build number $target <= counter $($counter.Value) -- counter blob unchanged"
|
||||
$buildNumber = $target
|
||||
break
|
||||
}
|
||||
Write-Host "Manual build number $target > counter $($counter.Value) -- advancing counter"
|
||||
} else {
|
||||
$target = $counter.Value + 1
|
||||
}
|
||||
try {
|
||||
$headers = @{ 'x-ms-blob-type' = 'BlockBlob'; 'If-Match' = $counter.ETag }
|
||||
Invoke-RestMethod -Uri "$counterUrl`?$sasToken" -Method Put -Headers $headers -Body "$target" -ContentType 'text/plain' | Out-Null
|
||||
$buildNumber = $target
|
||||
Write-Host "Reserved build number ${target} (CAS write ok, attempt $attempt)"
|
||||
break
|
||||
} catch {
|
||||
$status = if ($_.Exception.Response) { [int]$_.Exception.Response.StatusCode } else { 0 }
|
||||
if ($status -eq 412) {
|
||||
Write-Host "CAS conflict on attempt $attempt (another build reserved concurrently) -- retrying"
|
||||
Start-Sleep -Seconds ($attempt * 2)
|
||||
} else {
|
||||
throw
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($null -eq $buildNumber) {
|
||||
Write-Error "Could not reserve a build number after 5 CAS attempts"
|
||||
}
|
||||
$counterUrl = "https://autopilotmonitoreu.blob.core.windows.net/agent/buildcounter-v$($env:LINE_MAJOR).txt"
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl $counterUrl -SasToken '${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}' -Override $override
|
||||
|
||||
$csproj = Get-Content $env:CSPROJ_PATH -Raw
|
||||
if ($csproj -match '<VersionPrefix>(\d+\.\d+)</VersionPrefix>') {
|
||||
@@ -228,8 +177,13 @@ jobs:
|
||||
Write-Warning "Could not parse bootstrap `$ScriptVersion from $env:BOOTSTRAP_SOURCE"
|
||||
}
|
||||
|
||||
# Enrich the per-line manifest emitted by MSBuild (version-v2.json, read above)
|
||||
# Enrich the per-line manifest emitted by MSBuild (version-v2.json, read above).
|
||||
# commit/buildUtc are the same two fields the backend and MCP manifests carry —
|
||||
# additive, and the bootstrap reads this file by field name, so older installers
|
||||
# ignore them. sha256/bootstrapVersion stay: those are the customer contract.
|
||||
$manifest | Add-Member -NotePropertyName "sha256" -NotePropertyValue $sha256Hash -Force
|
||||
$manifest | Add-Member -NotePropertyName "commit" -NotePropertyValue '${{ github.sha }}'.Substring(0, 7) -Force
|
||||
$manifest | Add-Member -NotePropertyName "buildUtc" -NotePropertyValue ([DateTime]::UtcNow.ToString('o')) -Force
|
||||
if ($bootstrapVersion) {
|
||||
$manifest | Add-Member -NotePropertyName "bootstrapVersion" -NotePropertyValue $bootstrapVersion -Force
|
||||
}
|
||||
@@ -277,7 +231,12 @@ jobs:
|
||||
param([string]$Local, [string]$BlobName, [string]$ContentType)
|
||||
$url = "$containerUrl/$BlobName`?$sasToken"
|
||||
$bytes = [System.IO.File]::ReadAllBytes($Local)
|
||||
$headers = @{ "x-ms-blob-type" = "BlockBlob"; "Content-Type" = $ContentType }
|
||||
# no-cache on EVERY blob: they all rotate in place (even the per-line ZIP is
|
||||
# overwritten each build), and the download alias fronts them with Front Door.
|
||||
# Route caching is disabled there, but a stale manifest paired with a fresh ZIP
|
||||
# (independent cache entries) would fail the bootstrap SHA check -- this header
|
||||
# keeps that class of bug impossible even if caching is ever re-enabled.
|
||||
$headers = @{ "x-ms-blob-type" = "BlockBlob"; "Content-Type" = $ContentType; "x-ms-blob-cache-control" = "no-cache" }
|
||||
if ($ContentType -eq "application/zip") {
|
||||
$md5 = [System.Security.Cryptography.MD5]::Create()
|
||||
$headers["Content-MD5"] = [Convert]::ToBase64String($md5.ComputeHash($bytes))
|
||||
@@ -352,13 +311,13 @@ jobs:
|
||||
subject-path: ${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: AutopilotMonitor-Agent-v${{ steps.version.outputs.version }}
|
||||
path: ${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
|
||||
- name: Upload version metadata
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: version-metadata
|
||||
path: ${{ env.MANIFEST_NAME }}
|
||||
|
||||
+18
-14
@@ -1,8 +1,12 @@
|
||||
name: CI
|
||||
|
||||
# Runs the full test suite on every pull request and every push to main.
|
||||
# The deploy workflows build but do not test — this is the gate that decides
|
||||
# whether a change is allowed to be correct.
|
||||
# Tests, typechecks and drift-checks every push to main and every pull request:
|
||||
# the .NET suites, the web and MCP suites, and a guard that the generated rule
|
||||
# files still match rules/guardrails.json.
|
||||
#
|
||||
# The deploy workflows build but never test, so this is the only place the suites
|
||||
# actually run. It does not gate anything — main is deliberately unprotected, so a
|
||||
# red run reports rather than blocks. Read the result and follow up.
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -24,9 +28,9 @@ jobs:
|
||||
# windows-latest is required: the agent test projects target net48.
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-dotnet@v5
|
||||
- uses: actions/setup-dotnet@v6
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
@@ -55,11 +59,11 @@ jobs:
|
||||
run:
|
||||
working-directory: src/Web/autopilot-monitor-web
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-node@v5
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: src/Web/autopilot-monitor-web/package-lock.json
|
||||
|
||||
@@ -78,11 +82,11 @@ jobs:
|
||||
run:
|
||||
working-directory: src/McpServer/autopilot-monitor-mcp
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-node@v5
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: src/McpServer/autopilot-monitor-mcp/package-lock.json
|
||||
|
||||
@@ -98,11 +102,11 @@ jobs:
|
||||
name: Generated guardrails match rules/guardrails.json
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-node@v5
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
|
||||
- name: Regenerate
|
||||
run: node rules/scripts/combine.js
|
||||
|
||||
@@ -1,12 +1,26 @@
|
||||
name: CodeQL
|
||||
|
||||
# Documentation-only changes cannot affect what CodeQL analyzes (C# and
|
||||
# TypeScript), so they are filtered out rather than spending ~10 minutes to
|
||||
# re-report an unchanged result. A commit that touches docs AND code still runs:
|
||||
# GitHub skips only when EVERY changed path matches the filter.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- '**/*.md'
|
||||
- 'LICENSE'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- '**/*.md'
|
||||
- 'LICENSE'
|
||||
schedule:
|
||||
# Weekly, so advisories published after a merge still surface.
|
||||
# Weekly, so advisories published after a merge still surface. Deliberately
|
||||
# unfiltered — it re-analyzes the whole tree regardless of what changed, which
|
||||
# is also the backstop if the filters above ever skip too much.
|
||||
- cron: '17 3 * * 1'
|
||||
|
||||
permissions:
|
||||
@@ -35,24 +49,25 @@ jobs:
|
||||
build-mode: none
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-dotnet@v5
|
||||
- uses: actions/setup-dotnet@v6
|
||||
if: matrix.language == 'csharp'
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Build
|
||||
if: matrix.build-mode == 'manual'
|
||||
run: dotnet build AutopilotMonitor.sln --configuration Release
|
||||
|
||||
- name: Analyze
|
||||
uses: github/codeql-action/analyze@v3
|
||||
uses: github/codeql-action/analyze@v4
|
||||
with:
|
||||
category: /language:${{ matrix.language }}
|
||||
|
||||
@@ -22,11 +22,11 @@ jobs:
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-node@v5
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
|
||||
- name: Install ajv-cli
|
||||
run: npm install -g ajv-cli ajv-formats
|
||||
|
||||
@@ -7,13 +7,19 @@ permissions:
|
||||
id-token: write # OIDC federated login (secret-less)
|
||||
contents: read
|
||||
|
||||
env:
|
||||
FUNCTION_APP: autopilotmonitor-api-eu
|
||||
HEALTH_URL: https://autopilotmonitor-api-eu.azurewebsites.net/api/health
|
||||
VERSIONS_CONTAINER: https://autopilotmonitoreu.blob.core.windows.net/versions
|
||||
CSPROJ_PATH: src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build and Deploy Backend
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@v3
|
||||
@@ -23,21 +29,95 @@ jobs:
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Setup .NET 10
|
||||
uses: actions/setup-dotnet@v5
|
||||
uses: actions/setup-dotnet@v6
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
# Same counter mechanism as the agent line: reserve-before-build with ETag-CAS
|
||||
# against a per-component counter blob. BuildTimestampUtc is pinned here rather
|
||||
# than left to MSBuild's default so the published manifest and the value baked
|
||||
# into the assembly (reported by /api/health) are the same instant.
|
||||
- name: Reserve build number (counter blob CAS)
|
||||
id: version
|
||||
shell: pwsh
|
||||
run: |
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "${{ env.VERSIONS_CONTAINER }}/backend.txt" -SasToken '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'
|
||||
|
||||
$csproj = Get-Content ${{ env.CSPROJ_PATH }} -Raw
|
||||
if ($csproj -notmatch '<VersionPrefix>(\d+\.\d+)</VersionPrefix>') {
|
||||
throw "VersionPrefix not found in ${{ env.CSPROJ_PATH }}"
|
||||
}
|
||||
$version = "$($Matches[1]).$buildNumber"
|
||||
$buildUtc = [DateTime]::UtcNow.ToString('o')
|
||||
Write-Host "Backend version: $version (built $buildUtc)"
|
||||
|
||||
"build_number=$buildNumber" >> $env:GITHUB_OUTPUT
|
||||
"version=$version" >> $env:GITHUB_OUTPUT
|
||||
"build_utc=$buildUtc" >> $env:GITHUB_OUTPUT
|
||||
|
||||
- name: Restore dependencies
|
||||
run: dotnet restore src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj
|
||||
run: dotnet restore ${{ env.CSPROJ_PATH }}
|
||||
|
||||
- name: Build
|
||||
run: dotnet build src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj --configuration Release --no-restore -p:SourceRevisionId=${{ github.sha }}
|
||||
run: dotnet build ${{ env.CSPROJ_PATH }} --configuration Release --no-restore -p:SourceRevisionId=${{ github.sha }} -p:BuildNumber=${{ steps.version.outputs.build_number }} -p:BuildTimestampUtc=${{ steps.version.outputs.build_utc }}
|
||||
|
||||
- name: Publish
|
||||
run: dotnet publish src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj --configuration Release --output ./publish --no-build -p:SourceRevisionId=${{ github.sha }}
|
||||
run: dotnet publish ${{ env.CSPROJ_PATH }} --configuration Release --output ./publish --no-build -p:SourceRevisionId=${{ github.sha }} -p:BuildNumber=${{ steps.version.outputs.build_number }} -p:BuildTimestampUtc=${{ steps.version.outputs.build_utc }}
|
||||
|
||||
- name: Deploy to Azure Functions
|
||||
uses: Azure/functions-action@v1
|
||||
with:
|
||||
app-name: autopilotmonitor-api-eu
|
||||
app-name: ${{ env.FUNCTION_APP }}
|
||||
package: ./publish
|
||||
|
||||
# A green deploy step only means the package was accepted. This gate makes the
|
||||
# workflow assert what is actually being served, so the manifest published below
|
||||
# can be trusted as "this is live" rather than "this was uploaded".
|
||||
# /api/health is IP rate-limited at 30/min; one poll per 10s stays well under.
|
||||
- name: Verify deployed build is live
|
||||
shell: pwsh
|
||||
run: |
|
||||
$expectedCommit = '${{ github.sha }}'.Substring(0, 7)
|
||||
$expectedVersion = '${{ steps.version.outputs.version }}'
|
||||
$deadline = [DateTime]::UtcNow.AddMinutes(3)
|
||||
$attempt = 0
|
||||
|
||||
while ([DateTime]::UtcNow -lt $deadline) {
|
||||
$attempt++
|
||||
try {
|
||||
$health = Invoke-RestMethod -Uri '${{ env.HEALTH_URL }}' -Method Get -TimeoutSec 30
|
||||
if ($health.commitHash -eq $expectedCommit -and $health.version -eq $expectedVersion) {
|
||||
Write-Host "Live after $attempt attempt(s): $($health.version) @ $($health.commitHash), built $($health.buildUtc)"
|
||||
exit 0
|
||||
}
|
||||
Write-Host "Attempt ${attempt}: serving $($health.version) @ $($health.commitHash) -- waiting for $expectedVersion @ $expectedCommit"
|
||||
} catch {
|
||||
Write-Host "Attempt ${attempt}: /api/health not answering yet -- $($_.Exception.Message)"
|
||||
}
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
|
||||
throw "Backend did not report $expectedVersion @ $expectedCommit within 3 minutes -- deployment not verified, version manifest NOT published"
|
||||
|
||||
# The manifest is the cross-component record of what is deployed where. It is
|
||||
# written last and only on a verified deploy, so a difference between the
|
||||
# manifest and /api/health means the running app drifted after the fact
|
||||
# (rollback, stale instance) rather than a half-finished pipeline.
|
||||
- name: Publish version manifest
|
||||
shell: pwsh
|
||||
run: |
|
||||
$manifest = [ordered]@{
|
||||
component = 'backend'
|
||||
version = '${{ steps.version.outputs.version }}'
|
||||
commit = '${{ github.sha }}'.Substring(0, 7)
|
||||
buildUtc = '${{ steps.version.outputs.build_utc }}'
|
||||
deployedUtc = [DateTime]::UtcNow.ToString('o')
|
||||
runId = '${{ github.run_id }}'
|
||||
} | ConvertTo-Json
|
||||
|
||||
$sas = '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'.TrimStart('?')
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/backend.json?$sas"
|
||||
# no-cache: rotating pointer, must never be served stale by any fronting cache.
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' } -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Write-Host "Published versions/backend.json"
|
||||
Write-Host $manifest
|
||||
|
||||
@@ -12,6 +12,9 @@ env:
|
||||
CONTAINER_APP_NAME: autopilotmonitor-mcp
|
||||
RESOURCE_GROUP: ${{ secrets.AZURE_MCP_RESOURCE_GROUP }}
|
||||
IMAGE_TAG: ${{ github.sha }}
|
||||
HEALTH_URL: https://mcp.autopilotmonitor.com/health
|
||||
VERSIONS_CONTAINER: https://autopilotmonitoreu.blob.core.windows.net/versions
|
||||
PACKAGE_JSON: src/McpServer/autopilot-monitor-mcp/package.json
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
@@ -19,7 +22,7 @@ jobs:
|
||||
name: Build and Deploy MCP Server
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
# The published customer documentation lives in its own PUBLIC repo and is
|
||||
# baked into the image as the search_docs corpus (see the Dockerfile). No
|
||||
@@ -28,7 +31,7 @@ jobs:
|
||||
# RUNNING SERVER; this workflow must be re-run. /health reports the SHA
|
||||
# captured below so that staleness is checkable.
|
||||
- name: Check out documentation bundle
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
repository: autopilotmonitor/autopilotmonitor-docs
|
||||
path: mcp-docs
|
||||
@@ -44,6 +47,30 @@ jobs:
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# Same counter mechanism as agent and backend: reserve-before-build with
|
||||
# ETag-CAS against the component's own counter blob. major.minor stays curated
|
||||
# in package.json; only the build part is minted here.
|
||||
- name: Reserve build number (counter blob CAS)
|
||||
id: version
|
||||
shell: pwsh
|
||||
run: |
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "${{ env.VERSIONS_CONTAINER }}/mcp.txt" -SasToken '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'
|
||||
|
||||
$pkgVersion = (Get-Content ${{ env.PACKAGE_JSON }} -Raw | ConvertFrom-Json).version
|
||||
$prefix = ($pkgVersion -split '\.')[0..1] -join '.'
|
||||
$version = "$prefix.$buildNumber"
|
||||
$buildUtc = [DateTime]::UtcNow.ToString('o')
|
||||
Write-Host "MCP version: $version (built $buildUtc)"
|
||||
|
||||
# 7 chars, matching what the backend's BackendBuildInfo reports — one commit
|
||||
# shape across all components so the values can be compared as-is.
|
||||
$shortSha = '${{ github.sha }}'.Substring(0, 7)
|
||||
|
||||
"build_number=$buildNumber" >> $env:GITHUB_OUTPUT
|
||||
"version=$version" >> $env:GITHUB_OUTPUT
|
||||
"build_utc=$buildUtc" >> $env:GITHUB_OUTPUT
|
||||
"short_sha=$shortSha" >> $env:GITHUB_OUTPUT
|
||||
|
||||
- name: Login to ACR
|
||||
run: az acr login --name ${{ env.ACR_NAME }}
|
||||
|
||||
@@ -52,6 +79,9 @@ jobs:
|
||||
docker build \
|
||||
-f src/McpServer/autopilot-monitor-mcp/Dockerfile \
|
||||
--build-arg DOCS_COMMIT=${{ steps.docs.outputs.sha }} \
|
||||
--build-arg BUILD_NUMBER=${{ steps.version.outputs.build_number }} \
|
||||
--build-arg BUILD_COMMIT=${{ steps.version.outputs.short_sha }} \
|
||||
--build-arg BUILD_UTC=${{ steps.version.outputs.build_utc }} \
|
||||
-t ${{ env.ACR_NAME }}.azurecr.io/${{ env.CONTAINER_APP_NAME }}:${{ env.IMAGE_TAG }} \
|
||||
-t ${{ env.ACR_NAME }}.azurecr.io/${{ env.CONTAINER_APP_NAME }}:latest \
|
||||
.
|
||||
@@ -63,3 +93,55 @@ jobs:
|
||||
--name ${{ env.CONTAINER_APP_NAME }} \
|
||||
--resource-group ${{ env.RESOURCE_GROUP }} \
|
||||
--image ${{ env.ACR_NAME }}.azurecr.io/${{ env.CONTAINER_APP_NAME }}:${{ env.IMAGE_TAG }}
|
||||
|
||||
# `az containerapp update` returns once the revision is provisioned, not once
|
||||
# it serves traffic — and with minReplicas=0 the first request also pays a cold
|
||||
# start. Polling /health until it reports THIS build is what makes the manifest
|
||||
# below mean "live"; a full 5 minutes because the cold start alone is ~25s.
|
||||
- name: Verify deployed build is live
|
||||
shell: pwsh
|
||||
run: |
|
||||
$expectedCommit = '${{ steps.version.outputs.short_sha }}'
|
||||
$expectedVersion = '${{ steps.version.outputs.version }}'
|
||||
$deadline = [DateTime]::UtcNow.AddMinutes(5)
|
||||
$attempt = 0
|
||||
|
||||
while ([DateTime]::UtcNow -lt $deadline) {
|
||||
$attempt++
|
||||
try {
|
||||
$health = Invoke-RestMethod -Uri '${{ env.HEALTH_URL }}' -Method Get -TimeoutSec 60
|
||||
if ($health.commit -eq $expectedCommit -and $health.version -eq $expectedVersion) {
|
||||
Write-Host "Live after $attempt attempt(s): $($health.version) @ $($health.commit), docs $($health.docs.commit) ($($health.docs.chunks) chunks)"
|
||||
exit 0
|
||||
}
|
||||
Write-Host "Attempt ${attempt}: serving $($health.version) @ $($health.commit) -- waiting for $expectedVersion @ $expectedCommit"
|
||||
} catch {
|
||||
Write-Host "Attempt ${attempt}: /health not answering yet -- $($_.Exception.Message)"
|
||||
}
|
||||
Start-Sleep -Seconds 15
|
||||
}
|
||||
|
||||
throw "MCP did not report $expectedVersion @ $expectedCommit within 5 minutes -- deployment not verified, version manifest NOT published"
|
||||
|
||||
# Cross-component record of what is deployed where; written last and only on a
|
||||
# verified deploy. docsCommit rides along because for this component "which
|
||||
# build" and "which documentation" are two separate questions.
|
||||
- name: Publish version manifest
|
||||
shell: pwsh
|
||||
run: |
|
||||
$manifest = [ordered]@{
|
||||
component = 'mcp'
|
||||
version = '${{ steps.version.outputs.version }}'
|
||||
commit = '${{ steps.version.outputs.short_sha }}'
|
||||
buildUtc = '${{ steps.version.outputs.build_utc }}'
|
||||
deployedUtc = [DateTime]::UtcNow.ToString('o')
|
||||
runId = '${{ github.run_id }}'
|
||||
docsCommit = '${{ steps.docs.outputs.sha }}'.Substring(0, 7)
|
||||
} | ConvertTo-Json
|
||||
|
||||
$sas = '${{ secrets.AZURE_BLOB_VERSIONS_SAS_TOKEN }}'.TrimStart('?')
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/mcp.json?$sas"
|
||||
# no-cache: rotating pointer, must never be served stale by any fronting cache.
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' } -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Write-Host "Published versions/mcp.json"
|
||||
Write-Host $manifest
|
||||
|
||||
@@ -21,13 +21,21 @@ jobs:
|
||||
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.action != 'closed')
|
||||
runs-on: ubuntu-latest
|
||||
name: Build and Deploy Job
|
||||
# Dependabot PRs run without repo secrets, so the deploy token is absent.
|
||||
# Surface it as an env var so the deploy step can gate on it — secrets
|
||||
# cannot be referenced directly in an `if:` expression.
|
||||
env:
|
||||
DEPLOY_TOKEN: ${{ secrets.AZURE_SWA_DEPLOY_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
lfs: false
|
||||
fetch-depth: 0 # Full history needed for git-based sitemap lastmod dates
|
||||
- name: Build And Deploy
|
||||
id: builddeploy
|
||||
# Skip (green) instead of failing when the token is missing (Dependabot
|
||||
# PRs); real deploys on main and internal PRs still have it.
|
||||
if: env.DEPLOY_TOKEN != ''
|
||||
uses: Azure/static-web-apps-deploy@v1
|
||||
env:
|
||||
NEXT_PUBLIC_ENTRA_CLIENT_ID: ${{ vars.NEXT_PUBLIC_ENTRA_CLIENT_ID }}
|
||||
@@ -51,9 +59,12 @@ jobs:
|
||||
if: github.event_name == 'pull_request' && github.event.action == 'closed'
|
||||
runs-on: ubuntu-latest
|
||||
name: Close Pull Request Job
|
||||
env:
|
||||
DEPLOY_TOKEN: ${{ secrets.AZURE_SWA_DEPLOY_TOKEN }}
|
||||
steps:
|
||||
- name: Close Pull Request
|
||||
id: closepullrequest
|
||||
if: env.DEPLOY_TOKEN != ''
|
||||
uses: Azure/static-web-apps-deploy@v1
|
||||
with:
|
||||
azure_static_web_apps_api_token: ${{ secrets.AZURE_SWA_DEPLOY_TOKEN }}
|
||||
|
||||
@@ -22,26 +22,21 @@ attestation on top.
|
||||
|
||||
# Schema
|
||||
|
||||
- **Counter blob**: `https://autopilotmonitor.blob.core.windows.net/agent/buildcounter-v{LINE}.txt`
|
||||
(per release line, analogous to `version-v{LINE}.json`). Content: the **last used**
|
||||
build number as plain text. Public-read like the rest of the container (the number
|
||||
is visible in `version-v{LINE}.json` anyway); writes require the container SAS.
|
||||
- **Protocol — reserve before build, ETag-CAS**:
|
||||
1. Anonymous `GET` the blob, capture value `N` and the `ETag`.
|
||||
2. `PUT N+1` with `If-Match: <etag>` (quoted form — pwsh strips the quotes on read,
|
||||
they must be re-added or the request is rejected client-side).
|
||||
3. `412 Precondition Failed` → another build reserved concurrently → re-read, retry
|
||||
(5 attempts, linear backoff).
|
||||
4. Build with `BUILD_NUMBER=N+1` in the environment. The csproj honors
|
||||
`BUILD_NUMBER` and then skips its `PersistBuildCounter` target, so the repo file
|
||||
is not touched.
|
||||
- **Failed builds burn a number.** That is intentional: uniqueness matters, density
|
||||
does not. Reserve-before-build is what makes concurrent minting impossible.
|
||||
The reserve-before-build CAS protocol itself is shared with the backend and the MCP
|
||||
server and is documented once in [Version Contract](../versioning.md); the single
|
||||
implementation is `.github/scripts/Request-BuildNumber.ps1`. Agent-specific:
|
||||
|
||||
- **Counter blob**: `https://autopilotmonitoreu.blob.core.windows.net/agent/buildcounter-v{LINE}.txt`
|
||||
— one per release line, analogous to `version-v{LINE}.json`, and kept in the `agent`
|
||||
container next to the artifacts it numbers rather than in `versions/`. Content: the
|
||||
**last used** build number as plain text. Public-read like the rest of the container
|
||||
(the number is visible in `version-v{LINE}.json` anyway); writes require the SAS.
|
||||
- The reserved number reaches the build as `BUILD_NUMBER` in the environment. The
|
||||
csproj honors it and then skips its `PersistBuildCounter` target, so the repo file
|
||||
is not touched.
|
||||
- **CI manual override** (`build_number` input): numbers `<=` counter are re-builds
|
||||
and leave the blob unchanged; numbers `>` counter advance it (monotonic guard).
|
||||
- **Seeding** (one-time per line): `PUT <last used number>` with `If-None-Match: *`
|
||||
so an existing counter can never be clobbered. V2 was seeded with `1356` on
|
||||
2026-07-17.
|
||||
- V2 was seeded with `1356` on 2026-07-17.
|
||||
|
||||
# Consequences
|
||||
|
||||
@@ -62,7 +57,9 @@ attestation on top.
|
||||
|
||||
# Citations
|
||||
|
||||
- [Version Contract](../versioning.md) — the shared CAS protocol and manifest format
|
||||
- `.github/workflows/build-agent.yml` — "Reserve build number (counter blob CAS)" step
|
||||
- `scripts/Deployment/build.ps1` (gitignored, contains SAS) — `Request-BuildNumber`
|
||||
- `.github/scripts/Request-BuildNumber.ps1` — the shared implementation
|
||||
- `scripts/Deployment/build.ps1` (gitignored, contains SAS) — local build path
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2/AutopilotMonitor.Agent.V2.csproj` —
|
||||
`BUILD_NUMBER` property + `PersistBuildCounter` target
|
||||
|
||||
@@ -137,8 +137,17 @@ false` and an arm holds), `EspExiting`, `ImeUserSessionCompleted`,
|
||||
advisory backstop. All routes converge on `CompleteThroughFinalizingOrDefer`:
|
||||
|
||||
1. **Completion gates** (currently only the RealmJoin AND-gate): if RealmJoin was
|
||||
detected and is neither resolved nor timed-out (hard 60-min timeout), completion is
|
||||
*deferred* — the session stays in stage and emits `completion_waiting`.
|
||||
detected and has no terminal outcome yet, completion is *deferred* — the session
|
||||
stays in stage and emits `completion_waiting`. Three terminals open the gate:
|
||||
`Resolved` (DeploymentPhase 110), `FirstDeploymentIncomplete` (phase left the
|
||||
first-deployment window 100/101 for 200/210 without 110 — the RJ ESP was aborted,
|
||||
typically by an interactive logon reclassifying the run as secondary-user; emits
|
||||
the `realmjoin_first_deployment_incomplete` Warning, session 224b2087), and
|
||||
`Timeout` (hard 60-min deadline). The abort rule reads the *persisted*
|
||||
`RealmJoinFacts.LastDeploymentPhase` (updated by the typed `RealmJoinPhaseChanged`
|
||||
signal), so it survives agent restarts — the `RealmJoinDetected` replay path
|
||||
evaluates it too. A first observation of 210 does **not** release (RJ may have
|
||||
completed before agent boot and deploy afterwards).
|
||||
2. `Finalizing` stage + ~5 s `FinalizingGrace` deadline.
|
||||
3. Grace fires → `Completed`, `EnrollmentComplete` outcome, `enrollment_complete`
|
||||
timeline event with full audit payload.
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
---
|
||||
type: Concept
|
||||
title: ESP Policy-Provider Stall Detection — the no-timeout Setup/Apps wait and the co-management Sidecar gap
|
||||
description: Why the ESP's Apps wait is keyed to the Intune-registered "Sidecar" provider by name — a foreign provider (ConfigMgr) with TrackingPoliciesCreated=1 does not satisfy it — plus the classic never-completes stall, and how the always-on two-arm esp_policy_provider_stalled tripwire (reason=provider_incomplete | sidecar_provider_missing, 15-min wall-clock dwell, StartupEventGate cross-restart dedup) surfaces both.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspPolicyProviderProbe.cs
|
||||
tags:
|
||||
- agent
|
||||
- esp
|
||||
- enrollment-status-tracking
|
||||
- co-management
|
||||
- stall-detection
|
||||
timestamp: 2026-07-24T00:00:00+02:00
|
||||
---
|
||||
|
||||
# ESP Policy-Provider Stall Detection
|
||||
|
||||
## Schema
|
||||
|
||||
The Enrollment Status Page decides when its "Apps" category may progress from
|
||||
"Identifying" using the **EnrollmentStatusTracking CSP**
|
||||
(<https://learn.microsoft.com/en-us/windows/client-management/mdm/enrollmentstatustracking-csp>),
|
||||
mirrored in the registry under
|
||||
`HKLM\SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking\{Device|<UserSID>}\…`.
|
||||
Two contracts matter:
|
||||
|
||||
1. **`Setup/Apps/PolicyProviders/<Name>`** — per the CSP doc: *"Existence of this node
|
||||
indicates to the ESP that it shouldn't show the tracking status message until the
|
||||
TrackingPoliciesCreated node has been set to true."* This wait has **no timeout**. A
|
||||
registered provider that never sets `TrackingPoliciesCreated=1` parks the ESP at
|
||||
"Apps (Identifying)" indefinitely — observed for days in the field. Exists in device
|
||||
scope (`Device`) and user scope (`S-<SID>`, appears at sign-in).
|
||||
2. **`DevicePreparation/PolicyProviders/<Name>/InstallationState`** — must reach 2
|
||||
(NotRequired) or 3 (Completed); 1=NotInstalled, 4=Error. Device scope only. The ESP
|
||||
applies a default 15-minute timeout here, so these stalls self-surface as ESP errors —
|
||||
the detector still records them for the diagnostic trail.
|
||||
|
||||
Known provider names: **`Sidecar` is the Intune Management Extension** (the CSP doc names
|
||||
"Intune's agents, such as SideCar"); **`ConfigMgr`** is the Configuration Manager client,
|
||||
registered during Autopilot-into-co-management — the troubleshooting section of
|
||||
<https://learn.microsoft.com/en-us/intune/configmgr/comanage/autopilot-enrollment> queries
|
||||
exactly these registry paths. The same document states that ESP policy providers **are not
|
||||
aware of each other** and that **pre-provisioning is unsupported** with Autopilot into
|
||||
co-management.
|
||||
|
||||
## The field case (issue #106)
|
||||
|
||||
A tenant pre-provisions devices and installs the ConfigMgr client as a blocking app.
|
||||
Occasionally the user-phase ESP reaches "Apps (Identifying)" and never leaves it — for
|
||||
hours to days. On those devices `Device\Setup\Apps\PolicyProviders` contains **only a
|
||||
`ConfigMgr` key (no `Sidecar`)** — and, per the customer's follow-up, **`TrackingPoliciesCreated=1`
|
||||
IS set under it**. The provider is "complete" by the CSP value contract, yet the ESP
|
||||
still waits. Manually renaming the key to `Sidecar` — which creates no tracking entries,
|
||||
only satisfies the name — unblocks the ESP instantly.
|
||||
|
||||
The naive CSP-doc reading ("any provider with TrackingPoliciesCreated=1 satisfies the
|
||||
wait") is therefore wrong for this case. What the evidence supports instead: provider
|
||||
nodes are registered **by the Intune service at enrollment**, and its ESP-policy WAP
|
||||
payload always registers `Sidecar` (ConfigMgr joins only via a co-management settings
|
||||
policy — see the payload documented by Niehaus). The ESP waits on the provider it
|
||||
expects **by name**; a foreign provider occupying the slot does not satisfy it,
|
||||
regardless of its completion values. On the affected devices the CM client (installed
|
||||
as a plain Win32 app, no co-management settings policy) evidently displaced the
|
||||
Sidecar node — consistent with Microsoft's warning that providers "aren't currently
|
||||
aware of others" and that pre-provisioning is unsupported with co-management. The exact
|
||||
CloudExperienceHost mechanism is not publicly documented; the registry signature —
|
||||
setupApps providers present, `Sidecar` absent — is deterministic either way.
|
||||
|
||||
## Detection
|
||||
|
||||
`EspPolicyProviderProbe` (fail-soft, static, `ScopedOverride` test seam — structural clone
|
||||
of `EspTrackingInfoProbe`) enumerates all providers in both kinds and scopes.
|
||||
`EspPolicyProviderStallDetector` evaluates a pure transition function on a 60-second tick
|
||||
from the always-on `EspPolicyProviderStallHost` — **two arms**, same 15-minute wall-clock
|
||||
dwell, one shared event type discriminated by `reason`:
|
||||
|
||||
* **Arm 1, `reason=provider_incomplete`** — a provider **continuously incomplete for
|
||||
≥ 15 minutes**. The dwell clock starts at first incomplete observation and resets on
|
||||
completion or disappearance. Missing values parse as incomplete
|
||||
(fail-toward-observation). Catches e.g. Sidecar itself never reporting.
|
||||
* **Arm 2, `reason=sidecar_provider_missing`** — **≥ 1 `Setup\Apps` provider registered
|
||||
but none named `Sidecar`**, continuously for ≥ 15 minutes. Deliberately ignores
|
||||
provider completeness (the field case had `TrackingPoliciesCreated=1`). Sidecar
|
||||
registering later — the legitimate startup ordering — clears the condition and resets
|
||||
the clock. DevicePreparation-only snapshots don't activate it.
|
||||
* When both arms cross in the same round, ONE event is emitted and
|
||||
`sidecar_provider_missing` wins as `reason` (it explains the hang regardless of the
|
||||
foreign provider's state).
|
||||
* Missing root key / empty provider list = normal early enrollment: clears all dwell
|
||||
state (both arms), never fires.
|
||||
* One `esp_policy_provider_stalled` **Warning** per newly stalled condition set
|
||||
(`ImmediateUpload`), one-shot per condition in-process, cross-restart deduped via a
|
||||
`StartupEventGate` fingerprint over the sorted currently-active fired conditions
|
||||
(provider keys + a `sidecarMissing` sentinel) — a restart into the same stall stays
|
||||
silent, a new condition joining re-reports.
|
||||
* Payload: `reason`, `providers[]` (full table: name, scope, kind, raw values, complete
|
||||
flag), `stalledProviders[]` with `stalledForMinutes`, `sidecarMissingForMinutes`
|
||||
(arm 2 only), `sidecarRegistered`, `dwellMinutes`.
|
||||
* **Purely observational**: emitted via `InformationalEventPost` (dispatch-guard exempt
|
||||
pass-through) — never touches decision state. No config toggle, no ConfigVersion bump
|
||||
(precedent: `disk_space_low`).
|
||||
|
||||
Deliberately **not** built on `StallProbeCollector` (its idle clock resets on any session
|
||||
activity — a provider stall coexists with a chatty session) and not on the periodic
|
||||
collectors (idle-stopped after 15 minutes — exactly the dormant window the dwell must
|
||||
observe).
|
||||
|
||||
## Guardrail companion
|
||||
|
||||
`rules/guardrails.json` allow-lists `SOFTWARE\Microsoft\Windows\Autopilot` (category
|
||||
"Autopilot / OOBE / Setup") so tenants can build their own gather rules on the subtree —
|
||||
e.g. a `not_exists` registry rule on the `Sidecar` provider key. The subtree carries
|
||||
deployment state only (provider registrations, tracking lists, app IDs, per-user SIDs);
|
||||
sensitivity is at or below existing entries. Enforcement is agent-side (embedded
|
||||
resource), so both the detector and the guardrail change ship with the agent release.
|
||||
|
||||
## Examples
|
||||
|
||||
Customer-case event (abbreviated) — note the provider is "complete" by CSP values, the
|
||||
hang is carried entirely by `reason=sidecar_provider_missing`:
|
||||
|
||||
```json
|
||||
{
|
||||
"eventType": "esp_policy_provider_stalled",
|
||||
"severity": "Warning",
|
||||
"data": {
|
||||
"reason": "sidecar_provider_missing",
|
||||
"dwellMinutes": 15,
|
||||
"sidecarRegistered": false,
|
||||
"sidecarMissingForMinutes": 15.0,
|
||||
"providers": [
|
||||
{ "name": "ConfigMgr", "scope": "device", "kind": "setupApps",
|
||||
"trackingPoliciesCreated": 1, "installationState": null, "complete": true }
|
||||
],
|
||||
"stalledProviders": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`reason=sidecar_provider_missing` is the co-management signature;
|
||||
`reason=provider_incomplete` with `sidecarRegistered=true` covers a stalled Sidecar or
|
||||
third-party provider — the generalized contract violation.
|
||||
|
||||
# Citations
|
||||
|
||||
* EnrollmentStatusTracking CSP — <https://learn.microsoft.com/en-us/windows/client-management/mdm/enrollmentstatustracking-csp>
|
||||
* Windows Autopilot into co-management (requirements, provider-conflict warning, troubleshooting registry paths) — <https://learn.microsoft.com/en-us/intune/configmgr/comanage/autopilot-enrollment>
|
||||
* Troubleshoot the ESP (models app tracking exclusively via the `Sidecar` subkey) — <https://learn.microsoft.com/en-us/troubleshoot/mem/intune/device-enrollment/understand-troubleshoot-esp>
|
||||
* Niehaus: the Intune ESP-policy WAP payload registering the provider nodes (`Sidecar` always, `ConfigMgr` only with a co-management settings policy) — <https://oofhours.com/2023/08/25/run-an-sccm-task-sequence-during-autopilot/>
|
||||
* Niehaus: ESP tracks arbitrarily named providers when they are properly registered — <https://oofhours.com/2023/09/18/track-anything-using-esp/>
|
||||
* Agent code: `EspPolicyProviderProbe.cs`, `EspPolicyProviderStallDetector.cs`, `EspPolicyProviderStallHost.cs` (SystemSignals / Orchestration.Hosts)
|
||||
* Related: [MDM Reboot Coalescing](mdm-reboot-coalescing.md), [Gather-Rule Guardrails](../rules/gather-rule-guardrails.md)
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Secure Boot CA-2023 Detection — firmware evidence vs. Windows servicing status
|
||||
description: Why ANALYZE-SEC-001 v3 reads the UEFI db/KEK variables directly (GetFirmwareEnvironmentVariableExW + SeSystemEnvironmentPrivilege) instead of trusting the SecureBoot\Servicing registry key, and how the one-sided uefiCa2023FirmwareConfirmed marker plus a not_exists precondition give absence-tolerant rule suppression across old and new agents.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/UefiSecureBootCertReader.cs
|
||||
tags:
|
||||
- agent
|
||||
- secureboot
|
||||
- ca2023
|
||||
- analyze-rules
|
||||
- backward-compatibility
|
||||
timestamp: 2026-07-23T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Secure Boot CA-2023 Detection
|
||||
|
||||
## Schema
|
||||
|
||||
Two independent evidence sources describe the Windows UEFI CA 2023 rollout state:
|
||||
|
||||
1. **Servicing registry** (`HKLM\SYSTEM\CurrentControlSet\Control\SecureBoot\Servicing\UEFICA2023Status`)
|
||||
— reflects whether the *Windows Update* rollout ran and self-verified. Lags reality:
|
||||
devices whose firmware received the certificate another way (OEM factory image, WinCS
|
||||
CLI, manual deployment) or where WU simply has not run yet show `unknown`/`notfound`.
|
||||
2. **Firmware itself** — the UEFI variables `db` (signature database, GUID
|
||||
`{d719b2cb-3d3a-4596-a3bc-dad00e67656f}`) and `KEK` (GUID
|
||||
`{8be4df61-93ca-11d2-aa0d-00e098032b8c}`). Authoritative. Certificate subject CNs are
|
||||
ASCII bytes inside the DER signature lists, so a byte-substring search for
|
||||
`Windows UEFI CA 2023` / `Microsoft Corporation KEK 2K CA 2023` decides presence —
|
||||
the same check `Get-SecureBootUEFI` enables from PowerShell.
|
||||
|
||||
`UefiSecureBootCertReader` (agent, `Monitoring/Interop`) reads both variables natively via
|
||||
`GetFirmwareEnvironmentVariableExW` — no PowerShell runspace during OOBE. The call needs
|
||||
`SeSystemEnvironmentPrivilege`: SYSTEM holds it default-disabled, so the reader enables it
|
||||
on the process token (`AdjustTokenPrivileges`) and restores the previous state afterwards.
|
||||
Fail-soft contract like `OobeStateReader`: never throws; failures map to status strings
|
||||
(`not_uefi` = legacy BIOS via `ERROR_INVALID_FUNCTION`, `variable_not_found`,
|
||||
`privilege_denied`, `error_<code>`).
|
||||
|
||||
Fields land on the existing `secureboot_status` event (once per agent start,
|
||||
restart-deduped by the StartupEventGate): `uefiFirmwareReadStatus`, per-cert booleans
|
||||
(`uefiDbHasWindowsUefiCa2023`, `uefiDbHasWindowsProductionPca2011`,
|
||||
`uefiKekHasMicrosoftKek2kCa2023`, …) — emitted only when the respective read succeeded —
|
||||
plus the marker below.
|
||||
|
||||
## The one-sided marker (absence-tolerant suppression)
|
||||
|
||||
Rule-engine constraints (pinned by `RuleEnginePreconditionTests`):
|
||||
|
||||
* **Conditions** never match on an absent field (null guard) — new fields must stay
|
||||
`required: false` or the rule dies for every old agent.
|
||||
* **Preconditions**: `not_exists` passes while the field is absent/empty and fails as soon
|
||||
as the field is present with *any* value; comparison operators fail closed on absence.
|
||||
|
||||
A two-sided boolean therefore cannot drive suppression: `not_exists` on
|
||||
`uefiDbHasWindowsUefiCa2023` would also suppress the rule when the field is present with
|
||||
`false`. Hence the **one-sided marker** `uefiCa2023FirmwareConfirmed = true`, emitted
|
||||
ONLY when the db read succeeded AND the 2023 certificate was found — never as `false`.
|
||||
ANALYZE-SEC-001 v3 adds a `not_exists` precondition on it:
|
||||
|
||||
| Device state | Marker | Rule |
|
||||
|---|---|---|
|
||||
| Old agent (no firmware fields) | absent | evaluates exactly like v2 |
|
||||
| New agent, cert missing or firmware unreadable | absent | evaluates; optional conditions `uefiDbHasWindowsUefiCa2023=false` (+20) / `uefiKekHasMicrosoftKek2kCa2023=false` (+10) raise confidence |
|
||||
| New agent, firmware confirms cert | present | silently suppressed (the customer-reported false positive) |
|
||||
|
||||
Because the marker exists nowhere until the new agent ships, the v3 rule is safe to seed
|
||||
**before** the agent rollout.
|
||||
|
||||
## Examples
|
||||
|
||||
Customer-equivalent verification from PowerShell (authoritative, matches the reader):
|
||||
|
||||
```powershell
|
||||
[Text.Encoding]::ASCII.GetString((Get-SecureBootUEFI -Name db).Bytes).Contains('Windows UEFI CA 2023')
|
||||
[Text.Encoding]::ASCII.GetString((Get-SecureBootUEFI -Name KEK).Bytes).Contains('Microsoft Corporation KEK 2K CA 2023')
|
||||
```
|
||||
|
||||
## Citations
|
||||
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/UefiSecureBootCertReader.cs` — native reader
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/DeviceInfo/DeviceInfoCollector.NetworkAndSecurity.cs` — `CollectSecureBootStatus` / `AppendFirmwareCertFields`
|
||||
* `rules/analyze/ANALYZE-SEC-001.json` — v3.0.0 rule definition
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/RuleEnginePreconditionTests.cs` — pinned precondition semantics + v3 mirror tests
|
||||
* Microsoft: [Act now — Secure Boot certificates expire June 2026](https://techcommunity.microsoft.com/blog/windows-itpro-blog/act-now-secure-boot-certificates-expire-in-june-2026/4426856)
|
||||
@@ -12,6 +12,8 @@ https://docs.autopilotmonitor.com (separate repository).
|
||||
# Architecture
|
||||
|
||||
* [Architecture Guide](architecture.md) - High-level architecture overview and solution layout for contributors.
|
||||
* [Version Contract](versioning.md) - One version shape for agent, backend and MCP: the ETag-CAS counter blob that mints build numbers, and the manifests published only after a deploy is verified live.
|
||||
* [URL Registry](url-registry.md) - Every well-known host lives in one registry file per component (Constants.cs / utils/config.ts / src/config.ts), enforced by guard tests; includes the agent-download migration to the download alias and its deploy-order constraint.
|
||||
|
||||
# Agent
|
||||
|
||||
@@ -19,6 +21,8 @@ https://docs.autopilotmonitor.com (separate repository).
|
||||
* [Build Counter Blob](agent/build-counter-blob.md) - How agent build numbers are minted: shared counter blob with ETag-CAS so local and CI builds never collide; CI additionally attests build provenance (Sigstore).
|
||||
* [Agent Endpoint Migration](agent/endpoint-migration.md) - How the backend re-homes agents to a new API base URL via the config channel (backend move or per-tenant region move): live-fetch-only, allowlist-validated on both sides, one hop, kill wins.
|
||||
* [MDM Reboot Coalescing](agent/mdm-reboot-coalescing.md) - Attributing the mid-ESP coalesced reboot ("second sign-in") to the device-assigned policy URIs that forced it: aggregated DM-Enterprise 2800 observations (neutral, flush-time watermark) + the ANALYZE-ESP-005 advisory gated on an actually-observed reboot.
|
||||
* [Secure Boot CA-2023 Detection](agent/secureboot-ca2023-detection.md) - Why ANALYZE-SEC-001 v3 reads the UEFI db/KEK variables directly instead of trusting the SecureBoot\Servicing registry, and how the one-sided uefiCa2023FirmwareConfirmed marker plus a not_exists precondition give absence-tolerant suppression across old and new agents.
|
||||
* [ESP Policy-Provider Stall Detection](agent/esp-policy-provider-stall.md) - The EnrollmentStatusTracking CSP's no-timeout Setup/Apps wait is keyed to the Intune-registered "Sidecar" provider by NAME: co-management's "ConfigMgr" parks the user ESP at "Apps (Identifying)" for days even with TrackingPoliciesCreated=1 (issue #106); the always-on two-arm esp_policy_provider_stalled tripwire (provider_incomplete | sidecar_provider_missing) and the SOFTWARE\Microsoft\Windows\Autopilot gather-rule allowlist entry that surface it.
|
||||
|
||||
# Rules
|
||||
|
||||
|
||||
+16
@@ -1,7 +1,23 @@
|
||||
# Log
|
||||
|
||||
## 2026-07-24
|
||||
|
||||
* **Update**: `agent/esp-policy-provider-stall.md` — the customer's follow-up on issue #106 corrected the field-case mechanics: `TrackingPoliciesCreated=1` WAS set under the foreign `ConfigMgr` provider, so the ESP ignores a foreign provider's completion values and waits on the Intune-registered `Sidecar` provider by name (the rename to `Sidecar` creates no tracking entries yet unblocks instantly). The completeness-keyed detector would have missed exactly this — the incident it was built for. The tripwire is now two-armed under one event type discriminated by `reason`: `provider_incomplete` (unchanged dwell on continuously-incomplete providers) and `sidecar_provider_missing` (≥ 1 Setup/Apps provider registered without `Sidecar`, completeness deliberately ignored, clock reset when Sidecar registers later). Sidecar-missing wins the `reason` when both arms cross in one round; the cross-restart fingerprint gains a `sidecarMissing` sentinel; payload adds `reason` and `sidecarMissingForMinutes`. Evidence for the name-keyed wait: Niehaus' documented Intune WAP payload always registers `Sidecar` (ConfigMgr only via a co-management settings policy) and Microsoft's ESP troubleshooting models app tracking exclusively through the Sidecar subkey.
|
||||
|
||||
* **Creation**: Added `agent/esp-policy-provider-stall.md` — a customer diagnosed that pre-provisioned devices with the ConfigMgr client as a blocking app occasionally park the user ESP at "Apps (Identifying)" for days: only a `ConfigMgr` provider is registered under `EnrollmentStatusTracking\Device\Setup\Apps\PolicyProviders` (no `Sidecar` = IME) and `TrackingPoliciesCreated` never becomes 1 — a wait the CSP defines without any timeout; renaming the key to `Sidecar` unblocks immediately. The agent now ships the always-on `esp_policy_provider_stalled` tripwire (60-s tick, 15-min continuous-incompleteness dwell, one-shot per provider, `StartupEventGate` fingerprint across restarts, purely observational via the InformationalEvent rail — no toggle, no ConfigVersion bump), generalized to any provider in both `Setup/Apps` and `DevicePreparation` kinds rather than hardcoding the ConfigMgr case. Companion change: `SOFTWARE\Microsoft\Windows\Autopilot` joins the gather-rule registry allowlist so tenants can target the subtree with custom rules (e.g. `not_exists` on the Sidecar key).
|
||||
|
||||
* **Creation**: Added `agent/secureboot-ca2023-detection.md` — ANALYZE-SEC-001 warned on devices whose firmware already carried the 2023 certificate, because the agent only read the `SecureBoot\Servicing\UEFICA2023Status` registry value, which mirrors the Windows Update rollout rather than the firmware (customer report). The agent now reads the UEFI `db`/`KEK` variables natively (`GetFirmwareEnvironmentVariableExW` with `SeSystemEnvironmentPrivilege` enabled on the process token — first privilege-adjustment code in the repo) and emits per-certificate booleans on `secureboot_status`. Suppression rides on a one-sided marker, `uefiCa2023FirmwareConfirmed`, emitted only on positive confirmation: the engine's `not_exists` precondition fails on any present value, so a two-sided boolean could not distinguish "confirmed" from "checked and missing", while the marker's absence keeps old agents and firmware-missing devices on the v2 path. Rule v3.0.0 is therefore safe to seed before the agent release.
|
||||
|
||||
* **Update**: `url-registry.md` — the first release after the alias migration surfaced Front Door caching as a correctness hazard: the alias served the previous `version.json` (`X-Cache: TCP_REMOTE_HIT`) after the origin had rotated, and since manifest and ZIP are independent cache entries, a stale manifest with a fresh ZIP fails the bootstrap SHA-256 check. Route caching on `/agent/*` is now disabled and every blob upload stamps `Cache-Control: no-cache` (CI `Send-Blob`, local `build.ps1`, `versions/*.json` manifests) — all agent-container blobs rotate in place, so nothing there is safely cacheable.
|
||||
|
||||
* **Update**: `agent/decision-engine.md` — the RealmJoin completion gate has a third terminal, `FirstDeploymentIncomplete`. Session 224b2087 showed why: an interactive logon 17 s after DeploymentPhase 101 makes RealmJoin reclassify the run as a secondary-user deployment, whose branch writes 200/210 and never 110 — the gate stayed closed until the 60-min hard timeout although Hello and desktop were long resolved. `realmjoin_phase_changed` is now a typed `RealmJoinPhaseChanged` signal, so `RealmJoinFacts.LastDeploymentPhase` is persisted across agent restarts (this also fixes every `realmjoin_timeout` event claiming "last phase: 0"); a transition out of 100/101 into 200/210 releases the gate with the `realmjoin_first_deployment_incomplete` Warning, and the `RealmJoinDetected` replay path evaluates the same rule for restarts that swallow the transition. A first observation of 210 deliberately does not release — session 6f1959c0 started at 210 and only deployed afterwards.
|
||||
|
||||
* **Creation**: Added `url-registry.md` — all well-known hosts (own subdomains, both blob accounts, API origin, Graph, Entra login) are now defined once per component (`Constants.cs`, web `utils/config.ts`, MCP `src/config.ts`) and referenced everywhere else; three guard tests fail the build on any literal outside the registries. Written because the EU cutover missed two hardcoded copies of the legacy blob host — `LatestVersionsService` (feeds the portal's version badge) and `ValidateBootstrapCodeFunction` — which kept working only through the fail-soft legacy mirror in `build-agent.yml`; those two plus the agent `SelfUpdater` now use the `download.autopilotmonitor.com` alias (verified serving all four artifacts first). The portal's bootstrap allow-list accepts alias AND legacy host during the transition, which imposes a deploy order: web before (or with) backend. The sweep also surfaced a fourth copy of the www/portal host pair in `middleware.ts` (now derived from the registry via `hostRouting.ts`) and a backend short-link built on the bare apex domain, which only worked through a registrar-level 301.
|
||||
|
||||
## 2026-07-22
|
||||
|
||||
* **Creation**: Added `versioning.md` — agent, backend and MCP now share one version shape (`<major>.<minor>.<build>` + 7-char commit + build timestamp) and one implementation of the build-number reservation (`.github/scripts/Request-BuildNumber.ps1`, previously copy-pasted between `build-agent.yml` and `build.ps1`). The backend had been pinned at a hand-written `1.0.0` since inception and now starts at `1.5.1000`; the MCP server's `package.json` version was bumped by hand and now starts at `1.5.300`, and it reports its own commit on `/health` instead of only the documentation bundle's. Both deploy workflows poll their health endpoint until it reports the build just deployed and publish `versions/<component>.json` only then, so a manifest that disagrees with the live endpoint means the instance drifted rather than the pipeline failing halfway. `agent/build-counter-blob.md` keeps the agent-specific parts and now points at the shared protocol; its counter URL was still the pre-cutover storage account.
|
||||
|
||||
* **Creation**: Added `rules/gather-rule-guardrails.md` — the gather-rule allowlists are enforced only on the agent (the backend validates no `collectorType`, `target`, or `parameters`), so a collector that forgets to call its guard *is* the vulnerability. Written because two had: `LogParserCollector` read any path in `rule.Target`, bypassing even the `C:\Users` hard block that every other file collector honours, and `EventLogCollector` read any channel including `Security` — while the customer-facing documentation already promised both were allowlisted. Both now guard, with `logparser` checking the directory for wildcard targets (`Path.GetFullPath` throws on `*`) and re-checking each resolved path. Added an event-log channel allowlist (`eventLogChannels`) plus hard-blocked channels held in C# rather than JSON, so a parse error cannot lift them. Also fixed the command timeout, which was inert: `ReadToEnd()` blocks until the pipe closes, so `WaitForExit(30000)` was only reached after the process had already exited and a hung command pinned its worker forever — now read asynchronously and killed at the timeout.
|
||||
|
||||
* **Update**: Added `.github/workflows/ci.yml` — the full suite (backend, agent V2 Core, DecisionCore, SummaryDialog, web, MCP) now runs on every pull request and every push to main, plus a job that fails if the generated guardrail files drift from `rules/guardrails.json`. Previously **no** workflow ran tests on a PR: the deploy workflows build without testing, and `build-agent.yml` tested only `DecisionCore.Tests` on manual dispatch — while `trust/security-faq.md` already told customers that automated tests run in CI. Added CodeQL (C# and TypeScript) and Dependabot (nuget, npm ×2, actions). Marked `OfficeBinaryWatcherTests` `[Collection("SerialThreading")]`: its bounded re-probe races the shared ThreadPool and failed once under full-suite load.
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
type: Concept
|
||||
title: URL Registry — One Place per Component, Guard-Enforced
|
||||
description: Every well-known own or Microsoft host lives in exactly one registry file per component (Constants.cs, utils/config.ts, src/config.ts); guard tests fail the build on any literal outside them. Includes the agent-download host migration to the download alias.
|
||||
resource: /src/Shared/AutopilotMonitor.Shared/Constants.cs
|
||||
tags:
|
||||
- architecture
|
||||
- urls
|
||||
- ci
|
||||
- convention
|
||||
timestamp: 2026-07-23T00:00:00+02:00
|
||||
---
|
||||
|
||||
# URL Registry
|
||||
|
||||
Every well-known base URL — own hosts (portal, www, docs, download alias, MCP,
|
||||
API, both blob accounts) and Microsoft hosts (Graph, Entra login) — is defined
|
||||
in exactly one registry file per component. Code references the constant; the
|
||||
literal appears nowhere else.
|
||||
|
||||
| Component | Registry | Guard test |
|
||||
|---|---|---|
|
||||
| C# (Shared/Backend/Agent) | `AutopilotMonitor.Shared.Constants` (`*BaseUrl`) | `HardcodedUrlGuardTests` |
|
||||
| Web | `utils/config.ts` (`DOCS_URL`, `SITE_URL`, `PORTAL_URL`, `ENTRA_LOGIN_URL`, `API_URL_PROD`, `BLOB_URL_PROD`, `AGENT_DOWNLOAD_HOSTNAMES`) | `utils/__tests__/hardcodedUrls.guard.test.ts` |
|
||||
| MCP | `src/config.ts` (`API_BASE_URL`, `ENTRA_LOGIN_BASE_URL`, `DOCS_BASE_URL`) | `src/__tests__/hardcoded-urls.guard.test.ts` |
|
||||
|
||||
Derived registries build on these instead of repeating hosts:
|
||||
`lib/hostRouting.ts` computes `PUBLIC_HOST`/`PORTAL_HOST`/`APEX_HOST` from the
|
||||
URL constants, and `middleware.ts` imports from it.
|
||||
|
||||
# Schema
|
||||
|
||||
The guards scan non-test source for the enforced host list and fail with
|
||||
file:line on any occurrence outside the registry. Comment lines are ignored
|
||||
(docs may cite URLs). Test files are excluded on purpose: a literal in a test
|
||||
is an independent oracle — `ValidateBootstrapCodeResponseShapeTests` spells the
|
||||
full agent download URL out so a registry change must be acknowledged in two
|
||||
places, backend and portal allow-list, consciously.
|
||||
|
||||
Why enforced rather than agreed: the registry existed before (both
|
||||
`Constants.AgentBlobBaseUrl` and the MCP `config.ts` header document earlier
|
||||
drift), yet the EU cutover still missed two literal copies of the blob host —
|
||||
`LatestVersionsService` and `ValidateBootstrapCodeFunction` kept reading the
|
||||
legacy account, silently kept alive only by the fail-soft legacy mirror in
|
||||
`build-agent.yml`. A convention that is not checked decays; these guards make
|
||||
the next stray literal a red build instead of a production surprise.
|
||||
|
||||
# Agent-download host migration (2026-07-23)
|
||||
|
||||
The two legacy readers above plus the agent's `SelfUpdater` now use
|
||||
`Constants.AgentDownloadBaseUrl` (`download.autopilotmonitor.com/agent`, Front
|
||||
Door in front of the current blob origin — verified serving `version.json`,
|
||||
`version-v2.json` and both ZIPs before the switch). The portal's
|
||||
`bootstrapValidation.ts` allow-lists BOTH the alias and the legacy blob host
|
||||
(`AGENT_DOWNLOAD_HOSTNAMES`) because bootstrap scripts already deployed in
|
||||
customer Intune tenants still carry legacy URLs.
|
||||
|
||||
Deploy-order constraint: the web allow-list must be live BEFORE (or with) the
|
||||
backend that serves alias URLs — an old portal build rejects the alias host and
|
||||
the bootstrap flow dies client-side. Remove the legacy hostname from the
|
||||
allow-list only after the legacy account is torn down.
|
||||
|
||||
**Front Door caching is a correctness hazard here, not a staleness nuisance.**
|
||||
The first release after the migration exposed it: the alias kept serving the
|
||||
previous `version.json` (`X-Cache: TCP_REMOTE_HIT`) while the origin was already
|
||||
updated — and because manifest and ZIP are independent cache entries, a stale
|
||||
manifest paired with a fresh ZIP fails the bootstrap's SHA-256 check. Two
|
||||
defenses, both in place since 2026-07-23: route caching on `/agent/*` is
|
||||
DISABLED, and every blob upload (CI `Send-Blob`, local `build.ps1`, the
|
||||
`versions/*.json` manifests) stamps `Cache-Control: no-cache` — all these blobs
|
||||
rotate in place, even the per-line ZIP. If caching is ever re-enabled, the
|
||||
headers keep the mismatch class impossible.
|
||||
|
||||
Deliberately still on the legacy account: the `HealthCheckService` /
|
||||
`MaintenanceService` legacy-keepalive probes and the fail-soft mirror in
|
||||
`build-agent.yml` — they exist precisely to keep already-deployed customer
|
||||
bootstrap scripts working until the customer migration completes.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Shared/AutopilotMonitor.Shared/Constants.cs` — C# registry
|
||||
- `src/Web/autopilot-monitor-web/utils/config.ts` — web registry
|
||||
- `src/McpServer/autopilot-monitor-mcp/src/config.ts` — MCP registry
|
||||
- `src/Backend/AutopilotMonitor.Functions.Tests/HardcodedUrlGuardTests.cs`
|
||||
- [Version Contract](versioning.md) — the release pipeline this migration protects
|
||||
@@ -0,0 +1,108 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Version Contract — Agent, Backend, MCP
|
||||
description: One version shape for all three shipping components, the ETag-CAS counter blob that mints build numbers, and the published manifests that make "what is deployed" checkable from outside.
|
||||
resource: /.github/scripts/Request-BuildNumber.ps1
|
||||
tags:
|
||||
- build
|
||||
- ci
|
||||
- release
|
||||
- versioning
|
||||
timestamp: 2026-07-22T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Version Contract
|
||||
|
||||
Three components ship from this repository — the agent, the Functions backend and the
|
||||
MCP server. They share one version shape, one way of minting build numbers, and one
|
||||
manifest format, so that a version string means the same thing whoever emits it.
|
||||
|
||||
```
|
||||
version = <major>.<minor>.<build> major.minor curated by hand, build minted per build
|
||||
commit = 7-char short SHA of this repo
|
||||
buildUtc = ISO-8601 build moment
|
||||
```
|
||||
|
||||
| Component | major.minor from | Counter blob | Reported by |
|
||||
|---|---|---|---|
|
||||
| Agent V2 | `VersionPrefix` in `AutopilotMonitor.Agent.V2.csproj` | `agent/buildcounter-v2.txt` | `agent/version.json` |
|
||||
| Backend | `VersionPrefix` in `AutopilotMonitor.Functions.csproj` | `versions/backend.txt` | `GET /api/health` |
|
||||
| MCP | `version` in `package.json` | `versions/mcp.txt` | `GET /health`, MCP handshake |
|
||||
|
||||
# Schema
|
||||
|
||||
## Build numbers — reserve before build, ETag-CAS
|
||||
|
||||
`.github/scripts/Request-BuildNumber.ps1` is the only implementation; the CI
|
||||
workflows and the local `build.ps1` all call it.
|
||||
|
||||
1. Anonymous `GET` on the counter blob → value `N` and `ETag`. The blob holds the
|
||||
**last used** number as plain text and is public-read; only the write needs the
|
||||
container SAS.
|
||||
2. `PUT N+1` with `If-Match: <etag>`. The quoted ETag form is required — PowerShell
|
||||
strips the quotes on read, and an unquoted `If-Match` is rejected client-side.
|
||||
3. `412 Precondition Failed` → a concurrent build reserved first → re-read and retry
|
||||
(5 attempts, linear backoff).
|
||||
4. Build with the reserved number.
|
||||
|
||||
Reserving *before* the build is what makes concurrent minting impossible. **A failed
|
||||
build burns a number** — deliberate: uniqueness matters, density does not.
|
||||
|
||||
Why a counter and not the published manifest: manifests are written on publish, not on
|
||||
build. The agent writes `version.json` only for stable releases, so deriving the next
|
||||
number from it would hand two consecutive dev builds the same number and let the second
|
||||
overwrite the first's artifact. A state-free scheme such as `git rev-list --count HEAD`
|
||||
fails for the same reason — two builds of one commit collide.
|
||||
|
||||
**Seeding** (once per component): `PUT <last used number>` with `If-None-Match: *`, so
|
||||
an existing counter can never be clobbered. Agent V2 was seeded with `1356` on
|
||||
2026-07-17; `versions/backend.txt` with `999` and `versions/mcp.txt` with `299` on
|
||||
2026-07-22.
|
||||
|
||||
## Local builds
|
||||
|
||||
No counter is reached, the build number stays `0`, and the commit is empty or
|
||||
`unknown`. A `1.5.0` with no commit hash on `/api/health` is therefore a developer
|
||||
machine, not a release — the fallback is the signal, not a defect. The agent is the
|
||||
exception: `build.ps1` produces uploadable artifacts and reserves a real number unless
|
||||
`-SkipUpload` is passed.
|
||||
|
||||
## Manifests
|
||||
|
||||
`versions/backend.json` and `versions/mcp.json` (public-read) record what is deployed:
|
||||
|
||||
```json
|
||||
{ "component": "backend", "version": "1.5.1000", "commit": "abc1234",
|
||||
"buildUtc": "…", "deployedUtc": "…", "runId": "…" }
|
||||
```
|
||||
|
||||
The MCP manifest adds `docsCommit`, because for that component "which build" and "which
|
||||
documentation" are separate questions.
|
||||
|
||||
Each is written **last and only after the deploy workflow has polled the component's
|
||||
health endpoint until it reports that exact version and commit**. A green deploy step
|
||||
alone means the package was accepted, not that it is being served. Consequently a
|
||||
manifest that disagrees with the live endpoint means the running instance drifted after
|
||||
the fact (rollback, stale revision) rather than a half-finished pipeline.
|
||||
|
||||
The agent keeps its own manifest at `agent/version.json` — the bootstrap derives that
|
||||
URL from the download URL and verifies the ZIP against its `sha256`, so it cannot move.
|
||||
It carries the same `commit` and `buildUtc` fields; `sha256` and `bootstrapVersion`
|
||||
remain, being the customer-facing integrity contract.
|
||||
|
||||
# Consequences
|
||||
|
||||
- Bumping `major.minor` is a human decision, made in the component's own file
|
||||
(`VersionPrefix`, or `version` in `package.json`). Nothing bumps it automatically.
|
||||
- A new component joins by seeding a counter blob and calling the same script; no new
|
||||
mechanism.
|
||||
- The counter blobs and manifests live in the `versions` container on
|
||||
`autopilotmonitoreu` (writes via `AZURE_BLOB_VERSIONS_SAS_TOKEN`). The agent's
|
||||
counter stays in the `agent` container next to the artifacts it numbers.
|
||||
|
||||
# Citations
|
||||
|
||||
- `.github/scripts/Request-BuildNumber.ps1` — the CAS protocol
|
||||
- `.github/workflows/deploy-backend.yml`, `deploy-mcp.yml` — reserve, verify, publish
|
||||
- `.github/workflows/build-agent.yml`, `scripts/Deployment/build.ps1` — agent line
|
||||
- [Build Counter Blob](agent/build-counter-blob.md) — agent-specific consequences
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"$schema": "../schema/analyze-rule.schema.json",
|
||||
"ruleId": "ANALYZE-ESP-006",
|
||||
"title": "ESP Failure Recovered After User Retry",
|
||||
"description": "Detects the recovery story: the ESP reported a terminal failure (typically DeviceSetup/Apps), the user most likely pressed 'Try again' on the failure page, the failed step re-ran and completed successfully. The earlier failure did not stick — the enrollment continued past it.",
|
||||
"severity": "info",
|
||||
"category": "esp",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"baseConfidence": 85,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "advisory",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_advisory",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "recovered",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "retry",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_retry_detected",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "category",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "category",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "failedSubcategory",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "failedSubcategory",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "likelyCulpritApps",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_advisory",
|
||||
"dataField": "likelyCulpritApps",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "minutesSinceFailure",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "minutesSinceFailure",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [
|
||||
{ "signal": "retry", "condition": "exists", "weight": 10 },
|
||||
{ "signal": "likelyCulpritApps", "condition": "exists", "weight": 5 }
|
||||
],
|
||||
"explanation": "The Enrollment Status Page reported a **failure** in the `{{category}}` phase (`{{failedSubcategory}}` subcategory, most likely app(s): `{{likelyCulpritApps}}`) — and then **recovered**: the failed step left its failed state and completed successfully about `{{minutesSinceFailure}}` min later.\n\nThat pattern is consistent with the user pressing **'Try again'** on the ESP failure page (the profile allows failure-page actions; the agent cannot observe the click itself). The failure the user saw was real, but it did not stick — the ESP re-ran the step, it succeeded, and the enrollment continued.\n\nRead the session timeline as one arc: `esp_failure_advisory` (the failure, with the likely culprit app) → `esp_failure_retry_detected` (the retry) → `esp_failure_recovered` (the recovery). If this pattern repeats across devices for the same app, that app's install is flaky on first attempt — fix the app rather than the devices. Any literal `{{…}}` placeholder means that detail was not recorded for this session.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Make the first install attempt stick",
|
||||
"steps": [
|
||||
"Check the likely culprit app(s) (`{{likelyCulpritApps}}`) in Intune: an app that fails on the first ESP pass but succeeds on retry usually has a dependency/ordering or detection-timing problem.",
|
||||
"If the app shows 'never started installing' in esp_failure_settle_started, verify its assignment and applicability rules — ESP counted it as required, but IME never processed it.",
|
||||
"If the pattern repeats across the tenant, consider removing the app from the ESP blocking list until the first-attempt install is reliable — users are seeing an avoidable failure screen.",
|
||||
"No device-side action is needed for this session: the failure recovered and the enrollment continued."
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": ["esp", "recovery", "try-again", "retry", "continue-anyway", "enrollment"]
|
||||
}
|
||||
@@ -2,10 +2,10 @@
|
||||
"$schema": "../schema/analyze-rule.schema.json",
|
||||
"ruleId": "ANALYZE-SEC-001",
|
||||
"title": "Secure Boot UEFI CA 2023 certificate not deployed",
|
||||
"description": "Detects devices where the Windows UEFI CA 2023 certificate has not been deployed to the Secure Boot DB. The 2011 certificates expired in June 2026 — affected devices no longer receive Secure Boot security updates until the 2023 certificate is deployed.",
|
||||
"description": "Detects devices where the Windows UEFI CA 2023 certificate has not been deployed to the Secure Boot DB. The 2011 certificates expired in June 2026 — affected devices no longer receive Secure Boot security updates until the 2023 certificate is deployed. Since v3 the agent reads the firmware db/KEK variables directly: when the firmware itself confirms the 2023 certificate, the rule is suppressed even if the Windows servicing status has not caught up yet.",
|
||||
"severity": "warning",
|
||||
"category": "security",
|
||||
"version": "2.0.0",
|
||||
"version": "3.0.0",
|
||||
"enabled": true,
|
||||
"baseConfidence": 70,
|
||||
"preconditions": [
|
||||
@@ -16,6 +16,14 @@
|
||||
"operator": "equals",
|
||||
"value": "false",
|
||||
"description": "Skip on virtual machines — Secure Boot DB is host-controlled (Hyper-V/VMware/etc), the inside-the-VM UEFICA2023Status registry value is not the right indicator and not actionable for the VM admin."
|
||||
},
|
||||
{
|
||||
"source": "event_data",
|
||||
"eventType": "secureboot_status",
|
||||
"dataField": "uefiCa2023FirmwareConfirmed",
|
||||
"operator": "not_exists",
|
||||
"value": "",
|
||||
"description": "Skip when the firmware db itself confirms the Windows UEFI CA 2023 certificate — the marker is only emitted on positive firmware confirmation, so old agents (field absent) and unconfirmed devices keep evaluating the rule. The servicing registry status lags behind actual firmware state (OEM factory image, WinCS, manual deployment, or WU simply not run yet)."
|
||||
}
|
||||
],
|
||||
"conditions": [
|
||||
@@ -54,13 +62,33 @@
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "db_ca2023_firmware_missing",
|
||||
"source": "event_data",
|
||||
"eventType": "secureboot_status",
|
||||
"dataField": "uefiDbHasWindowsUefiCa2023",
|
||||
"operator": "equals",
|
||||
"value": "false",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "kek_2023_missing",
|
||||
"source": "event_data",
|
||||
"eventType": "secureboot_status",
|
||||
"dataField": "uefiKekHasMicrosoftKek2kCa2023",
|
||||
"operator": "equals",
|
||||
"value": "false",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [
|
||||
{ "signal": "ca2023_notfound", "condition": "exists", "weight": 20 },
|
||||
{ "signal": "ca2023_error", "condition": "exists", "weight": 10 }
|
||||
{ "signal": "ca2023_error", "condition": "exists", "weight": 10 },
|
||||
{ "signal": "db_ca2023_firmware_missing", "condition": "exists", "weight": 20 },
|
||||
{ "signal": "kek_2023_missing", "condition": "exists", "weight": 10 }
|
||||
],
|
||||
"explanation": "This device has Secure Boot enabled but the **Windows UEFI CA 2023** certificate has not been deployed to the Secure Boot DB.\n\nThe original Windows UEFI CA 2011 certificates **expired on June 27, 2026 — this deadline has passed**. The automatic pre-expiry rollout window is over: this device no longer receives Secure Boot security updates (DB/DBX) and cannot trust 2023-signed boot managers until the 2023 certificate is deployed. Deploying it now requires active remediation (Windows servicing opt-in and, on some models, an OEM firmware update) rather than waiting for an automatic update.\n\nCurrent UEFI CA 2023 status: `{{uefiCA2023Status}}`",
|
||||
"explanation": "This device has Secure Boot enabled but the **Windows UEFI CA 2023** certificate has not been deployed to the Secure Boot DB.\n\nThe original Windows UEFI CA 2011 certificates **expired on June 27, 2026 — this deadline has passed**. The automatic pre-expiry rollout window is over: this device no longer receives Secure Boot security updates (DB/DBX) and cannot trust 2023-signed boot managers until the 2023 certificate is deployed. Deploying it now requires active remediation (Windows servicing opt-in and, on some models, an OEM firmware update) rather than waiting for an automatic update.\n\nOn devices running a current agent this finding is verified against the firmware itself: the agent reads the UEFI `db` signature database directly, and this warning is automatically suppressed when the firmware already contains the 2023 certificate — regardless of the Windows servicing status below.\n\nCurrent UEFI CA 2023 servicing status: `{{uefiCA2023Status}}`",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Deploy UEFI CA 2023 certificate now",
|
||||
@@ -69,7 +97,7 @@
|
||||
"Use Intune policy to enable Secure Boot DB update (Windows UEFI CA 2023)",
|
||||
"Check with the device OEM whether a firmware update is required for the DB update to apply on this model",
|
||||
"Alternatively, use the WinCS CLI tool from Microsoft for manual deployment",
|
||||
"Verify deployment: check registry HKLM\\SYSTEM\\CurrentControlSet\\Control\\SecureBoot\\Servicing\\UEFICA2023Status = 'Updated'",
|
||||
"Verify deployment against the firmware directly: [Text.Encoding]::ASCII.GetString((Get-SecureBootUEFI -Name db).Bytes).Contains('Windows UEFI CA 2023') — this is authoritative; the registry value HKLM\\SYSTEM\\CurrentControlSet\\Control\\SecureBoot\\Servicing\\UEFICA2023Status = 'Updated' only reflects the Windows Update rollout",
|
||||
"After DB update: ensure the boot manager is updated to a 2023-signed version via monthly cumulative update"
|
||||
]
|
||||
}
|
||||
|
||||
+144
-4
@@ -1884,6 +1884,110 @@
|
||||
"user-experience"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-ESP-006",
|
||||
"title": "ESP Failure Recovered After User Retry",
|
||||
"description": "Detects the recovery story: the ESP reported a terminal failure (typically DeviceSetup/Apps), the user most likely pressed 'Try again' on the failure page, the failed step re-ran and completed successfully. The earlier failure did not stick — the enrollment continued past it.",
|
||||
"severity": "info",
|
||||
"category": "esp",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"baseConfidence": 85,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "advisory",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_advisory",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "recovered",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "retry",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_retry_detected",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "category",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "category",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "failedSubcategory",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "failedSubcategory",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "likelyCulpritApps",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_advisory",
|
||||
"dataField": "likelyCulpritApps",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "minutesSinceFailure",
|
||||
"source": "event_type",
|
||||
"eventType": "esp_failure_recovered",
|
||||
"dataField": "minutesSinceFailure",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [
|
||||
{
|
||||
"signal": "retry",
|
||||
"condition": "exists",
|
||||
"weight": 10
|
||||
},
|
||||
{
|
||||
"signal": "likelyCulpritApps",
|
||||
"condition": "exists",
|
||||
"weight": 5
|
||||
}
|
||||
],
|
||||
"explanation": "The Enrollment Status Page reported a **failure** in the `{{category}}` phase (`{{failedSubcategory}}` subcategory, most likely app(s): `{{likelyCulpritApps}}`) — and then **recovered**: the failed step left its failed state and completed successfully about `{{minutesSinceFailure}}` min later.\n\nThat pattern is consistent with the user pressing **'Try again'** on the ESP failure page (the profile allows failure-page actions; the agent cannot observe the click itself). The failure the user saw was real, but it did not stick — the ESP re-ran the step, it succeeded, and the enrollment continued.\n\nRead the session timeline as one arc: `esp_failure_advisory` (the failure, with the likely culprit app) → `esp_failure_retry_detected` (the retry) → `esp_failure_recovered` (the recovery). If this pattern repeats across devices for the same app, that app's install is flaky on first attempt — fix the app rather than the devices. Any literal `{{…}}` placeholder means that detail was not recorded for this session.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Make the first install attempt stick",
|
||||
"steps": [
|
||||
"Check the likely culprit app(s) (`{{likelyCulpritApps}}`) in Intune: an app that fails on the first ESP pass but succeeds on retry usually has a dependency/ordering or detection-timing problem.",
|
||||
"If the app shows 'never started installing' in esp_failure_settle_started, verify its assignment and applicability rules — ESP counted it as required, but IME never processed it.",
|
||||
"If the pattern repeats across the tenant, consider removing the app from the ESP blocking list until the first-attempt install is reliable — users are seeing an avoidable failure screen.",
|
||||
"No device-side action is needed for this session: the failure recovered and the enrollment continued."
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": [
|
||||
"esp",
|
||||
"recovery",
|
||||
"try-again",
|
||||
"retry",
|
||||
"continue-anyway",
|
||||
"enrollment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-ID-001",
|
||||
"title": "Expected Machine Certificate Not Found",
|
||||
@@ -2178,10 +2282,10 @@
|
||||
{
|
||||
"ruleId": "ANALYZE-SEC-001",
|
||||
"title": "Secure Boot UEFI CA 2023 certificate not deployed",
|
||||
"description": "Detects devices where the Windows UEFI CA 2023 certificate has not been deployed to the Secure Boot DB. The 2011 certificates expired in June 2026 — affected devices no longer receive Secure Boot security updates until the 2023 certificate is deployed.",
|
||||
"description": "Detects devices where the Windows UEFI CA 2023 certificate has not been deployed to the Secure Boot DB. The 2011 certificates expired in June 2026 — affected devices no longer receive Secure Boot security updates until the 2023 certificate is deployed. Since v3 the agent reads the firmware db/KEK variables directly: when the firmware itself confirms the 2023 certificate, the rule is suppressed even if the Windows servicing status has not caught up yet.",
|
||||
"severity": "warning",
|
||||
"category": "security",
|
||||
"version": "2.0.0",
|
||||
"version": "3.0.0",
|
||||
"enabled": true,
|
||||
"baseConfidence": 70,
|
||||
"preconditions": [
|
||||
@@ -2192,6 +2296,14 @@
|
||||
"operator": "equals",
|
||||
"value": "false",
|
||||
"description": "Skip on virtual machines — Secure Boot DB is host-controlled (Hyper-V/VMware/etc), the inside-the-VM UEFICA2023Status registry value is not the right indicator and not actionable for the VM admin."
|
||||
},
|
||||
{
|
||||
"source": "event_data",
|
||||
"eventType": "secureboot_status",
|
||||
"dataField": "uefiCa2023FirmwareConfirmed",
|
||||
"operator": "not_exists",
|
||||
"value": "",
|
||||
"description": "Skip when the firmware db itself confirms the Windows UEFI CA 2023 certificate — the marker is only emitted on positive firmware confirmation, so old agents (field absent) and unconfirmed devices keep evaluating the rule. The servicing registry status lags behind actual firmware state (OEM factory image, WinCS, manual deployment, or WU simply not run yet)."
|
||||
}
|
||||
],
|
||||
"conditions": [
|
||||
@@ -2230,6 +2342,24 @@
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "db_ca2023_firmware_missing",
|
||||
"source": "event_data",
|
||||
"eventType": "secureboot_status",
|
||||
"dataField": "uefiDbHasWindowsUefiCa2023",
|
||||
"operator": "equals",
|
||||
"value": "false",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "kek_2023_missing",
|
||||
"source": "event_data",
|
||||
"eventType": "secureboot_status",
|
||||
"dataField": "uefiKekHasMicrosoftKek2kCa2023",
|
||||
"operator": "equals",
|
||||
"value": "false",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [
|
||||
@@ -2242,9 +2372,19 @@
|
||||
"signal": "ca2023_error",
|
||||
"condition": "exists",
|
||||
"weight": 10
|
||||
},
|
||||
{
|
||||
"signal": "db_ca2023_firmware_missing",
|
||||
"condition": "exists",
|
||||
"weight": 20
|
||||
},
|
||||
{
|
||||
"signal": "kek_2023_missing",
|
||||
"condition": "exists",
|
||||
"weight": 10
|
||||
}
|
||||
],
|
||||
"explanation": "This device has Secure Boot enabled but the **Windows UEFI CA 2023** certificate has not been deployed to the Secure Boot DB.\n\nThe original Windows UEFI CA 2011 certificates **expired on June 27, 2026 — this deadline has passed**. The automatic pre-expiry rollout window is over: this device no longer receives Secure Boot security updates (DB/DBX) and cannot trust 2023-signed boot managers until the 2023 certificate is deployed. Deploying it now requires active remediation (Windows servicing opt-in and, on some models, an OEM firmware update) rather than waiting for an automatic update.\n\nCurrent UEFI CA 2023 status: `{{uefiCA2023Status}}`",
|
||||
"explanation": "This device has Secure Boot enabled but the **Windows UEFI CA 2023** certificate has not been deployed to the Secure Boot DB.\n\nThe original Windows UEFI CA 2011 certificates **expired on June 27, 2026 — this deadline has passed**. The automatic pre-expiry rollout window is over: this device no longer receives Secure Boot security updates (DB/DBX) and cannot trust 2023-signed boot managers until the 2023 certificate is deployed. Deploying it now requires active remediation (Windows servicing opt-in and, on some models, an OEM firmware update) rather than waiting for an automatic update.\n\nOn devices running a current agent this finding is verified against the firmware itself: the agent reads the UEFI `db` signature database directly, and this warning is automatically suppressed when the firmware already contains the 2023 certificate — regardless of the Windows servicing status below.\n\nCurrent UEFI CA 2023 servicing status: `{{uefiCA2023Status}}`",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Deploy UEFI CA 2023 certificate now",
|
||||
@@ -2253,7 +2393,7 @@
|
||||
"Use Intune policy to enable Secure Boot DB update (Windows UEFI CA 2023)",
|
||||
"Check with the device OEM whether a firmware update is required for the DB update to apply on this model",
|
||||
"Alternatively, use the WinCS CLI tool from Microsoft for manual deployment",
|
||||
"Verify deployment: check registry HKLM\\SYSTEM\\CurrentControlSet\\Control\\SecureBoot\\Servicing\\UEFICA2023Status = 'Updated'",
|
||||
"Verify deployment against the firmware directly: [Text.Encoding]::ASCII.GetString((Get-SecureBootUEFI -Name db).Bytes).Contains('Windows UEFI CA 2023') — this is authoritative; the registry value HKLM\\SYSTEM\\CurrentControlSet\\Control\\SecureBoot\\Servicing\\UEFICA2023Status = 'Updated' only reflects the Windows Update rollout",
|
||||
"After DB update: ensure the boot manager is updated to a 2023-signed version via monthly cumulative update"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -43,7 +43,8 @@
|
||||
"prefixes": [
|
||||
"SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Setup",
|
||||
"SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\OOBE",
|
||||
"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon"
|
||||
"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon",
|
||||
"SOFTWARE\\Microsoft\\Windows\\Autopilot"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -272,13 +272,16 @@ try {
|
||||
$downloadAttempt++
|
||||
try {
|
||||
Write-Log "Download attempt ${downloadAttempt}/${maxDownloadAttempts}"
|
||||
$downloadStopwatch = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
Invoke-WebRequest `
|
||||
-Uri $AgentDownloadUrl `
|
||||
-OutFile $zipPath `
|
||||
-UseBasicParsing `
|
||||
-TimeoutSec 30 `
|
||||
-ErrorAction Stop
|
||||
Write-Log "Downloaded agent to $zipPath"
|
||||
$downloadStopwatch.Stop()
|
||||
$downloadSeconds = [math]::Round($downloadStopwatch.Elapsed.TotalSeconds, 1)
|
||||
Write-Log "Downloaded agent to $zipPath (took ${downloadSeconds}s)"
|
||||
break
|
||||
}
|
||||
catch {
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
private const string CurrentUrl = "https://autopilotmonitor-api-eu.azurewebsites.net";
|
||||
private const string TargetUrl = "https://autopilotmonitor-api-us.azurewebsites.net";
|
||||
|
||||
private static AgentConfigResponse Config(string migrateUrl) =>
|
||||
private static AgentConfigResponse Config(string? migrateUrl) =>
|
||||
new AgentConfigResponse { MigrateToApiBaseUrl = migrateUrl };
|
||||
|
||||
[Fact]
|
||||
|
||||
+1
@@ -70,6 +70,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
// Read the file the production code actually wrote and deserialize with the same lib.
|
||||
var raw = File.ReadAllText(_cachePath);
|
||||
var persisted = JsonConvert.DeserializeObject<AgentConfigResponse>(raw);
|
||||
Assert.NotNull(persisted);
|
||||
|
||||
Assert.False(persisted.UnrestrictedMode);
|
||||
Assert.False(persisted.AllowAgentDowngrade);
|
||||
|
||||
+3
-3
@@ -193,7 +193,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
case BackendBehaviour.AlwaysAuthFailure401:
|
||||
throw new BackendAuthException("Unauthorized", 401);
|
||||
case BackendBehaviour.AlwaysReturnNull:
|
||||
return Task.FromResult<AgentConfigResponse>(null);
|
||||
return Task.FromResult<AgentConfigResponse>(null!);
|
||||
default:
|
||||
throw new InvalidOperationException("unknown behaviour");
|
||||
}
|
||||
@@ -204,9 +204,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
|
||||
// In-memory + per-instance cache so a prior test's successful fetch can't
|
||||
// bleed into a later test that expects UsedDefaults.
|
||||
private AgentConfigResponse _cached;
|
||||
private AgentConfigResponse? _cached;
|
||||
protected override void CacheConfig(AgentConfigResponse config) { _cached = config; }
|
||||
protected override AgentConfigResponse LoadCachedConfig() => _cached;
|
||||
protected override AgentConfigResponse LoadCachedConfig() => _cached!;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Harness
|
||||
logFolder: Tmp.Path,
|
||||
patterns: new List<ImeLogPattern>(),
|
||||
logger: Logger);
|
||||
// Mirror ImeLogHost wiring: the historic-replay guard judges staleness against the
|
||||
// (virtual) agent clock so tests stay deterministic.
|
||||
Tracker.UtcNowProvider = () => Clock.UtcNow;
|
||||
}
|
||||
|
||||
public IReadOnlyList<FakeSignalIngressSink.PostedSignal> DecisionSignals(DecisionSignalKind kind) =>
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Integration
|
||||
logger: logger,
|
||||
helloTracker: null);
|
||||
|
||||
EspExitedEventArgs captured = null;
|
||||
EspExitedEventArgs? captured = null;
|
||||
tracker.EspExited += (_, args) => captured = args;
|
||||
|
||||
// Source time IS in the past relative to the clock — verify the tracker carries the
|
||||
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
#nullable enable
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Analyzers
|
||||
{
|
||||
/// <summary>
|
||||
/// LocalAdminAnalyzer.MatchesAllowedEntry — the allowed-list matching contract.
|
||||
/// Entries without wildcard characters match exactly (case-insensitive); entries
|
||||
/// containing * or ? are glob patterns with the same semantics as the hardware
|
||||
/// whitelist. The same matcher is applied to local accounts and profile folders.
|
||||
/// </summary>
|
||||
public sealed class LocalAdminAnalyzerTests
|
||||
{
|
||||
// -------------------------------------------------------------- literal entries
|
||||
|
||||
[Theory]
|
||||
[InlineData("SupportAdmin", "SupportAdmin")]
|
||||
[InlineData("supportadmin", "SupportAdmin")]
|
||||
[InlineData("SUPPORTADMIN", "supportadmin")]
|
||||
public void Literal_entry_matches_exactly_case_insensitive(string name, string entry)
|
||||
{
|
||||
Assert.True(LocalAdminAnalyzer.MatchesAllowedEntry(name, entry));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("SupportAdmin2", "SupportAdmin")]
|
||||
[InlineData("SupportAdmin", "SupportAdmin2")]
|
||||
[InlineData("Support", "SupportAdmin")]
|
||||
public void Literal_entry_does_not_substring_match(string name, string entry)
|
||||
{
|
||||
Assert.False(LocalAdminAnalyzer.MatchesAllowedEntry(name, entry));
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------- glob patterns
|
||||
|
||||
[Theory]
|
||||
[InlineData("adm-12345", "adm-*")]
|
||||
[InlineData("adm-", "adm-*")]
|
||||
[InlineData("ADM-9F3B2C", "adm-*")]
|
||||
[InlineData("lapsAdmin01", "lapsAdmin??")]
|
||||
[InlineData("svc-backup-01", "svc-*-01")]
|
||||
public void Wildcard_entry_matches_glob(string name, string entry)
|
||||
{
|
||||
Assert.True(LocalAdminAnalyzer.MatchesAllowedEntry(name, entry));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("administrator2", "adm-*")] // no literal "adm-" prefix
|
||||
[InlineData("xadm-123", "adm-*")] // pattern is anchored at start
|
||||
[InlineData("lapsAdmin1", "lapsAdmin??")] // ? is exactly one char
|
||||
[InlineData("adm.123", "adm-*")] // "-" is literal, not a regex class
|
||||
public void Wildcard_entry_does_not_overmatch(string name, string entry)
|
||||
{
|
||||
Assert.False(LocalAdminAnalyzer.MatchesAllowedEntry(name, entry));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Regex_metacharacters_in_entry_are_treated_literally()
|
||||
{
|
||||
// "adm.123" as an entry has no wildcard chars → exact match only,
|
||||
// the dot must not act as a regex any-char.
|
||||
Assert.True(LocalAdminAnalyzer.MatchesAllowedEntry("adm.123", "adm.123"));
|
||||
Assert.False(LocalAdminAnalyzer.MatchesAllowedEntry("admX123", "adm.123"));
|
||||
|
||||
// Mixed: wildcard present, remaining metacharacters stay literal.
|
||||
Assert.True(LocalAdminAnalyzer.MatchesAllowedEntry("adm.123", "adm.*"));
|
||||
Assert.False(LocalAdminAnalyzer.MatchesAllowedEntry("admX123", "adm.*"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+139
@@ -405,5 +405,144 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
var started = trace.Events.Find(e => e.EventType == SharedEventTypes.DesktopDetectorStarted);
|
||||
Assert.Equal(0, started.Data["noCandidateThresholdMinutes"]);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// OOBE-state flip (oobe_state_completed) — observational one-shot on the poll
|
||||
// ============================================================================
|
||||
// The WinRT OutOfBoxExperienceState flip in_progress->completed is an owner-
|
||||
// independent desktop corroboration for DAD-starved sessions (4d5a0b78 class).
|
||||
// Tests drive polls synchronously via PollOnceForTest after Stop() so the real
|
||||
// timer cannot race, and pin the owner seams to an excluded user so a real
|
||||
// explorer.exe on the test machine can never set _desktopArrived (which would
|
||||
// short-circuit PollForDesktop before the flip check).
|
||||
|
||||
private static DesktopArrivalDetector BuildFlipDetector(TempDirectory tmp, TraceCapture trace, System.Func<string> oobeProvider)
|
||||
{
|
||||
var detector = new DesktopArrivalDetector(new AgentLogger(tmp.Path, AgentLogLevel.Info), noCandidateTimeoutMinutes: 0)
|
||||
{
|
||||
OnTraceEvent = trace.Sink,
|
||||
OobeStateProvider = oobeProvider,
|
||||
SessionOwnerResolver = _ => "SYSTEM",
|
||||
ProcessOwnerResolver = _ => "SYSTEM",
|
||||
};
|
||||
return detector;
|
||||
}
|
||||
|
||||
/// <summary>Returns the queued states in order, then repeats the last one forever.</summary>
|
||||
private static System.Func<string> StateQueue(params string[] states)
|
||||
{
|
||||
var queue = new Queue<string>(states);
|
||||
var last = states[states.Length - 1];
|
||||
return () => queue.Count > 0 ? queue.Dequeue() : last;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OobeFlip_emits_exactly_once_with_payload()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var trace = new TraceCapture();
|
||||
// Baseline sample happens synchronously inside Start()/ArmTimer.
|
||||
using var detector = BuildFlipDetector(tmp, trace, StateQueue("in_progress", "completed"));
|
||||
|
||||
detector.Start();
|
||||
detector.Stop();
|
||||
detector.PollOnceForTest(); // reads "completed" -> flip
|
||||
detector.PollOnceForTest(); // still "completed" -> no second emit
|
||||
|
||||
Assert.Equal(1, trace.CountOf(SharedEventTypes.OobeStateCompleted));
|
||||
var flip = trace.Events.Find(e => e.EventType == SharedEventTypes.OobeStateCompleted);
|
||||
Assert.Equal("in_progress", flip.Data["previousState"]);
|
||||
Assert.True(flip.Data.ContainsKey("pollCount"));
|
||||
Assert.True(flip.Data.ContainsKey("minutesSinceStart"));
|
||||
Assert.False((bool)flip.Data["desktopArrived"]);
|
||||
Assert.True(flip.Data.ContainsKey("wmiErrorCount"));
|
||||
Assert.True(flip.Data.ContainsKey("wtsErrorCount"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OobeFlip_does_not_emit_when_state_is_already_completed()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var trace = new TraceCapture();
|
||||
using var detector = BuildFlipDetector(tmp, trace, StateQueue("completed"));
|
||||
|
||||
detector.Start();
|
||||
detector.Stop();
|
||||
detector.PollOnceForTest();
|
||||
detector.PollOnceForTest();
|
||||
|
||||
// Productive machines read "completed" from the very first sample — the flip
|
||||
// event must never fire there (the agent_started field already records the state).
|
||||
Assert.Equal(0, trace.CountOf(SharedEventTypes.OobeStateCompleted));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OobeFlip_does_not_emit_when_in_progress_was_never_seen()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var trace = new TraceCapture();
|
||||
using var detector = BuildFlipDetector(tmp, trace, StateQueue("unavailable", "completed"));
|
||||
|
||||
detector.Start();
|
||||
detector.Stop();
|
||||
detector.PollOnceForTest();
|
||||
|
||||
// Old builds without the WinRT contract report "unavailable"; a later
|
||||
// "completed" without a positively observed "in_progress" is not a flip.
|
||||
Assert.Equal(0, trace.CountOf(SharedEventTypes.OobeStateCompleted));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OobeFlip_survives_transient_unavailable_between_polls()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var trace = new TraceCapture();
|
||||
using var detector = BuildFlipDetector(tmp, trace, StateQueue("in_progress", "unavailable", "completed"));
|
||||
|
||||
detector.Start();
|
||||
detector.Stop();
|
||||
detector.PollOnceForTest(); // transient reflection failure — must NOT clear the baseline
|
||||
detector.PollOnceForTest(); // "completed" -> flip still detected
|
||||
|
||||
Assert.Equal(1, trace.CountOf(SharedEventTypes.OobeStateCompleted));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OobeFlip_provider_throwing_never_emits_and_never_pollutes_wmi_error_count()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var trace = new TraceCapture();
|
||||
using var detector = BuildFlipDetector(tmp, trace, () => throw new System.InvalidOperationException("boom"));
|
||||
|
||||
detector.Start(); // baseline sample throws -> swallowed
|
||||
detector.Stop();
|
||||
detector.PollOnceForTest();
|
||||
detector.PollOnceForTest();
|
||||
|
||||
Assert.Equal(0, trace.CountOf(SharedEventTypes.OobeStateCompleted));
|
||||
// The flip check has its own try/catch — a throw must not fall into
|
||||
// PollForDesktop's outer catch, which increments the WMI error counter the
|
||||
// session-4d5a0b78 liveness diagnostics rely on. first_poll carries that counter.
|
||||
var firstPoll = trace.Events.Find(e => e.EventType == SharedEventTypes.DesktopDetectorFirstPoll);
|
||||
Assert.Equal(0, firstPoll.Data["wmiErrorCount"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OobeFlip_one_shot_flag_survives_ResetForRealUserSwitch()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var trace = new TraceCapture();
|
||||
using var detector = BuildFlipDetector(tmp, trace,
|
||||
StateQueue("in_progress", "completed", "in_progress", "completed"));
|
||||
|
||||
detector.Start();
|
||||
detector.Stop();
|
||||
detector.PollOnceForTest(); // flip #1
|
||||
detector.ResetForRealUserSwitch(); // re-arms + re-samples baseline ("in_progress")
|
||||
detector.Stop();
|
||||
detector.PollOnceForTest(); // "completed" again — must NOT re-emit
|
||||
|
||||
Assert.Equal(1, trace.CountOf(SharedEventTypes.OobeStateCompleted));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+140
-3
@@ -40,7 +40,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather
|
||||
try { Directory.Delete(_outsideUsers, recursive: true); } catch { /* best effort */ }
|
||||
}
|
||||
|
||||
private GatherRuleContext Context(bool unrestricted = false, string imeLogPathOverride = null)
|
||||
private GatherRuleContext Context(bool unrestricted = false, string? imeLogPathOverride = null)
|
||||
=> new GatherRuleContext(
|
||||
new AgentLogger(_tmp.Path, AgentLogLevel.Info),
|
||||
"sess", "tenant",
|
||||
@@ -52,13 +52,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather
|
||||
};
|
||||
|
||||
private static GatherRule Rule(string collectorType, string target,
|
||||
Dictionary<string, string> parameters = null) => new GatherRule
|
||||
Dictionary<string, string>? parameters = null) => new GatherRule
|
||||
{
|
||||
RuleId = "GATHER-GUARD-TEST",
|
||||
Title = "guard test",
|
||||
CollectorType = collectorType,
|
||||
Target = target,
|
||||
Parameters = parameters,
|
||||
// Match the model default (empty, never null) so the collectors see the
|
||||
// same shape a deserialized rule without parameters gives them.
|
||||
Parameters = parameters ?? new Dictionary<string, string>(),
|
||||
Trigger = "startup",
|
||||
OutputEventType = "guard_test",
|
||||
};
|
||||
@@ -266,6 +268,141 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather
|
||||
$"expected the hung command to be killed at the timeout; took {sw.Elapsed}");
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// Every remaining collector actually calls its guard
|
||||
//
|
||||
// The guard functions have their own unit tests, but those prove nothing about
|
||||
// whether a collector consults them before touching the registry, WMI or disk.
|
||||
// These drive the collectors themselves: each blocked target must come back empty,
|
||||
// emit exactly one security_warning naming the target, and produce no rule output.
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
private void AssertBlocked(Dictionary<string, object> result, string target)
|
||||
{
|
||||
Assert.Empty(result);
|
||||
var warning = SingleSecurityWarning();
|
||||
Assert.Equal(true, warning.Data["blocked"]);
|
||||
Assert.Equal(target, warning.Data["target"]);
|
||||
Assert.DoesNotContain(_events, e => e.EventType == "guard_test");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(@"SAM\SAM\Domains\Account\Users")]
|
||||
[InlineData(@"SECURITY\Policy\Secrets")]
|
||||
[InlineData(@"HKLM\SOFTWARE\Microsoft\EnrollmentsSomethingElse")] // prefix spoofing
|
||||
[InlineData(@"HKLM\SOFTWARE\Microsoft\Windows\AutopilotSomethingElse")] // sibling spoof of the Autopilot prefix
|
||||
public void RegistryCollector_OffAllowlistPath_IsBlockedBeforeTheHiveIsOpened(string target)
|
||||
=> AssertBlocked(new RegistryCollector().Execute(Rule("registry", target), Context()), target);
|
||||
|
||||
// Session 5d735290: the EnrollmentStatusTracking subtree (ESP policy providers / tracking
|
||||
// state) is allow-listed so tenants can build gather rules on the co-management stall
|
||||
// signature the esp_policy_provider_stalled detector reports (e.g. not_exists on the
|
||||
// Sidecar provider key).
|
||||
[Theory]
|
||||
[InlineData(@"SOFTWARE\Microsoft\Windows\Autopilot")]
|
||||
[InlineData(@"SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking\Device\Setup\Apps\PolicyProviders\Sidecar")]
|
||||
public void RegistryGuard_AutopilotEnrollmentTrackingSubtree_IsAllowed(string subPath)
|
||||
=> Assert.True(GatherRuleGuards.IsRegistryPathAllowed(subPath));
|
||||
|
||||
[Theory]
|
||||
[InlineData("SELECT * FROM Win32_UserAccount")]
|
||||
[InlineData("SELECT * FROM Win32_ShadowCopy")]
|
||||
[InlineData("SELECT * FROM Win32_OperatingSystemExtra")] // prefix spoofing
|
||||
public void WmiCollector_OffAllowlistQuery_IsBlockedBeforeTheQueryRuns(string query)
|
||||
=> AssertBlocked(new WmiCollector().Execute(Rule("wmi", query), Context()), query);
|
||||
|
||||
[Fact]
|
||||
public void JsonCollector_UserProfilePath_IsBlockedBeforeTheFileIsOpened()
|
||||
{
|
||||
const string target = @"C:\Users\Public\Documents\secrets.json";
|
||||
AssertBlocked(
|
||||
new JsonCollector().Execute(
|
||||
Rule("json", target, new Dictionary<string, string> { ["jsonpath"] = "$.token" }),
|
||||
Context()),
|
||||
target);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void XmlCollector_UserProfilePath_IsBlockedBeforeTheFileIsOpened()
|
||||
{
|
||||
const string target = @"C:\Users\Public\Documents\secrets.xml";
|
||||
AssertBlocked(
|
||||
new XmlCollector().Execute(
|
||||
Rule("xml", target, new Dictionary<string, string> { ["xpath"] = "//token" }),
|
||||
Context()),
|
||||
target);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FileCollector_UserProfilePath_IsBlockedBeforeTheFileIsOpened()
|
||||
{
|
||||
const string target = @"C:\Users\Public\Documents\secrets.txt";
|
||||
AssertBlocked(new FileCollector().Execute(Rule("file", target), Context()), target);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JsonCollector_AllowedPath_ReadsTheFile()
|
||||
{
|
||||
// Counterpart to the blocked cases: proves the guard is a gate, not a wall —
|
||||
// the collector does reach the file once the path is admitted.
|
||||
var path = Path.Combine(_outsideUsers, "inventory.json");
|
||||
File.WriteAllText(path, "{ \"agent\": { \"version\": \"2.4.0\" } }");
|
||||
|
||||
var result = new JsonCollector().Execute(
|
||||
Rule("json", path, new Dictionary<string, string> { ["jsonpath"] = "$.agent.version" }),
|
||||
Context(unrestricted: true));
|
||||
|
||||
Assert.DoesNotContain(_events, e => e.EventType == Constants.EventTypes.SecurityWarning);
|
||||
Assert.Contains("2.4.0", string.Join(",", result.Values));
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// Path traversal
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void FilePath_TraversalOutOfAnAllowedPrefix_IsBlocked()
|
||||
{
|
||||
// The raw string starts with an allowed prefix, so a naive StartsWith check would
|
||||
// admit it. Path.GetFullPath runs FIRST, so the allowlist sees C:\Secret.txt.
|
||||
const string traversal =
|
||||
@"C:\ProgramData\Microsoft\IntuneManagementExtension\Logs\..\..\..\..\Secret.txt";
|
||||
|
||||
Assert.Equal(@"C:\Secret.txt", Path.GetFullPath(traversal));
|
||||
Assert.False(GatherRuleGuards.IsFilePathAllowed(traversal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilePath_TraversalIntoTheUsersHardBlock_StaysBlockedEvenUnrestricted()
|
||||
{
|
||||
// Unrestricted mode lifts the allowlist but never the C:\Users privacy block —
|
||||
// and it must not be reachable by spelling the path with '..' either.
|
||||
const string traversal =
|
||||
@"C:\ProgramData\Microsoft\IntuneManagementExtension\Logs\..\..\..\..\Users\Public\ntuser.dat";
|
||||
|
||||
Assert.StartsWith(@"C:\Users\", Path.GetFullPath(traversal), StringComparison.OrdinalIgnoreCase);
|
||||
Assert.False(GatherRuleGuards.IsFilePathAllowed(traversal));
|
||||
Assert.False(GatherRuleGuards.IsFilePathAllowed(traversal, unrestrictedMode: true));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FilePath_TraversalIntoTheConfigHiveDirectory_StaysBlockedEvenUnrestricted()
|
||||
{
|
||||
const string traversal = @"C:\Windows\Logs\..\System32\config\SAM";
|
||||
|
||||
Assert.False(GatherRuleGuards.IsFilePathAllowed(traversal));
|
||||
Assert.False(GatherRuleGuards.IsFilePathAllowed(traversal, unrestrictedMode: true));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FileCollector_TraversalTarget_IsBlockedAndReported()
|
||||
{
|
||||
const string traversal =
|
||||
@"C:\ProgramData\Microsoft\IntuneManagementExtension\Logs\..\..\..\..\Users\Public\ntuser.dat";
|
||||
|
||||
AssertBlocked(new FileCollector().Execute(Rule("file", traversal), Context()), traversal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LogParser_ImeLogPathOverride_StillHonoursTheUsersHardBlock()
|
||||
{
|
||||
|
||||
+1
-1
@@ -192,7 +192,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather
|
||||
GatherRuleExecutor.ComputeCanonicalHash(new Dictionary<string, object> { ["v"] = "1" }),
|
||||
GatherRuleExecutor.ComputeCanonicalHash(new Dictionary<string, object> { ["v"] = 1 }));
|
||||
Assert.NotEqual(
|
||||
GatherRuleExecutor.ComputeCanonicalHash(new Dictionary<string, object> { ["v"] = null }),
|
||||
GatherRuleExecutor.ComputeCanonicalHash(new Dictionary<string, object> { ["v"] = null! }),
|
||||
GatherRuleExecutor.ComputeCanonicalHash(new Dictionary<string, object> { ["v"] = "<null>" }));
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -60,7 +60,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather
|
||||
}
|
||||
|
||||
private static GatherRule ExitRule(
|
||||
string id, string triggerPhase, List<string> activePhases = null, string fromPhase = null)
|
||||
string id, string triggerPhase, List<string>? activePhases = null, string? fromPhase = null)
|
||||
=> new GatherRule
|
||||
{
|
||||
RuleId = id,
|
||||
|
||||
+2
-2
@@ -63,7 +63,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather
|
||||
|
||||
/// <summary>Interval rule with a 1h period — the timer never ticks within a test run,
|
||||
/// so these rules exist purely for synchronous scope-matrix assertions.</summary>
|
||||
private static GatherRule ScopeProbeRule(string id, List<string> activePhases = null, string fromPhase = null)
|
||||
private static GatherRule ScopeProbeRule(string id, List<string>? activePhases = null, string? fromPhase = null)
|
||||
=> new GatherRule
|
||||
{
|
||||
RuleId = id,
|
||||
@@ -78,7 +78,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Gather
|
||||
ActiveFromPhase = fromPhase,
|
||||
};
|
||||
|
||||
private static GatherRule StartupRule(string id, List<string> activePhases = null, string fromPhase = null)
|
||||
private static GatherRule StartupRule(string id, List<string>? activePhases = null, string? fromPhase = null)
|
||||
=> new GatherRule
|
||||
{
|
||||
RuleId = id,
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
[InlineData(" ")] // whitespace only
|
||||
[InlineData("<![LOG[truncated line with no closing")] // truncated / missing fields
|
||||
[InlineData("garbage prefix <![LOG[msg]LOG]!><time=\"06:08:04.8834397\" date=\"2-8-2026\" component=\"C\" context=\"\" type=\"1\" thread=\"1\" file=\"\">")] // leading junk (non-ignorable) fails the StartsWith gate
|
||||
public void TryParseLine_returns_false_without_throwing_for_malformed_input(string line)
|
||||
public void TryParseLine_returns_false_without_throwing_for_malformed_input(string? line)
|
||||
{
|
||||
var ok = CmTraceLogParser.TryParseLine(line, out var entry);
|
||||
|
||||
|
||||
+3
-2
@@ -24,12 +24,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
/// </summary>
|
||||
public sealed class ImeLogPatternReDoSTests
|
||||
{
|
||||
private static ImeLogPattern Pattern(string id, string regex, string action = null) => new ImeLogPattern
|
||||
private static ImeLogPattern Pattern(string id, string regex, string? action = null) => new ImeLogPattern
|
||||
{
|
||||
PatternId = id,
|
||||
Pattern = regex,
|
||||
Category = "always",
|
||||
Action = action,
|
||||
// Action is optional on the wire (absent = null); the model declares it non-nullable.
|
||||
Action = action!,
|
||||
Enabled = true,
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
};
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
{
|
||||
/// <summary>
|
||||
/// Tracker-level historic-replay guard (session eaf3d8c4 part 2): source lines > 24 h older
|
||||
/// than now are content from a previous enrollment whose IME log survived on disk. The guard
|
||||
/// in <c>HandlePatternMatch</c> skips app-mutating actions for such lines so replayed apps
|
||||
/// never enter <c>_packageStates</c> / phase snapshots / persistence — keeping
|
||||
/// app_tracking_summary, culprit lists and final-status clean. Script actions pass through
|
||||
/// (the adapter suppresses their emissions), and SimulationMode bypasses the guard entirely.
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerHistoricReplayGuardTests
|
||||
{
|
||||
private static readonly DateTime Now = new DateTime(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc);
|
||||
|
||||
private static List<ImeLogPattern> TestPatterns() => new List<ImeLogPattern>
|
||||
{
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-POLICIES", Category = "always", Enabled = true,
|
||||
Pattern = @"policies discovered = (?<policies>\[.*\])",
|
||||
Action = "policiesDiscovered",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-INSTALLED", Category = "always", Enabled = true,
|
||||
Pattern = @"app (?<id>[\w-]+) installed",
|
||||
Action = "updateStateInstalled",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-ESP-PHASE", Category = "always", Enabled = true,
|
||||
Pattern = @"In EspPhase: (?<espPhase>\w+)",
|
||||
Action = "espPhaseDetected",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-SCRIPT-START", Category = "always", Enabled = true,
|
||||
Pattern = @"script (?<id>[\w-]+) started",
|
||||
Action = "scriptStarted",
|
||||
Parameters = new Dictionary<string, string> { ["scriptType"] = "platform" },
|
||||
},
|
||||
};
|
||||
|
||||
private static ImeLogTracker BuildTracker(TempDirectory tmp)
|
||||
{
|
||||
var tracker = new ImeLogTracker(
|
||||
logFolder: tmp.Path,
|
||||
patterns: TestPatterns(),
|
||||
logger: new AgentLogger(tmp.Path, AgentLogLevel.Info));
|
||||
tracker.UtcNowProvider = () => Now;
|
||||
return tracker;
|
||||
}
|
||||
|
||||
private const string PoliciesLine = @"policies discovered = [{""Id"":""app-1"",""Name"":""App One""}]";
|
||||
|
||||
[Fact]
|
||||
public void Stale_policies_line_does_not_populate_package_states()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddDays(-7));
|
||||
|
||||
Assert.Empty(tracker.GetAllKnownPackageStates());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Fresh_relog_of_same_policies_is_tracked_after_stale_skip()
|
||||
{
|
||||
// IME re-logs the full policy JSON on every app-policy check-in — skipping the
|
||||
// stale copy cannot starve the current enrollment.
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddDays(-7));
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddMinutes(-2));
|
||||
|
||||
var all = tracker.GetAllKnownPackageStates();
|
||||
var pkg = Assert.Single(all);
|
||||
Assert.Equal("app-1", pkg.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_state_update_against_fresh_app_is_skipped()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
var stateChanges = 0;
|
||||
tracker.OnAppStateChanged = (_, __, ___) => stateChanges++;
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddMinutes(-5)); // fresh discovery
|
||||
tracker.ProcessLogMessageForTest("app app-1 installed", Now.AddDays(-7)); // stale mutation
|
||||
|
||||
Assert.Equal(0, stateChanges);
|
||||
var pkg = Assert.Single(tracker.GetAllKnownPackageStates());
|
||||
Assert.NotEqual(AppInstallationState.Installed, pkg.InstallationState);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_esp_phase_line_does_not_block_fresh_phase_detection()
|
||||
{
|
||||
// A stale "AccountSetup" would advance _currentPhaseOrder and make the fresh
|
||||
// enrollment's DeviceSetup bounce as "backward" — the guard must skip it.
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
var phases = new List<string>();
|
||||
tracker.OnEspPhaseChanged = p => phases.Add(p);
|
||||
|
||||
tracker.ProcessLogMessageForTest("In EspPhase: AccountSetup", Now.AddDays(-7));
|
||||
tracker.ProcessLogMessageForTest("In EspPhase: DeviceSetup", Now.AddMinutes(-1));
|
||||
|
||||
Assert.Equal(new[] { "DeviceSetup" }, phases);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_script_line_still_reaches_script_handlers()
|
||||
{
|
||||
// Script tracker state is harmless for stale lines (adapter suppresses the
|
||||
// emissions; the stale-slot hardening covers leftovers) — the guard must not
|
||||
// swallow script actions.
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
var started = new List<ScriptStartedInfo>();
|
||||
tracker.OnScriptStarted = i => started.Add(i);
|
||||
|
||||
tracker.ProcessLogMessageForTest("script aaaa1111-0000-0000-0000-000000000000 started", Now.AddDays(-7));
|
||||
|
||||
Assert.Single(started);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimulationMode_bypasses_the_guard()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
tracker.SimulationMode = true;
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, Now.AddDays(-7));
|
||||
|
||||
Assert.Single(tracker.GetAllKnownPackageStates());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Lines_without_timestamp_are_not_treated_as_stale()
|
||||
{
|
||||
// Non-CMTrace lines carry no source timestamp (entry == null) — never stale.
|
||||
using var tmp = new TempDirectory();
|
||||
using var tracker = BuildTracker(tmp);
|
||||
|
||||
tracker.ProcessLogMessageForTest(PoliciesLine, sourceTimestampUtc: null);
|
||||
|
||||
Assert.Single(tracker.GetAllKnownPackageStates());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#nullable enable
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Smoke tests for the WinRT <c>SystemSetupInfo.OutOfBoxExperienceState</c> reader.
|
||||
/// The actual state on the test machine is unknown (CI runners are post-OOBE; Server
|
||||
/// SKUs may lack the UniversalApiContract entirely), so only the fail-soft contract
|
||||
/// is pinned: never throw, always return a member of the known value set.
|
||||
/// </summary>
|
||||
public sealed class OobeStateReaderTests
|
||||
{
|
||||
[Fact]
|
||||
public void Read_never_throws_and_returns_known_value()
|
||||
{
|
||||
var value = OobeStateReader.Read();
|
||||
|
||||
Assert.False(string.IsNullOrEmpty(value));
|
||||
Assert.True(
|
||||
value == "not_started"
|
||||
|| value == "in_progress"
|
||||
|| value == "completed"
|
||||
|| value == OobeStateReader.Unavailable
|
||||
|| value.StartsWith("unknown_"),
|
||||
$"unexpected reader value '{value}'");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Read_is_stable_across_repeated_calls()
|
||||
{
|
||||
// Exercises the cached-PropertyInfo path (second call skips resolution).
|
||||
var first = OobeStateReader.Read();
|
||||
var second = OobeStateReader.Read();
|
||||
|
||||
Assert.Equal(first, second);
|
||||
}
|
||||
}
|
||||
}
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
{
|
||||
/// <summary>
|
||||
/// Executes the generated cleanup script against REAL scratch trees with a real
|
||||
/// powershell.exe — end-to-end replays of the 2026-07-22 incident mechanics that the
|
||||
/// string pins in <see cref="CleanupServiceScriptTests"/> cannot prove:
|
||||
/// <list type="bullet">
|
||||
/// <item>a locked tree must abort WITHOUT deleting anything (old script force-deleted
|
||||
/// marker + state around the locks and orphaned the binaries),</item>
|
||||
/// <item>a straggler agent process (the incident's second instance) must be found by
|
||||
/// NAME, killed, and the cleanup must then complete — simulated via function
|
||||
/// shadowing, NEVER by launching a renamed executable (EDR isolation, see the test),</item>
|
||||
/// <item>concurrent cleanup scripts must be serialized by the lock file.</item>
|
||||
/// </list>
|
||||
/// Timings are dialed down via the test-only BuildCleanupScript parameters so each case
|
||||
/// runs in seconds. Every test uses GUID-unique process/task/tree names so nothing real
|
||||
/// (a dev-box agent, a real Scheduled Task) can ever be touched.
|
||||
/// </summary>
|
||||
[Collection("SerialThreading")]
|
||||
public class CleanupServiceScriptIntegrationTests : IDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
private readonly string _basePath;
|
||||
private readonly string _taskName = $"AmTest-NoSuchTask-{Guid.NewGuid():N}";
|
||||
private readonly string _fakeProcessName = $"AmTestAgent{Guid.NewGuid().ToString("N").Substring(0, 8)}";
|
||||
|
||||
public CleanupServiceScriptIntegrationTests()
|
||||
{
|
||||
_root = Path.Combine(Path.GetTempPath(), $"am-cleanup-it-{Guid.NewGuid():N}");
|
||||
_basePath = Path.Combine(_root, "AutopilotMonitor");
|
||||
Directory.CreateDirectory(Path.Combine(_basePath, "Agent"));
|
||||
Directory.CreateDirectory(Path.Combine(_basePath, "State"));
|
||||
Directory.CreateDirectory(Path.Combine(_basePath, "Logs"));
|
||||
File.WriteAllText(Path.Combine(_basePath, "Agent", "fake.exe"), "x");
|
||||
File.WriteAllText(Path.Combine(_basePath, "State", "enrollment-complete.marker"), "x");
|
||||
File.WriteAllText(Path.Combine(_basePath, "Logs", "agent.log"), "x");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try { Directory.Delete(_root, recursive: true); } catch { }
|
||||
}
|
||||
|
||||
private static string LockPath => Path.Combine(Path.GetTempPath(), "AutopilotMonitor-Cleanup.lock");
|
||||
|
||||
private string BuildScript(bool keepLogs = false, int processWaitSeconds = 1)
|
||||
=> CleanupService.BuildCleanupScript(
|
||||
agentProcessName: _fakeProcessName,
|
||||
agentBasePath: _basePath,
|
||||
keepLogs: keepLogs,
|
||||
scheduledTaskName: _taskName,
|
||||
rebootOnComplete: false,
|
||||
initialDelaySeconds: 0,
|
||||
processWaitSeconds: processWaitSeconds,
|
||||
renameRetries: 2,
|
||||
renameRetryDelaySeconds: 1);
|
||||
|
||||
/// <summary>Runs the script synchronously in a real powershell.exe. Returns the script path.</summary>
|
||||
private string RunScript(string script)
|
||||
{
|
||||
var scriptPath = Path.Combine(_root, $"cleanup-{Guid.NewGuid():N}.ps1");
|
||||
File.WriteAllText(scriptPath, script);
|
||||
using (var proc = Process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = SystemPaths.PowerShell,
|
||||
Arguments = $"-NoProfile -ExecutionPolicy Bypass -File \"{scriptPath}\"",
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
}))
|
||||
{
|
||||
Assert.True(proc.WaitForExit(90_000), "cleanup script did not finish within 90s");
|
||||
}
|
||||
return scriptPath;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Happy_path_deletes_tree_and_removes_lock_and_script()
|
||||
{
|
||||
var scriptPath = RunScript(BuildScript());
|
||||
|
||||
Assert.False(Directory.Exists(_basePath), "tree should be fully deleted");
|
||||
Assert.Empty(Directory.GetFileSystemEntries(_root, "*.del-*"));
|
||||
Assert.False(File.Exists(scriptPath), "script should self-delete");
|
||||
Assert.False(File.Exists(LockPath), "cleanup lock should be released");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Locked_tree_aborts_completely_marker_and_files_survive()
|
||||
{
|
||||
// The incident regression: with a file locked inside the tree the OLD script
|
||||
// deleted everything else (incl. the enrollment-complete marker) and left the
|
||||
// binaries orphaned with the task already gone. The hardened script must treat
|
||||
// the failed probe-rename as "someone still holds this" and walk away entirely.
|
||||
using (File.Open(Path.Combine(_basePath, "Agent", "fake.exe"),
|
||||
FileMode.Open, FileAccess.Read, FileShare.None))
|
||||
{
|
||||
RunScript(BuildScript());
|
||||
}
|
||||
|
||||
Assert.True(File.Exists(Path.Combine(_basePath, "Agent", "fake.exe")), "binaries must stay in place");
|
||||
Assert.True(File.Exists(Path.Combine(_basePath, "State", "enrollment-complete.marker")),
|
||||
"marker must survive so the next boot retries cleanup");
|
||||
Assert.Empty(Directory.GetFileSystemEntries(_root, "*.del-*"));
|
||||
Assert.False(File.Exists(LockPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Straggler_agent_process_is_waited_on_and_force_killed_via_name_backstop()
|
||||
{
|
||||
// Incident replay of the second-instance mechanics WITHOUT spawning a process:
|
||||
// an earlier version of this test copied powershell.exe under the fake agent name
|
||||
// and launched it — textbook MITRE T1036.003 ("Rename Legitimate Utilities"),
|
||||
// which Defender for Endpoint answered with a machine ISOLATION on the dev box
|
||||
// (see tasks/lessons.md). Instead, a harness script shadows Get-Process /
|
||||
// Stop-Process with functions (PowerShell resolves functions before cmdlets, so
|
||||
// the dot-sourced cleanup script binds to the stubs): Get-Process reports a live
|
||||
// agent-named straggler on every probe; Stop-Process records the kill. This pins
|
||||
// the same contract — the script polls by NAME until the deadline, then
|
||||
// force-kills by NAME before deleting — with zero EDR-visible behavior.
|
||||
var probeLog = Path.Combine(_root, "probe.log");
|
||||
var cleanupScriptPath = Path.Combine(_root, $"cleanup-{Guid.NewGuid():N}.ps1");
|
||||
File.WriteAllText(cleanupScriptPath, BuildScript(processWaitSeconds: 2));
|
||||
|
||||
var harness = @"
|
||||
$probeLog = 'LOG_PATH'
|
||||
$global:AmProbeCount = 0
|
||||
function Get-Process {
|
||||
[CmdletBinding()]
|
||||
param([string[]]$Name)
|
||||
if ($Name -contains 'FAKE_NAME') {
|
||||
$global:AmProbeCount++
|
||||
Add-Content -Path $probeLog -Value ('probe ' + $global:AmProbeCount)
|
||||
return [pscustomobject]@{ Name = 'FAKE_NAME'; Id = 424242 }
|
||||
}
|
||||
Microsoft.PowerShell.Management\Get-Process @PSBoundParameters
|
||||
}
|
||||
function Stop-Process {
|
||||
[CmdletBinding()]
|
||||
param([Parameter(ValueFromPipeline = $true)]$InputObject, [switch]$Force)
|
||||
process { Add-Content -Path $probeLog -Value ('kill ' + $InputObject.Id + ' force=' + $Force) }
|
||||
}
|
||||
. 'SCRIPT_PATH'
|
||||
"
|
||||
.Replace("LOG_PATH", probeLog)
|
||||
.Replace("FAKE_NAME", _fakeProcessName)
|
||||
.Replace("SCRIPT_PATH", cleanupScriptPath);
|
||||
|
||||
RunScript(harness);
|
||||
|
||||
var log = File.ReadAllLines(probeLog);
|
||||
Assert.Contains("kill 424242 force=True", log);
|
||||
Assert.True(Array.FindAll(log, l => l.StartsWith("probe")).Length >= 2,
|
||||
"script must poll by name in the wait loop AND re-probe for the kill backstop");
|
||||
Assert.False(Directory.Exists(_basePath), "after the kill the tree must be fully deleted");
|
||||
Assert.Empty(Directory.GetFileSystemEntries(_root, "*.del-*"));
|
||||
Assert.False(File.Exists(LockPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Fresh_foreign_lock_backs_off_without_touching_the_tree()
|
||||
{
|
||||
File.WriteAllText(LockPath, "held by another cleanup");
|
||||
try
|
||||
{
|
||||
var scriptPath = RunScript(BuildScript());
|
||||
|
||||
Assert.True(File.Exists(Path.Combine(_basePath, "Agent", "fake.exe")),
|
||||
"loser script must not touch the tree — the lock holder owns the deletion");
|
||||
Assert.True(File.Exists(LockPath), "foreign lock must not be deleted by the loser");
|
||||
Assert.False(File.Exists(scriptPath), "loser still self-deletes its script");
|
||||
}
|
||||
finally
|
||||
{
|
||||
try { File.Delete(LockPath); } catch { }
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_foreign_lock_is_taken_over_and_cleanup_proceeds()
|
||||
{
|
||||
File.WriteAllText(LockPath, "crashed cleanup from a previous boot");
|
||||
File.SetLastWriteTime(LockPath, DateTime.Now.AddMinutes(-15));
|
||||
|
||||
RunScript(BuildScript());
|
||||
|
||||
Assert.False(Directory.Exists(_basePath), "stale lock must be taken over, cleanup must run");
|
||||
Assert.False(File.Exists(LockPath));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void KeepLogs_variant_deletes_everything_except_logs()
|
||||
{
|
||||
RunScript(BuildScript(keepLogs: true));
|
||||
|
||||
Assert.True(File.Exists(Path.Combine(_basePath, "Logs", "agent.log")), "Logs must survive");
|
||||
Assert.False(Directory.Exists(Path.Combine(_basePath, "Agent")));
|
||||
Assert.False(Directory.Exists(Path.Combine(_basePath, "State")));
|
||||
Assert.Empty(Directory.GetFileSystemEntries(_basePath, "*.del-*"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
{
|
||||
/// <summary>
|
||||
/// Pins the LIFE-F7 hardening invariants of the self-destruct cleanup script
|
||||
/// (incident 2026-07-22: a second agent instance held exe + loaded DLLs while the
|
||||
/// old script — waiting on the launcher PID only — force-deleted around the locks
|
||||
/// AFTER removing the Scheduled Task, orphaning the binaries with no retry path).
|
||||
/// These are string-level pins on <see cref="CleanupService.BuildCleanupScript"/>:
|
||||
/// weakening any of them silently re-introduces the orphaned-binaries failure mode.
|
||||
/// </summary>
|
||||
public class CleanupServiceScriptTests
|
||||
{
|
||||
private const string ProcessName = "AutopilotMonitor.Agent";
|
||||
private const string BasePath = @"C:\ProgramData\AutopilotMonitor";
|
||||
private const string TaskName = "AutopilotMonitorAgent";
|
||||
|
||||
private static string Script(bool keepLogs = false, bool reboot = false)
|
||||
=> CleanupService.BuildCleanupScript(ProcessName, BasePath, keepLogs, TaskName, reboot);
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public void Waits_by_process_name_not_launcher_pid(bool keepLogs)
|
||||
{
|
||||
var script = Script(keepLogs);
|
||||
|
||||
// LIFE-F7 #2: name-based wait covers marker-retry / self-update / manual instances
|
||||
// and is immune to PID reuse. Wait-Process -Id saw exactly one PID and nothing else.
|
||||
Assert.Contains($"Get-Process -Name '{ProcessName}'", script);
|
||||
Assert.DoesNotContain("Wait-Process", script);
|
||||
|
||||
// Backstop: stragglers are force-killed — their images are deleted right after.
|
||||
Assert.Contains("Stop-Process -Force", script);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public void Never_deletes_the_original_path(bool keepLogs)
|
||||
{
|
||||
var script = Script(keepLogs);
|
||||
|
||||
// LIFE-F7 #3: deletion happens exclusively on successfully renamed targets
|
||||
// ($entry.Renamed). A Remove-Item against the original tree is what deleted
|
||||
// marker + state around locked binaries in the incident.
|
||||
Assert.DoesNotContain($"Remove-Item -Path '{BasePath}'", script);
|
||||
Assert.Contains("Remove-Item -Path $entry.Renamed", script);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public void Task_removal_is_gated_on_probe_and_deletion_on_task_gone(bool keepLogs)
|
||||
{
|
||||
var script = Script(keepLogs);
|
||||
|
||||
// Probe-rename first; only a successful probe reaches task removal, and only a
|
||||
// removed task reaches deletion. Both failure branches rename BACK so the next
|
||||
// boot retries with task + marker + files intact.
|
||||
Assert.Contains("if ($probeOk)", script);
|
||||
Assert.Contains("if ($taskGone)", script);
|
||||
Assert.Contains("Rename-Item -Path $entry.Renamed -NewName $entry.Original", script);
|
||||
|
||||
// LIFE-F6 task-removal loop + module-independent fallback stay intact.
|
||||
Assert.Contains("Unregister-ScheduledTask -TaskName $taskName", script);
|
||||
Assert.Contains("schtasks.exe /Delete /TN $taskName /F", script);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public void Concurrent_cleanup_scripts_are_serialized_by_lock_file(bool keepLogs)
|
||||
{
|
||||
var script = Script(keepLogs);
|
||||
|
||||
// LIFE-F7 #1: CreateNew is the atomic single-flight gate; the loser exits without
|
||||
// touching the tree. Stale locks (>10 min) are taken over.
|
||||
Assert.Contains("AutopilotMonitor-Cleanup.lock", script);
|
||||
Assert.Contains("[System.IO.FileMode]::CreateNew", script);
|
||||
Assert.Contains("$lockAge.TotalMinutes -gt 10", script);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public void Rename_targets_carry_a_guid_suffix(bool keepLogs)
|
||||
{
|
||||
// A fixed '.del' target collides with leftovers of earlier attempts and with a
|
||||
// concurrent script's rename — the GUID makes every attempt's target unique.
|
||||
Assert.Contains(".del-' + [Guid]::NewGuid().ToString('N')", Script(keepLogs));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void KeepLogs_variant_excludes_logs_directory()
|
||||
{
|
||||
var script = Script(keepLogs: true);
|
||||
Assert.Contains("-Exclude 'Logs'", script);
|
||||
Assert.DoesNotContain($"Rename-Item -Path '{BasePath}' -NewName", script);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Full_variant_renames_the_whole_tree()
|
||||
{
|
||||
var script = Script(keepLogs: false);
|
||||
Assert.Contains($"Rename-Item -Path '{BasePath}' -NewName $renamedPath", script);
|
||||
Assert.DoesNotContain("-Exclude 'Logs'", script);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public void Reboot_flag_controls_shutdown_invocation(bool reboot)
|
||||
{
|
||||
var script = Script(keepLogs: false, reboot: reboot);
|
||||
if (reboot)
|
||||
Assert.Contains("shutdown.exe /r /t 10", script);
|
||||
else
|
||||
Assert.DoesNotContain("shutdown.exe", script);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public void Script_and_lock_are_released_in_finally(bool keepLogs)
|
||||
{
|
||||
var script = Script(keepLogs);
|
||||
Assert.Contains("finally", script);
|
||||
Assert.Contains("$lockStream.Dispose()", script);
|
||||
Assert.Contains("Remove-Item -Path $scriptPath -Force", script);
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -256,7 +256,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
[InlineData(null, false)]
|
||||
[InlineData("", false)]
|
||||
[InlineData("0x8007", false)]
|
||||
public void IsAppxFamilyHresult_MatchesOnlyAppxDeploymentRange(string hresult, bool expected)
|
||||
public void IsAppxFamilyHresult_MatchesOnlyAppxDeploymentRange(string? hresult, bool expected)
|
||||
{
|
||||
Assert.Equal(expected, AppxFailureAnalyzer.IsAppxFamilyHresult(hresult));
|
||||
}
|
||||
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Session 4910a5a5 (2026-07-23) — post-terminal recovery story. Field sequence:
|
||||
/// DeviceSetup/Apps failed (culprit app never started installing) → settle window expired
|
||||
/// → terminal EspFailureDetected → the user pressed "Try again" ~23 min later → Apps left
|
||||
/// the failed state → 10/10 installed → DeviceSetup resolved. Pre-fix, that recovery only
|
||||
/// appeared as a generic <c>subcategory_state_change</c>. Coverage:
|
||||
/// <list type="bullet">
|
||||
/// <item><c>esp_failure_retry_detected</c> fires once when the failed subcategory leaves
|
||||
/// the failed state AFTER the terminal fire.</item>
|
||||
/// <item><c>esp_failure_recovered</c> fires once when the category then resolves to
|
||||
/// success.</item>
|
||||
/// <item>An in-settle-window recovery keeps emitting only
|
||||
/// <c>esp_failure_settle_recovered</c> — the post-terminal pair stays silent.</item>
|
||||
/// <item>Apps failures carry <c>LikelyCulpritApps</c> on the EspFailureDetected args,
|
||||
/// never-started apps ranked first.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class EspFailureRecoveryStoryTests
|
||||
{
|
||||
private static readonly DateTime Fixed = new DateTime(2026, 7, 23, 4, 30, 0, DateTimeKind.Utc);
|
||||
|
||||
private sealed class Fixture : IDisposable
|
||||
{
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public FakeSignalIngressSink Sink { get; } = new FakeSignalIngressSink();
|
||||
public List<EspFailureDetectedEventArgs> EspFailures { get; } = new List<EspFailureDetectedEventArgs>();
|
||||
public ProvisioningStatusTracker Tracker { get; }
|
||||
|
||||
public Fixture(Func<IReadOnlyList<AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState>>? packageStatesProbe = null)
|
||||
{
|
||||
var clock = new VirtualClock(Fixed);
|
||||
var post = new InformationalEventPost(Sink, clock);
|
||||
var logger = new AgentLogger(Tmp.Path, AgentLogLevel.Info);
|
||||
Tracker = new ProvisioningStatusTracker(
|
||||
sessionId: "S1",
|
||||
tenantId: "T1",
|
||||
post: post,
|
||||
logger: logger,
|
||||
appxScanner: new FakeAppxDeploymentFailureScanner(),
|
||||
backgroundDispatcher: action => action(),
|
||||
packageStatesProbe: packageStatesProbe);
|
||||
Tracker.EspFailureDetected += (_, args) => EspFailures.Add(args);
|
||||
}
|
||||
|
||||
public IReadOnlyList<(string EventType, IReadOnlyDictionary<string, object> Data)> CapturedEvents()
|
||||
{
|
||||
return Sink.Posted
|
||||
.Where(p => p.Kind == DecisionSignalKind.InformationalEvent && p.Payload != null)
|
||||
.Select(p =>
|
||||
{
|
||||
var eventType = p.Payload!.TryGetValue(SignalPayloadKeys.EventType, out var et) ? et : "";
|
||||
var data = p.TypedPayload as IReadOnlyDictionary<string, object>
|
||||
?? (IReadOnlyDictionary<string, object>)new Dictionary<string, object>();
|
||||
return (eventType, data);
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>Drive the session-4910a5a5 shape up to the terminal fire.</summary>
|
||||
public void DriveToTerminalDeviceSetupAppsFailure()
|
||||
{
|
||||
Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (9 of 10 installed)""}
|
||||
}");
|
||||
Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (Error)""}
|
||||
}");
|
||||
Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Tracker.Dispose();
|
||||
Tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PostTerminalRetry_EmitsRetryDetected_Once()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
f.DriveToTerminalDeviceSetupAppsFailure();
|
||||
Assert.Single(f.EspFailures);
|
||||
|
||||
// "Try again": Apps leaves the failed state well after the settle window.
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""}
|
||||
}");
|
||||
|
||||
var retry = Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_retry_detected");
|
||||
Assert.Equal("DeviceSetup", (string)retry.Data["category"]);
|
||||
Assert.Equal("Apps", (string)retry.Data["subcategory"]);
|
||||
Assert.Equal("failed", (string)retry.Data["previousState"]);
|
||||
Assert.Equal("inProgress", (string)retry.Data["newState"]);
|
||||
Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", (string)retry.Data["failureType"]);
|
||||
Assert.True(retry.Data.ContainsKey("minutesSinceFailure"));
|
||||
|
||||
// A later unrelated transition does not re-emit.
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed)""}
|
||||
}");
|
||||
Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_retry_detected");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PostTerminalRecovery_EmitsRecovered_OnCategorySuccess()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
f.DriveToTerminalDeviceSetupAppsFailure();
|
||||
|
||||
// Retry → all apps installed → Windows resolves the category boolean.
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""}
|
||||
}");
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": true,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed)""}
|
||||
}");
|
||||
|
||||
var recovered = Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_recovered");
|
||||
Assert.Equal("DeviceSetup", (string)recovered.Data["category"]);
|
||||
Assert.Equal("Apps", (string)recovered.Data["failedSubcategory"]);
|
||||
Assert.Equal("Provisioning_DeviceSetup_Apps_Failed", (string)recovered.Data["failureType"]);
|
||||
Assert.True(recovered.Data.ContainsKey("minutesSinceFailure"));
|
||||
|
||||
// Duplicate category-success writes do not re-emit (fire-once).
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": true,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Security policies (1 of 1 applied)""},
|
||||
""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (10 of 10 installed, take 2)""}
|
||||
}");
|
||||
Assert.Single(f.CapturedEvents(), e => e.EventType == "esp_failure_recovered");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InSettleWindowRecovery_KeepsPostTerminalPairSilent()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (0 of 1 installed)""}
|
||||
}");
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (Error)""}
|
||||
}");
|
||||
// Recovery INSIDE the settle window — terminal fire is suppressed.
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""AppsSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Apps (Working on it...)""}
|
||||
}");
|
||||
f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status");
|
||||
Assert.Empty(f.EspFailures);
|
||||
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": true,
|
||||
""AppsSubcategory"": {""subcategoryState"":""succeeded"",""subcategoryStatusText"":""Apps (1 of 1 installed)""}
|
||||
}");
|
||||
|
||||
var events = f.CapturedEvents();
|
||||
Assert.Contains(events, e => e.EventType == "esp_failure_settle_recovered");
|
||||
Assert.DoesNotContain(events, e => e.EventType == "esp_failure_retry_detected");
|
||||
Assert.DoesNotContain(events, e => e.EventType == "esp_failure_recovered");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AppsFailure_TerminalArgs_CarryLikelyCulprits_NeverStartedFirst()
|
||||
{
|
||||
var neverStarted = BuildApp("app-bg", "Teams Backgrounds v2",
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown,
|
||||
downloadingSeen: false);
|
||||
var startedButPending = BuildApp("app-slow", "Slow Installer",
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown,
|
||||
downloadingSeen: true);
|
||||
var installed = BuildApp("app-ok", "Falcon Sensor",
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Installed,
|
||||
downloadingSeen: true);
|
||||
using var f = new Fixture(() => new[] { startedButPending, neverStarted, installed });
|
||||
|
||||
f.DriveToTerminalDeviceSetupAppsFailure();
|
||||
|
||||
var fired = Assert.Single(f.EspFailures);
|
||||
Assert.Equal(2, fired.LikelyCulpritApps.Count);
|
||||
// Never-started app ranked first — it starved the ESP gate without any IME event.
|
||||
Assert.Equal("Teams Backgrounds v2", fired.LikelyCulpritApps[0]);
|
||||
Assert.Equal("Slow Installer", fired.LikelyCulpritApps[1]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonAppsFailure_TerminalArgs_CarryNoCulprits()
|
||||
{
|
||||
var neverStarted = BuildApp("app-bg", "Teams Backgrounds v2",
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown,
|
||||
downloadingSeen: false);
|
||||
using var f = new Fixture(() => new[] { neverStarted });
|
||||
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""inProgress"",""subcategoryStatusText"":""Working""}
|
||||
}");
|
||||
f.Tracker.ProcessCategoryStatusForTest("DeviceSetupCategory.Status", @"{
|
||||
""categorySucceeded"": null,
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Failed""}
|
||||
}");
|
||||
f.Tracker.TriggerSettleTimerForTest("DeviceSetupCategory.Status");
|
||||
|
||||
var fired = Assert.Single(f.EspFailures);
|
||||
Assert.Empty(fired.LikelyCulpritApps);
|
||||
}
|
||||
|
||||
private static AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState BuildApp(
|
||||
string id, string name,
|
||||
AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState state,
|
||||
bool downloadingSeen)
|
||||
{
|
||||
var pkg = new AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState(id, 0);
|
||||
pkg.UpdateName(name);
|
||||
pkg.UpdateIntent(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppIntent.Install);
|
||||
pkg.UpdateTargeted(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppTargeted.Device);
|
||||
// DownloadingOrInstallingSeen is derived inside UpdateState (Downloading..Installing
|
||||
// range) — drive it through a Downloading transition instead of setting it directly.
|
||||
if (downloadingSeen)
|
||||
pkg.UpdateState(AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Downloading);
|
||||
if (state != AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Unknown
|
||||
&& state != AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppInstallationState.Downloading)
|
||||
pkg.UpdateState(state);
|
||||
return pkg;
|
||||
}
|
||||
}
|
||||
}
|
||||
+465
@@ -0,0 +1,465 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Persistence;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Session 5d735290 — pure detection logic of <see cref="EspPolicyProviderStallDetector"/>:
|
||||
/// the CSP completion rule and the dwell/one-shot transition. No registry, no timers.
|
||||
/// </summary>
|
||||
public sealed class EspPolicyProviderStallLogicTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 7, 23, 10, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly TimeSpan Dwell = TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes);
|
||||
|
||||
private static PolicyProviderState SetupApps(string name, int? tracking, string scope = "device") =>
|
||||
new PolicyProviderState(name, scope, EspPolicyProviderProbe.KindSetupApps, tracking, null);
|
||||
|
||||
private static PolicyProviderState DevicePrep(string name, int? installState) =>
|
||||
new PolicyProviderState(name, "device", EspPolicyProviderProbe.KindDevicePreparation, null, installState);
|
||||
|
||||
private static EspPolicyProviderSnapshot Snapshot(params PolicyProviderState[] providers) =>
|
||||
new EspPolicyProviderSnapshot(providers);
|
||||
|
||||
// ------------------------------------------------- completion rule (CSP contract)
|
||||
|
||||
[Theory]
|
||||
[InlineData(1, true)] // TrackingPoliciesCreated=true
|
||||
[InlineData(0, false)] // explicit false (the CSP default)
|
||||
[InlineData(null, false)] // value missing — the field case
|
||||
[InlineData(2, false)] // anything else is not "true"
|
||||
public void SetupApps_provider_complete_only_at_tracking_1(int? tracking, bool expected)
|
||||
=> Assert.Equal(expected, EspPolicyProviderProbe.IsProviderComplete(
|
||||
EspPolicyProviderProbe.KindSetupApps, tracking, null));
|
||||
|
||||
[Theory]
|
||||
[InlineData(2, true)] // NotRequired
|
||||
[InlineData(3, true)] // Completed
|
||||
[InlineData(1, false)] // NotInstalled
|
||||
[InlineData(4, false)] // Error
|
||||
[InlineData(null, false)]
|
||||
public void DevicePreparation_provider_complete_at_state_2_or_3(int? state, bool expected)
|
||||
=> Assert.Equal(expected, EspPolicyProviderProbe.IsProviderComplete(
|
||||
EspPolicyProviderProbe.KindDevicePreparation, null, state));
|
||||
|
||||
[Fact]
|
||||
public void TryReadInt_tolerates_dword_string_and_bool_forms()
|
||||
{
|
||||
Assert.Equal(1, EspPolicyProviderProbe.TryReadInt(1));
|
||||
Assert.Equal(3, EspPolicyProviderProbe.TryReadInt(3L));
|
||||
Assert.Equal(1, EspPolicyProviderProbe.TryReadInt(" 1 "));
|
||||
Assert.Equal(1, EspPolicyProviderProbe.TryReadInt(true));
|
||||
Assert.Equal(0, EspPolicyProviderProbe.TryReadInt(false));
|
||||
Assert.Null(EspPolicyProviderProbe.TryReadInt(null));
|
||||
Assert.Null(EspPolicyProviderProbe.TryReadInt("not-a-number"));
|
||||
Assert.Null(EspPolicyProviderProbe.TryReadInt(new byte[] { 1 }));
|
||||
}
|
||||
|
||||
// ------------------------------------------------- dwell transition
|
||||
|
||||
[Fact]
|
||||
public void Empty_or_missing_snapshot_clears_dwell_state_and_never_stalls()
|
||||
{
|
||||
var previous = new Dictionary<string, DateTime> { ["setupApps|device|ConfigMgr"] = T0 };
|
||||
|
||||
var missing = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
EspPolicyProviderSnapshot.Empty, previous, new HashSet<string>(), T0, false, T0.AddHours(2), Dwell);
|
||||
Assert.Empty(missing.UpdatedFirstSeenUtc);
|
||||
Assert.Empty(missing.NewlyStalled);
|
||||
Assert.Null(missing.UpdatedSidecarMissingSinceUtc); // arm 2 resets too
|
||||
Assert.Null(missing.SidecarMissingStalledForMinutes);
|
||||
|
||||
var registeredButEmpty = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(), previous, new HashSet<string>(), T0, false, T0.AddHours(2), Dwell);
|
||||
Assert.Empty(registeredButEmpty.UpdatedFirstSeenUtc);
|
||||
Assert.Empty(registeredButEmpty.NewlyStalled);
|
||||
Assert.Null(registeredButEmpty.UpdatedSidecarMissingSinceUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Incomplete_provider_accrues_dwell_but_does_not_stall_before_threshold()
|
||||
{
|
||||
var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: null));
|
||||
|
||||
var round1 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
Assert.Equal(T0, round1.UpdatedFirstSeenUtc["setupApps|device|ConfigMgr"]);
|
||||
Assert.Empty(round1.NewlyStalled);
|
||||
|
||||
var round2 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, round1.UpdatedFirstSeenUtc, new HashSet<string>(),
|
||||
round1.UpdatedSidecarMissingSinceUtc, false, T0.AddMinutes(14), Dwell);
|
||||
Assert.Equal(T0, round2.UpdatedFirstSeenUtc["setupApps|device|ConfigMgr"]); // first-seen preserved
|
||||
Assert.Empty(round2.NewlyStalled);
|
||||
Assert.Null(round2.SidecarMissingStalledForMinutes); // 14 min < dwell on arm 2 as well
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Provider_stalls_at_dwell_threshold_with_correct_minutes()
|
||||
{
|
||||
var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: 0));
|
||||
var seen = new Dictionary<string, DateTime> { ["setupApps|device|ConfigMgr"] = T0 };
|
||||
|
||||
var eval = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, seen, new HashSet<string>(), null, false, T0.AddMinutes(15), Dwell);
|
||||
|
||||
var stalled = Assert.Single(eval.NewlyStalled);
|
||||
Assert.Equal("ConfigMgr", stalled.Provider.Name);
|
||||
Assert.Equal(15.0, stalled.StalledForMinutes, precision: 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Completion_or_disappearance_resets_the_dwell_clock()
|
||||
{
|
||||
var key = "setupApps|device|ConfigMgr";
|
||||
var seen = new Dictionary<string, DateTime> { [key] = T0 };
|
||||
|
||||
// Completes mid-dwell → dropped from the map.
|
||||
var completed = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)), seen, new HashSet<string>(), null, false, T0.AddMinutes(10), Dwell);
|
||||
Assert.Empty(completed.UpdatedFirstSeenUtc);
|
||||
|
||||
// Re-appears incomplete much later → dwell restarts from the new now, no stall yet.
|
||||
var reappeared = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 0)), completed.UpdatedFirstSeenUtc,
|
||||
new HashSet<string>(), null, false, T0.AddHours(3), Dwell);
|
||||
Assert.Equal(T0.AddHours(3), reappeared.UpdatedFirstSeenUtc[key]);
|
||||
Assert.Empty(reappeared.NewlyStalled);
|
||||
|
||||
// Disappearance behaves the same as completion.
|
||||
var disappeared = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("Sidecar", tracking: 1)), new Dictionary<string, DateTime> { [key] = T0 },
|
||||
new HashSet<string>(), null, false, T0.AddMinutes(20), Dwell);
|
||||
Assert.False(disappeared.UpdatedFirstSeenUtc.ContainsKey(key));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Already_fired_provider_is_never_reported_again()
|
||||
{
|
||||
var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: null));
|
||||
var seen = new Dictionary<string, DateTime> { ["setupApps|device|ConfigMgr"] = T0 };
|
||||
var fired = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "setupApps|device|ConfigMgr" };
|
||||
|
||||
var eval = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, seen, fired, null, false, T0.AddHours(5), Dwell);
|
||||
|
||||
Assert.Empty(eval.NewlyStalled);
|
||||
Assert.True(eval.UpdatedFirstSeenUtc.ContainsKey("setupApps|device|ConfigMgr")); // still tracked
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Mixed_kinds_and_scopes_are_tracked_under_distinct_keys()
|
||||
{
|
||||
var snapshot = Snapshot(
|
||||
SetupApps("ConfigMgr", tracking: null),
|
||||
SetupApps("ConfigMgr", tracking: null, scope: "user:S-1-5-21-1111"),
|
||||
DevicePrep("ConfigMgr", installState: 1));
|
||||
|
||||
var eval = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
|
||||
Assert.Equal(3, eval.UpdatedFirstSeenUtc.Count);
|
||||
Assert.Contains("setupApps|device|ConfigMgr", eval.UpdatedFirstSeenUtc.Keys);
|
||||
Assert.Contains("setupApps|user:S-1-5-21-1111|ConfigMgr", eval.UpdatedFirstSeenUtc.Keys);
|
||||
Assert.Contains("devicePreparation|device|ConfigMgr", eval.UpdatedFirstSeenUtc.Keys);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SidecarRegistered_matches_setupApps_sidecar_only()
|
||||
{
|
||||
Assert.True(Snapshot(SetupApps("Sidecar", tracking: 0)).SidecarRegistered);
|
||||
Assert.True(Snapshot(SetupApps("sidecar", tracking: 1, scope: "user:S-1-5-21-1")).SidecarRegistered);
|
||||
Assert.False(Snapshot(SetupApps("ConfigMgr", tracking: 1)).SidecarRegistered);
|
||||
Assert.False(Snapshot(DevicePrep("Sidecar", installState: 3)).SidecarRegistered); // wrong kind
|
||||
Assert.False(EspPolicyProviderSnapshot.Empty.SidecarRegistered);
|
||||
}
|
||||
|
||||
// ------------------------------------------------- arm 2: sidecar-missing transition
|
||||
|
||||
[Fact]
|
||||
public void Issue106_foreign_provider_complete_but_sidecar_missing_stalls_after_dwell()
|
||||
{
|
||||
// The actual field case: ConfigMgr with TrackingPoliciesCreated=1 — "complete" by the
|
||||
// CSP value contract, arm 1 never fires — yet the ESP waits on the absent Sidecar.
|
||||
var snapshot = Snapshot(SetupApps("ConfigMgr", tracking: 1));
|
||||
|
||||
var round1 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
Assert.Empty(round1.NewlyStalled); // arm 1 stays silent...
|
||||
Assert.Equal(T0, round1.UpdatedSidecarMissingSinceUtc); // ...but arm 2 starts its clock
|
||||
Assert.Null(round1.SidecarMissingStalledForMinutes);
|
||||
|
||||
var round2 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
snapshot, round1.UpdatedFirstSeenUtc, new HashSet<string>(),
|
||||
round1.UpdatedSidecarMissingSinceUtc, false, T0.AddMinutes(15), Dwell);
|
||||
Assert.Empty(round2.NewlyStalled);
|
||||
Assert.Equal(15.0, round2.SidecarMissingStalledForMinutes!.Value, precision: 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sidecar_registering_resets_the_sidecar_missing_clock()
|
||||
{
|
||||
// ConfigMgr first (legitimate startup ordering) → clock runs...
|
||||
var round1 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)),
|
||||
new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
Assert.Equal(T0, round1.UpdatedSidecarMissingSinceUtc);
|
||||
|
||||
// ...Sidecar joins → condition clears, state resets.
|
||||
var round2 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1), SetupApps("Sidecar", tracking: 0)),
|
||||
round1.UpdatedFirstSeenUtc, new HashSet<string>(),
|
||||
round1.UpdatedSidecarMissingSinceUtc, false, T0.AddMinutes(10), Dwell);
|
||||
Assert.Null(round2.UpdatedSidecarMissingSinceUtc);
|
||||
Assert.Null(round2.SidecarMissingStalledForMinutes);
|
||||
|
||||
// Sidecar gone again much later → clock restarts from the new now, no instant stall.
|
||||
var round3 = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)),
|
||||
round2.UpdatedFirstSeenUtc, new HashSet<string>(),
|
||||
round2.UpdatedSidecarMissingSinceUtc, false, T0.AddHours(3), Dwell);
|
||||
Assert.Equal(T0.AddHours(3), round3.UpdatedSidecarMissingSinceUtc);
|
||||
Assert.Null(round3.SidecarMissingStalledForMinutes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sidecar_missing_one_shot_and_deviceprep_only_scoping()
|
||||
{
|
||||
// Already fired → never re-reported, but the clock keeps running.
|
||||
var latched = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(SetupApps("ConfigMgr", tracking: 1)),
|
||||
new Dictionary<string, DateTime>(), new HashSet<string>(), T0, true, T0.AddHours(5), Dwell);
|
||||
Assert.Null(latched.SidecarMissingStalledForMinutes);
|
||||
Assert.Equal(T0, latched.UpdatedSidecarMissingSinceUtc);
|
||||
|
||||
// DevicePreparation providers alone don't constitute the Setup/Apps wait — arm 2 off.
|
||||
var devicePrepOnly = EspPolicyProviderStallDetector.EvaluateStallTransition(
|
||||
Snapshot(DevicePrep("ConfigMgr", installState: 1)),
|
||||
new Dictionary<string, DateTime>(), new HashSet<string>(), null, false, T0, Dwell);
|
||||
Assert.Null(devicePrepOnly.UpdatedSidecarMissingSinceUtc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Emission surface of the detector via the real <see cref="InformationalEventPost"/> rail.
|
||||
/// Static probe override → serial collection (same contract as
|
||||
/// <c>EspConfigDetectedTrackingFieldsTests</c>).
|
||||
/// </summary>
|
||||
[Collection("SerialThreading")]
|
||||
public sealed class EspPolicyProviderStallDetectorEmissionTests : IDisposable
|
||||
{
|
||||
private static readonly DateTime At = new DateTime(2026, 7, 23, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
private (EspPolicyProviderStallDetector sut, FakeSignalIngressSink sink, VirtualClock clock)
|
||||
BuildDetector(StartupEventGate? gate = null)
|
||||
{
|
||||
var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
|
||||
var sink = new FakeSignalIngressSink();
|
||||
var clock = new VirtualClock(At);
|
||||
var post = new InformationalEventPost(sink, clock);
|
||||
var sut = new EspPolicyProviderStallDetector("S1", "T1", post, logger, clock, gate);
|
||||
return (sut, sink, clock);
|
||||
}
|
||||
|
||||
private static EspPolicyProviderSnapshot ConfigMgrOnly() => new EspPolicyProviderSnapshot(new[]
|
||||
{
|
||||
new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindSetupApps,
|
||||
trackingPoliciesCreated: null, installationState: null),
|
||||
});
|
||||
|
||||
private static EspPolicyProviderSnapshot SidecarCompleteConfigMgrIncomplete() => new EspPolicyProviderSnapshot(new[]
|
||||
{
|
||||
new PolicyProviderState("Sidecar", "device", EspPolicyProviderProbe.KindSetupApps, 1, null),
|
||||
new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindSetupApps, 0, null),
|
||||
});
|
||||
|
||||
private static IReadOnlyList<FakeSignalIngressSink.PostedSignal> StalledEvents(FakeSignalIngressSink sink) =>
|
||||
sink.Posted.Where(p =>
|
||||
p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == "esp_policy_provider_stalled").ToList();
|
||||
|
||||
[Fact]
|
||||
public void CustomerScenario_configmgr_without_sidecar_emits_one_warning_after_dwell()
|
||||
{
|
||||
using var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => ConfigMgrOnly());
|
||||
var (sut, sink, clock) = BuildDetector();
|
||||
|
||||
sut.Tick(); // first sighting — dwell starts
|
||||
Assert.Empty(StalledEvents(sink));
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes));
|
||||
sut.Tick(); // threshold crossed → emit
|
||||
|
||||
var evt = Assert.Single(StalledEvents(sink));
|
||||
Assert.Equal(DecisionSignalKind.InformationalEvent, evt.Kind);
|
||||
Assert.Equal("Warning", evt.Payload![SignalPayloadKeys.Severity]);
|
||||
Assert.Equal("true", evt.Payload[SignalPayloadKeys.ImmediateUpload]);
|
||||
Assert.Equal(EspPolicyProviderStallDetector.SourceLabel, evt.Payload[SignalPayloadKeys.Source]);
|
||||
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(evt.TypedPayload);
|
||||
// Both arms cross in the same round (ConfigMgr incomplete AND Sidecar absent) →
|
||||
// ONE event; sidecar-missing is the dominant reason.
|
||||
Assert.Equal(EspPolicyProviderStallDetector.ReasonSidecarMissing, data["reason"]);
|
||||
Assert.Equal(false, data["sidecarRegistered"]);
|
||||
Assert.Equal(15.0, data["sidecarMissingForMinutes"]);
|
||||
Assert.Equal(EspPolicyProviderStallDetector.DwellMinutes, data["dwellMinutes"]);
|
||||
var stalled = Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object>>>(data["stalledProviders"]);
|
||||
var entry = Assert.Single(stalled);
|
||||
Assert.Equal("ConfigMgr", entry["name"]);
|
||||
Assert.Equal(EspPolicyProviderProbe.KindSetupApps, entry["kind"]);
|
||||
Assert.Equal(15.0, entry["stalledForMinutes"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Issue106_configmgr_complete_without_sidecar_still_emits_sidecar_missing()
|
||||
{
|
||||
// Regression test for the customer's follow-up on issue #106: TrackingPoliciesCreated=1
|
||||
// WAS present under ConfigMgr — the provider looks "complete", arm 1 never fires, yet
|
||||
// the ESP hangs because Sidecar is absent. Arm 2 must catch exactly this.
|
||||
var configMgrComplete = new EspPolicyProviderSnapshot(new[]
|
||||
{
|
||||
new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindSetupApps,
|
||||
trackingPoliciesCreated: 1, installationState: null),
|
||||
});
|
||||
using var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => configMgrComplete);
|
||||
var (sut, sink, clock) = BuildDetector();
|
||||
|
||||
sut.Tick();
|
||||
Assert.Empty(StalledEvents(sink));
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes));
|
||||
sut.Tick();
|
||||
|
||||
var evt = Assert.Single(StalledEvents(sink));
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(evt.TypedPayload);
|
||||
Assert.Equal(EspPolicyProviderStallDetector.ReasonSidecarMissing, data["reason"]);
|
||||
Assert.Equal(false, data["sidecarRegistered"]);
|
||||
Assert.Equal(15.0, data["sidecarMissingForMinutes"]);
|
||||
Assert.Empty(Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object>>>(data["stalledProviders"]));
|
||||
var providers = Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object?>>>(data["providers"]);
|
||||
Assert.Equal(true, Assert.Single(providers)["complete"]); // the misleading "complete" is visible
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sidecar_complete_configmgr_incomplete_reports_only_configmgr_with_sidecar_flag()
|
||||
{
|
||||
using var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => SidecarCompleteConfigMgrIncomplete());
|
||||
var (sut, sink, clock) = BuildDetector();
|
||||
|
||||
sut.Tick();
|
||||
clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes));
|
||||
sut.Tick();
|
||||
|
||||
var evt = Assert.Single(StalledEvents(sink));
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(evt.TypedPayload);
|
||||
Assert.Equal(EspPolicyProviderStallDetector.ReasonProviderIncomplete, data["reason"]); // arm 2 off — Sidecar is there
|
||||
Assert.Equal(true, data["sidecarRegistered"]);
|
||||
Assert.False(data.ContainsKey("sidecarMissingForMinutes"));
|
||||
var stalled = Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object>>>(data["stalledProviders"]);
|
||||
Assert.Equal("ConfigMgr", Assert.Single(stalled)["name"]);
|
||||
var providers = Assert.IsAssignableFrom<IReadOnlyList<Dictionary<string, object?>>>(data["providers"]);
|
||||
Assert.Equal(2, providers.Count); // full table always attached
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Latch_prevents_second_emission_for_the_same_provider()
|
||||
{
|
||||
using var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => ConfigMgrOnly());
|
||||
var (sut, sink, clock) = BuildDetector();
|
||||
|
||||
sut.Tick();
|
||||
clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes));
|
||||
sut.Tick();
|
||||
clock.Advance(TimeSpan.FromMinutes(30));
|
||||
sut.Tick();
|
||||
sut.Tick();
|
||||
|
||||
Assert.Single(StalledEvents(sink));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Startup_gate_suppresses_same_stall_across_restart_but_reports_new_provider_set()
|
||||
{
|
||||
var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
// Run 1: fires and persists the fingerprint.
|
||||
using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => ConfigMgrOnly()))
|
||||
{
|
||||
var (sut, sink, clock) = BuildDetector(new StartupEventGate(_tmp.Path, logger));
|
||||
sut.Tick();
|
||||
clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes));
|
||||
sut.Tick();
|
||||
Assert.Single(StalledEvents(sink));
|
||||
}
|
||||
|
||||
// Run 2 (simulated restart — fresh detector, gate reloaded from the same state dir):
|
||||
// identical stall re-accrues dwell but lands on the same fingerprint → suppressed.
|
||||
using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => ConfigMgrOnly()))
|
||||
{
|
||||
var (sut, sink, clock) = BuildDetector(new StartupEventGate(_tmp.Path, logger));
|
||||
sut.Tick();
|
||||
clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes));
|
||||
sut.Tick();
|
||||
Assert.Empty(StalledEvents(sink));
|
||||
}
|
||||
|
||||
// Run 3: a SECOND provider joins the stall → fingerprint changes → reported.
|
||||
var widened = new EspPolicyProviderSnapshot(new[]
|
||||
{
|
||||
new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindSetupApps, null, null),
|
||||
new PolicyProviderState("Contoso", "device", EspPolicyProviderProbe.KindSetupApps, 0, null),
|
||||
});
|
||||
using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => widened))
|
||||
{
|
||||
var (sut, sink, clock) = BuildDetector(new StartupEventGate(_tmp.Path, logger));
|
||||
sut.Tick();
|
||||
clock.Advance(TimeSpan.FromMinutes(EspPolicyProviderStallDetector.DwellMinutes));
|
||||
sut.Tick();
|
||||
Assert.Single(StalledEvents(sink));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Healthy_or_absent_tracking_never_emits()
|
||||
{
|
||||
var healthy = new EspPolicyProviderSnapshot(new[]
|
||||
{
|
||||
new PolicyProviderState("Sidecar", "device", EspPolicyProviderProbe.KindSetupApps, 1, null),
|
||||
new PolicyProviderState("ConfigMgr", "device", EspPolicyProviderProbe.KindDevicePreparation, null, 3),
|
||||
});
|
||||
using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => healthy))
|
||||
{
|
||||
var (sut, sink, clock) = BuildDetector();
|
||||
sut.Tick();
|
||||
clock.Advance(TimeSpan.FromHours(4));
|
||||
sut.Tick();
|
||||
Assert.Empty(StalledEvents(sink));
|
||||
}
|
||||
|
||||
using (var _probe = new EspPolicyProviderProbe.ScopedOverride(_ => EspPolicyProviderSnapshot.Empty))
|
||||
{
|
||||
var (sut, sink, clock) = BuildDetector();
|
||||
sut.Tick();
|
||||
clock.Advance(TimeSpan.FromHours(4));
|
||||
sut.Tick();
|
||||
Assert.Empty(StalledEvents(sink));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
-9
@@ -39,7 +39,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
public List<EspFailureDetectedEventArgs> EspFailures { get; } = new List<EspFailureDetectedEventArgs>();
|
||||
public ProvisioningStatusTracker Tracker { get; }
|
||||
|
||||
public Fixture(Func<IReadOnlyList<AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState>> packageStatesProbe = null)
|
||||
public Fixture(Func<IReadOnlyList<AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime.AppPackageState>>? packageStatesProbe = null)
|
||||
{
|
||||
var clock = new VirtualClock(Fixed);
|
||||
var post = new InformationalEventPost(Sink, clock);
|
||||
@@ -179,7 +179,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
}");
|
||||
|
||||
var captured = f.CapturedEvents();
|
||||
Assert.Single(captured.Where(e => e.EventType == "esp_failure_settle_started"));
|
||||
Assert.Single(captured, e => e.EventType == "esp_failure_settle_started");
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
@@ -214,8 +214,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
// No terminal fire — the failure was retracted.
|
||||
Assert.Empty(f.EspFailures);
|
||||
|
||||
var recovered = Assert.Single(f.CapturedEvents()
|
||||
.Where(e => e.EventType == "esp_failure_settle_recovered"));
|
||||
var recovered = Assert.Single(
|
||||
f.CapturedEvents(), e => e.EventType == "esp_failure_settle_recovered");
|
||||
Assert.Equal("AccountSetup", (string)recovered.Data["category"]);
|
||||
Assert.Equal("Apps", (string)recovered.Data["failedSubcategory"]);
|
||||
Assert.Equal("inProgress", (string)recovered.Data["observedState"]);
|
||||
@@ -280,7 +280,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
f.Tracker.TriggerSettleTimerForTest("AccountSetupCategory.Status");
|
||||
|
||||
Assert.Single(f.EspFailures);
|
||||
Assert.Empty(f.CapturedEvents().Where(e => e.EventType == "esp_failure_settle_recovered"));
|
||||
Assert.DoesNotContain(f.CapturedEvents(), e => e.EventType == "esp_failure_settle_recovered");
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
@@ -326,8 +326,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
""AppsSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Apps (Error)""}
|
||||
}");
|
||||
|
||||
var settleStarted = Assert.Single(f.CapturedEvents()
|
||||
.Where(e => e.EventType == "esp_failure_settle_started"));
|
||||
var settleStarted = Assert.Single(
|
||||
f.CapturedEvents(), e => e.EventType == "esp_failure_settle_started");
|
||||
Assert.Equal(1, (int)settleStarted.Data["trackedAppsNotCompletedCount"]);
|
||||
var apps = (List<Dictionary<string, object>>)settleStarted.Data["trackedAppsNotCompleted"];
|
||||
var app = Assert.Single(apps);
|
||||
@@ -354,8 +354,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
""SecurityPoliciesSubcategory"": {""subcategoryState"":""failed"",""subcategoryStatusText"":""Failed""}
|
||||
}");
|
||||
|
||||
var settleStarted = Assert.Single(f.CapturedEvents()
|
||||
.Where(e => e.EventType == "esp_failure_settle_started"));
|
||||
var settleStarted = Assert.Single(
|
||||
f.CapturedEvents(), e => e.EventType == "esp_failure_settle_started");
|
||||
Assert.False(settleStarted.Data.ContainsKey("trackedAppsNotCompleted"));
|
||||
Assert.False(settleStarted.Data.ContainsKey("trackedAppsNotCompletedCount"));
|
||||
}
|
||||
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// Pins the anti-flood semantics of <see cref="EmergencyReporter.TrySendAsync"/> after the
|
||||
/// delivery-hardening change (2026-07-23): the once-per-session dedup and the per-session
|
||||
/// budget count DELIVERED reports. The previous mark-before-send permanently suppressed an
|
||||
/// error key after its first transport failure — fatal for single-shot callers like the 48h
|
||||
/// emergency break, whose process exits right after the attempt.
|
||||
/// </summary>
|
||||
public class EmergencyReporterTests
|
||||
{
|
||||
private sealed class FakeApiClient : BackendApiClient
|
||||
{
|
||||
public readonly List<TimeSpan?> Timeouts = new List<TimeSpan?>();
|
||||
public readonly Queue<bool> Results = new Queue<bool>();
|
||||
|
||||
public FakeApiClient() : base(
|
||||
httpClient: new HttpClient(),
|
||||
baseUrl: "http://localhost",
|
||||
manufacturer: string.Empty,
|
||||
model: string.Empty,
|
||||
serialNumber: string.Empty,
|
||||
useBootstrapTokenAuth: false,
|
||||
bootstrapToken: null,
|
||||
agentVersion: "0.0.0",
|
||||
logger: null)
|
||||
{
|
||||
}
|
||||
|
||||
public int Calls => Timeouts.Count;
|
||||
|
||||
public override Task<bool> ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null)
|
||||
{
|
||||
Timeouts.Add(timeout);
|
||||
return Task.FromResult(Results.Count > 0 && Results.Dequeue());
|
||||
}
|
||||
}
|
||||
|
||||
private static EmergencyReporter Reporter(FakeApiClient client)
|
||||
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null);
|
||||
|
||||
[Fact]
|
||||
public async Task Failed_send_rolls_back_reservation_so_a_later_call_retries()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(false); // first call fails
|
||||
client.Results.Enqueue(true); // second call succeeds
|
||||
var reporter = Reporter(client);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m");
|
||||
await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m");
|
||||
|
||||
// Old behaviour: second call suppressed as "already reported" ⇒ 1 call.
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Delivered_send_is_deduplicated_per_key()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m");
|
||||
await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m");
|
||||
|
||||
Assert.Equal(1, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retries_stay_inside_one_reservation_and_stop_on_success()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(false);
|
||||
client.Results.Enqueue(true); // second attempt delivers — third must not fire
|
||||
var reporter = Reporter(client);
|
||||
|
||||
await reporter.TrySendAsync(
|
||||
AgentErrorType.SessionAgeEmergencyBreak, "m",
|
||||
attempts: 3, perAttemptTimeout: TimeSpan.FromMilliseconds(50), retryDelay: TimeSpan.Zero);
|
||||
|
||||
Assert.Equal(2, client.Calls);
|
||||
|
||||
// Delivered ⇒ dedup holds for the session.
|
||||
await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m");
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Per_attempt_timeout_is_forwarded_to_the_api_client()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client);
|
||||
|
||||
var timeout = TimeSpan.FromSeconds(15);
|
||||
await reporter.TrySendAsync(
|
||||
AgentErrorType.SessionAgeEmergencyBreak, "m",
|
||||
attempts: 1, perAttemptTimeout: timeout);
|
||||
|
||||
Assert.Equal(timeout, Assert.Single(client.Timeouts));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task All_attempts_failing_rolls_back_after_exhausting_attempts()
|
||||
{
|
||||
var client = new FakeApiClient(); // queue empty ⇒ every attempt fails
|
||||
var reporter = Reporter(client);
|
||||
|
||||
await reporter.TrySendAsync(
|
||||
AgentErrorType.SessionAgeEmergencyBreak, "m",
|
||||
attempts: 3, retryDelay: TimeSpan.Zero);
|
||||
Assert.Equal(3, client.Calls);
|
||||
|
||||
// Rolled back ⇒ a later call may try again (budget + dedup + cooldown restored).
|
||||
client.Results.Enqueue(true);
|
||||
await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m");
|
||||
Assert.Equal(4, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Different_keys_do_not_suppress_each_other_after_rollback()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(false); // break report fails → rollback (incl. cooldown restore)
|
||||
client.Results.Enqueue(true); // unrelated report must still go out
|
||||
var reporter = Reporter(client);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, "m");
|
||||
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
}
|
||||
}
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
#nullable enable
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Pins the pure parts of the UEFI Secure Boot certificate probe: the ASCII substring
|
||||
/// search over raw signature-list bytes and the <c>secureboot_status</c> payload shaping
|
||||
/// (in particular the one-sided <c>uefiCa2023FirmwareConfirmed</c> marker whose ABSENCE
|
||||
/// is load-bearing for ANALYZE-SEC-001's not_exists precondition). The actual firmware
|
||||
/// read is only smoke-tested: test machines may be VMs, non-elevated, or legacy BIOS,
|
||||
/// so only the fail-soft contract is asserted.
|
||||
/// </summary>
|
||||
public sealed class UefiSecureBootCertReaderTests
|
||||
{
|
||||
// --- ContainsAscii ---
|
||||
|
||||
[Fact]
|
||||
public void ContainsAscii_finds_needle_embedded_in_binary_noise()
|
||||
{
|
||||
var bytes = Concat(new byte[] { 0x00, 0xA1, 0x30, 0x82 },
|
||||
Encoding.ASCII.GetBytes("Windows UEFI CA 2023"),
|
||||
new byte[] { 0x00, 0xFF });
|
||||
|
||||
Assert.True(UefiSecureBootCertReader.ContainsAscii(bytes, "Windows UEFI CA 2023"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ContainsAscii_finds_needle_at_the_very_end()
|
||||
{
|
||||
var bytes = Concat(new byte[] { 0x01, 0x02 }, Encoding.ASCII.GetBytes("KEK 2K CA 2023"));
|
||||
|
||||
Assert.True(UefiSecureBootCertReader.ContainsAscii(bytes, "KEK 2K CA 2023"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ContainsAscii_rejects_partial_overlap_and_absent_needle()
|
||||
{
|
||||
var bytes = Encoding.ASCII.GetBytes("Windows UEFI CA 2011 something else");
|
||||
|
||||
Assert.False(UefiSecureBootCertReader.ContainsAscii(bytes, "Windows UEFI CA 2023"));
|
||||
// Repeated prefix bytes must not confuse the scan.
|
||||
var tricky = Encoding.ASCII.GetBytes("ababac");
|
||||
Assert.True(UefiSecureBootCertReader.ContainsAscii(tricky, "abac"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ContainsAscii_handles_null_empty_and_oversized_needle()
|
||||
{
|
||||
Assert.False(UefiSecureBootCertReader.ContainsAscii(null, "x"));
|
||||
Assert.False(UefiSecureBootCertReader.ContainsAscii(new byte[0], "x"));
|
||||
Assert.False(UefiSecureBootCertReader.ContainsAscii(new byte[] { 1, 2 }, ""));
|
||||
Assert.False(UefiSecureBootCertReader.ContainsAscii(new byte[] { 1, 2 }, "abc"));
|
||||
}
|
||||
|
||||
// --- Payload shaping ---
|
||||
|
||||
[Fact]
|
||||
public void Marker_emitted_only_when_db_ok_and_ca2023_present()
|
||||
{
|
||||
var data = new Dictionary<string, object>();
|
||||
DeviceInfoCollector.AppendFirmwareCertFields(data, new UefiSecureBootCertSnapshot
|
||||
{
|
||||
DbStatus = UefiSecureBootCertReader.StatusOk,
|
||||
KekStatus = UefiSecureBootCertReader.StatusOk,
|
||||
DbHasWindowsUefiCa2023 = true,
|
||||
KekHasMicrosoftKek2kCa2023 = true
|
||||
});
|
||||
|
||||
Assert.Equal(true, data["uefiCa2023FirmwareConfirmed"]);
|
||||
Assert.Equal("ok", data["uefiFirmwareReadStatus"]);
|
||||
Assert.Equal(true, data["uefiDbHasWindowsUefiCa2023"]);
|
||||
Assert.Equal(true, data["uefiKekHasMicrosoftKek2kCa2023"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Marker_absent_when_ca2023_missing_even_though_read_succeeded()
|
||||
{
|
||||
var data = new Dictionary<string, object>();
|
||||
DeviceInfoCollector.AppendFirmwareCertFields(data, new UefiSecureBootCertSnapshot
|
||||
{
|
||||
DbStatus = UefiSecureBootCertReader.StatusOk,
|
||||
KekStatus = UefiSecureBootCertReader.StatusOk,
|
||||
DbHasWindowsUefiCa2023 = false,
|
||||
DbHasWindowsProductionPca2011 = true
|
||||
});
|
||||
|
||||
// The marker must be OMITTED, not false — not_exists fails on any present value.
|
||||
Assert.False(data.ContainsKey("uefiCa2023FirmwareConfirmed"));
|
||||
Assert.Equal(false, data["uefiDbHasWindowsUefiCa2023"]);
|
||||
Assert.Equal(true, data["uefiDbHasWindowsProductionPca2011"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failed_db_read_emits_status_but_no_cert_booleans_and_no_marker()
|
||||
{
|
||||
var data = new Dictionary<string, object>();
|
||||
DeviceInfoCollector.AppendFirmwareCertFields(data, new UefiSecureBootCertSnapshot
|
||||
{
|
||||
DbStatus = UefiSecureBootCertReader.StatusPrivilegeDenied,
|
||||
KekStatus = UefiSecureBootCertReader.StatusPrivilegeDenied
|
||||
});
|
||||
|
||||
Assert.Equal(UefiSecureBootCertReader.StatusPrivilegeDenied, data["uefiFirmwareReadStatus"]);
|
||||
Assert.False(data.ContainsKey("uefiDbHasWindowsUefiCa2023"));
|
||||
Assert.False(data.ContainsKey("uefiKekHasMicrosoftKek2kCa2023"));
|
||||
Assert.False(data.ContainsKey("uefiCa2023FirmwareConfirmed"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Db_ok_but_kek_failed_keeps_db_booleans_and_reports_kek_status()
|
||||
{
|
||||
var data = new Dictionary<string, object>();
|
||||
DeviceInfoCollector.AppendFirmwareCertFields(data, new UefiSecureBootCertSnapshot
|
||||
{
|
||||
DbStatus = UefiSecureBootCertReader.StatusOk,
|
||||
KekStatus = UefiSecureBootCertReader.StatusVariableNotFound,
|
||||
DbHasWindowsUefiCa2023 = true
|
||||
});
|
||||
|
||||
Assert.Equal(UefiSecureBootCertReader.StatusVariableNotFound, data["uefiFirmwareReadStatus"]);
|
||||
Assert.Equal(true, data["uefiDbHasWindowsUefiCa2023"]);
|
||||
Assert.Equal(true, data["uefiCa2023FirmwareConfirmed"]);
|
||||
Assert.False(data.ContainsKey("uefiKekHasMicrosoftKek2kCa2023"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Null_snapshot_leaves_payload_untouched()
|
||||
{
|
||||
var data = new Dictionary<string, object> { { "uefiSecureBootEnabled", true } };
|
||||
DeviceInfoCollector.AppendFirmwareCertFields(data, null);
|
||||
|
||||
Assert.Single(data);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DescribeFirmwareVerdict_covers_all_shapes()
|
||||
{
|
||||
Assert.Equal("unavailable", DeviceInfoCollector.DescribeFirmwareVerdict(null));
|
||||
Assert.Equal(UefiSecureBootCertReader.StatusNotUefi, DeviceInfoCollector.DescribeFirmwareVerdict(
|
||||
new UefiSecureBootCertSnapshot { DbStatus = UefiSecureBootCertReader.StatusNotUefi }));
|
||||
Assert.Equal("CA2023 confirmed", DeviceInfoCollector.DescribeFirmwareVerdict(
|
||||
new UefiSecureBootCertSnapshot { DbStatus = UefiSecureBootCertReader.StatusOk, DbHasWindowsUefiCa2023 = true }));
|
||||
Assert.Equal("CA2023 missing", DeviceInfoCollector.DescribeFirmwareVerdict(
|
||||
new UefiSecureBootCertSnapshot { DbStatus = UefiSecureBootCertReader.StatusOk, DbHasWindowsUefiCa2023 = false }));
|
||||
}
|
||||
|
||||
// --- Reader smoke (real firmware; machine state unknown) ---
|
||||
|
||||
[Fact]
|
||||
public void Read_never_throws_and_reports_a_status_for_both_variables()
|
||||
{
|
||||
var snapshot = UefiSecureBootCertReader.Read();
|
||||
|
||||
Assert.NotNull(snapshot);
|
||||
Assert.False(string.IsNullOrEmpty(snapshot.DbStatus));
|
||||
Assert.False(string.IsNullOrEmpty(snapshot.KekStatus));
|
||||
}
|
||||
|
||||
private static byte[] Concat(params byte[][] parts)
|
||||
{
|
||||
var total = 0;
|
||||
foreach (var part in parts) total += part.Length;
|
||||
var result = new byte[total];
|
||||
var offset = 0;
|
||||
foreach (var part in parts)
|
||||
{
|
||||
System.Array.Copy(part, 0, result, offset, part.Length);
|
||||
offset += part.Length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
+15
-6
@@ -44,7 +44,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
private readonly FakeQuarantineSink _quarantine = new FakeQuarantineSink();
|
||||
private readonly FakeSignalIngressSink _sink = new FakeSignalIngressSink();
|
||||
private readonly DecisionEngine _engine = new DecisionEngine();
|
||||
private DecisionStepProcessor _sut;
|
||||
private DecisionStepProcessor? _sut;
|
||||
|
||||
public DecisionStepProcessorParkedTripwireTests()
|
||||
{
|
||||
@@ -107,7 +107,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
return (step, signal);
|
||||
}
|
||||
|
||||
private FakeSignalIngressSink.PostedSignal FindTripwirePost()
|
||||
private FakeSignalIngressSink.PostedSignal? FindTripwirePost()
|
||||
{
|
||||
foreach (var posted in _sink.Posted)
|
||||
{
|
||||
@@ -121,6 +121,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>The tripwire post, asserted present — for the assertions that only run
|
||||
/// after <c>WaitForTripwire()</c> has already confirmed one was emitted.</summary>
|
||||
private FakeSignalIngressSink.PostedSignal RequireTripwirePost()
|
||||
{
|
||||
var post = FindTripwirePost();
|
||||
Assert.NotNull(post);
|
||||
return post!;
|
||||
}
|
||||
|
||||
private int CountTripwirePosts()
|
||||
{
|
||||
var count = 0;
|
||||
@@ -165,9 +174,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
Thread.Sleep(SuppressWait);
|
||||
Assert.Equal(1, CountTripwirePosts());
|
||||
|
||||
var post = FindTripwirePost();
|
||||
var post = RequireTripwirePost();
|
||||
Assert.Equal(DecisionSignalKind.InformationalEvent, post.Kind);
|
||||
Assert.Equal(nameof(SessionStage.EspAccountSetup), post.Payload["stage"]);
|
||||
Assert.Equal(nameof(SessionStage.EspAccountSetup), post.Payload!["stage"]);
|
||||
Assert.Equal("Warning", post.Payload[SignalPayloadKeys.Severity]);
|
||||
Assert.Equal("true", post.Payload[SignalPayloadKeys.ImmediateUpload]);
|
||||
Assert.Contains("esp_final_exit", post.Payload["signalsSeen"]);
|
||||
@@ -222,7 +231,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
sut.ApplyStep(step, signal);
|
||||
|
||||
Assert.True(WaitForTripwire());
|
||||
Assert.Equal(DeadlineNames.ClassifierTick, FindTripwirePost().Payload["armedDeadlines"]);
|
||||
Assert.Equal(DeadlineNames.ClassifierTick, RequireTripwirePost().Payload!["armedDeadlines"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -237,7 +246,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
sut.ApplyStep(step, signal);
|
||||
|
||||
Assert.True(WaitForTripwire());
|
||||
Assert.Equal(nameof(SessionStage.AwaitingDesktop), FindTripwirePost().Payload["stage"]);
|
||||
Assert.Equal(nameof(SessionStage.AwaitingDesktop), RequireTripwirePost().Payload!["stage"]);
|
||||
}
|
||||
|
||||
// ===================================================================== Dwell semantics
|
||||
|
||||
+2
-2
@@ -256,8 +256,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
RemoteConfigFetchOutcome outcome,
|
||||
int configVersion = 0,
|
||||
int attempts = 1,
|
||||
string failureType = null,
|
||||
string failureMessage = null,
|
||||
string? failureType = null,
|
||||
string? failureMessage = null,
|
||||
int? authStatusCode = null)
|
||||
{
|
||||
// CurrentConfig is set via the protected SetConfig path — easiest:
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
[InlineData("Network connections (1 of 1 added)", null)]
|
||||
[InlineData("", null)]
|
||||
[InlineData(null, null)]
|
||||
public void TryExtractErrorCode_extracts_parenthesised_hex_tail(string statusText, string expected)
|
||||
public void TryExtractErrorCode_extracts_parenthesised_hex_tail(string? statusText, string? expected)
|
||||
{
|
||||
var actual = ProvisioningStatusTracker.TryExtractErrorCode(statusText);
|
||||
Assert.Equal(expected, actual);
|
||||
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
{
|
||||
/// <summary>
|
||||
/// Historic-replay suppression for APP events (session eaf3d8c4 part 2): 147 of 157 app
|
||||
/// events in that session were replayed from a previous enrollment's IME log — all 10
|
||||
/// "installed apps" were phantoms. Stale app-state transitions and DO telemetry must be
|
||||
/// suppressed BEFORE the sub-phase declaration, the timing bookkeeping and the terminal
|
||||
/// DecisionSignal dedup, so fire-once flags stay unconsumed and no phantom
|
||||
/// AppInstallCompleted/Failed reaches the engine (which could spuriously re-arm the
|
||||
/// AdvisoryCompletion window). The script twin lives in
|
||||
/// <see cref="ImeLogTrackerAdapterHistoricReplayTests"/>.
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerAdapterHistoricAppReplayTests
|
||||
{
|
||||
private static readonly DateTime ClockNow = new DateTime(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public void Stale_terminal_app_transition_is_fully_suppressed()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-old", 0);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.AppInstallComplete));
|
||||
Assert.Empty(f.DecisionSignals(DecisionSignalKind.AppInstallCompleted));
|
||||
// Shadow download_progress (status=completed) and the summary snapshot ride the
|
||||
// same emit — both must be gone too.
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.DownloadProgress));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.AppTrackingSummary));
|
||||
// Timing bookkeeping must not be poisoned (feeds FinalStatusBuilder).
|
||||
Assert.False(adapter.AppTimings.ContainsKey("app-old"));
|
||||
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Fresh_run_after_suppressed_stale_run_fires_signal_timing_and_subphase()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-X", 0);
|
||||
|
||||
// Fresh ESP phase so the sub-phase declaration is armed.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-5);
|
||||
adapter.TriggerEspPhaseFromTest("DeviceSetup");
|
||||
|
||||
// Stale replay of the same app — suppressed, must consume NEITHER the sub-phase
|
||||
// fire-once NOR the terminal-signal dedup for this appId.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Unknown, AppInstallationState.Installed);
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.PhaseTransition));
|
||||
Assert.Empty(f.DecisionSignals(DecisionSignalKind.AppInstallCompleted));
|
||||
|
||||
// The app genuinely runs in THIS enrollment.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-2);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Unknown, AppInstallationState.Installing);
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-1);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.PhaseTransition)); // sub-phase fired fresh
|
||||
Assert.Single(f.DecisionSignals(DecisionSignalKind.AppInstallCompleted)); // signal fired fresh
|
||||
var timing = adapter.AppTimings["app-X"];
|
||||
Assert.Equal(ClockNow.AddMinutes(-2), timing.StartedAtUtc); // fresh stamps, not stale
|
||||
Assert.Equal(ClockNow.AddMinutes(-1), timing.CompletedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_do_telemetry_is_suppressed_fresh_emits()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-do", 0);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerDoTelemetryFromTest(app);
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.DoTelemetry));
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-1);
|
||||
adapter.TriggerDoTelemetryFromTest(app);
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.DoTelemetry));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Oneshot_summary_is_shared_between_script_and_app_suppressions()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "aaa11111",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
adapter.TriggerAppStateFromTest(new AppPackageState("app-old", 0),
|
||||
AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FutureSkew_app_transition_still_emits()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-skew", 0);
|
||||
|
||||
// Mid-enrollment clock jump — genuine current activity, stamped to the clock.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddHours(2);
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.AppInstallComplete));
|
||||
Assert.Equal(ClockNow, info.OccurredAtUtc);
|
||||
Assert.Equal("true", info.Payload!["derivedTimestamp"]);
|
||||
Assert.Single(f.DecisionSignals(DecisionSignalKind.AppInstallCompleted));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SimulationMode_bypasses_suppression_for_apps_and_scripts()
|
||||
{
|
||||
// The --replay-log-dir dev tool intentionally feeds historic lines.
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
f.Tracker.SimulationMode = true;
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerAppStateFromTest(new AppPackageState("app-sim", 0),
|
||||
AppInstallationState.Installing, AppInstallationState.Installed);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "bbb22222",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.AppInstallComplete));
|
||||
Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
}
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
{
|
||||
/// <summary>
|
||||
/// Historic-replay handling (session eaf3d8c4): a previous enrollment's IME log surviving
|
||||
/// on disk made the agent replay week-old script activity — 156 phantom script_completed
|
||||
/// events with ~170 h durations (clock-clamped completion minus RAW stale start) and an
|
||||
/// immediate-upload flood. Script events whose source line is > 24 h stale are now
|
||||
/// suppressed entirely (one-shot <c>historic_ime_replay_detected</c> summary instead),
|
||||
/// and durations are computed from a timeline-consistent timestamp pair with a 24 h
|
||||
/// plausibility backstop.
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerAdapterHistoricReplayTests
|
||||
{
|
||||
private static readonly DateTime ClockNow = new DateTime(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc);
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Suppression: stale (> 24 h past) source lines are a previous enrollment
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Stale_completion_is_suppressed_and_emits_oneshot_summary()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
var ancient = ClockNow.AddDays(-7);
|
||||
f.Tracker.LastMatchedLogTimestamp = ancient;
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "d78c1822",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Failed",
|
||||
StartedAtUtc = ancient.AddMinutes(-30),
|
||||
// Bootstrap marker in stdout — week-old evidence must not drive the
|
||||
// bootstrap-detected trace either.
|
||||
Stdout = "Bootstrap script version: v2.0",
|
||||
});
|
||||
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptFailed));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.AgentTrace));
|
||||
|
||||
var summary = f.InfoEvent(SharedEventTypes.HistoricImeReplayDetected);
|
||||
Assert.Equal(ancient.ToString("o"), summary.Payload!["earliestRejectedSourceTimestamp"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Summary_is_oneshot_across_started_and_completed_suppressions()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerScriptStartedFromTest(new ScriptStartedInfo { PolicyId = "aaa11111", ScriptType = "platform" });
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7).AddMinutes(1);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "aaa11111",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7).AddMinutes(2);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "bbb22222",
|
||||
ScriptType = "remediation",
|
||||
ScriptPart = "detection",
|
||||
ExitCode = 0,
|
||||
ComplianceResult = "True",
|
||||
});
|
||||
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptStarted));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
// Exactly ONE summary regardless of how many replayed lines followed — and it dates
|
||||
// the replay window from the FIRST (earliest) suppressed line.
|
||||
var summary = Assert.Single(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
Assert.Equal(ClockNow.AddDays(-7).ToString("o"), summary.Payload!["earliestRejectedSourceTimestamp"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Fresh_events_after_suppression_emit_normally()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddDays(-7);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "aaa11111",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
|
||||
// The agent catches up to current log content — normal emission resumes.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-2);
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "ccc33333",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
StartedAtUtc = ClockNow.AddMinutes(-3),
|
||||
});
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Equal("ccc33333", info.Payload!["policyId"]);
|
||||
Assert.Equal("60.00", info.Payload["durationSeconds"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Clock_fallback_without_source_timestamp_still_emits()
|
||||
{
|
||||
// derivedFromClock with NO rejected source ts (synthetic/callback path) is not a
|
||||
// replay — must emit, and must not trigger the summary.
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "ddd44444",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
});
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Equal("true", info.Payload!["derivedTimestamp"]);
|
||||
Assert.False(info.Payload.ContainsKey("rejectedSourceTimestamp"));
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Duration: timeline-consistent pair + plausibility backstop
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void FutureSkew_completion_emits_with_clamped_stamp_and_raw_pair_duration()
|
||||
{
|
||||
// Mid-enrollment clock jump (WhiteGlove +1h CMTrace skew case): the source ts is
|
||||
// rejected as future-skewed — NOT suppressed — and the duration must come from the
|
||||
// raw source pair (both ends on the same skewed timeline), not clamped-now − raw-start.
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
var skewed = ClockNow.AddHours(2);
|
||||
f.Tracker.LastMatchedLogTimestamp = skewed;
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "eee55555",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Success",
|
||||
StartedAtUtc = skewed.AddSeconds(-360),
|
||||
});
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptCompleted));
|
||||
Assert.Equal(ClockNow, info.OccurredAtUtc); // stamp: clamped to clock
|
||||
Assert.Equal("true", info.Payload!["derivedTimestamp"]);
|
||||
Assert.Equal(skewed.ToString("o"), info.Payload["rejectedSourceTimestamp"]);
|
||||
Assert.Equal("360.00", info.Payload["durationSeconds"]); // duration: raw pair
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.HistoricImeReplayDetected));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FutureSkew_failed_platform_script_timeout_heuristic_uses_raw_pair()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
var skewed = ClockNow.AddHours(2);
|
||||
f.Tracker.LastMatchedLogTimestamp = skewed;
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "fff66666",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Failed",
|
||||
StartedAtUtc = skewed.AddMinutes(-26), // past the 25-min suspicion threshold
|
||||
});
|
||||
|
||||
var suspected = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected));
|
||||
Assert.Equal("1560.00", suspected.Payload!["durationSeconds"]);
|
||||
Assert.Equal(ClockNow, suspected.OccurredAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Implausible_duration_above_24h_is_omitted()
|
||||
{
|
||||
// Fresh (accepted) completion paired with an ancient start — e.g. a stale pending
|
||||
// slot a legacy path left behind. The duration would be a cross-run lie; omit it.
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddMinutes(-2);
|
||||
|
||||
adapter.TriggerScriptCompletedFromTest(new ScriptExecutionState
|
||||
{
|
||||
PolicyId = "abc12345",
|
||||
ScriptType = "platform",
|
||||
ExitCode = 0,
|
||||
Result = "Failed",
|
||||
StartedAtUtc = ClockNow.AddHours(-30),
|
||||
});
|
||||
|
||||
var info = Assert.Single(f.InfoEvents(SharedEventTypes.ScriptFailed));
|
||||
Assert.False(info.Payload!.ContainsKey("durationSeconds"));
|
||||
Assert.False(info.Payload.ContainsKey("durationBasis"));
|
||||
// The 30 h "duration" must not read as a hung script either.
|
||||
Assert.Empty(f.InfoEvents(SharedEventTypes.ScriptTimeoutSuspected));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Tracker slot hardening: stale pending start must not pair with a fresh run
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Fresh_start_discards_stale_pending_slot_from_previous_enrollment()
|
||||
{
|
||||
using var f = new ImeLogTrackerAdapterFixture(ClockNow);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
|
||||
// Previous enrollment: replayed start line whose matching result never appeared
|
||||
// (the script was mid-run when that enrollment ended).
|
||||
f.Tracker.SeedPendingPlatformScriptForTesting("beef7777", exitCode: null, exitObservedAtUtc: null,
|
||||
startedAtUtc: ClockNow.AddDays(-7));
|
||||
|
||||
// Same policy genuinely re-runs now — the fresh start must begin a NEW execution.
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow.AddSeconds(-90);
|
||||
f.Tracker.HandlePlatformScriptStarted("beef7777");
|
||||
|
||||
f.Tracker.LastMatchedLogTimestamp = ClockNow;
|
||||
f.Tracker.CompletePlatformScriptFromImeResultForTesting("beef7777", "Success", ClockNow);
|
||||
|
||||
var info = f.InfoEvent(SharedEventTypes.ScriptCompleted);
|
||||
// Without the hardening the slot kept the 7-day-old start: the duration would have
|
||||
// been dropped by the 24 h backstop (or, before that, shown as ~168 h).
|
||||
Assert.Equal("90.00", info.Payload!["durationSeconds"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -195,7 +195,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
var all = f.InfoEvents(SharedEventTypes.ScriptCompleted);
|
||||
Assert.Equal(2, all.Count);
|
||||
Assert.Equal("65.00", all[1].Payload!["durationSeconds"]);
|
||||
Assert.Equal("cycle_including_reporting_latency", all[1].Payload["durationBasis"]);
|
||||
Assert.Equal("cycle_including_reporting_latency", all[1].Payload!["durationBasis"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+8
-2
@@ -151,17 +151,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TriggerPhaseChangedFromTest_emits_informational_only_no_decision_signal()
|
||||
public void TriggerPhaseChangedFromTest_emits_typed_signal_and_informational_event()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var adapter = new RealmJoinWatcherAdapter(f.Watcher, f.Ingress, f.Clock);
|
||||
|
||||
adapter.TriggerPhaseChangedFromTest(prev: 100, curr: 200);
|
||||
|
||||
// Phase change is observability-only. There is no DecisionSignalKind.RealmJoinPhaseChanged.
|
||||
// 1) Typed signal — persists LastDeploymentPhase and drives the aborted-RJ-ESP
|
||||
// detection in the reducer (session 224b2087). Not Detected/Resolved.
|
||||
Assert.DoesNotContain(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.RealmJoinDetected);
|
||||
Assert.DoesNotContain(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.RealmJoinResolved);
|
||||
var decision = f.Ingress.Posted.Single(p => p.Kind == DecisionSignalKind.RealmJoinPhaseChanged);
|
||||
Assert.Equal("RealmJoinWatcher", decision.SourceOrigin);
|
||||
Assert.Equal("200", decision.Payload![DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase]);
|
||||
Assert.Equal("100", decision.Payload[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase]);
|
||||
|
||||
// 2) Dual-emit informational event keeps the existing timeline entry.
|
||||
var info = f.Ingress.Posted.Single(p =>
|
||||
p.Kind == DecisionSignalKind.InformationalEvent
|
||||
&& p.Payload != null
|
||||
|
||||
+12
-12
@@ -1,4 +1,4 @@
|
||||
using System;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic;
|
||||
using Xunit;
|
||||
@@ -19,7 +19,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
private static List<BandwidthJobSample> Jobs(params BandwidthJobSample[] jobs)
|
||||
=> new List<BandwidthJobSample>(jobs);
|
||||
|
||||
private static BandwidthJobSample Job(string fileId, long wan, long lan = 0)
|
||||
private static BandwidthJobSample Job(string? fileId, long wan, long lan = 0)
|
||||
=> new BandwidthJobSample { FileId = fileId, WanBytes = wan, LanBytes = lan };
|
||||
|
||||
[Fact]
|
||||
@@ -43,8 +43,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
var estimate = estimator.TryBuildEstimate();
|
||||
Assert.NotNull(estimate);
|
||||
Assert.Equal(1, estimate.WanSampleCount);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsMax.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90!.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsMax!.Value, 3);
|
||||
Assert.Equal(6_000_000, estimate.WanBytesObserved);
|
||||
Assert.Equal("10-50", estimate.Bucket);
|
||||
}
|
||||
@@ -58,7 +58,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
estimator.AddSnapshot(T0.AddSeconds(3), Jobs(Job("a", 3_000_000), Job("b", 3_000_000)));
|
||||
|
||||
var estimate = estimator.TryBuildEstimate();
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90!.Value, 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -74,7 +74,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
|
||||
var estimate = estimator.TryBuildEstimate();
|
||||
Assert.Equal(2, estimate.WanSampleCount);
|
||||
Assert.Equal(16.0, estimate.WanMbpsMax.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsMax!.Value, 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -91,7 +91,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
estimator.AddSnapshot(T0.AddSeconds(63), Jobs(Job("a", 106_000_000)));
|
||||
var estimate = estimator.TryBuildEstimate();
|
||||
Assert.Equal(1, estimate.WanSampleCount);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90!.Value, 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -130,8 +130,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
estimator.AddSnapshot(T0.AddSeconds(3), Jobs(Job("a", wan: 6_000_000, lan: 375_000_000)));
|
||||
|
||||
var estimate = estimator.TryBuildEstimate();
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90.Value, 3);
|
||||
Assert.Equal(1000.0, estimate.LanMbpsP90.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90!.Value, 3);
|
||||
Assert.Equal(1000.0, estimate.LanMbpsP90!.Value, 3);
|
||||
Assert.Equal("10-50", estimate.Bucket); // bucket keyed to WAN, not LAN
|
||||
}
|
||||
|
||||
@@ -182,8 +182,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
|
||||
var estimate = estimator.TryBuildEstimate();
|
||||
Assert.Equal(20, estimate.WanSampleCount);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90.Value, 3); // p90 stays on the steady plateau
|
||||
Assert.Equal(160.0, estimate.WanMbpsMax.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90!.Value, 3); // p90 stays on the steady plateau
|
||||
Assert.Equal(160.0, estimate.WanMbpsMax!.Value, 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -265,7 +265,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Telemetry
|
||||
|
||||
var estimate = estimator.TryBuildEstimate();
|
||||
Assert.Equal(1, estimate.WanSampleCount); // only the finite positive sample survived
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90.Value, 3);
|
||||
Assert.Equal(16.0, estimate.WanMbpsP90!.Value, 3);
|
||||
Assert.Equal(0, estimate.WanBytesObserved);
|
||||
}
|
||||
|
||||
|
||||
+51
-30
@@ -208,39 +208,60 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
}
|
||||
else
|
||||
{
|
||||
// Platform script: AgentExecutor.log entries create/enrich, IME log entries also create
|
||||
if (!_pendingPlatformScripts.ContainsKey(id))
|
||||
{
|
||||
// A fresh start for this policy (no pending entry) begins a NEW execution.
|
||||
// Clear any emitted-marker from a prior run so IME re-evaluations / retries of
|
||||
// the same platform-script policy within one agent lifetime are not silently
|
||||
// deduped away. Within a single run the start always precedes exit/result, so
|
||||
// this never clears the current run's own marker.
|
||||
_platformScriptResultEmitted.Remove(id);
|
||||
_pendingPlatformScripts[id] = new ScriptExecutionState
|
||||
{
|
||||
PolicyId = id,
|
||||
ScriptType = "platform",
|
||||
// Prefer the source CMTrace timestamp so a script that started before the
|
||||
// agent launched (replayed log content) is dated to its real start, not now.
|
||||
// Set only at slot creation: the start line fires twice (agentexecutor + ime
|
||||
// source) but the earliest wins.
|
||||
StartedAtUtc = LastMatchedLogTimestamp ?? DateTime.UtcNow,
|
||||
};
|
||||
// Live "running" indicator — same signal health scripts emit. Gated on slot
|
||||
// creation so the duplicate start line (agentexecutor + ime source) doesn't
|
||||
// double-emit for the same execution.
|
||||
try { OnScriptStarted?.Invoke(new ScriptStartedInfo { PolicyId = id, ScriptType = "platform" }); }
|
||||
catch (Exception ex) { _logger.Warning($"ImeLogTracker: OnScriptStarted handler threw: {ex.Message}"); }
|
||||
}
|
||||
_lastPlatformScriptPolicyId = id;
|
||||
// Started lines fire twice per script (agentexecutor + ime source) and carry no
|
||||
// outcome — the matching `platform script completed` line below carries result+exit
|
||||
// and stays on Info. Keep starts on Debug so Info reflects script outcomes only.
|
||||
_logger.Debug($"ImeLogTracker: platform script started: {id} (source: {source ?? "ime"})");
|
||||
HandlePlatformScriptStarted(id, source);
|
||||
}
|
||||
}
|
||||
|
||||
// Platform script: AgentExecutor.log entries create/enrich, IME log entries also create.
|
||||
// Internal so the stale-slot hardening is directly testable (the public path arrives via
|
||||
// a regex Match).
|
||||
internal void HandlePlatformScriptStarted(string id, string source = null)
|
||||
{
|
||||
// Stale-slot hardening (session eaf3d8c4): a replayed start line from a previous
|
||||
// enrollment whose matching result never appeared (script was mid-run when that
|
||||
// enrollment ended) leaves a pending slot with a days-old StartedAtUtc. When the
|
||||
// same policy genuinely re-runs now, the fresh start must begin a NEW execution —
|
||||
// otherwise the ancient start pairs with the fresh completion and the duration
|
||||
// lies. > 24 h mirrors the adapter's source-timestamp staleness clamp.
|
||||
var currentStartTs = LastMatchedLogTimestamp ?? DateTime.UtcNow;
|
||||
if (_pendingPlatformScripts.TryGetValue(id, out var pendingSlot)
|
||||
&& pendingSlot.StartedAtUtc.HasValue
|
||||
&& currentStartTs - pendingSlot.StartedAtUtc.Value > TimeSpan.FromHours(24))
|
||||
{
|
||||
_logger.Debug($"ImeLogTracker: discarding stale pending platform-script slot for {id} (started {pendingSlot.StartedAtUtc:o})");
|
||||
_pendingPlatformScripts.Remove(id);
|
||||
}
|
||||
if (!_pendingPlatformScripts.ContainsKey(id))
|
||||
{
|
||||
// A fresh start for this policy (no pending entry) begins a NEW execution.
|
||||
// Clear any emitted-marker from a prior run so IME re-evaluations / retries of
|
||||
// the same platform-script policy within one agent lifetime are not silently
|
||||
// deduped away. Within a single run the start always precedes exit/result, so
|
||||
// this never clears the current run's own marker.
|
||||
_platformScriptResultEmitted.Remove(id);
|
||||
_pendingPlatformScripts[id] = new ScriptExecutionState
|
||||
{
|
||||
PolicyId = id,
|
||||
ScriptType = "platform",
|
||||
// Prefer the source CMTrace timestamp so a script that started before the
|
||||
// agent launched (replayed log content) is dated to its real start, not now.
|
||||
// Set only at slot creation: the start line fires twice (agentexecutor + ime
|
||||
// source) but the earliest wins.
|
||||
StartedAtUtc = currentStartTs,
|
||||
};
|
||||
// Live "running" indicator — same signal health scripts emit. Gated on slot
|
||||
// creation so the duplicate start line (agentexecutor + ime source) doesn't
|
||||
// double-emit for the same execution.
|
||||
try { OnScriptStarted?.Invoke(new ScriptStartedInfo { PolicyId = id, ScriptType = "platform" }); }
|
||||
catch (Exception ex) { _logger.Warning($"ImeLogTracker: OnScriptStarted handler threw: {ex.Message}"); }
|
||||
}
|
||||
_lastPlatformScriptPolicyId = id;
|
||||
// Started lines fire twice per script (agentexecutor + ime source) and carry no
|
||||
// outcome — the matching `platform script completed` line below carries result+exit
|
||||
// and stays on Info. Keep starts on Debug so Info reflects script outcomes only.
|
||||
_logger.Debug($"ImeLogTracker: platform script started: {id} (source: {source ?? "ime"})");
|
||||
}
|
||||
|
||||
// Routes line-by-line script-data handlers (context / exitCode / output) to the right
|
||||
// accumulator based on the pattern's scriptType parameter. Health-script lines feed the
|
||||
// single-slot _pendingHealthScript; platform-script lines feed _pendingPlatformScripts.
|
||||
|
||||
+34
@@ -194,6 +194,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
}
|
||||
}
|
||||
|
||||
// Actions that mutate app/phase tracking state (_packageStates, _phasePackageSnapshots,
|
||||
// _currentPhaseOrder, DO telemetry) — the historic-replay guard skips these for source
|
||||
// lines from a previous enrollment so replayed apps never enter tracked state,
|
||||
// persistence, app_tracking_summary, culprit lists or final-status. Script actions are
|
||||
// deliberately NOT here: their tracker state is harmless (stale-slot hardening covers
|
||||
// leftovers) and the adapter suppresses their emissions. espphasedetected IS here — a
|
||||
// stale "In EspPhase: AccountSetup" would advance _currentPhaseOrder and make the fresh
|
||||
// enrollment's DeviceSetup lines bounce as "backward"; IME re-logs the phase constantly,
|
||||
// so fresh lines re-deliver it immediately.
|
||||
private static readonly HashSet<string> AppMutatingActions = new HashSet<string>(StringComparer.Ordinal)
|
||||
{
|
||||
"imeshutdown", "espphasedetected", "setcurrentapp",
|
||||
"updatestateinstalled", "updatestatedownloading", "updatestateinstalling",
|
||||
"updatestateskipped", "updatestateerror", "updatestatepostponed",
|
||||
"captureexitcode", "capturehresult", "captureappversion",
|
||||
"captureapptypewinget", "captureapptypemsi", "captureattemptnumber",
|
||||
"capturedetectionresult", "esptrackstatus", "policiesdiscovered",
|
||||
"ignorecompletedapp", "updatename", "updatewin32appstate",
|
||||
"cancelstuckandsetcurrent", "updatedotelemetry",
|
||||
};
|
||||
|
||||
private void HandlePatternMatch(CompiledPattern pattern, Match match, string message, CmTraceLogEntry entry)
|
||||
{
|
||||
LastMatchedPatternId = pattern.PatternId;
|
||||
@@ -206,6 +227,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
try { OnPatternMatched?.Invoke(pattern.PatternId); }
|
||||
catch (Exception ex) { _logger?.Warning($"OnPatternMatched handler threw: {ex.Message}"); }
|
||||
|
||||
// Historic-replay guard (session eaf3d8c4): a source line > 24 h older than now is
|
||||
// content from a previous enrollment whose IME log survived on disk. App-mutating
|
||||
// actions are skipped BEFORE _seenAppIds so replayed apps poison neither the tracked
|
||||
// state nor the phase-change ignore list. SimulationMode (--replay-log-dir dev tool)
|
||||
// replays historic logs on purpose and bypasses the guard.
|
||||
var isStaleReplayLine = !SimulationMode && entry != null
|
||||
&& (UtcNowProvider() - NormalizeUtc(entry.Timestamp)) > HistoricReplayThreshold;
|
||||
if (isStaleReplayLine && AppMutatingActions.Contains(pattern.Action?.ToLower() ?? string.Empty))
|
||||
{
|
||||
_logger.Debug($"ImeLogTracker: skipped app action '{pattern.Action}' for historic line ({entry.Timestamp:o})");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var id = match.Groups["id"]?.Value;
|
||||
|
||||
@@ -71,6 +71,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
public double SpeedFactor { get; set; } = 50;
|
||||
private DateTime _lastLogTimestamp = DateTime.MinValue;
|
||||
|
||||
// Source log lines older than this relative to now are content from a previous
|
||||
// enrollment (IME logs surviving a re-enrollment, session eaf3d8c4). Single source of
|
||||
// truth shared with the adapter's ResolveOccurredAt clamp and replay suppression.
|
||||
internal static readonly TimeSpan HistoricReplayThreshold = TimeSpan.FromHours(24);
|
||||
|
||||
// Deterministic-time seam for the historic-replay guard; production default is the
|
||||
// system clock, ImeLogHost rewires it to the agent IClock, tests to their VirtualClock.
|
||||
internal Func<DateTime> UtcNowProvider { get; set; } = () => DateTime.UtcNow;
|
||||
|
||||
/// <summary>
|
||||
/// CMTrace timestamps from IME are emitted in UTC, but DateTimeKind may arrive as
|
||||
/// Unspecified depending on the parser path. Normalize via SpecifyKind=Utc rather than
|
||||
/// ToUniversalTime() to avoid double-conversion when Kind is already UTC.
|
||||
/// </summary>
|
||||
internal static DateTime NormalizeUtc(DateTime value) =>
|
||||
value.Kind == DateTimeKind.Utc ? value : DateTime.SpecifyKind(value, DateTimeKind.Utc);
|
||||
|
||||
// Background task
|
||||
private Task _pollingTask;
|
||||
private CancellationTokenSource _cts;
|
||||
|
||||
+75
@@ -61,6 +61,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
internal Func<int, string> SessionOwnerResolver { get; set; }
|
||||
internal Func<int, string> ProcessOwnerResolver { get; set; }
|
||||
|
||||
// Test seam for the WinRT OOBE-state sample (null = OobeStateReader.Read).
|
||||
internal Func<string> OobeStateProvider { get; set; }
|
||||
|
||||
// OOBE-state flip tracking (observational only — see OobeStateReader contract).
|
||||
// _oobeCompletedEmitted is one-shot per agent lifetime and deliberately NOT reset
|
||||
// by ResetForRealUserSwitch: the OOBE flip is a global OS transition, not per-user.
|
||||
private string _lastOobeState;
|
||||
private bool _oobeCompletedEmitted;
|
||||
|
||||
/// <summary>
|
||||
/// System/service account names that should NOT be considered real users.
|
||||
/// </summary>
|
||||
@@ -161,6 +170,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
_startedAtUtc = DateTime.UtcNow;
|
||||
|
||||
// Baseline OOBE-state sample so the first poll can already detect a flip
|
||||
// (self-contained try/catch inside, like the Emit* helpers).
|
||||
CheckOobeStateFlip();
|
||||
|
||||
_pollingTimer?.Dispose();
|
||||
_pollingTimer = new Timer(
|
||||
PollForDesktop,
|
||||
@@ -186,6 +199,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
|
||||
_pollCount++;
|
||||
|
||||
CheckOobeStateFlip();
|
||||
|
||||
try
|
||||
{
|
||||
var explorerProcesses = Process.GetProcessesByName("explorer");
|
||||
@@ -381,6 +396,66 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
catch (Exception ex) { _logger.Verbose($"DesktopArrivalDetector: detector-started emit failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Samples the WinRT OOBE state (<see cref="OobeStateReader"/>) and emits the one-shot
|
||||
/// <c>oobe_state_completed</c> trace event on the in_progress->completed flip.
|
||||
/// Observational only — an owner-independent desktop corroboration for sessions where
|
||||
/// WTS+WMI owner resolution starves and <c>desktop_arrived</c> never fires. Often absent
|
||||
/// in healthy sessions: the polling timer is disposed at desktop arrival, which is when
|
||||
/// the flip lands (empirical probe, session 9c404ae9).
|
||||
/// <para>
|
||||
/// Self-contained try/catch: a throw here must NOT reach <see cref="PollForDesktop"/>'s
|
||||
/// outer catch, which would inflate <see cref="_wmiErrorCountSinceStart"/> and skew the
|
||||
/// session-4d5a0b78 liveness diagnostics. "unavailable" readings never overwrite the last
|
||||
/// known state, so a transient reflection failure between two polls cannot mask the flip.
|
||||
/// One-shot flag survives <see cref="ResetForRealUserSwitch"/> — the flip is a global OS
|
||||
/// transition, not per-user.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void CheckOobeStateFlip()
|
||||
{
|
||||
try
|
||||
{
|
||||
var current = (OobeStateProvider ?? OobeStateReader.Read)();
|
||||
if (string.IsNullOrEmpty(current) || current == OobeStateReader.Unavailable)
|
||||
return;
|
||||
|
||||
if (!_oobeCompletedEmitted
|
||||
&& _lastOobeState == "in_progress"
|
||||
&& current == "completed")
|
||||
{
|
||||
_oobeCompletedEmitted = true;
|
||||
var minutesSinceStart = Math.Round((DateTime.UtcNow - _startedAtUtc).TotalMinutes, 1);
|
||||
_logger.Info("DesktopArrivalDetector: OOBE state flipped in_progress->completed (WinRT SystemSetupInfo)");
|
||||
OnTraceEvent?.Invoke(
|
||||
SharedEventTypes.OobeStateCompleted,
|
||||
$"OOBE state flipped in_progress->completed (WinRT SystemSetupInfo, poll {_pollCount}) — owner-independent desktop corroboration.",
|
||||
new Dictionary<string, object>
|
||||
{
|
||||
{ "previousState", "in_progress" },
|
||||
{ "pollCount", _pollCount },
|
||||
{ "minutesSinceStart", minutesSinceStart },
|
||||
{ "desktopArrived", _desktopArrived },
|
||||
{ "wmiErrorCount", _wmiErrorCountSinceStart },
|
||||
{ "wtsErrorCount", _wtsErrorCountSinceStart },
|
||||
});
|
||||
}
|
||||
|
||||
_lastOobeState = current;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Verbose($"DesktopArrivalDetector: OOBE state check failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drives a single synchronous poll for unit tests — the production poll is private and
|
||||
/// timer-driven with a fixed 30 s interval (precedent: <see cref="ResolveOwner"/> is
|
||||
/// likewise internal for direct testing).
|
||||
/// </summary>
|
||||
internal void PollOnceForTest() => PollForDesktop(null);
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the owning user of an explorer.exe candidate. Session 4d5a0b78 fix
|
||||
/// (2026-06-11): the WMI <c>Win32_Process.GetOwner</c> call failed on every poll on
|
||||
|
||||
+16
-1
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
@@ -38,16 +39,30 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// </summary>
|
||||
public string Category { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Session 4910a5a5: names of the tracked apps most likely behind an Apps-subcategory
|
||||
/// failure (capped, never-started apps ranked first — those starve ESP's app gate
|
||||
/// without ever producing an IME install event). Snapshot from
|
||||
/// <c>ProvisioningStatusTracker.SnapshotTrackedAppsNotCompleted</c> at failure-arming
|
||||
/// time; empty for non-Apps failures and event-log-derived failures. Propagated onto
|
||||
/// the EspTerminalFailure signal payload so the DecisionEngine's
|
||||
/// <c>esp_failure_advisory</c> / <c>enrollment_failed</c> events can name the culprit
|
||||
/// directly.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> LikelyCulpritApps { get; }
|
||||
|
||||
public EspFailureDetectedEventArgs(
|
||||
string failureType,
|
||||
string errorCode = null,
|
||||
string failedSubcategory = null,
|
||||
string category = null)
|
||||
string category = null,
|
||||
IReadOnlyList<string> likelyCulpritApps = null)
|
||||
{
|
||||
FailureType = string.IsNullOrEmpty(failureType) ? "unknown" : failureType;
|
||||
ErrorCode = string.IsNullOrEmpty(errorCode) ? null : errorCode;
|
||||
FailedSubcategory = string.IsNullOrEmpty(failedSubcategory) ? null : failedSubcategory;
|
||||
Category = string.IsNullOrEmpty(category) ? null : category;
|
||||
LikelyCulpritApps = likelyCulpritApps ?? Array.Empty<string>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using Microsoft.Win32;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Session 5d735290 (2026-07-23) — reads the ESP's registered policy providers from
|
||||
/// <c>HKLM\SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking</c>, the registry
|
||||
/// mirror of the EnrollmentStatusTracking CSP
|
||||
/// (https://learn.microsoft.com/en-us/windows/client-management/mdm/enrollmentstatustracking-csp).
|
||||
/// <para>
|
||||
/// CSP contract the ESP enforces (quotes from the doc above):
|
||||
/// <list type="bullet">
|
||||
/// <item><c>Setup/Apps/PolicyProviders/<Name></c> — "Existence of this node indicates
|
||||
/// to the ESP that it shouldn't show the tracking status message until the
|
||||
/// TrackingPoliciesCreated node has been set to true." This wait has NO timeout: a
|
||||
/// registered provider that never sets <c>TrackingPoliciesCreated=1</c> parks the ESP at
|
||||
/// "Apps (Identifying)" indefinitely (observed for days in the field).</item>
|
||||
/// <item><c>DevicePreparation/PolicyProviders/<Name>/InstallationState</c> — must reach
|
||||
/// 2 (NotRequired) or 3 (Completed); 1=NotInstalled, 4=Error. ESP applies a default
|
||||
/// 15-minute timeout here, so stalls in this stage are self-surfacing — still reported
|
||||
/// for the diagnostic trail.</item>
|
||||
/// </list>
|
||||
/// Registry layout: <c>Device</c> subkey for device scope, <c>S-<SID></c> subkeys for user
|
||||
/// scope (appear at sign-in), each containing <c>Setup\Apps\PolicyProviders\<Name></c>;
|
||||
/// <c>DevicePreparation\PolicyProviders</c> exists in device scope only.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Known provider names: <c>Sidecar</c> is the Intune Management Extension ("Intune's agents,
|
||||
/// such as SideCar" per the CSP doc); <c>ConfigMgr</c> is the Configuration Manager client
|
||||
/// (registered during Autopilot-into-co-management, see the troubleshooting registry paths in
|
||||
/// https://learn.microsoft.com/en-us/intune/configmgr/comanage/autopilot-enrollment). Microsoft
|
||||
/// warns that ESP policy providers are not aware of each other and that pre-provisioning is
|
||||
/// unsupported with co-management. Field case that motivated this probe (issue #106): a
|
||||
/// pre-provisioned device with the ConfigMgr client installed as a blocking app ended up with
|
||||
/// ONLY <c>ConfigMgr</c> registered under <c>Device\Setup\Apps\PolicyProviders</c> (no
|
||||
/// <c>Sidecar</c>) — and although <c>TrackingPoliciesCreated=1</c> WAS set under it, the user
|
||||
/// ESP hung at "Apps (Identifying)" for hours to days; manually renaming the key to
|
||||
/// <c>Sidecar</c> unblocked it immediately. So the ESP's Apps wait is keyed to the provider
|
||||
/// the Intune service registered at enrollment (always <c>Sidecar</c>; ConfigMgr only joins
|
||||
/// via a co-management settings policy — see the WAP payload documented in
|
||||
/// https://oofhours.com/2023/08/25/run-an-sccm-task-sequence-during-autopilot/), and a
|
||||
/// foreign provider's completion values do not satisfy it. Detection therefore keys on the
|
||||
/// provider NAME topology, not only on completeness:
|
||||
/// <see cref="EspPolicyProviderStallDetector"/> turns this contract into the two-arm
|
||||
/// <c>esp_policy_provider_stalled</c> tripwire.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Fail-soft like <see cref="EspTrackingInfoProbe"/>: any failure is Debug-logged and yields
|
||||
/// <see cref="EspPolicyProviderSnapshot.Empty"/>; a missing root key (early enrollment,
|
||||
/// non-Autopilot device) is not an error. Values are parsed defensively — a missing or
|
||||
/// unparseable value counts as incomplete (fail-toward-observation).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static class EspPolicyProviderProbe
|
||||
{
|
||||
internal const string TrackingRootKeyPath =
|
||||
@"SOFTWARE\Microsoft\Windows\Autopilot\EnrollmentStatusTracking";
|
||||
|
||||
/// <summary>Provider kind: <c>Setup\Apps\PolicyProviders</c> (TrackingPoliciesCreated wait, no timeout).</summary>
|
||||
internal const string KindSetupApps = "setupApps";
|
||||
|
||||
/// <summary>Provider kind: <c>DevicePreparation\PolicyProviders</c> (InstallationState wait, 15-min OS timeout).</summary>
|
||||
internal const string KindDevicePreparation = "devicePreparation";
|
||||
|
||||
/// <summary>The IME's provider name — its absence while other providers are registered is the field-case signature.</summary>
|
||||
internal const string SidecarProviderName = "Sidecar";
|
||||
|
||||
internal const string DeviceScope = "device";
|
||||
|
||||
private const string TrackingPoliciesCreatedValueName = "TrackingPoliciesCreated";
|
||||
private const string InstallationStateValueName = "InstallationState";
|
||||
|
||||
/// <summary>
|
||||
/// Test seam — when non-null, <see cref="Read"/> delegates to this func instead of
|
||||
/// touching the live registry. Use <see cref="ScopedOverride"/> to guarantee cleanup.
|
||||
/// </summary>
|
||||
internal static Func<AgentLogger, EspPolicyProviderSnapshot> TestOverride;
|
||||
|
||||
/// <summary>
|
||||
/// Reads all registered ESP policy providers. <see cref="EspPolicyProviderSnapshot.HasData"/>
|
||||
/// is <c>true</c> when the EnrollmentStatusTracking root key existed and was readable
|
||||
/// (even with no providers registered); <see cref="EspPolicyProviderSnapshot.Empty"/> otherwise.
|
||||
/// </summary>
|
||||
/// <param name="logger">Optional logger — Debug-level trace only; no warn/error.</param>
|
||||
public static EspPolicyProviderSnapshot Read(AgentLogger logger = null)
|
||||
{
|
||||
var probe = TestOverride;
|
||||
if (probe != null) return probe(logger);
|
||||
|
||||
try
|
||||
{
|
||||
using (var rootKey = Registry.LocalMachine.OpenSubKey(TrackingRootKeyPath))
|
||||
{
|
||||
if (rootKey == null)
|
||||
{
|
||||
logger?.Debug("EspPolicyProviderProbe: EnrollmentStatusTracking registry key not found");
|
||||
return EspPolicyProviderSnapshot.Empty;
|
||||
}
|
||||
|
||||
var providers = new List<PolicyProviderState>();
|
||||
|
||||
foreach (var scopeName in rootKey.GetSubKeyNames())
|
||||
{
|
||||
var isDevice = string.Equals(scopeName, "Device", StringComparison.OrdinalIgnoreCase);
|
||||
// User scope repeats the Setup structure under the signed-in user's SID.
|
||||
var isUserSid = scopeName.StartsWith("S-", StringComparison.OrdinalIgnoreCase);
|
||||
if (!isDevice && !isUserSid) continue;
|
||||
|
||||
var scopeLabel = isDevice ? DeviceScope : "user:" + scopeName;
|
||||
using (var scopeKey = rootKey.OpenSubKey(scopeName))
|
||||
{
|
||||
if (scopeKey == null) continue;
|
||||
|
||||
CollectProviders(
|
||||
scopeKey, @"Setup\Apps\PolicyProviders", scopeLabel, KindSetupApps,
|
||||
TrackingPoliciesCreatedValueName, providers, logger);
|
||||
|
||||
if (isDevice)
|
||||
{
|
||||
CollectProviders(
|
||||
scopeKey, @"DevicePreparation\PolicyProviders", scopeLabel, KindDevicePreparation,
|
||||
InstallationStateValueName, providers, logger);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new EspPolicyProviderSnapshot(providers);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Debug($"EspPolicyProviderProbe: read threw: {ex.Message}");
|
||||
return EspPolicyProviderSnapshot.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private static void CollectProviders(
|
||||
RegistryKey scopeKey,
|
||||
string providersSubPath,
|
||||
string scopeLabel,
|
||||
string kind,
|
||||
string stateValueName,
|
||||
List<PolicyProviderState> providers,
|
||||
AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
using (var providersKey = scopeKey.OpenSubKey(providersSubPath))
|
||||
{
|
||||
if (providersKey == null) return;
|
||||
|
||||
foreach (var providerName in providersKey.GetSubKeyNames())
|
||||
{
|
||||
using (var providerKey = providersKey.OpenSubKey(providerName))
|
||||
{
|
||||
if (providerKey == null) continue;
|
||||
var value = TryReadInt(providerKey.GetValue(stateValueName));
|
||||
providers.Add(new PolicyProviderState(
|
||||
name: providerName,
|
||||
scope: scopeLabel,
|
||||
kind: kind,
|
||||
trackingPoliciesCreated: kind == KindSetupApps ? value : null,
|
||||
installationState: kind == KindDevicePreparation ? value : null));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Debug($"EspPolicyProviderProbe: '{providersSubPath}' ({scopeLabel}) read threw: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Defensive value parse: REG_DWORD expected per CSP (booleans stored as 0/1), but string
|
||||
/// forms are tolerated. Anything missing or unparseable is <c>null</c> — which the
|
||||
/// completion rule treats as incomplete.
|
||||
/// </summary>
|
||||
internal static int? TryReadInt(object rawValue)
|
||||
{
|
||||
switch (rawValue)
|
||||
{
|
||||
case null: return null;
|
||||
case int i: return i;
|
||||
case long l when l >= int.MinValue && l <= int.MaxValue: return (int)l;
|
||||
case string s when int.TryParse(s.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed):
|
||||
return parsed;
|
||||
case bool b: return b ? 1 : 0;
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure completion rule per the CSP contract: <c>setupApps</c> providers are complete when
|
||||
/// <c>TrackingPoliciesCreated == 1</c>; <c>devicePreparation</c> providers when
|
||||
/// <c>InstallationState</c> is 2 (NotRequired) or 3 (Completed).
|
||||
/// </summary>
|
||||
internal static bool IsProviderComplete(string kind, int? trackingPoliciesCreated, int? installationState)
|
||||
=> kind == KindSetupApps
|
||||
? trackingPoliciesCreated == 1
|
||||
: installationState == 2 || installationState == 3;
|
||||
|
||||
/// <summary>
|
||||
/// Disposable scope that sets <see cref="TestOverride"/> for the lifetime of the scope
|
||||
/// and restores the previous value on Dispose. Nestable. Internal test-only helper.
|
||||
/// </summary>
|
||||
internal sealed class ScopedOverride : IDisposable
|
||||
{
|
||||
private readonly Func<AgentLogger, EspPolicyProviderSnapshot> _previous;
|
||||
private int _disposed;
|
||||
|
||||
public ScopedOverride(Func<AgentLogger, EspPolicyProviderSnapshot> probe)
|
||||
{
|
||||
_previous = TestOverride;
|
||||
TestOverride = probe;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (System.Threading.Interlocked.Exchange(ref _disposed, 1) == 1) return;
|
||||
TestOverride = _previous;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One registered ESP policy provider with its completion-relevant state.
|
||||
/// <see cref="Key"/> is the stable identity used for dwell tracking and one-shot latching.
|
||||
/// </summary>
|
||||
internal readonly struct PolicyProviderState
|
||||
{
|
||||
public PolicyProviderState(string name, string scope, string kind, int? trackingPoliciesCreated, int? installationState)
|
||||
{
|
||||
Name = name ?? string.Empty;
|
||||
Scope = scope ?? string.Empty;
|
||||
Kind = kind ?? string.Empty;
|
||||
TrackingPoliciesCreated = trackingPoliciesCreated;
|
||||
InstallationState = installationState;
|
||||
IsComplete = EspPolicyProviderProbe.IsProviderComplete(kind, trackingPoliciesCreated, installationState);
|
||||
}
|
||||
|
||||
/// <summary>Provider key name, e.g. "Sidecar" or "ConfigMgr".</summary>
|
||||
public string Name { get; }
|
||||
|
||||
/// <summary><c>"device"</c> or <c>"user:S-1-5-21-…"</c>.</summary>
|
||||
public string Scope { get; }
|
||||
|
||||
/// <summary><see cref="EspPolicyProviderProbe.KindSetupApps"/> or <see cref="EspPolicyProviderProbe.KindDevicePreparation"/>.</summary>
|
||||
public string Kind { get; }
|
||||
|
||||
/// <summary>Raw <c>TrackingPoliciesCreated</c> value (setupApps only); <c>null</c> when missing/unparseable.</summary>
|
||||
public int? TrackingPoliciesCreated { get; }
|
||||
|
||||
/// <summary>Raw <c>InstallationState</c> value (devicePreparation only); <c>null</c> when missing/unparseable.</summary>
|
||||
public int? InstallationState { get; }
|
||||
|
||||
/// <summary>Completion per <see cref="EspPolicyProviderProbe.IsProviderComplete"/>.</summary>
|
||||
public bool IsComplete { get; }
|
||||
|
||||
/// <summary>Stable dwell/latch identity: <c>kind|scope|name</c>.</summary>
|
||||
public string Key => Kind + "|" + Scope + "|" + Name;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot of all registered ESP policy providers. <see cref="HasData"/> is <c>false</c> only
|
||||
/// when the EnrollmentStatusTracking root key was missing or unreadable — an empty provider
|
||||
/// list with <see cref="HasData"/> = <c>true</c> is valid data (nothing registered yet).
|
||||
/// </summary>
|
||||
internal readonly struct EspPolicyProviderSnapshot
|
||||
{
|
||||
public static readonly EspPolicyProviderSnapshot Empty = default;
|
||||
|
||||
public EspPolicyProviderSnapshot(IReadOnlyList<PolicyProviderState> providers)
|
||||
{
|
||||
HasData = true;
|
||||
Providers = providers ?? Array.Empty<PolicyProviderState>();
|
||||
}
|
||||
|
||||
/// <summary><c>true</c> when the EnrollmentStatusTracking root key existed and was readable.</summary>
|
||||
public bool HasData { get; }
|
||||
|
||||
/// <summary>All registered providers across scopes and kinds; empty when none registered.</summary>
|
||||
public IReadOnlyList<PolicyProviderState> Providers { get; }
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> when a <c>Setup\Apps</c> provider named "Sidecar" (the IME) is registered in
|
||||
/// any scope — its absence while other providers are registered is the co-management
|
||||
/// field-case signature.
|
||||
/// </summary>
|
||||
public bool SidecarRegistered
|
||||
{
|
||||
get
|
||||
{
|
||||
var providers = Providers;
|
||||
if (providers == null) return false;
|
||||
return providers.Any(p =>
|
||||
p.Kind == EspPolicyProviderProbe.KindSetupApps &&
|
||||
string.Equals(p.Name, EspPolicyProviderProbe.SidecarProviderName, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+315
@@ -0,0 +1,315 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Persistence;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Turns the ESP policy-provider contract read by <see cref="EspPolicyProviderProbe"/> into
|
||||
/// the one-shot <c>esp_policy_provider_stalled</c> Warning via two independent arms, each
|
||||
/// with the same <see cref="DwellMinutes"/>-minute continuous-dwell rule and the full
|
||||
/// provider table in the payload:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Arm 1 — <c>reason=provider_incomplete</c></b>: a registered provider stays
|
||||
/// continuously incomplete (e.g. Sidecar itself never sets TrackingPoliciesCreated).</item>
|
||||
/// <item><b>Arm 2 — <c>reason=sidecar_provider_missing</c></b>: at least one
|
||||
/// <c>Setup\Apps</c> provider is registered but none of them is <c>Sidecar</c> (the IME).
|
||||
/// This is the actual co-management field-case signature (issue #106): the foreign
|
||||
/// <c>ConfigMgr</c> provider had <c>TrackingPoliciesCreated=1</c> — i.e. was "complete"
|
||||
/// by the CSP value contract — yet the user ESP stayed parked at "Apps (Identifying)"
|
||||
/// until the key was renamed to <c>Sidecar</c>. The ESP's Apps wait is keyed to the
|
||||
/// Intune-registered provider by NAME, so provider completeness alone must not gate this
|
||||
/// detection. See the probe's header for sources.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// Dwell semantics: arm 1's clock per provider starts when it is first observed incomplete
|
||||
/// and RESETS when the provider completes or disappears; arm 2's clock starts when the
|
||||
/// sidecar-missing condition is first observed and RESETS when Sidecar registers (covers the
|
||||
/// legitimate "ConfigMgr registered before Sidecar" startup ordering) — only
|
||||
/// <i>continuously</i> bad states fire. A missing root key / empty provider list is normal
|
||||
/// early enrollment and clears all dwell state (probe D4). Purely observational: emission
|
||||
/// goes through <see cref="InformationalEventPost"/> (InformationalEvent pass-through,
|
||||
/// dispatch-guard exempt), never into decision state.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Duplicate suppression: in-process one-shot per provider key, plus a
|
||||
/// <see cref="StartupEventGate"/> fingerprint over the sorted set of currently-stalled keys so
|
||||
/// an agent restart into the same stall does not re-report it — while a NEW provider joining
|
||||
/// the stall changes the fingerprint and is reported (the event carries the full list).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal sealed class EspPolicyProviderStallDetector
|
||||
{
|
||||
/// <summary>
|
||||
/// Wall-clock minutes a provider must stay continuously incomplete before it is reported.
|
||||
/// Mirrors the CSP's own DevicePreparation default timeout; the Setup/Apps wait this
|
||||
/// tripwire mainly targets has no OS timeout at all.
|
||||
/// </summary>
|
||||
internal const int DwellMinutes = 15;
|
||||
|
||||
internal const string SourceLabel = "EspPolicyProviderStallDetector";
|
||||
|
||||
/// <summary>Payload <c>reason</c>: a registered provider stayed continuously incomplete (arm 1).</summary>
|
||||
internal const string ReasonProviderIncomplete = "provider_incomplete";
|
||||
|
||||
/// <summary>Payload <c>reason</c>: Setup\Apps providers registered but Sidecar absent (arm 2 — dominant when both arms fire in one round).</summary>
|
||||
internal const string ReasonSidecarMissing = "sidecar_provider_missing";
|
||||
|
||||
/// <summary>
|
||||
/// Sentinel token representing the arm-2 condition in the cross-restart fingerprint.
|
||||
/// Cannot collide with provider keys — those always contain <c>|</c> separators.
|
||||
/// </summary>
|
||||
private const string SidecarMissingFingerprintToken = "sidecarMissing";
|
||||
|
||||
private readonly string _sessionId;
|
||||
private readonly string _tenantId;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly IClock _clock;
|
||||
private readonly StartupEventGate? _startupGate;
|
||||
private readonly TimeSpan _dwell;
|
||||
|
||||
private readonly Dictionary<string, DateTime> _firstSeenIncompleteUtc =
|
||||
new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly HashSet<string> _firedKeys =
|
||||
new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Arm-2 dwell state: first observation of "setupApps providers without Sidecar"; null = condition not active.</summary>
|
||||
private DateTime? _sidecarMissingSinceUtc;
|
||||
|
||||
/// <summary>Arm-2 in-process one-shot latch (mirror of <see cref="_firedKeys"/>).</summary>
|
||||
private bool _sidecarMissingFired;
|
||||
|
||||
public EspPolicyProviderStallDetector(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
InformationalEventPost post,
|
||||
AgentLogger logger,
|
||||
IClock clock,
|
||||
StartupEventGate? startupGate = null,
|
||||
TimeSpan? dwell = null)
|
||||
{
|
||||
_sessionId = sessionId;
|
||||
_tenantId = tenantId;
|
||||
_post = post ?? throw new ArgumentNullException(nameof(post));
|
||||
_logger = logger;
|
||||
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
_startupGate = startupGate;
|
||||
_dwell = dwell ?? TimeSpan.FromMinutes(DwellMinutes);
|
||||
}
|
||||
|
||||
/// <summary>Host timer callback and test seam — one probe + evaluation round.</summary>
|
||||
internal void Tick()
|
||||
{
|
||||
var snapshot = EspPolicyProviderProbe.Read(_logger);
|
||||
var eval = EvaluateStallTransition(
|
||||
snapshot, _firstSeenIncompleteUtc, _firedKeys,
|
||||
_sidecarMissingSinceUtc, _sidecarMissingFired, _clock.UtcNow, _dwell);
|
||||
|
||||
_firstSeenIncompleteUtc.Clear();
|
||||
foreach (var kv in eval.UpdatedFirstSeenUtc)
|
||||
_firstSeenIncompleteUtc[kv.Key] = kv.Value;
|
||||
_sidecarMissingSinceUtc = eval.UpdatedSidecarMissingSinceUtc;
|
||||
|
||||
if (eval.NewlyStalled.Count > 0 || eval.SidecarMissingStalledForMinutes != null)
|
||||
EmitStalled(snapshot, eval);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure evaluation of one probe round — the entire detection decision, testable without
|
||||
/// registry, timer, or ingress (pattern: <c>PerformanceCollector.EvaluateDiskLowTransition</c>).
|
||||
/// </summary>
|
||||
/// <param name="snapshot">Current provider snapshot.</param>
|
||||
/// <param name="firstSeenIncompleteUtc">Arm-1 dwell state from the previous round (provider key → first-seen-incomplete).</param>
|
||||
/// <param name="alreadyFired">Provider keys already reported this run (never re-report).</param>
|
||||
/// <param name="sidecarMissingSinceUtc">Arm-2 dwell state from the previous round; null = condition was not active.</param>
|
||||
/// <param name="sidecarMissingAlreadyFired">Arm-2 one-shot latch (never re-report).</param>
|
||||
/// <param name="nowUtc">Evaluation timestamp.</param>
|
||||
/// <param name="dwell">Continuous-condition threshold (shared by both arms).</param>
|
||||
internal static StallEvaluation EvaluateStallTransition(
|
||||
EspPolicyProviderSnapshot snapshot,
|
||||
IReadOnlyDictionary<string, DateTime> firstSeenIncompleteUtc,
|
||||
ISet<string> alreadyFired,
|
||||
DateTime? sidecarMissingSinceUtc,
|
||||
bool sidecarMissingAlreadyFired,
|
||||
DateTime nowUtc,
|
||||
TimeSpan dwell)
|
||||
{
|
||||
var result = new StallEvaluation();
|
||||
|
||||
// Missing root key or no registered providers = normal early enrollment: clear all
|
||||
// dwell state (a condition must be continuously OBSERVED to accrue dwell).
|
||||
if (!snapshot.HasData || snapshot.Providers.Count == 0)
|
||||
return result;
|
||||
|
||||
// Arm 1 — a registered provider continuously incomplete.
|
||||
foreach (var provider in snapshot.Providers)
|
||||
{
|
||||
if (provider.IsComplete)
|
||||
continue; // completed → falls out of the dwell map (reset)
|
||||
|
||||
var firstSeen = firstSeenIncompleteUtc.TryGetValue(provider.Key, out var seen)
|
||||
? seen
|
||||
: nowUtc;
|
||||
result.UpdatedFirstSeenUtc[provider.Key] = firstSeen;
|
||||
|
||||
var stalledFor = nowUtc - firstSeen;
|
||||
if (stalledFor >= dwell && !alreadyFired.Contains(provider.Key))
|
||||
result.NewlyStalled.Add(new StalledProvider(provider, stalledFor.TotalMinutes));
|
||||
}
|
||||
|
||||
// Arm 2 — Setup\Apps providers registered, but none of them is Sidecar. Deliberately
|
||||
// ignores provider completeness: the field case (issue #106) had ConfigMgr with
|
||||
// TrackingPoliciesCreated=1 and the ESP still waited. Sidecar registering later
|
||||
// (legitimate startup ordering) clears the condition and resets the clock.
|
||||
if (HasSetupAppsProvider(snapshot) && !snapshot.SidecarRegistered)
|
||||
{
|
||||
var since = sidecarMissingSinceUtc ?? nowUtc;
|
||||
result.UpdatedSidecarMissingSinceUtc = since;
|
||||
|
||||
var missingFor = nowUtc - since;
|
||||
if (missingFor >= dwell && !sidecarMissingAlreadyFired)
|
||||
result.SidecarMissingStalledForMinutes = missingFor.TotalMinutes;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static bool HasSetupAppsProvider(EspPolicyProviderSnapshot snapshot)
|
||||
=> snapshot.Providers.Any(p => p.Kind == EspPolicyProviderProbe.KindSetupApps);
|
||||
|
||||
private void EmitStalled(EspPolicyProviderSnapshot snapshot, StallEvaluation eval)
|
||||
{
|
||||
foreach (var stalled in eval.NewlyStalled)
|
||||
_firedKeys.Add(stalled.Provider.Key);
|
||||
if (eval.SidecarMissingStalledForMinutes != null)
|
||||
_sidecarMissingFired = true;
|
||||
|
||||
// Cross-restart dedup: fingerprint = sorted set of all fired conditions that are
|
||||
// still active (incomplete providers + the sidecar-missing sentinel). A restart into
|
||||
// the same stall re-accrues dwell and lands on the same fingerprint → suppressed; a
|
||||
// new condition joining the stall changes it → re-emitted.
|
||||
var currentlyStalledKeys = snapshot.Providers
|
||||
.Where(p => !p.IsComplete && _firedKeys.Contains(p.Key))
|
||||
.Select(p => p.Key)
|
||||
.ToList();
|
||||
if (_sidecarMissingFired && HasSetupAppsProvider(snapshot) && !snapshot.SidecarRegistered)
|
||||
currentlyStalledKeys.Add(SidecarMissingFingerprintToken);
|
||||
currentlyStalledKeys.Sort(StringComparer.OrdinalIgnoreCase);
|
||||
var fingerprint = StartupEventGate.HashString(string.Join(";", currentlyStalledKeys));
|
||||
if (_startupGate != null &&
|
||||
!_startupGate.ShouldEmit(Constants.EventTypes.EspPolicyProviderStalled, fingerprint))
|
||||
{
|
||||
_logger?.Debug(
|
||||
"EspPolicyProviderStallDetector: stalled provider set already reported by a " +
|
||||
"previous agent run (fingerprint match) — suppressed.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Sidecar-missing is the dominant reason when both arms fire in one round: it
|
||||
// explains the ESP hang regardless of the foreign providers' completeness.
|
||||
var reason = eval.SidecarMissingStalledForMinutes != null
|
||||
? ReasonSidecarMissing
|
||||
: ReasonProviderIncomplete;
|
||||
|
||||
var messageParts = new List<string>();
|
||||
if (eval.SidecarMissingStalledForMinutes != null)
|
||||
{
|
||||
var registeredNames = string.Join(", ", snapshot.Providers
|
||||
.Where(p => p.Kind == EspPolicyProviderProbe.KindSetupApps)
|
||||
.Select(p => $"{p.Name} ({p.Scope})"));
|
||||
messageParts.Add(
|
||||
$"expected IME provider '{EspPolicyProviderProbe.SidecarProviderName}' not registered for " +
|
||||
$">= {DwellMinutes} min while Setup/Apps providers exist: {registeredNames}");
|
||||
}
|
||||
var stalledNames = string.Join(", ",
|
||||
eval.NewlyStalled.Select(s => $"{s.Provider.Name} ({s.Provider.Scope}/{s.Provider.Kind})"));
|
||||
if (eval.NewlyStalled.Count > 0)
|
||||
messageParts.Add($"provider(s) incomplete for >= {DwellMinutes} min: {stalledNames}");
|
||||
|
||||
var data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
["reason"] = reason,
|
||||
["dwellMinutes"] = DwellMinutes,
|
||||
["sidecarRegistered"] = snapshot.SidecarRegistered,
|
||||
["providers"] = snapshot.Providers.Select(p => new Dictionary<string, object?>(StringComparer.Ordinal)
|
||||
{
|
||||
["name"] = p.Name,
|
||||
["scope"] = p.Scope,
|
||||
["kind"] = p.Kind,
|
||||
["trackingPoliciesCreated"] = p.TrackingPoliciesCreated,
|
||||
["installationState"] = p.InstallationState,
|
||||
["complete"] = p.IsComplete,
|
||||
}).ToList(),
|
||||
["stalledProviders"] = eval.NewlyStalled.Select(s => new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
["name"] = s.Provider.Name,
|
||||
["scope"] = s.Provider.Scope,
|
||||
["kind"] = s.Provider.Kind,
|
||||
["stalledForMinutes"] = Math.Round(s.StalledForMinutes, 1),
|
||||
}).ToList(),
|
||||
};
|
||||
if (eval.SidecarMissingStalledForMinutes != null)
|
||||
data["sidecarMissingForMinutes"] = Math.Round(eval.SidecarMissingStalledForMinutes.Value, 1);
|
||||
|
||||
var message =
|
||||
"ESP Setup/Apps tracking stalled: " + string.Join("; ", messageParts) +
|
||||
" — the ESP waits on its app-tracking providers without any timeout";
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
Timestamp = _clock.UtcNow,
|
||||
EventType = Constants.EventTypes.EspPolicyProviderStalled,
|
||||
Severity = EventSeverity.Warning,
|
||||
Source = SourceLabel,
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
ImmediateUpload = true,
|
||||
Message = message,
|
||||
Data = data
|
||||
});
|
||||
_startupGate?.MarkEmitted(Constants.EventTypes.EspPolicyProviderStalled);
|
||||
|
||||
_logger?.Warning(
|
||||
$"EspPolicyProviderStallDetector: {string.Join("; ", messageParts)} " +
|
||||
$"(reason={reason}, sidecarRegistered={snapshot.SidecarRegistered}).");
|
||||
}
|
||||
|
||||
/// <summary>Result of one <see cref="EvaluateStallTransition"/> round.</summary>
|
||||
internal sealed class StallEvaluation
|
||||
{
|
||||
/// <summary>Next round's arm-1 dwell state — only currently-incomplete providers survive.</summary>
|
||||
public Dictionary<string, DateTime> UpdatedFirstSeenUtc { get; } =
|
||||
new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Providers that crossed the dwell threshold this round and were not yet reported (arm 1).</summary>
|
||||
public List<StalledProvider> NewlyStalled { get; } = new List<StalledProvider>();
|
||||
|
||||
/// <summary>Next round's arm-2 dwell state; null = sidecar-missing condition not active (reset).</summary>
|
||||
public DateTime? UpdatedSidecarMissingSinceUtc { get; set; }
|
||||
|
||||
/// <summary>Non-null when the sidecar-missing condition crossed the dwell threshold this round and was not yet reported (arm 2).</summary>
|
||||
public double? SidecarMissingStalledForMinutes { get; set; }
|
||||
}
|
||||
|
||||
internal readonly struct StalledProvider
|
||||
{
|
||||
public StalledProvider(PolicyProviderState provider, double stalledForMinutes)
|
||||
{
|
||||
Provider = provider;
|
||||
StalledForMinutes = stalledForMinutes;
|
||||
}
|
||||
|
||||
public PolicyProviderState Provider { get; }
|
||||
public double StalledForMinutes { get; }
|
||||
}
|
||||
}
|
||||
}
|
||||
+189
-1
@@ -116,6 +116,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// failure details that fire after the settle window expires.
|
||||
private Dictionary<string, Timer> _provisioningFailureSettleTimers;
|
||||
private Dictionary<string, EspFailureDetectedEventArgs> _provisioningFailureSettleArgs;
|
||||
// Session 4910a5a5 — post-terminal recovery story. Populated when the settle window
|
||||
// expires and EspFailureDetected actually fires (terminal path); consumed by the
|
||||
// esp_failure_retry_detected / esp_failure_recovered emitters so a late "Try again"
|
||||
// recovery (23 min after the failure in the field session) is visible on the timeline
|
||||
// instead of hiding inside a generic subcategory_state_change. Keyed by category-name.
|
||||
private Dictionary<string, EspFailureDetectedEventArgs> _provisioningFailureTerminalArgs;
|
||||
private Dictionary<string, DateTime> _provisioningFailureTerminalUtc;
|
||||
private HashSet<string> _postFailureRetryEmitted;
|
||||
private HashSet<string> _postFailureRecoveredEmitted;
|
||||
// Session 2bc884b6: AppX enrichment scan seams + fire-once guard. The scanner and the
|
||||
// dispatcher are injectable so tests never touch the real event log or the threadpool.
|
||||
private readonly IAppxDeploymentFailureScanner _appxScanner;
|
||||
@@ -285,6 +294,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_lastSubcategoryStates = new Dictionary<string, Dictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
|
||||
_provisioningFailureSettleTimers = new Dictionary<string, Timer>(StringComparer.OrdinalIgnoreCase);
|
||||
_provisioningFailureSettleArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
|
||||
_provisioningFailureTerminalArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
|
||||
_provisioningFailureTerminalUtc = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
|
||||
_postFailureRetryEmitted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
_postFailureRecoveredEmitted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
_deviceSetupProvisioningCompleteFired = false;
|
||||
_deviceSetupFallbackTimer = null;
|
||||
_accountSetupProvisioningCompleteFired = false;
|
||||
@@ -665,6 +678,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// null-ing here is correctness-neutral on the start-condition side.
|
||||
ResetAccountSetupFallbackTimer();
|
||||
_accountSetupProvisioningCompleteFired = true;
|
||||
MaybeEmitPostFailureRecovered("AccountSetupCategory.Status", "AccountSetup");
|
||||
try { AccountSetupProvisioningComplete?.Invoke(this, EventArgs.Empty); }
|
||||
catch (Exception ex) { _logger.Error("AccountSetupProvisioningComplete handler failed (fallback path)", ex); }
|
||||
}
|
||||
@@ -833,6 +847,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// null-ing here is correctness-neutral on the start-condition side.
|
||||
ResetDeviceSetupFallbackTimer();
|
||||
_deviceSetupProvisioningCompleteFired = true;
|
||||
MaybeEmitPostFailureRecovered("DeviceSetupCategory.Status", "DeviceSetup");
|
||||
try { DeviceSetupProvisioningComplete?.Invoke(this, EventArgs.Empty); }
|
||||
catch (Exception ex) { _logger.Error("DeviceSetupProvisioningComplete handler failed (fallback path)", ex); }
|
||||
}
|
||||
@@ -946,12 +961,25 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
var severity = categorySucceeded == false ? EventSeverity.Warning : EventSeverity.Info;
|
||||
EmitProvisioningEvent(categoryLabel, categorySucceeded, statusText, subcategories, severity);
|
||||
EmitRawRegistryDump(categoryLabel, jsonValue, $"category_resolved_{(categorySucceeded == true ? "success" : "failed")}");
|
||||
|
||||
// Session 4910a5a5: category resolving to success after a terminally
|
||||
// fired failure = the failure recovered (fallback-path successes call
|
||||
// this from their own timer handlers).
|
||||
if (categorySucceeded == true)
|
||||
{
|
||||
MaybeEmitPostFailureRecovered(categoryName, categoryLabel);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var transitions = DetectSubcategoryTransitions(categoryName, subcategories);
|
||||
StoreSubcategoryStates(categoryName, subcategories);
|
||||
|
||||
// Session 4910a5a5: after a terminally fired failure, the failed
|
||||
// subcategory leaving "failed" is the ESP re-running the step —
|
||||
// consistent with the user pressing "Try again" on the failure page.
|
||||
MaybeEmitPostFailureRetry(categoryName, categoryLabel, transitions);
|
||||
|
||||
var failureTransitions = transitions.Where(t => t.IsFailure).ToList();
|
||||
if (transitions.Count > 0)
|
||||
{
|
||||
@@ -1025,6 +1053,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
$"escalating as {failureTypeName} (errorCode={errorCode ?? "n/a"})");
|
||||
_logger.Warning($"Provisioning failure detected: {failureTypeName}");
|
||||
|
||||
// Session 4910a5a5: name the likely culprit app(s) directly on the failure args so
|
||||
// the DecisionEngine's advisory/terminal events carry them — never-started apps
|
||||
// first (they starve ESP's app gate without ever producing an IME install event),
|
||||
// capped at 3 names.
|
||||
IReadOnlyList<string> likelyCulpritApps = null;
|
||||
if (string.Equals(failedSubcategoryName, "Apps", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
likelyCulpritApps = SnapshotTrackedAppsNotCompleted()
|
||||
.OrderBy(a => a["everStartedInstalling"] is bool started && started ? 1 : 0)
|
||||
.Select(a => a["appName"]?.ToString())
|
||||
.Where(n => !string.IsNullOrEmpty(n))
|
||||
.Take(3)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
// Caller already holds _stateLock.
|
||||
ArmFailureSettleTimer(
|
||||
categoryName: categoryName,
|
||||
@@ -1032,7 +1075,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
failureType: failureTypeName,
|
||||
errorCode: errorCode,
|
||||
failedSubcategory: failedSubcategoryName,
|
||||
category: categoryLabel));
|
||||
category: categoryLabel,
|
||||
likelyCulpritApps: likelyCulpritApps));
|
||||
|
||||
// Settle-window owns the actual EspFailureDetected fire. The legacy "fire-after-emit"
|
||||
// path in CheckProvisioningStatus is now a no-op for this code-path; it stays in place
|
||||
@@ -1292,6 +1336,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_provisioningCategoriesResolved.Remove(categoryName);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Session 4910a5a5: remember the terminally fired failure so a LATE
|
||||
// recovery (outside the settle window — e.g. user pressing "Try again"
|
||||
// minutes later) can be told as esp_failure_retry_detected /
|
||||
// esp_failure_recovered instead of hiding in a generic state-change.
|
||||
_provisioningFailureTerminalArgs[categoryName] = args;
|
||||
_provisioningFailureTerminalUtc[categoryName] = DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
|
||||
if (recovered)
|
||||
@@ -1382,6 +1435,133 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Session 4910a5a5: after a terminally fired failure (settle window expired,
|
||||
/// EspFailureDetected raised), the failed subcategory leaving the <c>failed</c> state
|
||||
/// means the ESP re-ran the step — consistent with the user pressing "Try again" on
|
||||
/// the ESP failure page (the agent cannot observe the click itself). Emits
|
||||
/// <c>esp_failure_retry_detected</c> once per terminally fired category. Caller holds
|
||||
/// <see cref="_stateLock"/>.
|
||||
/// </summary>
|
||||
private void MaybeEmitPostFailureRetry(string categoryName, string categoryLabel, List<SubcategoryTransition> transitions)
|
||||
{
|
||||
if (transitions == null || transitions.Count == 0) return;
|
||||
if (_postFailureRetryEmitted.Contains(categoryName)) return;
|
||||
if (_provisioningFailureTerminalArgs == null
|
||||
|| !_provisioningFailureTerminalArgs.TryGetValue(categoryName, out var terminalArgs))
|
||||
{
|
||||
return; // No terminal fire on record — in-window recovery is settle_recovered's job.
|
||||
}
|
||||
|
||||
SubcategoryTransition? retryTransition = null;
|
||||
foreach (var t in transitions)
|
||||
{
|
||||
if (string.Equals(t.OldState, "failed", StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(t.NewState, "failed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
retryTransition = t;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (retryTransition == null) return;
|
||||
var retry = retryTransition.Value;
|
||||
|
||||
_postFailureRetryEmitted.Add(categoryName);
|
||||
|
||||
var minutesSinceFailure = MinutesSinceTerminalFailure(categoryName);
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "category", categoryLabel },
|
||||
{ "subcategory", retry.SubcategoryName },
|
||||
{ "previousState", retry.OldState },
|
||||
{ "newState", retry.NewState },
|
||||
{ "failureType", terminalArgs.FailureType },
|
||||
{ "minutesSinceFailure", minutesSinceFailure },
|
||||
};
|
||||
if (!string.IsNullOrEmpty(terminalArgs.ErrorCode))
|
||||
data["errorCode"] = terminalArgs.ErrorCode;
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.EspFailureRetryDetected,
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "EspAndHelloTracker",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"ESP retry detected — {categoryLabel}/{retry.SubcategoryName} left the failed state " +
|
||||
$"({retry.OldState} → {retry.NewState}) {minutesSinceFailure} min after the failure — " +
|
||||
"consistent with the user pressing 'Try again' on the ESP failure page.",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Session 4910a5a5: the category behind a terminally fired failure reached success
|
||||
/// (categorySucceeded=true or the all-subcategories-succeeded fallback) — the failure
|
||||
/// demonstrably recovered. Emits <c>esp_failure_recovered</c> once per terminally
|
||||
/// fired category so the timeline tells "failed → retried → recovered" instead of
|
||||
/// leaving the stale failure as the last word. Caller holds <see cref="_stateLock"/>.
|
||||
/// </summary>
|
||||
private void MaybeEmitPostFailureRecovered(string categoryName, string categoryLabel)
|
||||
{
|
||||
if (_postFailureRecoveredEmitted == null || _postFailureRecoveredEmitted.Contains(categoryName)) return;
|
||||
if (_provisioningFailureTerminalArgs == null
|
||||
|| !_provisioningFailureTerminalArgs.TryGetValue(categoryName, out var terminalArgs))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_postFailureRecoveredEmitted.Add(categoryName);
|
||||
|
||||
var minutesSinceFailure = MinutesSinceTerminalFailure(categoryName);
|
||||
string subcategoryStatus = null;
|
||||
if (terminalArgs.FailedSubcategory != null
|
||||
&& _lastSubcategoryStates != null
|
||||
&& _lastSubcategoryStates.TryGetValue(categoryName, out var subs)
|
||||
&& subs.TryGetValue(terminalArgs.FailedSubcategory, out var currentState))
|
||||
{
|
||||
subcategoryStatus = currentState;
|
||||
}
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "category", categoryLabel },
|
||||
{ "failedSubcategory", terminalArgs.FailedSubcategory ?? "category-level" },
|
||||
{ "failureType", terminalArgs.FailureType },
|
||||
{ "minutesSinceFailure", minutesSinceFailure },
|
||||
};
|
||||
if (!string.IsNullOrEmpty(terminalArgs.ErrorCode))
|
||||
data["errorCode"] = terminalArgs.ErrorCode;
|
||||
if (subcategoryStatus != null)
|
||||
data["subcategoryState"] = subcategoryStatus;
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.EspFailureRecovered,
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "EspAndHelloTracker",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"ESP failure recovered — {categoryLabel}/{terminalArgs.FailedSubcategory ?? "category-level"} " +
|
||||
$"completed successfully {minutesSinceFailure} min after the failure; the earlier ESP failure no longer applies.",
|
||||
Data = data,
|
||||
ImmediateUpload = true
|
||||
});
|
||||
}
|
||||
|
||||
private int MinutesSinceTerminalFailure(string categoryName)
|
||||
{
|
||||
if (_provisioningFailureTerminalUtc != null
|
||||
&& _provisioningFailureTerminalUtc.TryGetValue(categoryName, out var firedAt))
|
||||
{
|
||||
return Math.Max(0, (int)Math.Round((DateTime.UtcNow - firedAt).TotalMinutes));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private void EmitFailureSettleStarted(EspFailureDetectedEventArgs args)
|
||||
{
|
||||
var data = new Dictionary<string, object>
|
||||
@@ -1863,6 +2043,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_provisioningFailureSettleTimers = new Dictionary<string, Timer>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_provisioningFailureSettleArgs == null)
|
||||
_provisioningFailureSettleArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_provisioningFailureTerminalArgs == null)
|
||||
_provisioningFailureTerminalArgs = new Dictionary<string, EspFailureDetectedEventArgs>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_provisioningFailureTerminalUtc == null)
|
||||
_provisioningFailureTerminalUtc = new Dictionary<string, DateTime>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_postFailureRetryEmitted == null)
|
||||
_postFailureRetryEmitted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (_postFailureRecoveredEmitted == null)
|
||||
_postFailureRecoveredEmitted = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
using System;
|
||||
using System.Reflection;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads WinRT <c>Windows.System.Profile.SystemSetupInfo.OutOfBoxExperienceState</c>
|
||||
/// (NotStarted=0 / InProgress=1 / Completed=2) via the CLR's built-in WinRT projection —
|
||||
/// no NuGet package, no winmd reference; the type is resolved at runtime with
|
||||
/// <c>ContentType=WindowsRuntime</c>, the same mechanism the bootstrap script uses from
|
||||
/// PowerShell 5.1.
|
||||
/// <para>
|
||||
/// <b>Contract:</b> observational only — never feed this into decision-engine logic.
|
||||
/// Requires Windows 10 1809+ (build 17763, UniversalApiContract 7.0) and is callable
|
||||
/// from the SYSTEM service context (empirically validated, probe session 9c404ae9).
|
||||
/// On older builds or SKUs without the contract the type resolves to null and
|
||||
/// <see cref="Read"/> returns <c>"unavailable"</c>. <see cref="Read"/> never throws.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Empirical timing (Win11 24H2 probe): the state flips InProgress→Completed at desktop
|
||||
/// arrival — it is a corroboration signal, not an early one. WDP reseal and placeholder
|
||||
/// (fooUser) flip semantics are unverified; another reason this must stay observational.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static class OobeStateReader
|
||||
{
|
||||
public const string Unavailable = "unavailable";
|
||||
|
||||
// Resolved once per process. Benign race: concurrent first calls resolve the same
|
||||
// PropertyInfo and the assignment is idempotent — no lock needed.
|
||||
private static PropertyInfo _stateProperty;
|
||||
private static bool _initTried;
|
||||
|
||||
/// <summary>
|
||||
/// Returns <c>"not_started"</c> / <c>"in_progress"</c> / <c>"completed"</c>,
|
||||
/// <c>"unknown_<n>"</c> for unexpected enum values, or <c>"unavailable"</c>
|
||||
/// when the WinRT contract is absent or any part of the read fails. Never throws.
|
||||
/// </summary>
|
||||
public static string Read()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!_initTried)
|
||||
{
|
||||
var type = Type.GetType(
|
||||
"Windows.System.Profile.SystemSetupInfo, Windows.System.Profile, ContentType=WindowsRuntime",
|
||||
throwOnError: false);
|
||||
_stateProperty = type?.GetProperty(
|
||||
"OutOfBoxExperienceState",
|
||||
BindingFlags.Public | BindingFlags.Static);
|
||||
_initTried = true;
|
||||
}
|
||||
|
||||
if (_stateProperty == null)
|
||||
return Unavailable;
|
||||
|
||||
var value = Convert.ToInt32(_stateProperty.GetValue(null));
|
||||
switch (value)
|
||||
{
|
||||
case 0: return "not_started";
|
||||
case 1: return "in_progress";
|
||||
case 2: return "completed";
|
||||
default: return "unknown_" + value;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
return Unavailable;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads the UEFI Secure Boot signature databases <c>db</c> and <c>KEK</c> directly from
|
||||
/// firmware via <c>GetFirmwareEnvironmentVariableExW</c> and checks which Microsoft
|
||||
/// signing certificates they contain. This is the authoritative source for the 2023
|
||||
/// certificate rollout state — the <c>SecureBoot\Servicing</c> registry key only reflects
|
||||
/// whether the *Windows Update* rollout ran and verified, and lags behind when the
|
||||
/// certificate reached the firmware another way (OEM factory image, WinCS, manual).
|
||||
/// <para>
|
||||
/// The certificate subject CNs are stored as ASCII bytes inside the DER-encoded
|
||||
/// signature lists, so a plain byte-substring search is sufficient — the same check
|
||||
/// Microsoft documents via <c>Get-SecureBootUEFI</c> (which wraps this exact API).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Contract:</b> observational only — never feed this into decision-engine logic.
|
||||
/// <see cref="Read"/> never throws. Reading firmware variables requires
|
||||
/// <c>SeSystemEnvironmentPrivilege</c>, which SYSTEM holds but must be enabled on the
|
||||
/// process token first; the previous privilege state is restored after the read.
|
||||
/// On legacy BIOS the API fails with <c>ERROR_INVALID_FUNCTION</c> → status
|
||||
/// <c>not_uefi</c>; every failure maps to a status string instead of an exception.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static class UefiSecureBootCertReader
|
||||
{
|
||||
public const string StatusOk = "ok";
|
||||
public const string StatusNotUefi = "not_uefi";
|
||||
public const string StatusVariableNotFound = "variable_not_found";
|
||||
public const string StatusPrivilegeDenied = "privilege_denied";
|
||||
|
||||
// EFI_IMAGE_SECURITY_DATABASE_GUID — namespace of the "db" variable.
|
||||
private const string DbVariableGuid = "{d719b2cb-3d3a-4596-a3bc-dad00e67656f}";
|
||||
// EFI_GLOBAL_VARIABLE — namespace of the "KEK" variable.
|
||||
private const string KekVariableGuid = "{8be4df61-93ca-11d2-aa0d-00e098032b8c}";
|
||||
|
||||
// Subject CNs searched for in db. 2023 set per Microsoft's June-2026 expiry guidance;
|
||||
// 2011 set kept for context (which legacy CAs are still present).
|
||||
internal const string CertWindowsUefiCa2023 = "Windows UEFI CA 2023";
|
||||
internal const string CertMicrosoftUefiCa2023 = "Microsoft UEFI CA 2023";
|
||||
internal const string CertMicrosoftOptionRomCa2023 = "Microsoft Option ROM UEFI CA 2023";
|
||||
internal const string CertWindowsProductionPca2011 = "Microsoft Windows Production PCA 2011";
|
||||
internal const string CertMicrosoftUefiCa2011 = "Microsoft Corporation UEFI CA 2011";
|
||||
// Subject CNs searched for in KEK.
|
||||
internal const string CertMicrosoftKek2kCa2023 = "Microsoft Corporation KEK 2K CA 2023";
|
||||
internal const string CertMicrosoftKekCa2011 = "Microsoft Corporation KEK CA 2011";
|
||||
|
||||
private const int ErrorInvalidFunction = 1; // legacy BIOS / no UEFI
|
||||
private const int ErrorInsufficientBuffer = 122;
|
||||
private const int ErrorEnvvarNotFound = 203;
|
||||
private const int ErrorPrivilegeNotHeld = 1314;
|
||||
private const int ErrorNotAllAssigned = 1300;
|
||||
|
||||
private const int InitialBufferSize = 64 * 1024;
|
||||
private const int RetryBufferSize = 256 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Reads db and KEK and evaluates the certificate presence flags. Never throws;
|
||||
/// per-variable failures surface as <see cref="UefiSecureBootCertSnapshot.DbStatus"/> /
|
||||
/// <see cref="UefiSecureBootCertSnapshot.KekStatus"/>. Certificate flags are only
|
||||
/// meaningful when the corresponding status is <see cref="StatusOk"/>.
|
||||
/// </summary>
|
||||
public static UefiSecureBootCertSnapshot Read()
|
||||
{
|
||||
var snapshot = new UefiSecureBootCertSnapshot();
|
||||
try
|
||||
{
|
||||
using (var privilege = SystemEnvironmentPrivilege.TryEnable())
|
||||
{
|
||||
if (!privilege.Enabled)
|
||||
{
|
||||
snapshot.DbStatus = StatusPrivilegeDenied;
|
||||
snapshot.KekStatus = StatusPrivilegeDenied;
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
byte[] db;
|
||||
snapshot.DbStatus = TryReadVariable("db", DbVariableGuid, out db);
|
||||
if (snapshot.DbStatus == StatusOk)
|
||||
{
|
||||
snapshot.DbHasWindowsUefiCa2023 = ContainsAscii(db, CertWindowsUefiCa2023);
|
||||
snapshot.DbHasMicrosoftUefiCa2023 = ContainsAscii(db, CertMicrosoftUefiCa2023);
|
||||
snapshot.DbHasMicrosoftOptionRomCa2023 = ContainsAscii(db, CertMicrosoftOptionRomCa2023);
|
||||
snapshot.DbHasWindowsProductionPca2011 = ContainsAscii(db, CertWindowsProductionPca2011);
|
||||
snapshot.DbHasMicrosoftUefiCa2011 = ContainsAscii(db, CertMicrosoftUefiCa2011);
|
||||
}
|
||||
|
||||
byte[] kek;
|
||||
snapshot.KekStatus = TryReadVariable("KEK", KekVariableGuid, out kek);
|
||||
if (snapshot.KekStatus == StatusOk)
|
||||
{
|
||||
snapshot.KekHasMicrosoftKek2kCa2023 = ContainsAscii(kek, CertMicrosoftKek2kCa2023);
|
||||
snapshot.KekHasMicrosoftKekCa2011 = ContainsAscii(kek, CertMicrosoftKekCa2011);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var status = "error_" + ex.GetType().Name;
|
||||
if (snapshot.DbStatus == null) snapshot.DbStatus = status;
|
||||
if (snapshot.KekStatus == null) snapshot.KekStatus = status;
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads one firmware variable into <paramref name="bytes"/> (trimmed to the actual
|
||||
/// length). Returns a status string; <paramref name="bytes"/> is null unless the
|
||||
/// status is <see cref="StatusOk"/>.
|
||||
/// </summary>
|
||||
private static string TryReadVariable(string name, string guid, out byte[] bytes)
|
||||
{
|
||||
bytes = null;
|
||||
var buffer = new byte[InitialBufferSize];
|
||||
var length = NativeMethods.GetFirmwareEnvironmentVariableExW(name, guid, buffer, (uint)buffer.Length, IntPtr.Zero);
|
||||
if (length == 0)
|
||||
{
|
||||
var error = Marshal.GetLastWin32Error();
|
||||
if (error == ErrorInsufficientBuffer)
|
||||
{
|
||||
buffer = new byte[RetryBufferSize];
|
||||
length = NativeMethods.GetFirmwareEnvironmentVariableExW(name, guid, buffer, (uint)buffer.Length, IntPtr.Zero);
|
||||
if (length == 0)
|
||||
error = Marshal.GetLastWin32Error();
|
||||
}
|
||||
|
||||
if (length == 0)
|
||||
{
|
||||
switch (error)
|
||||
{
|
||||
case ErrorInvalidFunction: return StatusNotUefi;
|
||||
case ErrorEnvvarNotFound: return StatusVariableNotFound;
|
||||
case ErrorPrivilegeNotHeld: return StatusPrivilegeDenied;
|
||||
default: return "error_" + error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bytes = new byte[length];
|
||||
Array.Copy(buffer, bytes, (int)length);
|
||||
return StatusOk;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Byte-level ASCII substring search (ordinal, case-sensitive). The signature lists
|
||||
/// are at most a few hundred KB and read once per agent start — a naive scan is fine.
|
||||
/// </summary>
|
||||
internal static bool ContainsAscii(byte[] haystack, string needle)
|
||||
{
|
||||
if (haystack == null || string.IsNullOrEmpty(needle) || needle.Length > haystack.Length)
|
||||
return false;
|
||||
|
||||
for (var i = 0; i <= haystack.Length - needle.Length; i++)
|
||||
{
|
||||
var j = 0;
|
||||
while (j < needle.Length && haystack[i + j] == (byte)needle[j])
|
||||
j++;
|
||||
if (j == needle.Length)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enables <c>SeSystemEnvironmentPrivilege</c> on the process token for the lifetime
|
||||
/// of the instance and restores the previous privilege state on dispose. SYSTEM holds
|
||||
/// the privilege (default-disabled); without elevation <see cref="Enabled"/> stays
|
||||
/// false and the caller reports <see cref="StatusPrivilegeDenied"/>.
|
||||
/// </summary>
|
||||
private sealed class SystemEnvironmentPrivilege : IDisposable
|
||||
{
|
||||
public bool Enabled { get; private set; }
|
||||
|
||||
private IntPtr _token = IntPtr.Zero;
|
||||
private NativeMethods.TOKEN_PRIVILEGES _previousState;
|
||||
private bool _restoreNeeded;
|
||||
|
||||
public static SystemEnvironmentPrivilege TryEnable()
|
||||
{
|
||||
var instance = new SystemEnvironmentPrivilege();
|
||||
try
|
||||
{
|
||||
if (!NativeMethods.OpenProcessToken(NativeMethods.GetCurrentProcess(),
|
||||
NativeMethods.TOKEN_ADJUST_PRIVILEGES | NativeMethods.TOKEN_QUERY, out instance._token))
|
||||
return instance;
|
||||
|
||||
NativeMethods.LUID luid;
|
||||
if (!NativeMethods.LookupPrivilegeValueW(null, "SeSystemEnvironmentPrivilege", out luid))
|
||||
return instance;
|
||||
|
||||
var newState = new NativeMethods.TOKEN_PRIVILEGES
|
||||
{
|
||||
PrivilegeCount = 1,
|
||||
Luid = luid,
|
||||
Attributes = NativeMethods.SE_PRIVILEGE_ENABLED
|
||||
};
|
||||
|
||||
uint returnLength;
|
||||
if (!NativeMethods.AdjustTokenPrivileges(instance._token, false, ref newState,
|
||||
(uint)Marshal.SizeOf(typeof(NativeMethods.TOKEN_PRIVILEGES)),
|
||||
out instance._previousState, out returnLength))
|
||||
return instance;
|
||||
|
||||
// AdjustTokenPrivileges returns TRUE even when the privilege is not held —
|
||||
// the real verdict is ERROR_NOT_ALL_ASSIGNED in the last error.
|
||||
if (Marshal.GetLastWin32Error() == ErrorNotAllAssigned)
|
||||
return instance;
|
||||
|
||||
// Restore only when the privilege was not already enabled before.
|
||||
instance._restoreNeeded = instance._previousState.PrivilegeCount > 0
|
||||
&& (instance._previousState.Attributes & NativeMethods.SE_PRIVILEGE_ENABLED) == 0;
|
||||
instance.Enabled = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Fail-soft: Enabled stays false.
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_token != IntPtr.Zero)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (_restoreNeeded)
|
||||
{
|
||||
NativeMethods.TOKEN_PRIVILEGES ignored;
|
||||
uint returnLength;
|
||||
NativeMethods.AdjustTokenPrivileges(_token, false, ref _previousState,
|
||||
(uint)Marshal.SizeOf(typeof(NativeMethods.TOKEN_PRIVILEGES)),
|
||||
out ignored, out returnLength);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Best effort — the process token privilege state is not load-bearing
|
||||
// for anything else the agent does.
|
||||
}
|
||||
NativeMethods.CloseHandle(_token);
|
||||
_token = IntPtr.Zero;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class NativeMethods
|
||||
{
|
||||
public const uint TOKEN_ADJUST_PRIVILEGES = 0x0020;
|
||||
public const uint TOKEN_QUERY = 0x0008;
|
||||
public const uint SE_PRIVILEGE_ENABLED = 0x0002;
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct LUID
|
||||
{
|
||||
public uint LowPart;
|
||||
public int HighPart;
|
||||
}
|
||||
|
||||
// Single-privilege variant of the variable-length native struct — sufficient here
|
||||
// because exactly one privilege is adjusted per call.
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct TOKEN_PRIVILEGES
|
||||
{
|
||||
public uint PrivilegeCount;
|
||||
public LUID Luid;
|
||||
public uint Attributes;
|
||||
}
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
||||
public static extern uint GetFirmwareEnvironmentVariableExW(
|
||||
string lpName, string lpGuid, byte[] pBuffer, uint nSize, IntPtr pdwAttributes);
|
||||
|
||||
[DllImport("kernel32.dll")]
|
||||
public static extern IntPtr GetCurrentProcess();
|
||||
|
||||
[DllImport("advapi32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool OpenProcessToken(IntPtr processHandle, uint desiredAccess, out IntPtr tokenHandle);
|
||||
|
||||
[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool LookupPrivilegeValueW(string lpSystemName, string lpName, out LUID lpLuid);
|
||||
|
||||
[DllImport("advapi32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool AdjustTokenPrivileges(IntPtr tokenHandle, bool disableAllPrivileges,
|
||||
ref TOKEN_PRIVILEGES newState, uint bufferLength, out TOKEN_PRIVILEGES previousState, out uint returnLength);
|
||||
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool CloseHandle(IntPtr handle);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of one firmware read. Certificate flags are only meaningful when the
|
||||
/// corresponding status equals <see cref="UefiSecureBootCertReader.StatusOk"/>.
|
||||
/// </summary>
|
||||
internal sealed class UefiSecureBootCertSnapshot
|
||||
{
|
||||
public string DbStatus { get; set; }
|
||||
public string KekStatus { get; set; }
|
||||
|
||||
public bool DbHasWindowsUefiCa2023 { get; set; }
|
||||
public bool DbHasMicrosoftUefiCa2023 { get; set; }
|
||||
public bool DbHasMicrosoftOptionRomCa2023 { get; set; }
|
||||
public bool DbHasWindowsProductionPca2011 { get; set; }
|
||||
public bool DbHasMicrosoftUefiCa2011 { get; set; }
|
||||
|
||||
public bool KekHasMicrosoftKek2kCa2023 { get; set; }
|
||||
public bool KekHasMicrosoftKekCa2011 { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -31,84 +31,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
{
|
||||
_logger.Info($"Executing FULL SELF-DESTRUCT (Scheduled Task + Files{(_configuration.RebootOnComplete ? " + Reboot" : "")})");
|
||||
|
||||
var currentProcessId = Process.GetCurrentProcess().Id;
|
||||
var currentProcess = Process.GetCurrentProcess();
|
||||
var agentBasePath = Path.GetFullPath(Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData),
|
||||
"AutopilotMonitor"
|
||||
));
|
||||
|
||||
var logDir = Path.GetFullPath(Path.Combine(agentBasePath, "Logs"));
|
||||
var keepLogs = _configuration.KeepLogFile;
|
||||
|
||||
// Create a self-deleting PowerShell script
|
||||
var cleanupScript = $@"
|
||||
$scriptPath = $MyInvocation.MyCommand.Path
|
||||
|
||||
# Wait for agent process to exit
|
||||
Start-Sleep -Seconds 2
|
||||
Wait-Process -Id {currentProcessId} -ErrorAction SilentlyContinue -Timeout 30
|
||||
Start-Sleep -Seconds 1
|
||||
|
||||
# Remove Scheduled Task — verify removal BEFORE deleting files (LIFE-F6). An orphan task firing
|
||||
# against a deleted exe spams Task Scheduler errors every boot forever and, once the files (incl.
|
||||
# the enrollment-complete marker) are gone, has no cleanup-retry path. So confirm the task is gone
|
||||
# first; if it cannot be removed, SKIP the file deletion and leave everything intact for a clean
|
||||
# cleanup retry on the next boot (the marker survives, so bootstrap re-runs cleanup).
|
||||
$taskName = '{_configuration.ScheduledTaskName}'
|
||||
$taskGone = $false
|
||||
for ($i = 1; $i -le 5; $i++) {{
|
||||
try {{ Stop-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
try {{ Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
$still = $null
|
||||
try {{ $still = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
if (-not $still) {{ $taskGone = $true; break }}
|
||||
Start-Sleep -Seconds 1
|
||||
}}
|
||||
if (-not $taskGone) {{
|
||||
# Fallback: schtasks.exe /Delete does not depend on the ScheduledTasks PowerShell module.
|
||||
try {{ & schtasks.exe /Delete /TN $taskName /F 2>$null | Out-Null }} catch {{ }}
|
||||
try {{ $taskGone = -not (Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue) }} catch {{ }}
|
||||
}}
|
||||
|
||||
{(keepLogs ? $@"
|
||||
if ($taskGone) {{
|
||||
# Delete everything except Logs directory
|
||||
Get-ChildItem -Path '{agentBasePath}' -Exclude 'Logs' | ForEach-Object {{
|
||||
$dest = $_.FullName + '.del'
|
||||
try {{ Rename-Item -Path $_.FullName -NewName $dest -Force -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
Remove-Item -Path $dest -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}}
|
||||
}}
|
||||
" : $@"
|
||||
if ($taskGone) {{
|
||||
# Rename folder then delete. Retry a few times to let the OS release handles.
|
||||
$renamedPath = '{agentBasePath}.del'
|
||||
$renamed = $false
|
||||
for ($i = 1; $i -le 10; $i++) {{
|
||||
try {{
|
||||
Rename-Item -Path '{agentBasePath}' -NewName $renamedPath -Force -ErrorAction Stop
|
||||
$renamed = $true
|
||||
break
|
||||
}} catch {{
|
||||
Start-Sleep -Seconds 2
|
||||
}}
|
||||
}}
|
||||
if ($renamed) {{
|
||||
Remove-Item -Path $renamedPath -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}} else {{
|
||||
Remove-Item -Path '{agentBasePath}' -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}}
|
||||
}}
|
||||
")}
|
||||
{(_configuration.RebootOnComplete ? @"
|
||||
# shutdown.exe (NOT Restart-Computer): Restart-Computer has no -Comment parameter and -Delay is
|
||||
# only honoured with -Wait, so the previous invocation failed parameter binding and the device
|
||||
# silently never rebooted. shutdown.exe /r /t 10 mirrors the standalone reboot path and reboots
|
||||
# after a 10 s delay (lets this script finish + the agent exit). Review LIFE-F2.
|
||||
shutdown.exe /r /t 10 /c 'Autopilot enrollment completed - Autopilot Monitor is rebooting'
|
||||
" : "")}
|
||||
Remove-Item -Path $scriptPath -Force -ErrorAction SilentlyContinue
|
||||
";
|
||||
var cleanupScript = BuildCleanupScript(
|
||||
agentProcessName: currentProcess.ProcessName,
|
||||
agentBasePath: agentBasePath,
|
||||
keepLogs: _configuration.KeepLogFile,
|
||||
scheduledTaskName: _configuration.ScheduledTaskName,
|
||||
rebootOnComplete: _configuration.RebootOnComplete);
|
||||
|
||||
// Write cleanup script to temp location (outside of agent folder)
|
||||
var tempScriptPath = Path.Combine(Path.GetTempPath(), $"AutopilotMonitor-Cleanup-{Guid.NewGuid()}.ps1");
|
||||
@@ -143,5 +77,171 @@ Remove-Item -Path $scriptPath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the self-deleting cleanup PowerShell script. Extracted (internal static, pure)
|
||||
/// so tests can pin the hardened invariants without launching processes.
|
||||
/// <para>
|
||||
/// LIFE-F7 hardening (incident 2026-07-22 — exe + loaded DLLs orphaned forever):
|
||||
/// </para>
|
||||
/// <list type="number">
|
||||
/// <item><b>Single-flight lock</b> — the enrollment-complete marker is written BEFORE
|
||||
/// cleanup launches, so any agent start during an in-flight cleanup spawns a SECOND
|
||||
/// cleanup script. Two unsynchronized scripts interleave their rename/delete phases
|
||||
/// and leave partial trees behind. A CreateNew lock file in the SYSTEM temp dir
|
||||
/// serializes them; a stale lock (>10 min) is taken over.</item>
|
||||
/// <item><b>Wait by process NAME, not launcher PID</b> — marker-retry instances,
|
||||
/// self-update restarts and manual runs all load the same images but have PIDs the
|
||||
/// old <c>Wait-Process -Id</c> never saw (and an already-freed PID can be reused by
|
||||
/// an unrelated boot-time process). After the deadline any straggler is force-killed:
|
||||
/// its images are about to be deleted anyway.</item>
|
||||
/// <item><b>Probe-rename BEFORE task removal, never delete the original path</b> — the
|
||||
/// rename is the atomic lock probe. If it fails, something still holds the tree: abort
|
||||
/// with task + files + marker intact so the next boot retries (file-side symmetry to
|
||||
/// the LIFE-F6 task-side rule). If the task cannot be removed afterwards, rename BACK
|
||||
/// and abort for the same reason. Only a fully renamed tree with the task gone is
|
||||
/// deleted — the old <c>Remove-Item</c>-on-original fallback deleted marker + state
|
||||
/// around locked binaries AFTER the task was gone, converting a transient lock into a
|
||||
/// permanently orphaned exe.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The timing parameters exist ONLY so integration tests can replay the incident
|
||||
/// scenarios (locked tree, straggler process) in seconds instead of minutes —
|
||||
/// production call sites always use the defaults.
|
||||
/// </remarks>
|
||||
internal static string BuildCleanupScript(
|
||||
string agentProcessName,
|
||||
string agentBasePath,
|
||||
bool keepLogs,
|
||||
string scheduledTaskName,
|
||||
bool rebootOnComplete,
|
||||
int initialDelaySeconds = 2,
|
||||
int processWaitSeconds = 60,
|
||||
int renameRetries = 10,
|
||||
int renameRetryDelaySeconds = 2)
|
||||
{
|
||||
return $@"
|
||||
$scriptPath = $MyInvocation.MyCommand.Path
|
||||
|
||||
# Single-flight lock (LIFE-F7 #1): serialize concurrent cleanup scripts. CreateNew is
|
||||
# atomic; the loser backs off and lets the winner own the deletion.
|
||||
$lockPath = Join-Path ([System.IO.Path]::GetTempPath()) 'AutopilotMonitor-Cleanup.lock'
|
||||
$lockStream = $null
|
||||
try {{
|
||||
if (Test-Path $lockPath) {{
|
||||
$lockAge = (Get-Date) - (Get-Item $lockPath -ErrorAction Stop).LastWriteTime
|
||||
if ($lockAge.TotalMinutes -gt 10) {{ Remove-Item -Path $lockPath -Force -ErrorAction SilentlyContinue }}
|
||||
}}
|
||||
$lockStream = [System.IO.File]::Open($lockPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None)
|
||||
}} catch {{
|
||||
Remove-Item -Path $scriptPath -Force -ErrorAction SilentlyContinue
|
||||
exit 0
|
||||
}}
|
||||
|
||||
try {{
|
||||
# Wait for EVERY agent process to exit, by NAME (LIFE-F7 #2) — covers marker-retry
|
||||
# instances, self-update restarts and manual runs, and is immune to PID reuse.
|
||||
Start-Sleep -Seconds {initialDelaySeconds}
|
||||
$deadline = (Get-Date).AddSeconds({processWaitSeconds})
|
||||
while ((Get-Date) -lt $deadline) {{
|
||||
if (-not (Get-Process -Name '{agentProcessName}' -ErrorAction SilentlyContinue)) {{ break }}
|
||||
Start-Sleep -Seconds 2
|
||||
}}
|
||||
# Backstop: force-kill stragglers — their images are about to be deleted anyway.
|
||||
Get-Process -Name '{agentProcessName}' -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 1
|
||||
|
||||
{(keepLogs ? $@"
|
||||
# Probe-rename all children except Logs (LIFE-F7 #3). Any failure ⇒ rename back and
|
||||
# abort with the Scheduled Task intact so the next boot retries cleanly.
|
||||
$renamed = @()
|
||||
$probeOk = $true
|
||||
foreach ($item in @(Get-ChildItem -Path '{agentBasePath}' -Exclude 'Logs' -ErrorAction SilentlyContinue)) {{
|
||||
$dest = $item.FullName + '.del-' + [Guid]::NewGuid().ToString('N')
|
||||
$itemOk = $false
|
||||
for ($i = 1; $i -le {renameRetries}; $i++) {{
|
||||
try {{
|
||||
Rename-Item -Path $item.FullName -NewName $dest -Force -ErrorAction Stop
|
||||
$itemOk = $true
|
||||
break
|
||||
}} catch {{
|
||||
Start-Sleep -Seconds {renameRetryDelaySeconds}
|
||||
}}
|
||||
}}
|
||||
if ($itemOk) {{ $renamed += @{{ Original = $item.FullName; Renamed = $dest }} }}
|
||||
else {{ $probeOk = $false; break }}
|
||||
}}
|
||||
" : $@"
|
||||
# Probe-rename the whole tree (LIFE-F7 #3). Failure ⇒ abort with task + files + marker
|
||||
# intact so the next boot retries cleanly. GUID suffix avoids collisions with leftovers
|
||||
# of earlier attempts.
|
||||
$renamedPath = '{agentBasePath}.del-' + [Guid]::NewGuid().ToString('N')
|
||||
$probeOk = $false
|
||||
for ($i = 1; $i -le {renameRetries}; $i++) {{
|
||||
try {{
|
||||
Rename-Item -Path '{agentBasePath}' -NewName $renamedPath -Force -ErrorAction Stop
|
||||
$probeOk = $true
|
||||
break
|
||||
}} catch {{
|
||||
Start-Sleep -Seconds {renameRetryDelaySeconds}
|
||||
}}
|
||||
}}
|
||||
$renamed = @(@{{ Original = '{agentBasePath}'; Renamed = $renamedPath }})
|
||||
")}
|
||||
if ($probeOk) {{
|
||||
# Remove Scheduled Task — verify removal BEFORE deleting files (LIFE-F6). An orphan
|
||||
# task firing against a deleted exe spams Task Scheduler errors every boot forever
|
||||
# and, once the files (incl. the enrollment-complete marker) are gone, has no
|
||||
# cleanup-retry path.
|
||||
$taskName = '{scheduledTaskName}'
|
||||
$taskGone = $false
|
||||
for ($i = 1; $i -le 5; $i++) {{
|
||||
try {{ Stop-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
try {{ Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
$still = $null
|
||||
try {{ $still = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
if (-not $still) {{ $taskGone = $true; break }}
|
||||
Start-Sleep -Seconds 1
|
||||
}}
|
||||
if (-not $taskGone) {{
|
||||
# Fallback: schtasks.exe /Delete does not depend on the ScheduledTasks PowerShell module.
|
||||
try {{ & schtasks.exe /Delete /TN $taskName /F 2>$null | Out-Null }} catch {{ }}
|
||||
try {{ $taskGone = -not (Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue) }} catch {{ }}
|
||||
}}
|
||||
|
||||
if ($taskGone) {{
|
||||
# Task gone + tree renamed ⇒ safe to delete. Never touches the original path.
|
||||
foreach ($entry in $renamed) {{
|
||||
Remove-Item -Path $entry.Renamed -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}}
|
||||
}} else {{
|
||||
# Task survived ⇒ rename back and leave everything for the next-boot retry
|
||||
# (the marker survives, so bootstrap re-runs cleanup).
|
||||
foreach ($entry in $renamed) {{
|
||||
try {{ Rename-Item -Path $entry.Renamed -NewName $entry.Original -Force -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
}}
|
||||
}}
|
||||
}} else {{
|
||||
# Probe failed ⇒ something still holds the tree. Rename back whatever succeeded and
|
||||
# abort WITHOUT deleting anything — task + marker stay, next boot retries.
|
||||
foreach ($entry in $renamed) {{
|
||||
try {{ Rename-Item -Path $entry.Renamed -NewName $entry.Original -Force -ErrorAction SilentlyContinue }} catch {{ }}
|
||||
}}
|
||||
}}
|
||||
{(rebootOnComplete ? @"
|
||||
# shutdown.exe (NOT Restart-Computer): Restart-Computer has no -Comment parameter and -Delay is
|
||||
# only honoured with -Wait, so the previous invocation failed parameter binding and the device
|
||||
# silently never rebooted. shutdown.exe /r /t 10 mirrors the standalone reboot path and reboots
|
||||
# after a 10 s delay (lets this script finish + the agent exit). Review LIFE-F2.
|
||||
shutdown.exe /r /t 10 /c 'Autopilot enrollment completed - Autopilot Monitor is rebooting'
|
||||
" : "")}
|
||||
}} finally {{
|
||||
if ($lockStream) {{ $lockStream.Dispose() }}
|
||||
Remove-Item -Path $lockPath -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path $scriptPath -Force -ErrorAction SilentlyContinue
|
||||
}}
|
||||
";
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -370,7 +370,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
{
|
||||
try
|
||||
{
|
||||
var versionUrl = $"{Constants.AgentBlobBaseUrl}/{Constants.AgentVersionFileNameForLine(2)}";
|
||||
// Download alias, not the legacy blob account: the legacy mirror in the
|
||||
// release pipeline is fail-soft, so it can silently lag behind eu.
|
||||
var versionUrl = $"{Constants.AgentDownloadBaseUrl}/{Constants.AgentVersionFileNameForLine(2)}";
|
||||
|
||||
using (var handler = new HttpClientHandler())
|
||||
using (var client = new HttpClient(handler) { Timeout = TimeSpan.FromMilliseconds(VersionCheckTimeoutMs) })
|
||||
@@ -497,7 +499,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
{
|
||||
try
|
||||
{
|
||||
var zipUrl = $"{Constants.AgentBlobBaseUrl}/{Constants.AgentZipFileNameForLine(2)}";
|
||||
var zipUrl = $"{Constants.AgentDownloadBaseUrl}/{Constants.AgentZipFileNameForLine(2)}";
|
||||
|
||||
// Clean up any previous download (fragment from aborted run, or symlink/junction).
|
||||
// Pre-delete + FileMode.CreateNew below: CreateNew refuses to open an existing
|
||||
|
||||
+43
-4
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Management;
|
||||
using System.Text.RegularExpressions;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
@@ -28,6 +29,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
///
|
||||
/// Emits a single "local_admin_analysis" event at startup and at shutdown,
|
||||
/// enabling delta detection between pre- and post-enrollment state.
|
||||
///
|
||||
/// Tenant-supplied allowed accounts may contain glob wildcards (e.g. "adm-*");
|
||||
/// see <see cref="MatchesAllowedEntry"/>.
|
||||
/// </summary>
|
||||
public class LocalAdminAnalyzer : IAgentAnalyzer
|
||||
{
|
||||
@@ -246,6 +250,43 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Allowed-list matching
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Matches an account or profile-folder name against one allowed-list entry.
|
||||
/// Entries containing <c>*</c> or <c>?</c> are glob patterns (same semantics as the
|
||||
/// hardware whitelist: <c>*</c> = any run, <c>?</c> = one char, case-insensitive);
|
||||
/// all other entries match exactly. No ambiguity with literal names — <c>*</c> and
|
||||
/// <c>?</c> are invalid characters in Windows account names.
|
||||
/// Internal for direct unit-testing via InternalsVisibleTo.
|
||||
/// </summary>
|
||||
internal static bool MatchesAllowedEntry(string name, string entry)
|
||||
{
|
||||
if (entry.IndexOf('*') < 0 && entry.IndexOf('?') < 0)
|
||||
return string.Equals(entry, name, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
var regexPattern = "^" + Regex.Escape(entry)
|
||||
.Replace("\\*", ".*")
|
||||
.Replace("\\?", ".") + "$";
|
||||
|
||||
try
|
||||
{
|
||||
// Timeout guards against pathological patterns (ReDoS)
|
||||
return Regex.IsMatch(name, regexPattern, RegexOptions.IgnoreCase, TimeSpan.FromMilliseconds(100));
|
||||
}
|
||||
catch (RegexMatchTimeoutException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsAllowed(string name, List<string> allowedAccounts)
|
||||
{
|
||||
return allowedAccounts.Any(a => MatchesAllowedEntry(name, a));
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Individual checks
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -308,8 +349,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
if (disabled)
|
||||
continue;
|
||||
|
||||
if (!allowedAccounts.Any(a =>
|
||||
string.Equals(a, name, StringComparison.OrdinalIgnoreCase)))
|
||||
if (!IsAllowed(name, allowedAccounts))
|
||||
{
|
||||
unexpected.Add(name);
|
||||
_logger.Debug($"{Name}: Unexpected local account: {name}");
|
||||
@@ -349,8 +389,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Analyzers
|
||||
|
||||
allFound.Add(folderName);
|
||||
|
||||
if (!allowedAccounts.Any(a =>
|
||||
string.Equals(a, folderName, StringComparison.OrdinalIgnoreCase)))
|
||||
if (!IsAllowed(folderName, allowedAccounts))
|
||||
{
|
||||
unexpected.Add(folderName);
|
||||
_logger.Debug($"{Name}: Unexpected profile folder: {folderName}");
|
||||
|
||||
+55
-1
@@ -8,6 +8,7 @@ using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
@@ -493,7 +494,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo
|
||||
}
|
||||
}
|
||||
|
||||
EmitDeviceInfoEvent(Constants.EventTypes.SecureBootStatus, $"SecureBoot: {data["uefiSecureBootEnabled"]}, CA2023: {data["uefiCA2023Status"]}", data);
|
||||
// Direct firmware evidence (UEFI db/KEK variables) — authoritative for the
|
||||
// CA-2023 rollout state. The Servicing key above only mirrors whether the
|
||||
// Windows Update rollout ran; it lags behind when the certificate reached
|
||||
// the firmware another way (OEM factory image, WinCS, manual deployment).
|
||||
var firmware = UefiSecureBootCertReader.Read();
|
||||
AppendFirmwareCertFields(data, firmware);
|
||||
|
||||
EmitDeviceInfoEvent(Constants.EventTypes.SecureBootStatus,
|
||||
$"SecureBoot: {data["uefiSecureBootEnabled"]}, CA2023: {data["uefiCA2023Status"]}, firmware: {DescribeFirmwareVerdict(firmware)}",
|
||||
data);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -501,6 +511,50 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps a <see cref="UefiSecureBootCertSnapshot"/> onto the <c>secureboot_status</c>
|
||||
/// payload. Certificate booleans are only emitted when the respective variable read
|
||||
/// succeeded — an absent boolean means "firmware state unknown", never "missing".
|
||||
/// </summary>
|
||||
internal static void AppendFirmwareCertFields(Dictionary<string, object> data, UefiSecureBootCertSnapshot firmware)
|
||||
{
|
||||
if (firmware == null) return;
|
||||
|
||||
data["uefiFirmwareReadStatus"] = firmware.DbStatus != UefiSecureBootCertReader.StatusOk
|
||||
? firmware.DbStatus
|
||||
: firmware.KekStatus;
|
||||
|
||||
if (firmware.DbStatus == UefiSecureBootCertReader.StatusOk)
|
||||
{
|
||||
data["uefiDbHasWindowsUefiCa2023"] = firmware.DbHasWindowsUefiCa2023;
|
||||
data["uefiDbHasMicrosoftUefiCa2023"] = firmware.DbHasMicrosoftUefiCa2023;
|
||||
data["uefiDbHasMicrosoftOptionRomCa2023"] = firmware.DbHasMicrosoftOptionRomCa2023;
|
||||
data["uefiDbHasWindowsProductionPca2011"] = firmware.DbHasWindowsProductionPca2011;
|
||||
data["uefiDbHasMicrosoftUefiCa2011"] = firmware.DbHasMicrosoftUefiCa2011;
|
||||
|
||||
// One-sided marker consumed by ANALYZE-SEC-001's not_exists precondition.
|
||||
// The engine's not_exists passes only while the field is absent/empty, so the
|
||||
// marker must be OMITTED (never emitted as false) unless the firmware db
|
||||
// positively confirms the 2023 certificate — old agents and unconfirmed
|
||||
// devices then keep evaluating the rule.
|
||||
if (firmware.DbHasWindowsUefiCa2023)
|
||||
data["uefiCa2023FirmwareConfirmed"] = true;
|
||||
}
|
||||
|
||||
if (firmware.KekStatus == UefiSecureBootCertReader.StatusOk)
|
||||
{
|
||||
data["uefiKekHasMicrosoftKek2kCa2023"] = firmware.KekHasMicrosoftKek2kCa2023;
|
||||
data["uefiKekHasMicrosoftKekCa2011"] = firmware.KekHasMicrosoftKekCa2011;
|
||||
}
|
||||
}
|
||||
|
||||
internal static string DescribeFirmwareVerdict(UefiSecureBootCertSnapshot firmware)
|
||||
{
|
||||
if (firmware == null) return "unavailable";
|
||||
if (firmware.DbStatus != UefiSecureBootCertReader.StatusOk) return firmware.DbStatus ?? "unavailable";
|
||||
return firmware.DbHasWindowsUefiCa2023 ? "CA2023 confirmed" : "CA2023 missing";
|
||||
}
|
||||
|
||||
private void CollectBitLockerStatus()
|
||||
{
|
||||
try
|
||||
|
||||
+2
-2
@@ -45,10 +45,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
/// </summary>
|
||||
private static readonly (string Name, string Url)[] MdmEndpoints =
|
||||
{
|
||||
("Entra ID", "https://login.microsoftonline.com"),
|
||||
("Entra ID", Constants.EntraLoginBaseUrl),
|
||||
("Device Registration", "https://enterpriseregistration.windows.net"),
|
||||
("Intune Portal", "https://portal.manage.microsoft.com"),
|
||||
("Microsoft Graph", "https://graph.microsoft.com"),
|
||||
("Microsoft Graph", Constants.GraphBaseUrl),
|
||||
};
|
||||
|
||||
public NetworkChangeDetector(
|
||||
|
||||
@@ -169,9 +169,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// <summary>
|
||||
/// Sends a critical error report to the emergency channel endpoint.
|
||||
/// Fire-and-forget: swallows all exceptions so a failure here never cascades.
|
||||
/// Uses a 5-second per-request timeout so it cannot block the upload loop.
|
||||
/// Default 5-second per-request timeout so it cannot block the upload loop; the
|
||||
/// emergency-break path passes a longer <paramref name="timeout"/> because its first
|
||||
/// request of a freshly booted process pays cold TLS + DNS + a possible backend
|
||||
/// cold start (observed 503s take well over 5 s).
|
||||
/// Returns <c>true</c> when the HTTP round-trip completed (the endpoint always
|
||||
/// answers 200), <c>false</c> on timeout/transport failure — the caller-side
|
||||
/// anti-flood layer uses this to decide whether the report counts as delivered.
|
||||
/// Virtual for tests.
|
||||
/// </summary>
|
||||
public async Task ReportAgentErrorAsync(AgentErrorReport report)
|
||||
public virtual async Task<bool> ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -190,16 +197,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
httpRequest.Headers.Add("X-Tenant-Id", report.TenantId);
|
||||
AddSecurityHeaders(httpRequest);
|
||||
|
||||
using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)))
|
||||
using (var cts = new CancellationTokenSource(timeout ?? TimeSpan.FromSeconds(5)))
|
||||
using (var response = await _httpClient.SendAsync(httpRequest, cts.Token).ConfigureAwait(false))
|
||||
{
|
||||
// Response status is deliberately ignored — endpoint always returns 200.
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger?.Debug($"ReportAgentErrorAsync: emergency channel failed: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,19 +57,33 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// (intended to be called as fire-and-forget via <c>_ = reporter.TrySendAsync(...)</c>).
|
||||
///
|
||||
/// All anti-flood checks are applied synchronously before the async HTTP call,
|
||||
/// so repeated calls are cheap when suppressed.
|
||||
/// so repeated calls are cheap when suppressed. The anti-flood reservation is
|
||||
/// taken up front (so concurrent same-key calls cannot double-send) but ROLLED
|
||||
/// BACK when every attempt fails: the budget and the once-per-session dedup are
|
||||
/// meant to count DELIVERED reports — the previous mark-before-send made the very
|
||||
/// first transport failure permanently suppress that error key for the session
|
||||
/// (found via the 48h emergency-break report, which is a single shot at boot).
|
||||
///
|
||||
/// <paramref name="attempts"/>/<paramref name="perAttemptTimeout"/>/<paramref name="retryDelay"/>
|
||||
/// exist for callers whose process is about to exit and cannot come back later
|
||||
/// (emergency break): retries stay INSIDE the one reservation, so they never
|
||||
/// multiply against the per-session budget.
|
||||
/// </summary>
|
||||
public virtual async Task TrySendAsync(
|
||||
AgentErrorType errorType,
|
||||
string message,
|
||||
int? httpStatusCode = null,
|
||||
long? sequenceNumber = null)
|
||||
long? sequenceNumber = null,
|
||||
int attempts = 1,
|
||||
TimeSpan? perAttemptTimeout = null,
|
||||
TimeSpan? retryDelay = null)
|
||||
{
|
||||
// Deduplicate by error type + status code: same failure category sent only once per session.
|
||||
// All anti-flood checks are protected by a lock to prevent race conditions when
|
||||
// TrySendAsync is called concurrently (fire-and-forget from multiple threads).
|
||||
var key = $"{errorType}:{httpStatusCode}";
|
||||
int currentReportCount;
|
||||
DateTime? previousLastReportTime;
|
||||
|
||||
lock (_antiFloodLock)
|
||||
{
|
||||
@@ -92,6 +106,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
return;
|
||||
}
|
||||
|
||||
previousLastReportTime = _lastReportTime;
|
||||
_reportedKeys.Add(key);
|
||||
_reportCount++;
|
||||
_lastReportTime = DateTime.UtcNow;
|
||||
@@ -113,15 +128,40 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
Timestamp = DateTime.UtcNow
|
||||
};
|
||||
|
||||
var delivered = false;
|
||||
var totalAttempts = attempts < 1 ? 1 : attempts;
|
||||
try
|
||||
{
|
||||
await _apiClient.ReportAgentErrorAsync(report);
|
||||
for (var attempt = 1; attempt <= totalAttempts && !delivered; attempt++)
|
||||
{
|
||||
if (attempt > 1 && retryDelay.HasValue)
|
||||
{
|
||||
await Task.Delay(retryDelay.Value).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
delivered = await _apiClient.ReportAgentErrorAsync(report, perAttemptTimeout).ConfigureAwait(false);
|
||||
if (!delivered && totalAttempts > 1)
|
||||
{
|
||||
_logger?.Debug($"[EmergencyChannel] Attempt {attempt}/{totalAttempts} failed: {key}");
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Swallow — emergency channel must never cascade failures into the caller
|
||||
_logger?.Debug($"[EmergencyChannel] Failed to send report: {ex.Message}");
|
||||
}
|
||||
|
||||
if (!delivered)
|
||||
{
|
||||
lock (_antiFloodLock)
|
||||
{
|
||||
_reportedKeys.Remove(key);
|
||||
_reportCount--;
|
||||
_lastReportTime = previousLastReportTime;
|
||||
}
|
||||
_logger?.Debug($"[EmergencyChannel] Report not delivered — reservation rolled back: {key}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// <item><b>StallProbeHost</b> — <see cref="StallProbeCollector"/> +
|
||||
/// <see cref="StallProbeCollectorAdapter"/>. Owns its 60-s idle-check timer (the
|
||||
/// collector itself has no timer — it's a pure probe invoked from outside).</item>
|
||||
/// <item><b>EspPolicyProviderStallHost</b> — always-on 60-s wall-clock tick for the
|
||||
/// two-arm <c>esp_policy_provider_stalled</c> tripwire (EnrollmentStatusTracking CSP
|
||||
/// provider continuously incomplete ≥ 15 min, OR Setup\Apps providers registered
|
||||
/// without the expected <c>Sidecar</c>/IME provider — the no-timeout ESP wait).</item>
|
||||
/// </list>
|
||||
/// Optional peripheral hosts (driven by <see cref="CollectorConfiguration"/> toggles):
|
||||
/// <list type="bullet">
|
||||
@@ -303,6 +307,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
harmlessModernDeploymentEventIds: collectors.ModernDeploymentHarmlessEventIds));
|
||||
}
|
||||
|
||||
// ESP policy-provider stall tripwire — always-on kernel host (no config gate, like
|
||||
// disk_space_low): a provider that never completes, or co-management "ConfigMgr"
|
||||
// registered without the expected "Sidecar"/IME provider (issue #106 — even with
|
||||
// TrackingPoliciesCreated=1 the ESP ignores foreign provider names), parks the user
|
||||
// ESP at "Apps (Identifying)" WITHOUT any OS timeout. Wall-clock dwell, deliberately
|
||||
// outside the StallProbe gate: the idle clock there resets on any session activity,
|
||||
// and the stall must accrue regardless of it.
|
||||
hosts.Add(new EspPolicyProviderStallHost(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
logger: logger,
|
||||
ingress: ingress,
|
||||
clock: clock,
|
||||
startupGate: _startupEventGate));
|
||||
|
||||
// Deterministic update corroboration — compares the persisted OS build (CurrentBuild.UBR)
|
||||
// across agent restarts and emits os_build_changed when it differs (session 7443317c:
|
||||
// build jump proved an OOBE quality update the WU channel never showed). Must be added
|
||||
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Persistence;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// Drives the <see cref="EspPolicyProviderStallDetector"/> from a 60-s wall-clock tick.
|
||||
/// Deliberately NOT idle-coupled (a policy-provider stall coexists with a chatty session —
|
||||
/// <see cref="StallProbeHost"/>'s idle clock would keep resetting) and NOT idle-stopped
|
||||
/// (<see cref="PeriodicCollectorLifecycleHost"/> would sleep through exactly the dormant
|
||||
/// window the dwell must observe). Always-on kernel host, no config gate — precedent:
|
||||
/// the <c>disk_space_low</c> tripwire.
|
||||
/// <para>
|
||||
/// First tick fires after 60 s, not at Start: the EnrollmentStatusTracking key legitimately
|
||||
/// doesn't exist at agent start, and nothing can be 15 min stale at t=0.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal sealed class EspPolicyProviderStallHost : ICollectorHost
|
||||
{
|
||||
public string Name => "EspPolicyProviderStallDetector";
|
||||
|
||||
private static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(60);
|
||||
|
||||
private readonly EspPolicyProviderStallDetector _detector;
|
||||
private readonly AgentLogger _logger;
|
||||
private Timer? _tickTimer;
|
||||
private int _disposed;
|
||||
|
||||
public EspPolicyProviderStallHost(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
AgentLogger logger,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
StartupEventGate? startupGate)
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
_logger = logger;
|
||||
var post = new InformationalEventPost(ingress, clock);
|
||||
_detector = new EspPolicyProviderStallDetector(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
post: post,
|
||||
logger: logger,
|
||||
clock: clock,
|
||||
startupGate: startupGate);
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
_tickTimer = new Timer(
|
||||
_ => SafeTick(),
|
||||
state: null,
|
||||
dueTime: PollInterval,
|
||||
period: PollInterval);
|
||||
_logger.Info($"EspPolicyProviderStallHost: started (tick every {PollInterval.TotalSeconds}s).");
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
try
|
||||
{
|
||||
_tickTimer?.Dispose();
|
||||
_tickTimer = null;
|
||||
}
|
||||
catch (Exception ex) { _logger.Warning($"EspPolicyProviderStallHost: timer dispose failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) == 1) return;
|
||||
Stop();
|
||||
}
|
||||
|
||||
private void SafeTick()
|
||||
{
|
||||
try
|
||||
{
|
||||
_detector.Tick();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error("EspPolicyProviderStallHost: tick failed.", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
logger: logger,
|
||||
matchLogPath: expandedMatchLogPath,
|
||||
stateDirectory: stateDirectory);
|
||||
// Historic-replay guard runs against the agent clock, not the raw system clock —
|
||||
// keeps the tracker's staleness verdicts consistent with the adapter's clamp.
|
||||
_tracker.UtcNowProvider = () => clock.UtcNow;
|
||||
|
||||
if (simulationMode)
|
||||
{
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
@@ -267,6 +268,14 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
payload["failedSubcategory"] = failedSubcategory!;
|
||||
if (!string.IsNullOrEmpty(category))
|
||||
payload["category"] = category!;
|
||||
// Session 4910a5a5: culprit-app names ride the signal so the DecisionEngine's
|
||||
// esp_failure_advisory / enrollment_failed events can name them directly.
|
||||
var culprits = args?.LikelyCulpritApps;
|
||||
if (culprits != null && culprits.Count > 0)
|
||||
{
|
||||
payload["likelyCulpritApps"] = string.Join(", ", culprits);
|
||||
payload["likelyCulpritAppCount"] = culprits.Count.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
_ingress.Post(
|
||||
kind: DecisionSignalKind.EspTerminalFailure,
|
||||
|
||||
@@ -97,12 +97,24 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
// bootstrap marker line. Lets MCP report "which device runs which bootstrap version".
|
||||
private bool _bootstrapDetectedPosted;
|
||||
|
||||
// One-shot: historic script replay detected (a previous enrollment's IME log content
|
||||
// surviving on disk) — its script events are suppressed, and a single
|
||||
// historic_ime_replay_detected Info event explains the gap in the timeline.
|
||||
private bool _historicReplayNoted;
|
||||
private int _suppressedHistoricReplayEvents; // forensic count for the log line
|
||||
|
||||
// A platform script whose observed run duration reaches this is treated as having hit the
|
||||
// IME script-execution timeout (default ~30 min). 25 min leaves headroom below 30 so clock
|
||||
// jitter / the agent's late replay window can't miss it, while staying well above any
|
||||
// legitimately long install wrapper that completes on its own.
|
||||
private static readonly TimeSpan ScriptTimeoutSuspectedThreshold = TimeSpan.FromMinutes(25);
|
||||
|
||||
// Source log lines older than this relative to the clock are pathological — shared by
|
||||
// ResolveOccurredAt's staleness clamp, the historic-replay suppression, the duration
|
||||
// plausibility backstop AND the tracker's app-mutation guard (single source of truth
|
||||
// in ImeLogTracker so the layers can never drift apart).
|
||||
private static readonly TimeSpan StaleSourceThreshold = ImeLogTracker.HistoricReplayThreshold;
|
||||
|
||||
// Matches the deterministic marker line the bootstrap script writes via Write-Log,
|
||||
// e.g. "Bootstrap script version: v2.0" — see scripts/Bootstrap/Install-AutopilotMonitor.ps1.
|
||||
// Same shape as the web-side regex in utils/bootstrapVersion.ts; kept in sync intentionally.
|
||||
@@ -289,17 +301,11 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
rawSourceTimestamp = sourceTsOverride ?? _tracker.LastMatchedLogTimestamp;
|
||||
if (rawSourceTimestamp.HasValue)
|
||||
{
|
||||
// CMTrace log timestamps from IME are emitted in UTC, but DateTimeKind may
|
||||
// arrive as Unspecified depending on the parser path. Normalize via
|
||||
// SpecifyKind=Utc rather than ToUniversalTime() to avoid double-conversion
|
||||
// when Kind is already UTC.
|
||||
var asUtc = rawSourceTimestamp.Value.Kind == DateTimeKind.Utc
|
||||
? rawSourceTimestamp.Value
|
||||
: DateTime.SpecifyKind(rawSourceTimestamp.Value, DateTimeKind.Utc);
|
||||
var asUtc = NormalizeUtc(rawSourceTimestamp.Value);
|
||||
|
||||
var clockNow = _clock.UtcNow;
|
||||
var ageHours = (clockNow - asUtc).TotalHours;
|
||||
if (ageHours > 24.0 || ageHours < -1.0)
|
||||
if (ageHours > StaleSourceThreshold.TotalHours || ageHours < -1.0)
|
||||
{
|
||||
// Pathologically stale (>24h old) or in the future (>1h skew) — log,
|
||||
// fall back, and let the caller flag the event.
|
||||
@@ -325,6 +331,67 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
data["rejectedSourceTimestamp"] = rawSourceTs.Value.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
// CMTrace Kind normalization — shared implementation lives on the tracker (used by its
|
||||
// historic-replay guard too).
|
||||
private static DateTime NormalizeUtc(DateTime value) => ImeLogTracker.NormalizeUtc(value);
|
||||
|
||||
/// <summary>
|
||||
/// True when an event's source log line is a historic replay: the source timestamp
|
||||
/// was rejected as stale (> <see cref="StaleSourceThreshold"/> in the PAST). Those lines
|
||||
/// are script runs and app installs from a previous enrollment whose IME log survived on
|
||||
/// disk (session eaf3d8c4: 156 replayed script completions + 147 replayed app events
|
||||
/// shown as current activity, immediate-upload flood → ingress_backpressure). Future-skew
|
||||
/// rejections (negative age — mid-enrollment clock jump) and clock-only fallbacks (no
|
||||
/// source timestamp at all) are NOT replay and must still emit. Emits the one-shot
|
||||
/// <c>historic_ime_replay_detected</c> summary on the first suppression; replay is
|
||||
/// chronological, so that first suppressed line carries the earliest rejected source
|
||||
/// timestamp. Bypassed in <see cref="ImeLogTracker.SimulationMode"/> — the
|
||||
/// --replay-log-dir dev tool feeds historic lines on purpose.
|
||||
/// </summary>
|
||||
private bool SuppressIfHistoricReplay(
|
||||
bool derivedFromClock, DateTime? rawSourceTs, string eventLabel, string entityId)
|
||||
{
|
||||
if (_tracker.SimulationMode) return false;
|
||||
if (!derivedFromClock || !rawSourceTs.HasValue) return false;
|
||||
var asUtc = NormalizeUtc(rawSourceTs.Value);
|
||||
var clockNow = _clock.UtcNow;
|
||||
var age = clockNow - asUtc;
|
||||
if (age <= StaleSourceThreshold) return false; // future-skew or fresh → emit
|
||||
|
||||
bool emitSummary;
|
||||
int suppressedSoFar;
|
||||
lock (_lock)
|
||||
{
|
||||
suppressedSoFar = ++_suppressedHistoricReplayEvents;
|
||||
emitSummary = !_historicReplayNoted;
|
||||
_historicReplayNoted = true;
|
||||
}
|
||||
|
||||
if (emitSummary)
|
||||
{
|
||||
var culture = System.Globalization.CultureInfo.InvariantCulture;
|
||||
// Standard batch upload on purpose — no immediateUpload; the whole point of the
|
||||
// suppression is to stop the replayed batch from flooding the ingest.
|
||||
_post.Emit(
|
||||
eventType: SharedEventTypes.HistoricImeReplayDetected,
|
||||
source: SourceLabel,
|
||||
message: "Historic IME activity from a previous enrollment detected in the log "
|
||||
+ $"(earliest source timestamp {asUtc:o}) — its script and app events are suppressed for this session.",
|
||||
severity: EventSeverity.Info,
|
||||
data: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["decision"] = "historic_ime_replay_suppressed",
|
||||
["earliestRejectedSourceTimestamp"] = asUtc.ToString("o", culture),
|
||||
["staleThresholdHours"] = StaleSourceThreshold.TotalHours.ToString("F0", culture),
|
||||
});
|
||||
}
|
||||
|
||||
_logger?.Debug(
|
||||
$"ImeAdapter: suppressed historic {eventLabel} for {entityId} "
|
||||
+ $"(source ts {asUtc:o}, age {age.TotalHours:F1}h, suppressed so far: {suppressedSoFar})");
|
||||
return true;
|
||||
}
|
||||
|
||||
private void EmitEspPhase(string phase)
|
||||
{
|
||||
if (string.IsNullOrEmpty(phase)) return;
|
||||
@@ -504,6 +571,14 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
|
||||
var now = ResolveOccurredAt(out var derivedFromClock, out var rawSourceTs);
|
||||
|
||||
// Historic replay (a previous enrollment's log) — do not emit at all. Placed BEFORE
|
||||
// the sub-phase declaration, the timing bookkeeping and the terminal-signal dedup so
|
||||
// a replayed app neither consumes the fire-once flags nor poisons _appTimings (which
|
||||
// feeds FinalStatusBuilder) nor posts a phantom DecisionSignal. The early return also
|
||||
// covers the shadow download_progress and the app_tracking_summary snapshot below.
|
||||
if (SuppressIfHistoricReplay(derivedFromClock, rawSourceTs, $"app-state {oldState}->{newState}", app.Id))
|
||||
return;
|
||||
|
||||
// Plan §5 Fix 2: first app activity in a given ESP phase opens the sub-phase on
|
||||
// the UI timeline via a `phase_transition` declaration event. Fire-once per ESP
|
||||
// phase; emission happens BEFORE the actual app_* event so the UI sees the phase
|
||||
@@ -819,6 +894,11 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
if (!string.IsNullOrEmpty(patternId)) data["patternId"] = patternId!;
|
||||
|
||||
var now = ResolveOccurredAt(out var derivedFromClock, out var rawSourceTs);
|
||||
|
||||
// Historic replay — a week-old DO summary is not this enrollment's download.
|
||||
if (SuppressIfHistoricReplay(derivedFromClock, rawSourceTs, SharedEventTypes.DoTelemetry, app.Id))
|
||||
return;
|
||||
|
||||
TagDerivedTimestamp(data, derivedFromClock, rawSourceTs);
|
||||
|
||||
var label = string.IsNullOrEmpty(app.Name) ? app.Id : app.Name;
|
||||
@@ -910,13 +990,27 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
now = ResolveOccurredAt(script.ExitObservedAtUtc, out derivedFromClock, out rawSourceTs);
|
||||
else
|
||||
now = ResolveOccurredAt(out derivedFromClock, out rawSourceTs);
|
||||
|
||||
// Historic replay (a previous enrollment's log) — do not emit at all. The early
|
||||
// return also skips MaybeEmitBootstrapDetected / MaybeEmitScriptTimeoutSuspected,
|
||||
// which would otherwise fire on week-old evidence.
|
||||
if (SuppressIfHistoricReplay(derivedFromClock, rawSourceTs, eventType, script.PolicyId))
|
||||
return;
|
||||
|
||||
TagDerivedTimestamp(data, derivedFromClock, rawSourceTs);
|
||||
|
||||
// Per-script run duration (start line → this completion, `now`). Surfaced on every
|
||||
// Per-script run duration (start line → this completion). Surfaced on every
|
||||
// script completion so the UI can flag slow / inefficient scripts, not just timed-out
|
||||
// ones. Omitted when the start line was never observed (StartedAtUtc null); clamped at 0
|
||||
// against clock skew between the (possibly clock-derived) completion stamp and start.
|
||||
//
|
||||
// When the completion timestamp was clock-derived because the source line's timestamp
|
||||
// was rejected (future-skew CMTrace clock jump — the stale case never reaches here),
|
||||
// `now` lives on the clock's timeline while StartedAtUtc was captured RAW from the
|
||||
// source log. Mixing the two produced absurd durations (session eaf3d8c4: 170 h), so
|
||||
// the duration uses the raw source pair — both ends on the same timeline, the
|
||||
// difference is the true runtime. The event stamp stays `now` (timeline position).
|
||||
//
|
||||
// L19 (delta review 2026-07-02): the same field name carries two different semantics —
|
||||
// start→own-end-signal (≈ script runtime) vs HS-SCRIPT-START→HS-NEW-RESULT (the whole
|
||||
// cycle INCLUDING IME's 30 s–3 min batched reporting latency, i.e. systematically
|
||||
@@ -924,10 +1018,16 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
// consumers never compare the two as equals. The emitting handler stamps the basis on
|
||||
// the state (health-script early-signal completions are script_runtime too); the
|
||||
// scriptType fallback covers states from paths that predate the field.
|
||||
var completionForDuration = derivedFromClock && rawSourceTs.HasValue
|
||||
? NormalizeUtc(rawSourceTs.Value)
|
||||
: now;
|
||||
if (script.StartedAtUtc.HasValue)
|
||||
{
|
||||
var durationSeconds = (now - script.StartedAtUtc.Value).TotalSeconds;
|
||||
if (durationSeconds >= 0)
|
||||
var durationSeconds = (completionForDuration - script.StartedAtUtc.Value).TotalSeconds;
|
||||
// Plausibility backstop: IME's script execution timeout is ~30 min; anything above
|
||||
// 24 h is a cross-run artifact (e.g. a stale pending start slot from a previous
|
||||
// enrollment paired with a fresh completion) — omit rather than lie.
|
||||
if (durationSeconds >= 0 && durationSeconds <= StaleSourceThreshold.TotalSeconds)
|
||||
{
|
||||
data["durationSeconds"] = durationSeconds.ToString("F2", culture);
|
||||
data["durationBasis"] = script.DurationBasis
|
||||
@@ -979,7 +1079,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
$"ImeAdapter: script completed policyId={shortId} type={script.ScriptType ?? "?"} result={script.Result ?? "?"} exit={(script.ExitCode.HasValue ? script.ExitCode.Value.ToString() : "n/a")}");
|
||||
|
||||
MaybeEmitBootstrapDetected(script, now);
|
||||
MaybeEmitScriptTimeoutSuspected(script, now);
|
||||
MaybeEmitScriptTimeoutSuspected(script, now, completionForDuration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -991,15 +1091,19 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
/// late start / pre-failed ESP. Advisory only: no state mutation, the underlying
|
||||
/// <c>script_failed</c> is still emitted separately. Health (remediation) scripts are
|
||||
/// excluded — they re-run on their own cadence and have a different timeout regime.
|
||||
/// <paramref name="durationEndUtc"/> is the timeline-consistent completion for duration
|
||||
/// math (raw source ts on rejected-timestamp emissions); <paramref name="occurredAtUtc"/>
|
||||
/// stamps the emitted event.
|
||||
/// </summary>
|
||||
private void MaybeEmitScriptTimeoutSuspected(ScriptExecutionState script, DateTime completionUtc)
|
||||
private void MaybeEmitScriptTimeoutSuspected(ScriptExecutionState script, DateTime occurredAtUtc, DateTime durationEndUtc)
|
||||
{
|
||||
if (IsRemediation(script.ScriptType)) return; // platform scripts only
|
||||
if (!script.StartedAtUtc.HasValue) return; // no start ⇒ no duration
|
||||
if (!string.Equals(script.Result, "Failed", StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
var duration = completionUtc - script.StartedAtUtc.Value;
|
||||
var duration = durationEndUtc - script.StartedAtUtc.Value;
|
||||
if (duration < ScriptTimeoutSuspectedThreshold) return;
|
||||
if (duration > StaleSourceThreshold) return; // implausible — cross-run artifact, not a hung script
|
||||
|
||||
if (!_tracker.TryClaimScriptTimeoutSuspected(script.PolicyId)) return; // restart-safe dedup per policyId (persisted tracker state)
|
||||
|
||||
@@ -1026,7 +1130,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
message: $"Platform script {shortId} ran {durationMinutes:F0} min and was marked Failed — suspected IME script timeout blocking the enrollment.",
|
||||
severity: EventSeverity.Warning,
|
||||
data: data,
|
||||
occurredAtUtc: completionUtc);
|
||||
occurredAtUtc: occurredAtUtc);
|
||||
|
||||
_logger?.Info(
|
||||
$"ImeAdapter: script_timeout_suspected policyId={shortId} duration={durationMinutes:F1}min result={script.Result} espPhase={espPhase ?? "(none)"}");
|
||||
@@ -1081,6 +1185,12 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
if (!string.IsNullOrEmpty(patternId)) data["patternId"] = patternId!;
|
||||
|
||||
var now = ResolveOccurredAt(out var derivedFromClock, out var rawSourceTs);
|
||||
|
||||
// Historic replay — suppressing here also kills the immediateUpload flood the
|
||||
// replayed batch caused (session eaf3d8c4: 3× ingress_backpressure).
|
||||
if (SuppressIfHistoricReplay(derivedFromClock, rawSourceTs, SharedEventTypes.ScriptStarted, info.PolicyId))
|
||||
return;
|
||||
|
||||
TagDerivedTimestamp(data, derivedFromClock, rawSourceTs);
|
||||
|
||||
var label = IsRemediation(info.ScriptType) ? "Health script" : "Platform script";
|
||||
|
||||
@@ -137,7 +137,24 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
|
||||
private void OnPhaseChanged(object? sender, RealmJoinPhaseChangedEventArgs e)
|
||||
{
|
||||
// Phase change is observability-only: no DecisionSignalKind, just a timeline event.
|
||||
// Dual-emission like OnDetected/OnResolved: (1) the typed signal persists the phase
|
||||
// into RealmJoinFacts.LastDeploymentPhase (restart-safe) and lets the reducer detect
|
||||
// the aborted-RJ-ESP shape (phase left 100/101 for 200/210 without 110 — session
|
||||
// 224b2087); (2) the InformationalEvent keeps the existing timeline entry.
|
||||
var payload = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = e.CurrentPhase.ToString(),
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = e.PreviousPhase.ToString(),
|
||||
};
|
||||
_ingress.Post(
|
||||
kind: DecisionSignalKind.RealmJoinPhaseChanged,
|
||||
occurredAtUtc: _clock.UtcNow,
|
||||
sourceOrigin: SourceLabel,
|
||||
evidence: BuildEvidence(
|
||||
"realmjoin-phase-changed-v1",
|
||||
$"RealmJoin DeploymentPhase {e.PreviousPhase} -> {e.CurrentPhase}"),
|
||||
payload: payload);
|
||||
|
||||
// Both raw int (deploymentPhase / previousPhase) and human-readable name
|
||||
// (deploymentPhaseName / previousPhaseName) flow through — int stays for downstream
|
||||
// filters / KQL, name fields make the message + UI rows readable without a lookup.
|
||||
|
||||
@@ -6,6 +6,7 @@ using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Runtime;
|
||||
@@ -93,6 +94,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
// correction between agent start and termination no longer skews bootToStartSeconds.
|
||||
private readonly ulong _bootAnchorTickMs;
|
||||
private readonly DateTime _bootAnchorUtc;
|
||||
// WinRT OOBE state at construction (handler is built at agent start) — corroborates
|
||||
// the late-start heuristic: "completed" here means OOBE was already over on arrival.
|
||||
private readonly string _oobeStateAtAgentStart;
|
||||
private int _handled;
|
||||
|
||||
public EnrollmentTerminationHandler(
|
||||
@@ -129,6 +133,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
|
||||
_bootAnchorTickMs = ObservationCoverage.CurrentTickMs();
|
||||
_bootAnchorUtc = DateTime.UtcNow;
|
||||
_oobeStateAtAgentStart = OobeStateReader.Read();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -658,6 +663,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
{ "agentUptimeSeconds", Math.Round(uptimeSeconds, 1) },
|
||||
{ "deviceBootUtc", deviceBootUtc.ToString("o") },
|
||||
{ "agentStartUtc", _agentStartTimeUtc.ToString("o") },
|
||||
{ "oobeStateAtAgentStart", _oobeStateAtAgentStart },
|
||||
{ "outcome", args.Outcome.ToString() },
|
||||
{ "note", "The agent's bootstrap (an Intune platform script) ran only near the end of the enrollment, so the agent observed the already-decided end-state rather than the failure window. Treat the diagnosis as a post-mortem; check for a platform/remediation script that hung ahead of the bootstrap (see script_timeout_suspected)." },
|
||||
},
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using System;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -15,28 +17,78 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// classifier terminalize the session instead of waiting out the grace. Fully swallowed — a send
|
||||
/// failure (no network, no cert) must never delay the cleanup/exit the break exists to perform.
|
||||
/// See tasks/enrollment-status-reclassification.md.
|
||||
/// <para>
|
||||
/// Delivery hardening (2026-07-23): this is the FIRST request of a freshly booted process and
|
||||
/// the process never comes back — a miss is gone forever. The old single shot with a 5 s cap
|
||||
/// died on boot-time realities (NIC still associating, cold TLS/DNS, backend cold starts
|
||||
/// observed as >5 s 503s). Now: wait briefly for the NIC, then up to
|
||||
/// <see cref="SendAttempts"/> attempts with a per-attempt timeout that survives a Functions
|
||||
/// cold start. The session is 48 h dead — spending up to ~75 s here is irrelevant, losing the
|
||||
/// only signal that the agent gave up is not.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static class EmergencyBreakReporter
|
||||
{
|
||||
private static readonly TimeSpan NetworkWaitMax = TimeSpan.FromSeconds(15);
|
||||
private const int SendAttempts = 3;
|
||||
private static readonly TimeSpan PerAttemptTimeout = TimeSpan.FromSeconds(15);
|
||||
private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(5);
|
||||
// Upper bound for the blocking wait: NIC wait + 3×15s attempts + 2×5s delays + slack.
|
||||
private static readonly TimeSpan OverallBudget = TimeSpan.FromSeconds(75);
|
||||
|
||||
public static void TrySend(AgentConfiguration agentConfig, string agentVersion, AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
WaitForNetwork(logger);
|
||||
|
||||
var auth = BackendClientFactory.BuildAuthClients(agentConfig, agentVersion, logger);
|
||||
var message =
|
||||
$"Agent absolute session-age emergency break fired (cap {agentConfig.AbsoluteMaxSessionHours}h) — cleaning up and exiting.";
|
||||
|
||||
// The process is about to exit, so block briefly on the otherwise fire-and-forget send
|
||||
// rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP errors;
|
||||
// the auth bundle's HttpClients are intentionally left for process teardown.
|
||||
// The process is about to exit, so block on the otherwise fire-and-forget send
|
||||
// rather than let it be abandoned mid-flight. TrySendAsync swallows its own HTTP
|
||||
// errors and keeps the retries inside one anti-flood reservation; the auth
|
||||
// bundle's HttpClients are intentionally left for process teardown.
|
||||
auth.EmergencyReporter
|
||||
.TrySendAsync(AgentErrorType.SessionAgeEmergencyBreak, message)
|
||||
.Wait(TimeSpan.FromSeconds(5));
|
||||
.TrySendAsync(
|
||||
AgentErrorType.SessionAgeEmergencyBreak,
|
||||
message,
|
||||
attempts: SendAttempts,
|
||||
perAttemptTimeout: PerAttemptTimeout,
|
||||
retryDelay: RetryDelay)
|
||||
.Wait(OverallBudget);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Debug($"Emergency-break report send failed (best-effort): {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Boot-time NIC grace: the break fires during bootstrap, often seconds after boot while
|
||||
/// Wi-Fi is still associating. Polls the cheap link-level signal — it cannot prove backend
|
||||
/// reachability (the send attempts handle that), it only avoids burning the first attempts
|
||||
/// into a known-dead link. Best-effort: probe errors abort the wait, never the send.
|
||||
/// </summary>
|
||||
private static void WaitForNetwork(AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (NetworkInterface.GetIsNetworkAvailable()) return;
|
||||
|
||||
logger?.Info($"Emergency-break report: no network link yet — waiting up to {NetworkWaitMax.TotalSeconds:F0}s.");
|
||||
var deadline = DateTime.UtcNow + NetworkWaitMax;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
if (NetworkInterface.GetIsNetworkAvailable()) return;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Debug($"Emergency-break report: network probe failed ({ex.Message}) — sending anyway.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
@@ -68,6 +69,10 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{ "configVersion", configVersion },
|
||||
{ "remoteConfigFetched", outcome == RemoteConfigFetchOutcome.Succeeded },
|
||||
{ "remoteConfigOutcome", outcome.ToString() },
|
||||
// WinRT OOBE state sampled at this start. Together with previousExitType
|
||||
// this makes late starts provable: "completed" on a first_run means OOBE
|
||||
// was already over when the agent arrived (e.g. IME queue blocked).
|
||||
{ "oobeStateAtAgentStart", OobeStateReader.Read() },
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(previousExit?.CrashExceptionType))
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Serialization;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.DecisionCore.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// Session 4910a5a5 (2026-07-23) — advisory recovery. Field failure: DeviceSetup/Apps
|
||||
/// failed (culprit app never started installing), the advisory defang armed the
|
||||
/// <c>advisory_completion</c> window, the user pressed "Try again" ~23 min later, the apps
|
||||
/// re-ran to 10/10 and <c>DeviceSetupProvisioningComplete</c> arrived — yet the window
|
||||
/// fired 30 min after the advisory and un-defanged the stale failure
|
||||
/// (<c>enrollment_failed: esp_terminal_failure</c>) on a live enrollment mid-AccountSetup,
|
||||
/// because the advisory variant was exempt from ALL re-arm/rebase guards. Coverage:
|
||||
/// <list type="bullet">
|
||||
/// <item>The advisory records the failed category + culprit apps.</item>
|
||||
/// <item>A ProvisioningComplete of the SAME category sets the resolved fact + emits
|
||||
/// <c>esp_failure_advisory_resolved</c>; a different category does not.</item>
|
||||
/// <item>Deadline fire after recovery + enforcement progress → re-arm, not Failed.</item>
|
||||
/// <item>Deadline fire after recovery without progress → Failed with the truthful
|
||||
/// <c>esp_recovered_without_completion_evidence</c> reason and
|
||||
/// LastFailureTrigger=DeadlineFired (likely-stuck promotion off).</item>
|
||||
/// <item>Unrecovered advisory keeps the legacy un-defang exactly (regression guard).</item>
|
||||
/// <item>Reboot re-bases a recovered-advisory window (and still does NOT touch an
|
||||
/// unrecovered one).</item>
|
||||
/// <item>Snapshot round-trip + census surfacing of the new facts.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class AdvisoryRecoveryTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 7, 23, 3, 30, 0, DateTimeKind.Utc);
|
||||
|
||||
private static DecisionSignal MakeSignal(
|
||||
long ordinal,
|
||||
DecisionSignalKind kind,
|
||||
DateTime occurredAtUtc,
|
||||
IReadOnlyDictionary<string, string>? payload = null)
|
||||
{
|
||||
return new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: kind,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: occurredAtUtc,
|
||||
sourceOrigin: "test",
|
||||
evidence: new Evidence(EvidenceKind.Synthetic, $"{kind}-{ordinal}", "test"),
|
||||
payload: payload);
|
||||
}
|
||||
|
||||
private static DecisionSignal DeadlineFired(long ordinal, DateTime occurredAtUtc) =>
|
||||
MakeSignal(ordinal, DecisionSignalKind.DeadlineFired, occurredAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.AdvisoryCompletion });
|
||||
|
||||
private static DecisionEffect? FindTimelineEffect(DecisionStep step, string eventType) =>
|
||||
step.Effects.FirstOrDefault(e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry
|
||||
&& e.Parameters != null
|
||||
&& e.Parameters.TryGetValue("eventType", out var et)
|
||||
&& et == eventType);
|
||||
|
||||
private static ActiveDeadline? FindDeadline(DecisionState state, string name) =>
|
||||
state.Deadlines.FirstOrDefault(d => d.Name == name);
|
||||
|
||||
/// <summary>
|
||||
/// Replays the session-4910a5a5 shape up to (but excluding) the ESP terminal failure:
|
||||
/// Classic UserDriven, ContinueAnyway profile, Hello disabled, DeviceSetup entered,
|
||||
/// AccountSetup entered (04:18, BEFORE the 04:34 DeviceSetup/Apps failure — the IME
|
||||
/// phase line landed while device apps were still at 9/10).
|
||||
/// </summary>
|
||||
private static DecisionState SetupSession(DecisionEngine engine)
|
||||
{
|
||||
var state = DecisionState.CreateInitial("sess-4910a5a5", "tenant-e46bc88e", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
5, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[SignalPayloadKeys.SkipUserEsp] = "false",
|
||||
[SignalPayloadKeys.SkipDeviceEsp] = "false",
|
||||
[SignalPayloadKeys.EspSyncFailureTimeoutMinutes] = "60",
|
||||
[SignalPayloadKeys.EspAllowContinueAnyway] = "true",
|
||||
})).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
8, DecisionSignalKind.HelloPolicyDetected, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.HelloEnabled] = "false" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
10, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(6),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(48),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
return state;
|
||||
}
|
||||
|
||||
private static DecisionStep ApplyDeviceSetupAppsFailure(DecisionEngine engine, DecisionState state, long ordinal = 50) =>
|
||||
engine.Reduce(state, MakeSignal(
|
||||
ordinal, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(64),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_DeviceSetup_Apps_Failed",
|
||||
["failedSubcategory"] = "Apps",
|
||||
["category"] = "DeviceSetup",
|
||||
["likelyCulpritApps"] = "Oriflame Cosmectics - Teams Backgroundsv v2",
|
||||
["likelyCulpritAppCount"] = "1",
|
||||
}));
|
||||
|
||||
private static DecisionStep ApplyDeviceSetupResolved(DecisionEngine engine, DecisionState state, long ordinal = 60) =>
|
||||
engine.Reduce(state, MakeSignal(
|
||||
ordinal, DecisionSignalKind.DeviceSetupProvisioningComplete, T0.AddMinutes(87)));
|
||||
|
||||
// ================================================== advisory arming context ====
|
||||
|
||||
[Fact]
|
||||
public void Advisory_RecordsFailedCategory_AndCulpritApps()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
|
||||
var step = ApplyDeviceSetupAppsFailure(engine, state);
|
||||
|
||||
Assert.NotNull(step.NewState.EspAdvisoryFailureRecordedUtc);
|
||||
Assert.Equal("DeviceSetup", step.NewState.EspAdvisoryFailureCategory?.Value);
|
||||
Assert.Null(step.NewState.EspAdvisoryFailureResolvedUtc);
|
||||
|
||||
var advisory = FindTimelineEffect(step, "esp_failure_advisory");
|
||||
Assert.NotNull(advisory);
|
||||
Assert.Equal("Oriflame Cosmectics - Teams Backgroundsv v2", advisory!.Parameters!["likelyCulpritApps"]);
|
||||
}
|
||||
|
||||
// ===================================================== recovery hook ====
|
||||
|
||||
[Fact]
|
||||
public void DeviceSetupComplete_SameCategory_ResolvesAdvisory_AndEmitsStoryEvent()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
|
||||
var step = ApplyDeviceSetupResolved(engine, state);
|
||||
|
||||
Assert.NotNull(step.NewState.EspAdvisoryFailureResolvedUtc);
|
||||
Assert.Equal(60, step.NewState.EspAdvisoryFailureResolvedUtc!.SourceSignalOrdinal);
|
||||
// The anchor itself stays — the advisory happened; it just no longer blocks.
|
||||
Assert.NotNull(step.NewState.EspAdvisoryFailureRecordedUtc);
|
||||
|
||||
var resolved = FindTimelineEffect(step, "esp_failure_advisory_resolved");
|
||||
Assert.NotNull(resolved);
|
||||
Assert.Equal("DeviceSetup", resolved!.Parameters!["category"]);
|
||||
Assert.Equal("23", resolved.Parameters["minutesSinceAdvisory"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeviceSetupComplete_DifferentCategory_DoesNotResolve()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
// Advisory for AccountSetup — DeviceSetup resolving must not defuse it.
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
50, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(64),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_AccountSetup_Apps_Failed",
|
||||
["failedSubcategory"] = "Apps",
|
||||
["category"] = "AccountSetup",
|
||||
})).NewState;
|
||||
|
||||
var step = ApplyDeviceSetupResolved(engine, state);
|
||||
|
||||
Assert.Null(step.NewState.EspAdvisoryFailureResolvedUtc);
|
||||
Assert.Null(FindTimelineEffect(step, "esp_failure_advisory_resolved"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeviceSetupComplete_Duplicate_ResolvesOnlyOnce()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
// Post-reboot duplicate (AnchorAlreadySet passthrough branch).
|
||||
var duplicate = engine.Reduce(state, MakeSignal(
|
||||
65, DecisionSignalKind.DeviceSetupProvisioningComplete, T0.AddMinutes(92)));
|
||||
|
||||
Assert.Equal(60, duplicate.NewState.EspAdvisoryFailureResolvedUtc!.SourceSignalOrdinal);
|
||||
Assert.Null(FindTimelineEffect(duplicate, "esp_failure_advisory_resolved"));
|
||||
}
|
||||
|
||||
// ================================================ deadline fire semantics ====
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFire_AfterRecovery_WithEnforcementProgress_ReArmsInsteadOfFailing()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
// Post-reboot IME re-assert of AccountSetup (session shape: 05:02:59, ordinal 654)
|
||||
// — user-phase enforcement demonstrably progressed since the arming at ordinal 50.
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
70, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(92),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94)));
|
||||
|
||||
// Not failed — the window re-arms with fresh baselines instead.
|
||||
Assert.Null(step.NewState.Outcome);
|
||||
Assert.NotEqual(SessionStage.Failed, step.NewState.Stage);
|
||||
var rearmed = FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion);
|
||||
Assert.NotNull(rearmed);
|
||||
Assert.Equal(T0.AddMinutes(94).AddMinutes(30), rearmed!.DueAtUtc);
|
||||
Assert.Contains(step.Effects, e => e.Kind == DecisionEffectKind.ScheduleDeadline);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFire_AfterRecovery_NoProgress_FailsWithRecoveredReason()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94)));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome);
|
||||
|
||||
var failed = FindTimelineEffect(step, "enrollment_failed");
|
||||
Assert.NotNull(failed);
|
||||
// Truthful reason: the original failure recovered — do not blame it.
|
||||
Assert.Equal("esp_recovered_without_completion_evidence", failed!.Parameters!["reason"]);
|
||||
Assert.Equal("true", failed.Parameters["failureRecovered"]);
|
||||
Assert.Equal("advisory_recovered_window_expired_without_completion_evidence", failed.Parameters["advisoryReason"]);
|
||||
// Likely-stuck app promotion stays off: apps are not stuck, they succeeded.
|
||||
Assert.Equal(nameof(DecisionSignalKind.DeadlineFired), step.NewState.LastFailureTrigger?.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFire_WithoutRecovery_KeepsLegacyUndefang()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
// No recovery — even with enforcement progress the advisory variant must
|
||||
// un-defang exactly as before (its anchor failure never un-happened).
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
70, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(92),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(80, T0.AddMinutes(94)));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
var failed = FindTimelineEffect(step, "enrollment_failed");
|
||||
Assert.NotNull(failed);
|
||||
Assert.Equal("esp_terminal_failure", failed!.Parameters!["reason"]);
|
||||
Assert.Equal(nameof(DecisionSignalKind.EspTerminalFailure), step.NewState.LastFailureTrigger?.Value);
|
||||
}
|
||||
|
||||
// ======================================================= reboot rebase ====
|
||||
|
||||
[Fact]
|
||||
public void Reboot_AfterRecovery_RebasesAdvisoryWindow()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
70, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92)));
|
||||
|
||||
Assert.Contains(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.CancelDeadline
|
||||
&& e.CancelDeadlineName == DeadlineNames.AdvisoryCompletion);
|
||||
var rebased = FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion);
|
||||
Assert.NotNull(rebased);
|
||||
Assert.Equal(T0.AddMinutes(92).AddMinutes(30), rebased!.DueAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reboot_WithoutRecovery_DoesNotTouchAdvisoryWindow()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
var originalDue = FindDeadline(state, DeadlineNames.AdvisoryCompletion)!.DueAtUtc;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(
|
||||
70, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92)));
|
||||
|
||||
Assert.DoesNotContain(step.Effects, e => e.Kind == DecisionEffectKind.CancelDeadline);
|
||||
Assert.Equal(originalDue, FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion)!.DueAtUtc);
|
||||
}
|
||||
|
||||
// ============================================== full session-4910a5a5 replay ====
|
||||
|
||||
[Fact]
|
||||
public void Session4910a5a5_FailureRecoveryRebootFire_DoesNotFailTheLiveEnrollment()
|
||||
{
|
||||
// Field ordinals: 527 EspTerminalFailure → 611 DeviceSetupProvisioningComplete
|
||||
// (post "Try again", apps 10/10) → 635 SystemRebootObserved → 654 AccountSetup
|
||||
// re-assert → 671 DeadlineFired. Pre-fix outcome: enrollment_failed at the fire.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state, ordinal: 527).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state, ordinal: 611).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
635, DecisionSignalKind.SystemRebootObserved, T0.AddMinutes(92))).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
654, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(93),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(671, T0.AddMinutes(94)));
|
||||
|
||||
Assert.Null(step.NewState.Outcome);
|
||||
Assert.NotEqual(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.NotNull(FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion));
|
||||
}
|
||||
|
||||
// =============================================== serialization + census ====
|
||||
|
||||
[Fact]
|
||||
public void SnapshotRoundTrip_PreservesRecoveryFacts()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var roundtripped = StateSerializer.Deserialize(StateSerializer.Serialize(state));
|
||||
|
||||
Assert.Equal("DeviceSetup", roundtripped.EspAdvisoryFailureCategory?.Value);
|
||||
Assert.Equal(state.EspAdvisoryFailureResolvedUtc!.Value, roundtripped.EspAdvisoryFailureResolvedUtc?.Value);
|
||||
Assert.Equal(state.EspAdvisoryFailureResolvedUtc.SourceSignalOrdinal, roundtripped.EspAdvisoryFailureResolvedUtc?.SourceSignalOrdinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LegacySnapshot_WithoutRecoveryFacts_DeserializesToNull()
|
||||
{
|
||||
// Simulate a snapshot written by a pre-fix agent: strip the new properties from
|
||||
// the JSON entirely — deserialization must default them to null (additive compat).
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var legacy = Newtonsoft.Json.Linq.JObject.Parse(StateSerializer.Serialize(state));
|
||||
foreach (var prop in legacy.Properties()
|
||||
.Where(p => p.Name.IndexOf("EspAdvisoryFailureResolved", StringComparison.OrdinalIgnoreCase) >= 0
|
||||
|| p.Name.IndexOf("EspAdvisoryFailureCategory", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
.ToList())
|
||||
{
|
||||
prop.Remove();
|
||||
}
|
||||
|
||||
var roundtripped = StateSerializer.Deserialize(legacy.ToString());
|
||||
Assert.Null(roundtripped.EspAdvisoryFailureResolvedUtc);
|
||||
Assert.Null(roundtripped.EspAdvisoryFailureCategory);
|
||||
// The pre-existing advisory anchor still round-trips.
|
||||
Assert.NotNull(roundtripped.EspAdvisoryFailureRecordedUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Census_SurfacesAdvisoryResolved()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupSession(engine);
|
||||
state = ApplyDeviceSetupAppsFailure(engine, state).NewState;
|
||||
state = ApplyDeviceSetupResolved(engine, state).NewState;
|
||||
|
||||
var census = DecisionStateSignalCensus.Build(state);
|
||||
|
||||
Assert.Contains("esp_advisory_failure_resolved", census.SignalsSeen);
|
||||
Assert.True(census.SignalTimestamps.ContainsKey("espAdvisoryFailureResolved"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,10 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
// state-change-only dedupe anchor for the completion_waiting event → 17.
|
||||
// Session 772fe502 (2026-07-13) added `helloWizardStartedUtc` — genuine
|
||||
// Shell-Core 62404 wizard launch; vetoes the policy-disabled Hello skip → 18.
|
||||
Assert.Equal(18, facts.Count);
|
||||
// Session 4910a5a5 (2026-07-23) added `espAdvisoryFailureResolvedUtc` (the failed
|
||||
// category recovered — lifts the advisory's re-arm/rebase guard exemption) and
|
||||
// `espAdvisoryFailureCategory` (which category the advisory was for) → 20.
|
||||
Assert.Equal(20, facts.Count);
|
||||
Assert.All(facts.Values, v => Assert.Null(v));
|
||||
}
|
||||
|
||||
@@ -236,10 +239,12 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
// `completionWaitingFingerprint` as the state-change-only dedupe anchor for
|
||||
// the completion_waiting event → 17. Session 772fe502 (2026-07-13) added
|
||||
// `helloWizardStartedUtc` — the genuine Shell-Core 62404 wizard-launch fact
|
||||
// that vetoes/retracts the policy-disabled Hello skip → 18. If this number
|
||||
// ever changes, both the count expectation AND the actual snapshot output
|
||||
// need to evolve in lockstep.
|
||||
Assert.Equal(18, expectedFactKeys.Count);
|
||||
// that vetoes/retracts the policy-disabled Hello skip → 18. Session 4910a5a5
|
||||
// (2026-07-23) added `espAdvisoryFailureResolvedUtc` + `espAdvisoryFailureCategory`
|
||||
// — advisory-recovery facts that lift the AdvisoryCompletion guard exemption → 20.
|
||||
// If this number ever changes, both the count expectation AND the actual snapshot
|
||||
// output need to evolve in lockstep.
|
||||
Assert.Equal(20, expectedFactKeys.Count);
|
||||
|
||||
var snapshot = DecisionStateSnapshotBuilder.Build(DecisionState.CreateInitial("s", "t"));
|
||||
var facts = (Dictionary<string, object?>)snapshot["facts"]!;
|
||||
|
||||
@@ -241,6 +241,199 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_timeout");
|
||||
}
|
||||
|
||||
// ============================================================== Aborted first deployment (session 224b2087)
|
||||
|
||||
[Fact]
|
||||
public void PhaseChanged_leaving_first_deployment_for_200_releases_gate_with_warning()
|
||||
{
|
||||
// Session 224b2087: interactive logon 17 s after phase 101 → RJ reclassifies the run
|
||||
// as secondary-user deployment and writes 200/210, never 110. The 101 -> 200
|
||||
// transition is impossible in a healthy first deployment, so it releases the gate
|
||||
// with Outcome=FirstDeploymentIncomplete and a Warning timeline entry.
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.DesktopArrived, T0.AddMinutes(6))).NewState;
|
||||
Assert.NotEqual(SessionStage.Finalizing, state.Stage);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(10),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "200",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "101",
|
||||
}));
|
||||
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Equal("FirstDeploymentIncomplete", step.NewState.RealmJoinFacts.Outcome!.Value);
|
||||
Assert.NotNull(step.NewState.RealmJoinFacts.ResolvedUtc);
|
||||
Assert.Equal(200, step.NewState.RealmJoinFacts.LastDeploymentPhase!.Value);
|
||||
|
||||
// Hard-timeout deadline cancelled in state AND as a scheduler effect.
|
||||
Assert.DoesNotContain(step.NewState.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
Assert.Contains(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.CancelDeadline &&
|
||||
e.CancelDeadlineName == DeadlineNames.RealmJoinTimeout);
|
||||
|
||||
// Warning timeline entry names both phases.
|
||||
var warning = Assert.Single(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_first_deployment_incomplete");
|
||||
Assert.Equal("Warning", warning.Parameters!["severity"]);
|
||||
Assert.Equal("101", warning.Parameters["previousPhase"]);
|
||||
Assert.Equal("200", warning.Parameters["deploymentPhase"]);
|
||||
Assert.Contains("110", warning.Parameters["message"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PhaseChanged_within_first_deployment_window_is_bookkeeping_only()
|
||||
{
|
||||
// 100 -> 101 stays inside the first-deployment window — no release, but the phase
|
||||
// fact must advance (it is the restart-safe witness for the abort rule).
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "100" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(6),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "100",
|
||||
}));
|
||||
|
||||
Assert.Equal(101, step.NewState.RealmJoinFacts.LastDeploymentPhase!.Value);
|
||||
Assert.Null(step.NewState.RealmJoinFacts.Outcome);
|
||||
Assert.Contains(step.NewState.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
Assert.Empty(step.Effects);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PhaseChanged_from_completed_deployment_start_does_not_release()
|
||||
{
|
||||
// Session 6f1959c0 protection: RJ already stood at CompletedDeployment (210) when
|
||||
// the agent booted and only deployed AFTERWARDS (210 -> 200 -> 210). Neither
|
||||
// transition starts from the first-deployment window, so the abort rule must not
|
||||
// fire — this ambiguous shape keeps waiting for 110 / the hard timeout (status quo).
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "210" })).NewState;
|
||||
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(6),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "200",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "210",
|
||||
})).NewState;
|
||||
Assert.Null(state.RealmJoinFacts.Outcome);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(7),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "210",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "200",
|
||||
}));
|
||||
|
||||
Assert.Null(step.NewState.RealmJoinFacts.Outcome);
|
||||
Assert.Contains(step.NewState.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Detected_replay_after_restart_with_deployment_phase_releases_gate()
|
||||
{
|
||||
// Agent restarted between the 101 and 200 observations: the watcher re-fires
|
||||
// Detected with the CURRENT phase (200) and no PhaseChanged ever carries the
|
||||
// transition. The persisted LastDeploymentPhase=101 is the witness — the Detected
|
||||
// replay path must release the gate exactly like the PhaseChanged path.
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.DesktopArrived, T0.AddMinutes(6))).NewState;
|
||||
Assert.NotEqual(SessionStage.Finalizing, state.Stage);
|
||||
Assert.Equal(101, state.RealmJoinFacts.LastDeploymentPhase!.Value);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(12),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "200" }));
|
||||
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Equal("FirstDeploymentIncomplete", step.NewState.RealmJoinFacts.Outcome!.Value);
|
||||
Assert.Contains(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_first_deployment_incomplete");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Timeout_message_reports_persisted_last_phase_instead_of_zero()
|
||||
{
|
||||
// Before RealmJoinPhaseChanged became a typed signal, LastDeploymentPhase was only
|
||||
// written at detection time — every realmjoin_timeout claimed "last phase: 0" no
|
||||
// matter how far RJ actually got, hiding this whole failure class from ops.
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "100" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(6),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "100",
|
||||
})).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.DesktopArrived, T0.AddMinutes(7))).NewState;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.DeadlineFired, T0.AddMinutes(65),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout }));
|
||||
|
||||
var timeout = Assert.Single(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_timeout");
|
||||
Assert.Contains("last phase: 101", timeout.Parameters!["message"]);
|
||||
Assert.Equal("101", timeout.Parameters["lastSeenPhase"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelfDeploying_deferred_terminal_released_by_first_deployment_incomplete()
|
||||
{
|
||||
// Same deferred-release contract as RealmJoinResolved: when the SelfDeploying
|
||||
// terminal was deferred on the closed RJ gate, the aborted-first-deployment release
|
||||
// must complete the session directly (Completed + enrollment_complete), with the
|
||||
// Warning entry leading the effects.
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("rj-sd-3", "rj-tenant", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(1, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(1),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(2, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(3, DecisionSignalKind.DeviceSetupProvisioningComplete, T0.AddMinutes(3))).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(4, DecisionSignalKind.DeadlineFired, T0.AddMinutes(8),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.DeviceOnlyEspDetection })).NewState;
|
||||
Assert.True(state.RealmJoinFacts.SelfDeployingDeferredCompletion?.Value);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(10),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "200",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "101",
|
||||
}));
|
||||
|
||||
Assert.Equal(SessionStage.Completed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentComplete, step.NewState.Outcome);
|
||||
Assert.Equal("FirstDeploymentIncomplete", step.NewState.RealmJoinFacts.Outcome!.Value);
|
||||
Assert.Contains(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_first_deployment_incomplete");
|
||||
Assert.Contains(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "enrollment_complete");
|
||||
}
|
||||
|
||||
// ============================================================== SelfDeploying flow
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Apps;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the terminal-only failure-rate convention on the Apps dashboard aggregations
|
||||
/// (<see cref="AppsAnalyticsHelper"/>): every failureRate is Failed / (Failed + Succeeded) —
|
||||
/// finished installs only. "InProgress" rows (still installing, or orphaned by a session that
|
||||
/// died mid-install) sit in the install counts but never dilute a rate, matching the enrollment
|
||||
/// success-rate convention. Responses are anonymous projections, so tests assert on the
|
||||
/// JSON wire shape like the sibling suites.
|
||||
/// </summary>
|
||||
public class AppsAnalyticsFailureRateTests
|
||||
{
|
||||
private const string TenantId = "00000000-0000-0000-0000-000000000fa1";
|
||||
|
||||
private static AppInstallSummary A(
|
||||
string status,
|
||||
string appName = "Contoso App",
|
||||
DateTime? startedAt = null,
|
||||
string sessionId = "s-1",
|
||||
string appVersion = "")
|
||||
=> new()
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = sessionId,
|
||||
AppName = appName,
|
||||
AppVersion = appVersion,
|
||||
Status = status,
|
||||
StartedAt = startedAt ?? DateTime.UtcNow.AddHours(-2),
|
||||
DurationSeconds = status == "Succeeded" ? 30 : 0,
|
||||
};
|
||||
|
||||
// ── /apps/list ──────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void List_FailureRate_IsOverFinishedInstalls_InProgressExcluded()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
A("Failed"),
|
||||
A("Succeeded"),
|
||||
A("InProgress"),
|
||||
A("InProgress"),
|
||||
};
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
var app = root.GetProperty("apps")[0];
|
||||
Assert.Equal(4, app.GetProperty("totalInstalls").GetInt32());
|
||||
// 1 / (1 + 1) = 50% (over finished), NOT 1 / 4 = 25% (over all installs).
|
||||
Assert.Equal(50d, app.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void List_FailureRate_IsZero_WhenNothingFinishedYet()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary> { A("InProgress"), A("InProgress") };
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
Assert.Equal(0d, root.GetProperty("apps")[0].GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void List_Trend_UsesFinishedRates_AndGatesOnFinishedSamples()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var firstHalf = now.AddDays(-20); // days: 30 → midpoint at now - 15d
|
||||
var secondHalf = now.AddDays(-5);
|
||||
|
||||
var summaries = new List<AppInstallSummary>();
|
||||
// First half: 1 failed / 5 finished = 20%.
|
||||
summaries.Add(A("Failed", startedAt: firstHalf));
|
||||
for (var i = 0; i < 4; i++) summaries.Add(A("Succeeded", startedAt: firstHalf));
|
||||
// Second half: 3 failed / 5 finished = 60% — plus 10 in-flight installs that
|
||||
// previously dragged the half's rate down to 20% and masked the regression.
|
||||
for (var i = 0; i < 3; i++) summaries.Add(A("Failed", startedAt: secondHalf));
|
||||
for (var i = 0; i < 2; i++) summaries.Add(A("Succeeded", startedAt: secondHalf));
|
||||
for (var i = 0; i < 10; i++) summaries.Add(A("InProgress", startedAt: secondHalf));
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
var app = root.GetProperty("apps")[0];
|
||||
Assert.Equal("worsening", app.GetProperty("trend").GetString());
|
||||
Assert.Equal(40d, app.GetProperty("trendDelta").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void List_Trend_StaysStable_WhenAHalfLacksFiveFinishedInstalls()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var summaries = new List<AppInstallSummary>();
|
||||
// First half: 5 finished. Second half: 4 finished + many in-flight — the old
|
||||
// install-count gate would have passed; the finished-sample gate must not.
|
||||
for (var i = 0; i < 5; i++) summaries.Add(A("Succeeded", startedAt: now.AddDays(-20)));
|
||||
for (var i = 0; i < 4; i++) summaries.Add(A("Failed", startedAt: now.AddDays(-5)));
|
||||
for (var i = 0; i < 10; i++) summaries.Add(A("InProgress", startedAt: now.AddDays(-5)));
|
||||
|
||||
var root = JsonSerializer.SerializeToElement(
|
||||
AppsAnalyticsHelper.BuildAppsListResponse(summaries, days: 30));
|
||||
|
||||
var app = root.GetProperty("apps")[0];
|
||||
Assert.Equal("stable", app.GetProperty("trend").GetString());
|
||||
Assert.Equal(JsonValueKind.Null, app.GetProperty("trendDelta").ValueKind);
|
||||
}
|
||||
|
||||
// ── /apps/{appName}/analytics ───────────────────────────────────────────
|
||||
|
||||
private static async Task<JsonElement> BuildAnalyticsAsync(
|
||||
List<AppInstallSummary> summaries, ISessionRepository? sessionRepo = null)
|
||||
{
|
||||
if (sessionRepo == null)
|
||||
{
|
||||
var mock = new Mock<ISessionRepository>();
|
||||
mock.Setup(r => r.GetSessionAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.ReturnsAsync((SessionSummary?)null);
|
||||
sessionRepo = mock.Object;
|
||||
}
|
||||
var result = await AppsAnalyticsHelper.BuildAnalyticsResponseAsync(
|
||||
summaries, sessionRepo, "Contoso App", days: 30);
|
||||
return JsonSerializer.SerializeToElement(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Analytics_SummaryVersionAndTimeSeriesRates_AreOverFinishedInstalls()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
A("Failed", appVersion: "1.0"),
|
||||
A("Succeeded", appVersion: "1.0"),
|
||||
A("InProgress", appVersion: "1.0"),
|
||||
A("InProgress", appVersion: "1.0"),
|
||||
};
|
||||
|
||||
var root = await BuildAnalyticsAsync(summaries);
|
||||
|
||||
// Summary: 1 / (1 + 1) = 50%, NOT 1 / 4 = 25%.
|
||||
var summary = root.GetProperty("summary");
|
||||
Assert.Equal(4, summary.GetProperty("totalInstalls").GetInt32());
|
||||
Assert.Equal(50d, summary.GetProperty("failureRate").GetDouble());
|
||||
|
||||
var version = root.GetProperty("versionBreakdown")[0];
|
||||
Assert.Equal(4, version.GetProperty("installs").GetInt32());
|
||||
Assert.Equal(50d, version.GetProperty("failureRate").GetDouble());
|
||||
|
||||
// All four installs land in today's bucket → same terminal-only rate there.
|
||||
var buckets = root.GetProperty("timeSeries").EnumerateArray().ToList();
|
||||
var todayBucket = buckets.Single(b => b.GetProperty("installs").GetInt32() > 0);
|
||||
Assert.Equal(50d, todayBucket.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Analytics_DeviceModelBreakdown_RatesAndSampleFloor_UseFinishedInstalls()
|
||||
{
|
||||
var sessionRepo = new Mock<ISessionRepository>();
|
||||
sessionRepo.Setup(r => r.GetSessionAsync(TenantId, It.IsAny<string>()))
|
||||
.ReturnsAsync((string _, string sessionId) => new SessionSummary
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = sessionId,
|
||||
Manufacturer = "Contoso",
|
||||
Model = sessionId.StartsWith("s-a") ? "ModelA" : "ModelB",
|
||||
});
|
||||
|
||||
var summaries = new List<AppInstallSummary>();
|
||||
// ModelA: 5 finished (1 failed → 20%) + 3 in-flight → ranked, rate undiluted.
|
||||
summaries.Add(A("Failed", sessionId: "s-a-0"));
|
||||
for (var i = 1; i <= 4; i++) summaries.Add(A("Succeeded", sessionId: $"s-a-{i}"));
|
||||
for (var i = 5; i <= 7; i++) summaries.Add(A("InProgress", sessionId: $"s-a-{i}"));
|
||||
// ModelB: 4 finished + 6 in-flight — 10 installs, but below the 5-finished floor.
|
||||
for (var i = 0; i <= 3; i++) summaries.Add(A("Failed", sessionId: $"s-b-{i}"));
|
||||
for (var i = 4; i <= 9; i++) summaries.Add(A("InProgress", sessionId: $"s-b-{i}"));
|
||||
|
||||
var root = await BuildAnalyticsAsync(summaries, sessionRepo.Object);
|
||||
|
||||
var breakdown = root.GetProperty("deviceModelBreakdown");
|
||||
var model = Assert.Single(breakdown.EnumerateArray());
|
||||
Assert.Equal("ModelA", model.GetProperty("model").GetString());
|
||||
Assert.Equal(8, model.GetProperty("installs").GetInt32());
|
||||
Assert.Equal(20d, model.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ public class AuthFunctionSideEffectTests
|
||||
{ CallBase = false };
|
||||
|
||||
var previewMock = new Mock<PreviewWhitelistService>(
|
||||
configRepo, cache, Mock.Of<ILogger<PreviewWhitelistService>>())
|
||||
configRepo, cache, Mock.Of<ILogger<PreviewWhitelistService>>(), _tenantConfigMock.Object)
|
||||
{ CallBase = false };
|
||||
|
||||
_telegramMock = new Mock<TelegramNotificationService>(
|
||||
|
||||
@@ -113,9 +113,11 @@ public class EnrollmentTimeoutClassifierTests
|
||||
// Negative Hello terminals leave the agent waiting — must NOT count as resolved.
|
||||
Assert.False(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("hello_provisioning_failed") }).HelloResolved);
|
||||
Assert.False(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("hello_completion_timeout") }).HelloResolved);
|
||||
// Both RealmJoin gate terminals count as resolved (phase 110 or 60-min hard timeout).
|
||||
// All RealmJoin gate terminals count as resolved (phase 110, aborted first
|
||||
// deployment — session 224b2087 — or 60-min hard timeout).
|
||||
Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("realmjoin_resolved") }).RealmJoinResolved);
|
||||
Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("realmjoin_timeout") }).RealmJoinResolved);
|
||||
Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { Evt("realmjoin_first_deployment_incomplete") }).RealmJoinResolved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -10,9 +10,10 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// <summary>
|
||||
/// Covers the server-side Fleet Health aggregation in
|
||||
/// <see cref="MetricsMath.BuildFleetHealthPayload"/>, the single source of truth shared by the
|
||||
/// tenant (metrics/fleet-health) and global (global/metrics/fleet-health) functions. These assert
|
||||
/// the semantics ported verbatim from the previous client-side aggregation — notably success rate
|
||||
/// over ALL sessions and average duration over non-in-progress sessions (failures included).
|
||||
/// tenant (metrics/fleet-health) and global (global/metrics/fleet-health) functions. Success and
|
||||
/// failure rates follow the SLA convention — over finished enrollments (Succeeded + Failed) only,
|
||||
/// so in-flight sessions never dilute them. Average duration stays over non-in-progress sessions
|
||||
/// that carry a duration (failures included).
|
||||
/// </summary>
|
||||
public class FleetHealthPayloadTests
|
||||
{
|
||||
@@ -55,7 +56,7 @@ public class FleetHealthPayloadTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsOverAllSessions_NotJustTerminal()
|
||||
public void SuccessRate_IsOverFinishedSessions_InFlightAndIncompleteExcluded()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
@@ -63,16 +64,34 @@ public class FleetHealthPayloadTests
|
||||
S(SessionStatus.Succeeded),
|
||||
S(SessionStatus.Failed),
|
||||
S(SessionStatus.InProgress),
|
||||
S(SessionStatus.Pending),
|
||||
S(SessionStatus.Incomplete), // terminal but non-failure — stays out of the rate
|
||||
};
|
||||
|
||||
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
|
||||
|
||||
Assert.Equal(4, payload.Stats.Total);
|
||||
Assert.Equal(6, payload.Stats.Total);
|
||||
Assert.Equal(2, payload.Stats.Succeeded);
|
||||
Assert.Equal(1, payload.Stats.Failed);
|
||||
Assert.Equal(1, payload.Stats.InProgress);
|
||||
// 2 / 4 = 50% (over total), NOT 2 / 3 = 66.7% (over terminal).
|
||||
Assert.Equal(50d, payload.Stats.SuccessRate);
|
||||
Assert.Equal(1, payload.Stats.Incomplete);
|
||||
// 2 / (2 + 1) = 66.7% (over finished), NOT 2 / 6 = 33.3% (over total).
|
||||
Assert.Equal(66.7d, payload.Stats.SuccessRate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsZero_WhenNothingFinishedYet()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S(SessionStatus.InProgress),
|
||||
S(SessionStatus.Pending),
|
||||
};
|
||||
|
||||
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
|
||||
|
||||
// No finished enrollments → no rate; clients render "—" off Succeeded + Failed == 0.
|
||||
Assert.Equal(0d, payload.Stats.SuccessRate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -111,6 +130,27 @@ public class FleetHealthPayloadTests
|
||||
Assert.DoesNotContain(payload.ModelHealth, m => m.Model == "Contoso Model1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModelHealth_CountsFailedSeparately_InFlightInTotalOnly()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S(SessionStatus.Succeeded, model: "X"),
|
||||
S(SessionStatus.Failed, model: "X"),
|
||||
S(SessionStatus.InProgress, model: "X"),
|
||||
};
|
||||
|
||||
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 7);
|
||||
|
||||
var m = Assert.Single(payload.ModelHealth);
|
||||
// Total keeps the full device count; the client derives the rate from
|
||||
// Succeeded / (Succeeded + Failed), so the in-flight session must land
|
||||
// in Total but in neither outcome bucket.
|
||||
Assert.Equal(3, m.Total);
|
||||
Assert.Equal(1, m.Succeeded);
|
||||
Assert.Equal(1, m.Failed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModelKey_FallsBackToUnknown_WhenManufacturerAndModelBlank()
|
||||
{
|
||||
@@ -147,10 +187,13 @@ public class FleetHealthPayloadTests
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
// Model A: 1 failed of 2 → 50%
|
||||
// Model A: 1 failed / 2 finished → 50%; the 2 in-flight sessions widen Total
|
||||
// but must not dilute the rate (over-total it would read 25%).
|
||||
S(SessionStatus.Failed, model: "A"),
|
||||
S(SessionStatus.Succeeded, model: "A"),
|
||||
// Model B: 2 failed of 2 → 100%
|
||||
S(SessionStatus.InProgress, model: "A"),
|
||||
S(SessionStatus.Pending, model: "A"),
|
||||
// Model B: 2 failed of 2 finished → 100%
|
||||
S(SessionStatus.Failed, model: "B"),
|
||||
S(SessionStatus.Failed, model: "B"),
|
||||
// Model C: no failures → excluded entirely
|
||||
@@ -164,35 +207,37 @@ public class FleetHealthPayloadTests
|
||||
Assert.Equal(2, payload.TopFailingModels[0].Failed);
|
||||
Assert.Equal(100, payload.TopFailingModels[0].FailureRate);
|
||||
Assert.Equal("Contoso A", payload.TopFailingModels[1].Model);
|
||||
Assert.Equal(4, payload.TopFailingModels[1].Total);
|
||||
Assert.Equal(50, payload.TopFailingModels[1].FailureRate);
|
||||
Assert.DoesNotContain(payload.TopFailingModels, m => m.Model == "Contoso C");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FailureReasons_TruncateLongReasons_AndTakeTopFive()
|
||||
public void FailureReasons_GroupByPrefix_KeepFullestText_AndTakeTopFive()
|
||||
{
|
||||
var sessions = new List<SessionSummary>();
|
||||
// 6 distinct reasons with descending frequency 6..1 → only top 5 kept.
|
||||
for (int freq = 6; freq >= 1; freq--)
|
||||
// 5 distinct short reasons with descending frequency 5..1.
|
||||
for (int freq = 5; freq >= 1; freq--)
|
||||
for (int i = 0; i < freq; i++)
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: $"Reason {freq}"));
|
||||
// A null reason folds into "Unknown".
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: null));
|
||||
// An over-long reason is collapsed to a 50-char prefix + ellipsis.
|
||||
var longReason = new string('x', 60);
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: longReason));
|
||||
|
||||
// Two variants that share the first 50 characters collapse into one group;
|
||||
// the longest variant is kept for display so the UI can show the full text.
|
||||
var prefix = new string('x', 50);
|
||||
var shortVariant = prefix + " short";
|
||||
var longVariant = prefix + " a much longer tail that must survive intact";
|
||||
for (int i = 0; i < 6; i++)
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: shortVariant));
|
||||
sessions.Add(S(SessionStatus.Failed, failureReason: longVariant));
|
||||
|
||||
var payload = MetricsMath.BuildFleetHealthPayload(sessions, days: 30);
|
||||
|
||||
Assert.Equal(5, payload.FailureReasons.Count);
|
||||
Assert.Equal("Reason 6", payload.FailureReasons[0].Reason);
|
||||
Assert.Equal(6, payload.FailureReasons[0].Count);
|
||||
// The collapsed long reason exists in the tally (50 chars + "...").
|
||||
var collapsed = new string('x', 50) + "...";
|
||||
Assert.True(
|
||||
sessions.Count(s => s.FailureReason == longReason) == 1,
|
||||
"sanity: exactly one long-reason session");
|
||||
Assert.Equal(53, collapsed.Length);
|
||||
// The grouped cluster (6 + 1 = 7) outranks every "Reason N".
|
||||
Assert.Equal(7, payload.FailureReasons[0].Count);
|
||||
// The full, longest text survives end-to-end — no 50-char truncation.
|
||||
Assert.Equal(longVariant, payload.FailureReasons[0].Reason);
|
||||
Assert.DoesNotContain("...", payload.FailureReasons[0].Reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Functions.Functions.Metrics;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the success-rate semantics of the geographic aggregation
|
||||
/// (<see cref="GetGeographicMetricsFunction.ComputeGeographicMetrics"/>): the rate is an outcome
|
||||
/// quota over finished enrollments (Succeeded + Failed) following the SLA convention. In-flight
|
||||
/// sessions (InProgress/Pending/Stalled/AwaitingUser) and Incomplete (terminal, non-failure) sit
|
||||
/// in SessionCount but never dilute the rate — the customer-visible symptom this pins against was
|
||||
/// a site mid-rollout showing a poor rate despite zero actual failures.
|
||||
/// </summary>
|
||||
public class GeographicMetricsSuccessRateTests
|
||||
{
|
||||
private static SessionSummary S(SessionStatus status, string city = "Frankfurt")
|
||||
=> new()
|
||||
{
|
||||
TenantId = "00000000-0000-0000-0000-000000000fe0",
|
||||
SessionId = Guid.NewGuid().ToString(),
|
||||
Status = status,
|
||||
StartedAt = DateTime.UtcNow.AddHours(-2),
|
||||
GeoCountry = "DE",
|
||||
GeoRegion = "Hessen",
|
||||
GeoCity = city,
|
||||
GeoLoc = "50.11,8.68",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsOverFinishedSessions_InFlightAndIncompleteExcluded()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S(SessionStatus.Succeeded),
|
||||
S(SessionStatus.Succeeded),
|
||||
S(SessionStatus.Succeeded),
|
||||
S(SessionStatus.Failed),
|
||||
S(SessionStatus.InProgress),
|
||||
S(SessionStatus.Pending),
|
||||
S(SessionStatus.AwaitingUser),
|
||||
S(SessionStatus.Incomplete), // terminal but non-failure — stays out of the rate
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "city");
|
||||
|
||||
var loc = Assert.Single(result.Locations);
|
||||
Assert.Equal(8, loc.SessionCount);
|
||||
Assert.Equal(3, loc.Succeeded);
|
||||
Assert.Equal(1, loc.Failed);
|
||||
// 3 / (3 + 1) = 75% (over finished), NOT 3 / 8 = 37.5% (over all sessions).
|
||||
Assert.Equal(75d, loc.SuccessRate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsZero_WhenNothingFinishedYet()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
S(SessionStatus.InProgress),
|
||||
S(SessionStatus.Pending),
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "city");
|
||||
|
||||
var loc = Assert.Single(result.Locations);
|
||||
// No finished enrollments → no rate; the frontend renders "—" off Succeeded + Failed == 0.
|
||||
Assert.Equal(0d, loc.SuccessRate);
|
||||
Assert.Equal(0, loc.Succeeded + loc.Failed);
|
||||
Assert.Equal(2, loc.SessionCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SuccessRate_IsPerLocation_NotGlobal()
|
||||
{
|
||||
var sessions = new List<SessionSummary>
|
||||
{
|
||||
// Frankfurt: 1 / 1 finished → 100% despite the in-flight session.
|
||||
S(SessionStatus.Succeeded, city: "Frankfurt"),
|
||||
S(SessionStatus.InProgress, city: "Frankfurt"),
|
||||
// Berlin: 1 / 2 finished → 50%.
|
||||
S(SessionStatus.Succeeded, city: "Berlin"),
|
||||
S(SessionStatus.Failed, city: "Berlin"),
|
||||
};
|
||||
|
||||
var result = GetGeographicMetricsFunction.ComputeGeographicMetrics(
|
||||
sessions, new List<AppInstallSummary>(), "city");
|
||||
|
||||
var frankfurt = result.Locations.Single(l => l.City == "Frankfurt");
|
||||
var berlin = result.Locations.Single(l => l.City == "Berlin");
|
||||
Assert.Equal(100d, frankfurt.SuccessRate);
|
||||
Assert.Equal(50d, berlin.SuccessRate);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Enforces the URL registry: every well-known own or Microsoft host must be
|
||||
/// referenced through <c>AutopilotMonitor.Shared.Constants</c>, never as a
|
||||
/// repeated string literal. The EU cutover missed two hardcoded copies of the
|
||||
/// blob host (LatestVersionsService, ValidateBootstrapCodeFunction) precisely
|
||||
/// because they did not go through the registry — this test makes that class
|
||||
/// of drift a build failure instead of a production surprise.
|
||||
///
|
||||
/// Scope: non-test C# source under src/. Comment lines are ignored (docs and
|
||||
/// examples may cite URLs); test projects are excluded because tests use
|
||||
/// literals deliberately as independent oracles.
|
||||
/// </summary>
|
||||
public class HardcodedUrlGuardTests
|
||||
{
|
||||
private static readonly string[] EnforcedHosts =
|
||||
{
|
||||
"portal.autopilotmonitor.com",
|
||||
"www.autopilotmonitor.com",
|
||||
"docs.autopilotmonitor.com",
|
||||
"download.autopilotmonitor.com",
|
||||
"mcp.autopilotmonitor.com",
|
||||
"autopilotmonitor-api-eu.azurewebsites.net",
|
||||
"autopilotmonitor.blob.core.windows.net",
|
||||
"autopilotmonitoreu.blob.core.windows.net",
|
||||
"graph.microsoft.com",
|
||||
"login.microsoftonline.com",
|
||||
};
|
||||
|
||||
/// <summary>Files allowed to carry the literals — the registry itself.</summary>
|
||||
private static readonly string[] RegistryFiles =
|
||||
{
|
||||
Path.Combine("src", "Shared", "AutopilotMonitor.Shared", "Constants.cs"),
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void WellKnownHosts_OnlyAppearInTheRegistry()
|
||||
{
|
||||
var repoRoot = FindRepoRoot();
|
||||
var srcDir = Path.Combine(repoRoot, "src");
|
||||
var violations = new List<string>();
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(srcDir, "*.cs", SearchOption.AllDirectories))
|
||||
{
|
||||
var relative = Path.GetRelativePath(repoRoot, file);
|
||||
var normalized = relative.Replace('/', Path.DirectorySeparatorChar);
|
||||
|
||||
// bin/obj artifacts and generated files
|
||||
if (normalized.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}") ||
|
||||
normalized.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}"))
|
||||
continue;
|
||||
|
||||
// Test projects: literals there are deliberate independent oracles.
|
||||
if (normalized.Contains("Tests"))
|
||||
continue;
|
||||
|
||||
if (RegistryFiles.Any(r => normalized.EndsWith(r, StringComparison.OrdinalIgnoreCase)))
|
||||
continue;
|
||||
|
||||
var lines = File.ReadAllLines(file);
|
||||
for (var i = 0; i < lines.Length; i++)
|
||||
{
|
||||
var trimmed = lines[i].TrimStart();
|
||||
// Comment approximation: full-line // and /// comments plus the
|
||||
// body lines of XML docs and block comments. A URL after code on
|
||||
// the same line still counts — that is the case being guarded.
|
||||
if (trimmed.StartsWith("//") || trimmed.StartsWith("*") || trimmed.StartsWith("/*"))
|
||||
continue;
|
||||
|
||||
foreach (var host in EnforcedHosts)
|
||||
{
|
||||
if (trimmed.Contains(host, StringComparison.OrdinalIgnoreCase))
|
||||
violations.Add($"{relative}:{i + 1}: {host}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(violations.Count == 0,
|
||||
"Hardcoded well-known host(s) found outside the URL registry " +
|
||||
"(use AutopilotMonitor.Shared.Constants.*BaseUrl instead):\n " +
|
||||
string.Join("\n ", violations));
|
||||
}
|
||||
|
||||
private static string FindRepoRoot()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AutopilotMonitor.sln")))
|
||||
{
|
||||
dir = dir.Parent;
|
||||
}
|
||||
Assert.NotNull(dir);
|
||||
return dir!.FullName;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Session eaf3d8c4 (2026-07-23): a previous enrollment's IME log surviving on disk made the
|
||||
/// agent replay a week-old script history — 156 replayed <c>script_completed</c> events
|
||||
/// inflated the session's RemediationScriptCount. Newer agents suppress the replay at the
|
||||
/// source; this predicate covers the fleet of not-yet-updated agents by skipping the script
|
||||
/// counter increment for events whose <c>rejectedSourceTimestamp</c> (the source log line's
|
||||
/// own time, preserved by the agent's staleness clamp) is more than 24 h older than the
|
||||
/// event's clock-clamped stamp.
|
||||
/// </summary>
|
||||
public class IngestHistoricImeReplayTests
|
||||
{
|
||||
private static readonly DateTime EventStamp = new(2026, 7, 23, 15, 42, 0, DateTimeKind.Utc);
|
||||
|
||||
private static EnrollmentEvent ScriptEvent(object? rejectedSourceTimestamp, bool includeKey = true)
|
||||
{
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = "script_completed",
|
||||
Timestamp = EventStamp,
|
||||
Data = new Dictionary<string, object> { ["scriptType"] = "remediation" },
|
||||
};
|
||||
if (includeKey)
|
||||
evt.Data["rejectedSourceTimestamp"] = rejectedSourceTimestamp!;
|
||||
return evt;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleRejectedSourceTimestamp_IsHistoricReplay()
|
||||
{
|
||||
var evt = ScriptEvent(EventStamp.AddDays(-7).ToString("o"));
|
||||
Assert.True(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleAppInstallEvent_IsHistoricReplay()
|
||||
{
|
||||
// The predicate is event-type-agnostic — it also guards AggregateAppInstallEvent
|
||||
// (AppInstallSummaries would otherwise be overwritten with week-old runs).
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = "app_install_completed",
|
||||
Timestamp = EventStamp,
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
["appName"] = "Contoso App",
|
||||
["rejectedSourceTimestamp"] = EventStamp.AddDays(-7).ToString("o"),
|
||||
},
|
||||
};
|
||||
Assert.True(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JustOver24h_IsHistoricReplay()
|
||||
{
|
||||
var evt = ScriptEvent(EventStamp.AddHours(-25).ToString("o"));
|
||||
Assert.True(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Within24h_IsNotHistoricReplay()
|
||||
{
|
||||
var evt = ScriptEvent(EventStamp.AddHours(-23).ToString("o"));
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FutureSkewRejection_IsNotHistoricReplay()
|
||||
{
|
||||
// A rejected source timestamp AHEAD of the event stamp is a mid-enrollment clock
|
||||
// jump (WhiteGlove +1h CMTrace skew), not replayed history.
|
||||
var evt = ScriptEvent(EventStamp.AddHours(2).ToString("o"));
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingKey_IsNotHistoricReplay()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(ScriptEvent(null, includeKey: false)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullData_IsNotHistoricReplay()
|
||||
{
|
||||
var evt = new EnrollmentEvent { EventType = "script_completed", Timestamp = EventStamp, Data = null! };
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(evt));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData("not-a-date")]
|
||||
public void UnparseableValue_IsNotHistoricReplay(string raw)
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(ScriptEvent(raw)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullEvent_IsNotHistoricReplay()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsHistoricImeReplay(null));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
public class LogSanitizerTests
|
||||
{
|
||||
[Fact]
|
||||
public void Clean_WithCrlf_StripsNewlines()
|
||||
{
|
||||
// The core log-forging vector: a value that injects a fake log line.
|
||||
var forged = "/api/sessions\r\n[Auth Middleware] FAKE: admin granted";
|
||||
var cleaned = LogSanitizer.Clean(forged);
|
||||
Assert.Equal("/api/sessions[Auth Middleware] FAKE: admin granted", cleaned);
|
||||
Assert.DoesNotContain('\r', cleaned!);
|
||||
Assert.DoesNotContain('\n', cleaned!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Clean_WithBareLineFeedOrCarriageReturn_StripsBoth()
|
||||
{
|
||||
Assert.Equal("ab", LogSanitizer.Clean("a\nb"));
|
||||
Assert.Equal("ab", LogSanitizer.Clean("a\rb"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Clean_WithOtherControlChars_StripsThem()
|
||||
{
|
||||
// Tab (9), NUL (0) and ESC (27, terminal escape) are all removed. Built
|
||||
// from char codes so the source file carries no literal control bytes.
|
||||
var input = $"a{(char)9}b{(char)0}c{(char)27}de";
|
||||
Assert.Equal("abcde", LogSanitizer.Clean(input));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Clean_WithUnicodeLineSeparators_StripsThem()
|
||||
{
|
||||
// U+2028 LINE SEPARATOR and U+2029 PARAGRAPH SEPARATOR are Unicode categories
|
||||
// Zl/Zp, not Cc — so a \p{Cc}-only pass would let them through, and a log viewer
|
||||
// or JavaScript-side reader would still break the line on them. Built from char
|
||||
// codes so the source file carries no literal separator bytes.
|
||||
var forged = $"/api/sessions{(char)0x2028}[Auth Middleware] FAKE: admin granted";
|
||||
var cleaned = LogSanitizer.Clean(forged);
|
||||
|
||||
Assert.Equal("/api/sessions[Auth Middleware] FAKE: admin granted", cleaned);
|
||||
Assert.DoesNotContain((char)0x2028, cleaned!);
|
||||
Assert.Equal("ab", LogSanitizer.Clean($"a{(char)0x2029}b"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("/api/sessions/abc-123/events")]
|
||||
[InlineData("user@contoso.com")]
|
||||
[InlineData("11111111-2222-3333-4444-555555555555")]
|
||||
[InlineData("GET")]
|
||||
public void Clean_WithLegitimateValue_ReturnsUnchanged(string value)
|
||||
{
|
||||
// Real paths, UPNs, GUIDs and methods never contain control chars, so
|
||||
// tracing fidelity is fully preserved.
|
||||
Assert.Equal(value, LogSanitizer.Clean(value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Clean_WithNull_ReturnsNull()
|
||||
{
|
||||
Assert.Null(LogSanitizer.Clean(null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Clean_WithEmpty_ReturnsEmpty()
|
||||
{
|
||||
Assert.Equal("", LogSanitizer.Clean(""));
|
||||
}
|
||||
}
|
||||
+58
@@ -1,4 +1,5 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
@@ -87,4 +88,61 @@ public class MaintenanceServiceExcessiveEventAutoActionTests
|
||||
// — better to no-op than block/kill on an unintended config.
|
||||
Assert.Null(MaintenanceService.DecideAutoAction(eventCount: 5000, autoMode: mode, autoThreshold: 2500));
|
||||
}
|
||||
|
||||
// ── Status eligibility ────────────────────────────────────────────────────
|
||||
// A device may only be blocked over a session that can still upload. Without this gate
|
||||
// a session that finished days ago with a high EventCount got its device blocked on the
|
||||
// next sweep, long after it stopped sending.
|
||||
|
||||
[Fact]
|
||||
public void IsAutoActionEligible_InProgress_IsEligible()
|
||||
{
|
||||
Assert.True(MaintenanceService.IsAutoActionEligible(SessionStatus.InProgress));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(SessionStatus.Succeeded)]
|
||||
[InlineData(SessionStatus.Failed)]
|
||||
[InlineData(SessionStatus.Incomplete)]
|
||||
[InlineData(SessionStatus.Stalled)]
|
||||
[InlineData(SessionStatus.AwaitingUser)]
|
||||
[InlineData(SessionStatus.Pending)]
|
||||
[InlineData(SessionStatus.Unknown)]
|
||||
public void IsAutoActionEligible_EverythingElse_IsRejected(SessionStatus status)
|
||||
{
|
||||
// Matches the time-window watchdog's `Status eq 'InProgress'` filter, so neither
|
||||
// detector can block a device over a session that no longer sends. Stalled and
|
||||
// AwaitingUser can heal back to InProgress — then the next sweep acts.
|
||||
Assert.False(MaintenanceService.IsAutoActionEligible(status));
|
||||
}
|
||||
|
||||
// ── Status parsing feeding that gate ──────────────────────────────────────
|
||||
// The runaway query projects a subset of columns. SessionStatus.InProgress is ordinal 0,
|
||||
// so an unparsed Status would leave every row looking in-progress and the gate above
|
||||
// silently always-true — these pin the fail-closed mapping.
|
||||
|
||||
[Theory]
|
||||
[InlineData("InProgress", SessionStatus.InProgress)]
|
||||
[InlineData("inprogress", SessionStatus.InProgress)]
|
||||
[InlineData("Succeeded", SessionStatus.Succeeded)]
|
||||
[InlineData("AwaitingUser", SessionStatus.AwaitingUser)]
|
||||
public void ParseStatusForAutoAction_KnownNames_RoundTrip(string stored, SessionStatus expected)
|
||||
{
|
||||
Assert.Equal(expected, TableStorageService.ParseStatusForAutoAction(stored));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)] // column absent from the projection or the row
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("Enrolling")] // status name that never existed
|
||||
[InlineData("0")] // Enum.TryParse would map this onto InProgress and open the gate
|
||||
[InlineData("3")]
|
||||
[InlineData("-1")]
|
||||
[InlineData("InProgress,Failed")] // flags-style combination is not a real status
|
||||
public void ParseStatusForAutoAction_UnusableInput_IsUnknown(string? stored)
|
||||
{
|
||||
Assert.Equal(SessionStatus.Unknown, TableStorageService.ParseStatusForAutoAction(stored));
|
||||
Assert.False(MaintenanceService.IsAutoActionEligible(TableStorageService.ParseStatusForAutoAction(stored)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +119,24 @@ public class MetricsMathAppPayloadTests
|
||||
Assert.Equal(2, topCode.GetProperty("count").GetInt32());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FailureRate_IsOverFinishedInstalls_InProgressExcluded()
|
||||
{
|
||||
var summaries = new List<AppInstallSummary>
|
||||
{
|
||||
new() { AppName = "App A", Status = "Failed" },
|
||||
new() { AppName = "App A", Status = "Succeeded", DurationSeconds = 10 },
|
||||
new() { AppName = "App A", Status = "InProgress" },
|
||||
new() { AppName = "App A", Status = "InProgress" },
|
||||
};
|
||||
|
||||
var app = Build(summaries).GetProperty("topFailingApps")[0];
|
||||
|
||||
Assert.Equal(4, app.GetProperty("totalInstalls").GetInt32());
|
||||
// 1 / (1 + 1) = 50% (over finished), NOT 1 / 4 = 25% (over all installs).
|
||||
Assert.Equal(50d, app.GetProperty("failureRate").GetDouble());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SlowestApps_DropsAppsBelowSampleFloor()
|
||||
{
|
||||
|
||||
@@ -117,7 +117,6 @@ public class OffboardingSessionEnumeratorTests
|
||||
public Task<List<SessionSummary>> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours) => throw new NotImplementedException();
|
||||
public Task<List<string>> GetAllTenantIdsAsync() => throw new NotImplementedException();
|
||||
public Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
|
||||
@@ -868,4 +868,149 @@ public class PolicyEnforcementMiddlewareTests
|
||||
Assert.False(result.Allowed);
|
||||
Assert.Equal(403, result.StatusCode);
|
||||
}
|
||||
|
||||
// ── Empty-UPN contract for device / anonymous routes ───────────────────────────
|
||||
//
|
||||
// UserRateLimitMiddleware buckets on RequestContext.UserPrincipalName and skips only when it is
|
||||
// EMPTY. Publishing the "anonymous" placeholder there collapsed every agent of every tenant into a
|
||||
// single shared user_ratelimit_anonymous bucket, which 429'd agent telemetry fleet-wide once the
|
||||
// per-user limit was reached. These tests pin the contract: no JWT ⇒ empty UPN.
|
||||
|
||||
[Theory]
|
||||
[InlineData("POST", "/api/agent/telemetry")]
|
||||
[InlineData("GET", "/api/agent/config")]
|
||||
[InlineData("POST", "/api/agent/register-session")]
|
||||
[InlineData("POST", "/api/agent/upload-url")]
|
||||
[InlineData("POST", "/api/agent/error")]
|
||||
[InlineData("POST", "/api/bootstrap/register-session")]
|
||||
[InlineData("GET", "/api/bootstrap/config")]
|
||||
[InlineData("POST", "/api/bootstrap/error")]
|
||||
public async Task DeviceRoute_WithoutJwt_PublishesEmptyUpn(string method, string path)
|
||||
{
|
||||
var h = BuildHarness();
|
||||
|
||||
var result = await h.Middleware.DecideAsync(method, path, null, principal: null);
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
// The rate-limit bucket key: empty means "no user bucket applies to this call".
|
||||
Assert.Equal(string.Empty, result.Context!.UserPrincipalName);
|
||||
// The placeholder still renders in log lines — it just never reaches the RequestContext.
|
||||
Assert.Equal("anonymous", result.UserIdentifier);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("GET", "/api/health")]
|
||||
[InlineData("GET", "/api/stats/platform")]
|
||||
[InlineData("POST", "/api/agent/distress")]
|
||||
[InlineData("GET", "/api/diagnostics/download")]
|
||||
public async Task AnonymousRoute_WithoutJwt_PublishesEmptyUpn(string method, string path)
|
||||
{
|
||||
var h = BuildHarness();
|
||||
|
||||
var result = await h.Middleware.DecideAsync(method, path, null, principal: null);
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
Assert.Equal(string.Empty, result.Context!.UserPrincipalName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AuthenticatedRoute_StillPublishesTheRealUpn()
|
||||
{
|
||||
// Guard the other direction: the fix must not blank the UPN for genuine JWT callers, or
|
||||
// per-user rate limiting and presence tracking would silently stop working.
|
||||
const string upn = "admin@contoso.com";
|
||||
var h = BuildHarness();
|
||||
h.AsTenantAdmin(TenantA, upn);
|
||||
|
||||
var result = await h.Middleware.DecideAsync("GET", "/api/notifications", null, AuthedPrincipal(TenantA, upn));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
Assert.Equal(upn, result.Context!.UserPrincipalName);
|
||||
Assert.Equal(upn, result.Context!.CallerId);
|
||||
}
|
||||
|
||||
// ── CallerId: throttle identity survives tokens that carry no UPN ──────────────
|
||||
//
|
||||
// AuthenticationMiddleware validates issuer/audience/lifetime/signature but requires NO upn and no
|
||||
// scp, so an app-only (client-credentials) token authenticates and reaches every AuthenticatedUser
|
||||
// route (feedback, realtime/negotiate, progress/*, health/detailed, …). UserRateLimitMiddleware
|
||||
// skips on an EMPTY throttle identity — so if CallerId fell back to the (absent) UPN, those tokens
|
||||
// would bypass rate limiting entirely. These tests pin the fallback chain.
|
||||
|
||||
/// <summary>Principal with a tid but NO upn/preferred_username — shaped like an app-only token.</summary>
|
||||
private static ClaimsPrincipal AppOnlyPrincipal(string tenantId, params (string Type, string Value)[] claims)
|
||||
{
|
||||
var all = new List<Claim> { new("tid", tenantId) };
|
||||
all.AddRange(claims.Select(c => new Claim(c.Type, c.Value)));
|
||||
return new ClaimsPrincipal(new ClaimsIdentity(all, "TestAuth"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("oid", "aaaaaaaa-0000-0000-0000-000000000001")]
|
||||
[InlineData("http://schemas.microsoft.com/identity/claims/objectidentifier", "aaaaaaaa-0000-0000-0000-000000000002")]
|
||||
[InlineData("sub", "subject-abc")]
|
||||
[InlineData("appid", "bbbbbbbb-0000-0000-0000-000000000003")]
|
||||
[InlineData("azp", "cccccccc-0000-0000-0000-000000000004")]
|
||||
public async Task AuthenticatedUserRoute_TokenWithoutUpn_StillGetsAThrottleIdentity(string claimType, string claimValue)
|
||||
{
|
||||
var h = BuildHarness();
|
||||
|
||||
// /api/feedback/status is AuthenticatedUser — admits ANY valid principal, role unresolved.
|
||||
var result = await h.Middleware.DecideAsync(
|
||||
"GET", "/api/feedback/status", null, AppOnlyPrincipal(TenantA, (claimType, claimValue)));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
// No UPN to report — audit/presence correctly see nothing...
|
||||
Assert.Equal(string.Empty, result.Context!.UserPrincipalName);
|
||||
// ...but the throttle still has something stable to bucket on.
|
||||
Assert.Equal(claimValue, result.Context!.CallerId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AuthenticatedUserRoute_TokenWithoutUpn_PrefersOidOverSubAndAppId()
|
||||
{
|
||||
var h = BuildHarness();
|
||||
var principal = AppOnlyPrincipal(TenantA,
|
||||
("appid", "cccccccc-0000-0000-0000-000000000009"),
|
||||
("sub", "subject-xyz"),
|
||||
("oid", "aaaaaaaa-0000-0000-0000-00000000000a"));
|
||||
|
||||
var result = await h.Middleware.DecideAsync("GET", "/api/feedback/status", null, principal);
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
Assert.Equal("aaaaaaaa-0000-0000-0000-00000000000a", result.Context!.CallerId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AuthenticatedUserRoute_TokenWithNoIdentifyingClaim_FailsClosedToASharedBucket()
|
||||
{
|
||||
// Pathological (every real Entra token has a sub): must NOT resolve to empty, because empty
|
||||
// means "skip the limiter". A shared bucket is the safe answer.
|
||||
var h = BuildHarness();
|
||||
|
||||
var result = await h.Middleware.DecideAsync(
|
||||
"GET", "/api/feedback/status", null, AppOnlyPrincipal(TenantA));
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
Assert.NotEqual(string.Empty, result.Context!.CallerId);
|
||||
Assert.Equal(PolicyEnforcementMiddleware.UnidentifiedCallerBucket, result.Context!.CallerId);
|
||||
// Never reuse the log placeholder as a bucket key — that was the original fleet-wide collapse.
|
||||
Assert.NotEqual("anonymous", result.Context!.CallerId);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("POST", "/api/agent/telemetry")]
|
||||
[InlineData("GET", "/api/health")]
|
||||
[InlineData("GET", "/api/stats/platform")]
|
||||
public async Task UnauthenticatedRoute_HasEmptyCallerId(string method, string path)
|
||||
{
|
||||
// Empty CallerId is the ONLY signal that skips the user limiter — it must mean
|
||||
// "no JWT at all", never "authenticated but unnamed".
|
||||
var h = BuildHarness();
|
||||
|
||||
var result = await h.Middleware.DecideAsync(method, path, null, principal: null);
|
||||
|
||||
Assert.True(result.Allowed);
|
||||
Assert.Equal(string.Empty, result.Context!.CallerId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,15 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Ingest;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
@@ -80,4 +88,116 @@ public class ReportAgentErrorFunctionTests
|
||||
var evt = ReportAgentErrorFunction.BuildAgentEmergencyBreakEvent(Report(Now, message: ""), TenantId, null!, Now);
|
||||
Assert.Contains("emergency break", evt.Message, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// MaterializeEmergencyBreakArtifactsAsync — the artifact set per break report
|
||||
// (2026-07-23 hardening): timeline event + cross-session EventType index row +
|
||||
// AgentEmergencyBreak ops event, once per session. The index upsert is load-bearing:
|
||||
// StoreEventsBatchAsync alone leaves the event invisible to every search-by-eventType
|
||||
// surface (found in the 2026-07-22 incident analysis).
|
||||
// =========================================================================
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public readonly Mock<ISessionRepository> SessionRepo = new();
|
||||
public readonly List<OpsEventEntry> OpsEvents = new();
|
||||
public readonly OpsEventService OpsService;
|
||||
|
||||
public Harness(List<EnrollmentEvent>? existingEvents = null)
|
||||
{
|
||||
SessionRepo.Setup(r => r.GetSessionEventsAsync(TenantId, SessionId, It.IsAny<int>()))
|
||||
.ReturnsAsync(existingEvents ?? new List<EnrollmentEvent>());
|
||||
SessionRepo.Setup(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()))
|
||||
.ReturnsAsync((List<EnrollmentEvent> e) => e);
|
||||
SessionRepo.Setup(r => r.UpsertEventTypeIndexBatchAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<IEnumerable<EnrollmentEvent>>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => { lock (OpsEvents) OpsEvents.Add(e); })
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions()));
|
||||
var alertDispatch = new OpsAlertDispatchService(
|
||||
adminConfig.Object,
|
||||
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
|
||||
NullLogger<TelegramNotificationService>.Instance),
|
||||
new WebhookNotificationService(new HttpClient(),
|
||||
NullLogger<WebhookNotificationService>.Instance),
|
||||
NullLogger<OpsAlertDispatchService>.Instance);
|
||||
OpsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
}
|
||||
|
||||
public Task RunAsync(AgentErrorReport report)
|
||||
=> ReportAgentErrorFunction.MaterializeEmergencyBreakArtifactsAsync(
|
||||
report, TenantId, SessionRepo.Object, OpsService, NullLogger<ReportAgentErrorFunction>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task First_break_report_materializes_event_index_row_and_ops_event()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e => e.Count == 1
|
||||
&& e[0].EventType == AutopilotMonitor.Shared.Constants.EventTypes.AgentEmergencyBreak)), Times.Once);
|
||||
h.SessionRepo.Verify(r => r.UpsertEventTypeIndexBatchAsync(
|
||||
TenantId, SessionId, It.IsAny<IEnumerable<EnrollmentEvent>>()), Times.Once);
|
||||
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal("AgentEmergencyBreak", ops.EventType);
|
||||
Assert.Equal(TenantId, ops.TenantId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Already_materialized_session_produces_no_second_artifact_set()
|
||||
{
|
||||
var h = new Harness(new List<EnrollmentEvent>
|
||||
{
|
||||
new() { EventType = AutopilotMonitor.Shared.Constants.EventTypes.AgentEmergencyBreak, Sequence = 45 },
|
||||
});
|
||||
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()), Times.Never);
|
||||
h.SessionRepo.Verify(r => r.UpsertEventTypeIndexBatchAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<IEnumerable<EnrollmentEvent>>()), Times.Never);
|
||||
Assert.Empty(h.OpsEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Non_break_error_types_and_missing_session_id_are_ignored()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
var other = Report(Now);
|
||||
other.ErrorType = AgentErrorType.ConfigFetchFailed;
|
||||
await h.RunAsync(other);
|
||||
|
||||
var noSession = Report(Now);
|
||||
noSession.SessionId = null!;
|
||||
await h.RunAsync(noSession);
|
||||
|
||||
h.SessionRepo.Verify(r => r.GetSessionEventsAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>()), Times.Never);
|
||||
Assert.Empty(h.OpsEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Storage_failure_is_swallowed_and_never_throws()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionRepo.Setup(r => r.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()))
|
||||
.ThrowsAsync(new InvalidOperationException("storage down"));
|
||||
|
||||
// Must not throw — the always-200 emergency channel must never turn into a retry loop.
|
||||
await h.RunAsync(Report(Now));
|
||||
|
||||
Assert.Empty(h.OpsEvents);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,6 +235,80 @@ public class RuleEnginePreconditionTests
|
||||
Assert.Single(outcome.Results);
|
||||
}
|
||||
|
||||
// ===== ANALYZE-SEC-001 v3: firmware-confirmation marker (uefiCa2023FirmwareConfirmed) =====
|
||||
|
||||
/// <summary>
|
||||
/// v3 core fix: when the agent's direct UEFI db read confirms the 2023 certificate, the
|
||||
/// one-sided marker is present and the not_exists precondition suppresses the rule — even
|
||||
/// though the servicing status ("unknown") would still satisfy both required conditions.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Sec001V3_FirmwareConfirmed_MarkerSuppressesRule()
|
||||
{
|
||||
var rule = MakeSecureBootRuleV3();
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
HardwareSpecEvent(isVirtualMachine: "false"),
|
||||
SecureBootStatusEvent(uefiCA2023Status: "unknown", secureBootEnabled: "true",
|
||||
extraData: new Dictionary<string, object>
|
||||
{
|
||||
["uefiDbHasWindowsUefiCa2023"] = true,
|
||||
["uefiCa2023FirmwareConfirmed"] = true
|
||||
})
|
||||
};
|
||||
|
||||
var outcome = await RunAsync(rule, events);
|
||||
|
||||
Assert.Empty(outcome.Results);
|
||||
Assert.Single(outcome.EvaluatedRules);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Backward compatibility: an old-agent payload (no firmware fields at all) must behave
|
||||
/// exactly like v2 — marker absent → precondition passes → rule fires on the servicing
|
||||
/// status, at base confidence (no firmware factors).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Sec001V3_OldAgentPayload_FiresLikeV2()
|
||||
{
|
||||
var rule = MakeSecureBootRuleV3();
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
HardwareSpecEvent(isVirtualMachine: "false"),
|
||||
SecureBootStatusEvent(uefiCA2023Status: "unknown", secureBootEnabled: "true")
|
||||
};
|
||||
|
||||
var outcome = await RunAsync(rule, events);
|
||||
|
||||
Assert.Single(outcome.Results);
|
||||
Assert.Equal(70, outcome.Results[0].ConfidenceScore);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// New agent, firmware-verified missing: the marker is omitted (never emitted as false),
|
||||
/// so the rule fires and the firmware evidence raises confidence (+20 db, +10 KEK).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Sec001V3_FirmwareVerifiedMissing_RaisesConfidence()
|
||||
{
|
||||
var rule = MakeSecureBootRuleV3();
|
||||
var events = new List<EnrollmentEvent>
|
||||
{
|
||||
HardwareSpecEvent(isVirtualMachine: "false"),
|
||||
SecureBootStatusEvent(uefiCA2023Status: "unknown", secureBootEnabled: "true",
|
||||
extraData: new Dictionary<string, object>
|
||||
{
|
||||
["uefiDbHasWindowsUefiCa2023"] = false,
|
||||
["uefiKekHasMicrosoftKek2kCa2023"] = false
|
||||
})
|
||||
};
|
||||
|
||||
var outcome = await RunAsync(rule, events);
|
||||
|
||||
Assert.Single(outcome.Results);
|
||||
Assert.Equal(100, outcome.Results[0].ConfidenceScore);
|
||||
}
|
||||
|
||||
// ===== Helpers =====
|
||||
|
||||
private static AnalyzeRule MakeSecureBootRule(List<RulePrecondition>? preconditions)
|
||||
@@ -258,6 +332,29 @@ public class RuleEnginePreconditionTests
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mirrors the shipped ANALYZE-SEC-001 v3.0.0 (rules/analyze/ANALYZE-SEC-001.json):
|
||||
/// VM-skip + firmware-confirmed-marker preconditions, the two required servicing
|
||||
/// conditions, and the optional firmware conditions with their confidence weights.
|
||||
/// Keep in sync when the rule JSON changes.
|
||||
/// </summary>
|
||||
private static AnalyzeRule MakeSecureBootRuleV3()
|
||||
{
|
||||
var rule = MakeSecureBootRule(preconditions: new List<RulePrecondition>
|
||||
{
|
||||
new() { Source = "event_data", EventType = "hardware_spec", DataField = "isVirtualMachine", Operator = "equals", Value = "false" },
|
||||
new() { Source = "event_data", EventType = "secureboot_status", DataField = "uefiCa2023FirmwareConfirmed", Operator = "not_exists", Value = "" }
|
||||
});
|
||||
rule.Conditions.Add(new RuleCondition { Signal = "db_ca2023_firmware_missing", Source = "event_data", EventType = "secureboot_status", DataField = "uefiDbHasWindowsUefiCa2023", Operator = "equals", Value = "false", Required = false });
|
||||
rule.Conditions.Add(new RuleCondition { Signal = "kek_2023_missing", Source = "event_data", EventType = "secureboot_status", DataField = "uefiKekHasMicrosoftKek2kCa2023", Operator = "equals", Value = "false", Required = false });
|
||||
rule.ConfidenceFactors = new List<ConfidenceFactor>
|
||||
{
|
||||
new() { Signal = "db_ca2023_firmware_missing", Condition = "exists", Weight = 20 },
|
||||
new() { Signal = "kek_2023_missing", Condition = "exists", Weight = 10 }
|
||||
};
|
||||
return rule;
|
||||
}
|
||||
|
||||
private static EnrollmentEvent HardwareSpecEvent(string isVirtualMachine) => new()
|
||||
{
|
||||
EventId = Guid.NewGuid().ToString(),
|
||||
@@ -280,20 +377,29 @@ public class RuleEnginePreconditionTests
|
||||
Data = new Dictionary<string, object> { ["edition"] = edition }
|
||||
};
|
||||
|
||||
private static EnrollmentEvent SecureBootStatusEvent(string uefiCA2023Status, string secureBootEnabled) => new()
|
||||
private static EnrollmentEvent SecureBootStatusEvent(string uefiCA2023Status, string secureBootEnabled,
|
||||
Dictionary<string, object>? extraData = null)
|
||||
{
|
||||
EventId = Guid.NewGuid().ToString(),
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
EventType = "secureboot_status",
|
||||
Timestamp = DateTime.UtcNow,
|
||||
Sequence = 3,
|
||||
Data = new Dictionary<string, object>
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
["uefiCA2023Status"] = uefiCA2023Status,
|
||||
["uefiSecureBootEnabled"] = secureBootEnabled
|
||||
}
|
||||
};
|
||||
};
|
||||
if (extraData != null)
|
||||
foreach (var kvp in extraData)
|
||||
data[kvp.Key] = kvp.Value;
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
EventId = Guid.NewGuid().ToString(),
|
||||
TenantId = TenantId,
|
||||
SessionId = SessionId,
|
||||
EventType = "secureboot_status",
|
||||
Timestamp = DateTime.UtcNow,
|
||||
Sequence = 3,
|
||||
Data = data
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<AnalysisOutcome> RunAsync(AnalyzeRule rule, List<EnrollmentEvent> events)
|
||||
{
|
||||
|
||||
+49
-10
@@ -78,15 +78,16 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
[Fact]
|
||||
public async Task RunCore_emits_LongRunning_when_body_runs_past_warning_threshold()
|
||||
{
|
||||
// Set watchdog thresholds to sub-second; make the fanout sleep long enough to trip warning
|
||||
// but not severe.
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(500));
|
||||
// Deterministic (no wall-clock race): the fanout body blocks until the WARN watchdog has
|
||||
// actually recorded its OpsEvent, then returns. Severe sits minutes out, so it can never
|
||||
// fire in the moment between warn firing and the body completing (which cancels it).
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(20), watchdogSevere: TimeSpan.FromMinutes(10));
|
||||
harness.SetKillSwitch(false);
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) =>
|
||||
{
|
||||
r.TenantsProcessed = 1;
|
||||
await Task.Delay(150, ct);
|
||||
await harness.WaitForEventAsync("SessionDeletionMaintenanceLongRunning");
|
||||
});
|
||||
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
@@ -102,14 +103,17 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
// assigned only AFTER the fanout returned, so every LongRunning warning showed zeros.
|
||||
// The fanout now mutates the caller-supplied result incrementally; the watchdog snapshot
|
||||
// must reflect that progress mid-run.
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromSeconds(5));
|
||||
// Deterministic: counters are set at fanout entry, then the body blocks until the WARN
|
||||
// watchdog fires and snapshots them. Severe sits minutes out and never fires — no
|
||||
// body-sleep-vs-threshold race.
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMinutes(10));
|
||||
harness.SetKillSwitch(false);
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(async (SessionRetentionFanoutService.FanoutResult r, DateTime _, CancellationToken ct) =>
|
||||
{
|
||||
r.TenantsProcessed = 3;
|
||||
r.SessionsEnqueued = 7;
|
||||
await Task.Delay(200, ct); // long enough for the 50ms watchdog to fire mid-run
|
||||
await harness.WaitForEventAsync("SessionDeletionMaintenanceLongRunning");
|
||||
});
|
||||
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
@@ -122,17 +126,20 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
[Fact]
|
||||
public async Task RunCore_emits_LongRunningSevere_when_body_runs_past_severe_threshold()
|
||||
{
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(50), watchdogSevere: TimeSpan.FromMilliseconds(150));
|
||||
// Deterministic: the body blocks until the SEVERE watchdog records its OpsEvent. Warn has a
|
||||
// smaller threshold, so it is guaranteed to have fired first — neither depends on a body
|
||||
// sleep outlasting a fixed wall-clock threshold.
|
||||
var harness = new Harness(watchdogWarn: TimeSpan.FromMilliseconds(20), watchdogSevere: TimeSpan.FromMilliseconds(60));
|
||||
harness.SetKillSwitch(false);
|
||||
harness.Fanout.Setup(f => f.RunAsync(It.IsAny<SessionRetentionFanoutService.FanoutResult>(), It.IsAny<DateTime>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(async (SessionRetentionFanoutService.FanoutResult _, DateTime _, CancellationToken ct) =>
|
||||
{
|
||||
await Task.Delay(300, ct);
|
||||
await harness.WaitForEventAsync("SessionDeletionMaintenanceLongRunningSevere");
|
||||
});
|
||||
|
||||
await harness.Sut.RunCoreAsync("Test", CancellationToken.None);
|
||||
|
||||
// Both watchdogs fire: warning at 50ms, severe at 150ms.
|
||||
// Both watchdogs fire: warning first (smaller threshold), then severe (which the body waited for).
|
||||
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunning");
|
||||
Assert.Contains(harness.OpsEvents, e => e.EventName == "SessionDeletionMaintenanceLongRunningSevere");
|
||||
}
|
||||
@@ -345,6 +352,12 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
private readonly Dictionary<string, List<TableEntity>> _sessionsByState = new(StringComparer.Ordinal);
|
||||
private readonly List<DeletionManifestBlobSummary> _ttlBlobs = new();
|
||||
|
||||
// Watchdog OpsEvents are recorded from concurrent watchdog tasks; _opsSync serializes the
|
||||
// OpsEvents append + waiter signalling, and _eventWaiters lets a test's fanout body block
|
||||
// until a specific watchdog OpsEvent has actually fired (see WaitForEventAsync).
|
||||
private readonly object _opsSync = new();
|
||||
private readonly Dictionary<string, TaskCompletionSource> _eventWaiters = new(StringComparer.Ordinal);
|
||||
|
||||
public Harness(TimeSpan? watchdogWarn = null, TimeSpan? watchdogSevere = null)
|
||||
{
|
||||
StorageMock = new Mock<TableStorageService>(Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance);
|
||||
@@ -376,7 +389,12 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Returns<OpsEventEntry>(e =>
|
||||
{
|
||||
OpsEvents.Add(new CapturedOpsEvent(e.EventType, e.Severity, e.Message, e.Details ?? string.Empty));
|
||||
lock (_opsSync)
|
||||
{
|
||||
OpsEvents.Add(new CapturedOpsEvent(e.EventType, e.Severity, e.Message, e.Details ?? string.Empty));
|
||||
if (_eventWaiters.TryGetValue(e.EventType, out var waiter))
|
||||
waiter.TrySetResult();
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
var alertDispatch = new OpsAlertDispatchService(
|
||||
@@ -426,6 +444,27 @@ public class SessionDeletionMaintenanceFunctionTests
|
||||
public void SetKillSwitch(bool active) =>
|
||||
AdminConfig.Setup(a => a.IsSessionDeletionKillSwitchActiveAsync()).ReturnsAsync(active);
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic watchdog gate: returns a task that completes once an OpsEvent whose type is
|
||||
/// <paramref name="eventName"/> has been recorded. A test's fanout body awaits this instead of
|
||||
/// sleeping a fixed span, so the watchdog is guaranteed to have fired before the body returns —
|
||||
/// removing the wall-clock race (body sleep vs sub-second thresholds) that flaked under CI load.
|
||||
/// </summary>
|
||||
public Task WaitForEventAsync(string eventName)
|
||||
{
|
||||
lock (_opsSync)
|
||||
{
|
||||
if (OpsEvents.Any(e => e.EventName == eventName))
|
||||
return Task.CompletedTask;
|
||||
if (!_eventWaiters.TryGetValue(eventName, out var waiter))
|
||||
{
|
||||
waiter = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
_eventWaiters[eventName] = waiter;
|
||||
}
|
||||
return waiter.Task;
|
||||
}
|
||||
}
|
||||
|
||||
public void SeedTtlBlobs(params (string TenantId, string SessionId, string ManifestId, DateTime LastModified)[] entries)
|
||||
{
|
||||
foreach (var e in entries)
|
||||
|
||||
@@ -62,7 +62,7 @@ public class SessionDeletionWorkerTests
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: QueuePollingWorkerBase.DefaultMaxDequeueCount + 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
// Poison queue received the body and main queue deleted the original.
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(
|
||||
@@ -121,7 +121,7 @@ public class SessionDeletionWorkerTests
|
||||
},
|
||||
"etag-mock"));
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned");
|
||||
Assert.Contains($"\"observedResidualCount\":{capObservation}", poisoned.RawDetails);
|
||||
@@ -168,7 +168,7 @@ public class SessionDeletionWorkerTests
|
||||
},
|
||||
"etag-mock"));
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned");
|
||||
Assert.Contains("\"observedResidualCount\":3", poisoned.RawDetails);
|
||||
@@ -307,7 +307,7 @@ public class SessionDeletionWorkerTests
|
||||
},
|
||||
"etag-mock"));
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned");
|
||||
Assert.True(poisoned.RawDetails.Length < 4096,
|
||||
@@ -357,7 +357,7 @@ public class SessionDeletionWorkerTests
|
||||
return Task.FromResult(Response.FromValue(receipt, new Mock<Response>().Object));
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
var casIdx = callOrder.IndexOf("cas-running-to-poisoned");
|
||||
var poisonIdx = callOrder.IndexOf("poison-queue-send");
|
||||
@@ -402,7 +402,7 @@ public class SessionDeletionWorkerTests
|
||||
CurrentState = SessionDeletionState.Poisoned,
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
// Both CAS attempts fired, and the poison-queue send still completed.
|
||||
harness.StorageMock.Verify(s => s.CasSetSessionDeletionStateAsync(
|
||||
@@ -444,7 +444,7 @@ public class SessionDeletionWorkerTests
|
||||
CurrentState = "Unknown",
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.OpsEventRecorded());
|
||||
|
||||
harness.PoisonQueue.Verify(q => q.SendMessageAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
@@ -457,7 +457,7 @@ public class SessionDeletionWorkerTests
|
||||
var harness = new Harness();
|
||||
harness.EnqueueMessage("{ \"this is not valid JSON for an envelope: garbage", dequeueCount: 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.MainQueueDeleted());
|
||||
|
||||
harness.HandlerMock.Verify(h => h.HandleAsync(
|
||||
It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()),
|
||||
@@ -491,7 +491,7 @@ public class SessionDeletionWorkerTests
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(350), ct);
|
||||
});
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(800));
|
||||
await harness.RunUntilAsync(() => harness.HeartbeatCount() >= 3);
|
||||
|
||||
// Heartbeat called at least 3× while the handler was busy (350ms / 50ms ≈ 7, minus a few
|
||||
// due to scheduling jitter). One call is enough to prove the heartbeat task is wired.
|
||||
@@ -512,7 +512,7 @@ public class SessionDeletionWorkerTests
|
||||
};
|
||||
harness.EnqueueMessage(JsonConvert.SerializeObject(envelope), dequeueCount: 1);
|
||||
|
||||
await harness.RunForAsync(TimeSpan.FromMilliseconds(500));
|
||||
await harness.RunUntilAsync(() => harness.MainQueueDeleted());
|
||||
|
||||
harness.HandlerMock.Verify(h => h.HandleAsync(
|
||||
It.IsAny<SessionDeletionEnvelope>(), It.IsAny<CancellationToken>()),
|
||||
@@ -663,7 +663,9 @@ public class SessionDeletionWorkerTests
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Returns<OpsEventEntry>(e =>
|
||||
{
|
||||
OpsEventCalls.Add(CapturedOpsEvent.From(e));
|
||||
// Locked so RunUntilAsync's condition poll can read Count from the test
|
||||
// thread while the worker thread appends.
|
||||
lock (OpsEventCalls) OpsEventCalls.Add(CapturedOpsEvent.From(e));
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
var alertDispatch = new OpsAlertDispatchService(
|
||||
@@ -721,6 +723,41 @@ public class SessionDeletionWorkerTests
|
||||
try { await Sut.StopAsync(CancellationToken.None); }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// De-flake (CI run 30047512186): <see cref="RunForAsync"/>'s fixed wall-clock window
|
||||
/// raced the worker's background poll loop on loaded CI runners — positive assertions
|
||||
/// (delete issued, ops event recorded) could fire before the loop ever processed the
|
||||
/// message. Runs the worker until <paramref name="condition"/> observes the awaited
|
||||
/// side effect (polled every 25 ms); the ceiling only bounds a genuinely broken worker,
|
||||
/// the asserts after this call remain the source of truth. Same pattern as the watchdog
|
||||
/// event-gate de-flake. Only for POSITIVE expectations — Times.Never-style tests keep
|
||||
/// the fixed window (there is no event to wait for).
|
||||
/// </summary>
|
||||
public async Task RunUntilAsync(Func<bool> condition, TimeSpan? timeout = null)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(10));
|
||||
using var cts = new CancellationTokenSource();
|
||||
try { await Sut.StartAsync(cts.Token); }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
while (!condition() && DateTime.UtcNow < deadline)
|
||||
await Task.Delay(25);
|
||||
cts.Cancel();
|
||||
try { await Sut.StopAsync(CancellationToken.None); }
|
||||
catch (OperationCanceledException) { /* expected */ }
|
||||
}
|
||||
|
||||
// Condition probes for RunUntilAsync. Moq's Invocations list snapshots under its
|
||||
// internal lock, so polling it from the test thread while the worker invokes the
|
||||
// mock is safe. OpsEventCalls is guarded by its own lock (see the repo callback).
|
||||
public bool MainQueueDeleted() =>
|
||||
MainQueue.Invocations.Any(i => i.Method.Name == nameof(QueueClient.DeleteMessageAsync));
|
||||
public int HeartbeatCount() =>
|
||||
MainQueue.Invocations.Count(i => i.Method.Name == nameof(QueueClient.UpdateMessageAsync));
|
||||
public bool OpsEventRecorded()
|
||||
{
|
||||
lock (OpsEventCalls) return OpsEventCalls.Count > 0;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record AuditEntry(
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user