mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 05:00:07 +00:00
Compare commits
91 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 93de130cd1 | |||
| b8b95c7e76 | |||
| e57e081cec | |||
| 3a60186824 | |||
| c35595503f | |||
| 838da4cc8a | |||
| a9937e0f5d | |||
| 1dc2cc9a54 | |||
| dd7ae299a8 | |||
| d96d10ccec | |||
| 457fff65f5 | |||
| dc45694611 | |||
| 7569061983 | |||
| 0f5e36db94 | |||
| e1de5a8971 | |||
| a2d93e884c | |||
| 425229deee | |||
| 8675e7ee06 | |||
| e857e99436 | |||
| 6435d5f290 | |||
| 445ce4e75b | |||
| 4e54ffe4b4 | |||
| 65d45a616e | |||
| 4aa1f07fbb | |||
| ede616444a | |||
| 61a359ee54 | |||
| babc89c776 | |||
| a2f448cad0 | |||
| f15e115dea | |||
| 977c5fb447 | |||
| 27fdd19568 | |||
| 72f7adcb3d | |||
| 06cf1c4c12 | |||
| e612c69178 | |||
| 829a841d9b | |||
| 10eedded13 | |||
| a9f6ccd86d | |||
| 7ba67df1a1 | |||
| e214e8dc1a | |||
| f8d0ccd2f5 | |||
| c9701301ea | |||
| ba2a390a83 | |||
| dbd7d2ac6f | |||
| 8f24876682 | |||
| 2d5ee887d7 | |||
| 81c14bb3ed | |||
| 3a0bd74b12 | |||
| 50e7590060 | |||
| 61e53a2e9b | |||
| 7e7b6f8086 | |||
| 2e20476559 | |||
| 990641b19e | |||
| 3ca2f3b499 | |||
| b9fa760404 | |||
| ab61839685 | |||
| 34fd40c3ad | |||
| a0eb1b9e35 | |||
| 9d677db2a8 | |||
| c728a0bd41 | |||
| 64336c0c89 | |||
| 59c944618b | |||
| 3c25172dfb | |||
| eda61eaade | |||
| 04b1a7c6e7 | |||
| 3b59dab2df | |||
| 389426f8f2 | |||
| e9dba11b96 | |||
| 828cbb5cd0 | |||
| 93a42ba5e4 | |||
| aaacc6ecb2 | |||
| bbc7a1d8b2 | |||
| 5f00124a86 | |||
| d43cfe2bb3 | |||
| 78cc038426 | |||
| ff8b847dc0 | |||
| aab7ae12ff | |||
| 935f0e8c71 | |||
| 541dc32ffd | |||
| 022cc0354b | |||
| 3aa8b95676 | |||
| ec945a93d2 | |||
| b36cf8ff36 | |||
| c6ee7808f1 | |||
| b61d7023bc | |||
| e1b03de5b3 | |||
| 25bac34acb | |||
| c0be389a95 | |||
| 91f9d656ae | |||
| b1e7077d0a | |||
| d8c84f41c1 | |||
| 12b51bbc24 |
+1
-1
Submodule .claude/commands updated: 3f98090747...2866e07882
@@ -0,0 +1,367 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Publishes the customer-facing PowerShell scripts behind download.autopilotmonitor.com/agent/.
|
||||
|
||||
.DESCRIPTION
|
||||
Single owner of the script-publishing chain. Called from two places:
|
||||
|
||||
* publish-scripts.yml -- on every push to main that touches the sources,
|
||||
so a script fix reaches customers without waiting
|
||||
for an agent release.
|
||||
* build-agent.yml -- on a publish_as_stable run, so a release always
|
||||
lands a consistent set (ZIP + manifest + scripts).
|
||||
|
||||
Both callers run the same code. The scripts on the download alias are the live
|
||||
copy the WDP bootstrap MSI (Invoke-BootstrapDownload.ps1) downloads on every
|
||||
enrolling device, so "current" is a correctness property, not cosmetics.
|
||||
|
||||
Steps, in order:
|
||||
1. Parse $ScriptVersion from the bootstrap source.
|
||||
2. Render the -Dev variant by literal substitution of the two URL/manifest
|
||||
defaults. A missing anchor is a hard failure -- a silently un-substituted
|
||||
dev script would point the dev fleet at the stable agent.
|
||||
3. Version-bump guard: if the published bootstrap differs from the source
|
||||
but carries the same $ScriptVersion, abort. Otherwise the docs badge,
|
||||
version.json.bootstrapVersion and the portal's "your script is outdated"
|
||||
hint would all keep asserting a version that is no longer what ships.
|
||||
4. Upload every script blob with Cache-Control: no-cache (they rotate in
|
||||
place), mirrored fail-soft to the legacy account.
|
||||
5. Reconcile the two version oracles: version.json.bootstrapVersion
|
||||
(read-modify-write under If-Match, so a concurrent agent release cannot
|
||||
lose its agent fields) and AdminConfiguration.LatestBootstrapV2ScriptVersion.
|
||||
6. Verify through the alias -- re-download each blob and compare SHA-256.
|
||||
|
||||
.PARAMETER SasToken
|
||||
Container SAS for the eu storage account with write permission. Leading '?' tolerated.
|
||||
|
||||
.PARAMETER LegacySasToken
|
||||
Container SAS for the legacy account. Optional -- a mirror failure warns, never fails.
|
||||
|
||||
.PARAMETER TableSasToken
|
||||
Table SAS for AdminConfiguration. Optional -- omitted or 'XXX' skips the oracle update.
|
||||
|
||||
.PARAMETER RepoRoot
|
||||
Repository root. Defaults to the parent of .github/scripts.
|
||||
|
||||
.PARAMETER DryRun
|
||||
Run every check and render, upload nothing.
|
||||
|
||||
.EXAMPLE
|
||||
./Publish-BootstrapScripts.ps1 -DryRun
|
||||
|
||||
.EXAMPLE
|
||||
./Publish-BootstrapScripts.ps1 -SasToken $env:SAS -TableSasToken $env:TABLE_SAS
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$SasToken,
|
||||
[string]$LegacySasToken,
|
||||
[string]$TableSasToken,
|
||||
[string]$RepoRoot,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not $RepoRoot) {
|
||||
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
|
||||
}
|
||||
if (-not $DryRun -and [string]::IsNullOrWhiteSpace($SasToken)) {
|
||||
throw 'SasToken is required unless -DryRun is used.'
|
||||
}
|
||||
|
||||
$ContainerUrl = 'https://autopilotmonitoreu.blob.core.windows.net/agent'
|
||||
$LegacyContainerUrl = 'https://autopilotmonitor.blob.core.windows.net/agent'
|
||||
$AliasUrl = 'https://download.autopilotmonitor.com/agent'
|
||||
$TableUrl = "https://autopilotmonitoreu.table.core.windows.net/AdminConfiguration(PartitionKey='GlobalConfig',RowKey='config')"
|
||||
|
||||
$BootstrapSource = Join-Path $RepoRoot 'scripts/Bootstrap/Install-AutopilotMonitor.ps1'
|
||||
$BootstrapBlob = 'Install-AutopilotMonitor.ps1'
|
||||
$DevBlob = 'Install-AutopilotMonitor-Dev.ps1'
|
||||
$ScriptContentType = 'text/plain; charset=utf-8'
|
||||
|
||||
function Get-HttpStatus {
|
||||
param($ErrorRecord)
|
||||
if ($ErrorRecord.Exception.PSObject.Properties['Response'] -and $ErrorRecord.Exception.Response) {
|
||||
return [int]$ErrorRecord.Exception.Response.StatusCode
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function Get-Sha256 {
|
||||
param([byte[]]$Bytes)
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try { return [BitConverter]::ToString($sha.ComputeHash($Bytes)).Replace('-', '').ToLowerInvariant() }
|
||||
finally { $sha.Dispose() }
|
||||
}
|
||||
|
||||
# Raw bytes of a published blob, or $null when it does not exist yet.
|
||||
function Get-PublishedBytes {
|
||||
param([string]$Url)
|
||||
try {
|
||||
$resp = Invoke-WebRequest -Uri $Url -Method Get -UseBasicParsing
|
||||
} catch {
|
||||
if ((Get-HttpStatus $_) -eq 404) { return $null }
|
||||
throw
|
||||
}
|
||||
# Comma operator: PowerShell unrolls an array returned from a function into the output
|
||||
# stream and reassembles it as Object[], which would silently defeat every [byte[]] use
|
||||
# downstream. ',' wraps it so the byte[] survives the return.
|
||||
if ($resp.Content -is [byte[]]) { return ,$resp.Content }
|
||||
return ,[System.Text.Encoding]::UTF8.GetBytes([string]$resp.Content)
|
||||
}
|
||||
|
||||
# Deterministic publish form: CRLF, UTF-8 without BOM. Git normalises line endings on
|
||||
# checkout (a Windows runner yields CRLF, a core.autocrlf=false clone yields LF), so
|
||||
# publishing the raw working-copy bytes would emit different blobs for identical source
|
||||
# and make every byte comparison -- guard and verify alike -- environment-dependent.
|
||||
# CRLF is also what PS 5.1 on the device expects and what is published today.
|
||||
function Get-PublishBytes {
|
||||
param([string]$Text)
|
||||
$normalised = $Text.Replace("`r`n", "`n").Replace("`n", "`r`n")
|
||||
# ',' keeps this a byte[]: without it PowerShell unrolls the array into the output stream
|
||||
# and the caller receives Object[]. Invoke-RestMethod then serialises an Object[] body as
|
||||
# space-separated decimals instead of raw bytes -- a silent corruption of the upload.
|
||||
return ,(New-Object System.Text.UTF8Encoding $false).GetBytes($normalised)
|
||||
}
|
||||
|
||||
function Get-BootstrapScriptVersion {
|
||||
param([string]$Content, [string]$Origin, [switch]$Optional)
|
||||
if ($Content -match '\$ScriptVersion\s*=\s*"([\d\.\-a-zA-Z]+)"') { return $Matches[1] }
|
||||
# Unreadable SOURCE is fatal. An unreadable PUBLISHED copy must not be: that is exactly
|
||||
# the state a republish is meant to repair, and a guard that blocks the repair is worse
|
||||
# than no guard.
|
||||
if ($Optional) { return $null }
|
||||
throw "Could not parse ScriptVersion from $Origin"
|
||||
}
|
||||
|
||||
# What the script actually DOES, with comments and layout removed: the PowerShell token
|
||||
# stream minus comments, newlines and line continuations. Lets the version-bump guard tell
|
||||
# a behaviour change (must be versioned -- consumers act on the version) apart from a
|
||||
# comment fix or reflow (nobody needs to know). Tokenising rather than regex-stripping
|
||||
# keeps '#' inside strings from being mistaken for a comment.
|
||||
function Get-CodeFingerprint {
|
||||
param([string]$Text, [string]$Origin)
|
||||
$tokens = $null
|
||||
$errors = $null
|
||||
[System.Management.Automation.Language.Parser]::ParseInput($Text, [ref]$tokens, [ref]$errors) | Out-Null
|
||||
if ($errors -and $errors.Count -gt 0) {
|
||||
throw "Parse error in ${Origin}: $($errors[0])"
|
||||
}
|
||||
$ignored = @('Comment', 'NewLine', 'LineContinuation', 'EndOfInput')
|
||||
return (($tokens | Where-Object { $ignored -notcontains $_.Kind.ToString() } | ForEach-Object { $_.Text }) -join "`n")
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 1. source
|
||||
$sourceContent = Get-Content $BootstrapSource -Raw
|
||||
$sourceBytes = Get-PublishBytes $sourceContent
|
||||
$scriptVersion = Get-BootstrapScriptVersion -Content $sourceContent -Origin $BootstrapSource
|
||||
Write-Host "Bootstrap script version: $scriptVersion"
|
||||
|
||||
# ------------------------------------------------------------------ 2. dev render
|
||||
$stableUrlLiteral = '$AgentDownloadUrl = "https://download.autopilotmonitor.com/agent/AutopilotMonitor-Agent.zip"'
|
||||
$devUrlLiteral = '$AgentDownloadUrl = "https://download.autopilotmonitor.com/agent/AutopilotMonitor-Agent-dev.zip"'
|
||||
$stableManLiteral = '$VersionJsonName = "version.json"'
|
||||
$devManLiteral = '$VersionJsonName = "version-dev.json"'
|
||||
|
||||
if ($sourceContent.IndexOf($stableUrlLiteral) -lt 0 -or $sourceContent.IndexOf($stableManLiteral) -lt 0) {
|
||||
throw "Bootstrap dev-render: anchor literal missing in $BootstrapSource"
|
||||
}
|
||||
$devContent = $sourceContent.Replace($stableUrlLiteral, $devUrlLiteral).Replace($stableManLiteral, $devManLiteral)
|
||||
Write-Host "Rendered $DevBlob (dev agent URL + version-dev.json)"
|
||||
|
||||
# ------------------------------------------------------------------ 3. bump guard
|
||||
$publishedBytes = Get-PublishedBytes "$AliasUrl/$BootstrapBlob"
|
||||
if ($null -eq $publishedBytes) {
|
||||
Write-Host 'No published bootstrap yet -- version-bump guard skipped (first publish)'
|
||||
} elseif ((Get-Sha256 $publishedBytes) -eq (Get-Sha256 $sourceBytes)) {
|
||||
Write-Host 'Published bootstrap is already byte-identical to the source'
|
||||
} else {
|
||||
$publishedText = [System.Text.Encoding]::UTF8.GetString($publishedBytes)
|
||||
$publishedVersion = Get-BootstrapScriptVersion -Content $publishedText -Origin "$AliasUrl/$BootstrapBlob" -Optional
|
||||
|
||||
if ($null -eq $publishedVersion) {
|
||||
Write-Host "::warning::The published $BootstrapBlob is not a readable bootstrap script (no ScriptVersion found). Republishing over it."
|
||||
} elseif ($publishedVersion -ne $scriptVersion) {
|
||||
Write-Host "Version bump: $publishedVersion -> $scriptVersion"
|
||||
} elseif ((Get-CodeFingerprint -Text $publishedText -Origin "$AliasUrl/$BootstrapBlob") -eq
|
||||
(Get-CodeFingerprint -Text $sourceContent -Origin $BootstrapSource)) {
|
||||
# Comments, typos in comments, reflow -- nothing a consumer of the version can act on.
|
||||
Write-Host "::warning::$BootstrapBlob changed in comments or formatting only; publishing under the unchanged version $scriptVersion."
|
||||
} else {
|
||||
throw ("Version-bump guard: $BootstrapBlob changed behaviour but ScriptVersion is still $scriptVersion. " +
|
||||
"Bump it in $BootstrapSource -- the docs badge, version.json.bootstrapVersion and the " +
|
||||
'portal outdated-script hint all read that value, and customers decide from it whether ' +
|
||||
'to re-upload their Intune copy. Comment-only and formatting changes do not need a bump.')
|
||||
}
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 4. upload
|
||||
# Sources beyond the bootstrap pair, read straight from the repo.
|
||||
$extraSources = @(
|
||||
@{ Path = 'scripts/Bootstrap/Test-ShouldBootstrapAgent.ps1'; BlobName = 'Test-ShouldBootstrapAgent.ps1' }
|
||||
# Customer-side Graph add-on grant script -- docs and admin UI link it directly.
|
||||
@{ Path = 'scripts/CustomerSetup/Grant-AutopilotMonitorAddOn.ps1'; BlobName = 'Grant-AutopilotMonitorAddOn.ps1' }
|
||||
)
|
||||
# Uninstall-AutopilotMonitor.ps1 is deliberately NOT published: nothing links it, and
|
||||
# an unauthenticated uninstall script on the public download host is not a feature.
|
||||
|
||||
$publishSet = [System.Collections.Generic.List[object]]::new()
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $BootstrapBlob; Bytes = $sourceBytes })
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $DevBlob; Bytes = (Get-PublishBytes $devContent) })
|
||||
foreach ($extra in $extraSources) {
|
||||
$extraPath = Join-Path $RepoRoot $extra.Path
|
||||
if (-not (Test-Path $extraPath)) {
|
||||
throw "Publish source missing: $extraPath"
|
||||
}
|
||||
$publishSet.Add([pscustomobject]@{ BlobName = $extra.BlobName; Bytes = (Get-PublishBytes (Get-Content $extraPath -Raw)) })
|
||||
}
|
||||
|
||||
$legacySas = if ($LegacySasToken) { $LegacySasToken.TrimStart('?') } else { '' }
|
||||
$writeSas = if ($SasToken) { $SasToken.TrimStart('?') } else { '' }
|
||||
|
||||
foreach ($item in $publishSet) {
|
||||
# Explicit type, not a convenience: an Object[] body is uploaded as space-separated
|
||||
# decimals rather than raw bytes, and every hash check still passes because [byte[]]
|
||||
# parameter coercion repairs it everywhere EXCEPT the wire.
|
||||
[byte[]]$bytes = $item.Bytes
|
||||
$item | Add-Member -NotePropertyName Sha256 -NotePropertyValue (Get-Sha256 $bytes) -Force
|
||||
|
||||
if ($DryRun) {
|
||||
# Doubles as a drift check: says per blob whether the alias already serves this source.
|
||||
$served = Get-PublishedBytes "$AliasUrl/$($item.BlobName)"
|
||||
$state = if ($null -eq $served) { 'MISSING on the alias' }
|
||||
elseif ((Get-Sha256 $served) -eq $item.Sha256) { 'already current' }
|
||||
else { 'STALE on the alias' }
|
||||
Write-Host " [dry-run] $($item.BlobName) ($($bytes.Length) bytes, sha256 $($item.Sha256)) -- $state"
|
||||
continue
|
||||
}
|
||||
|
||||
# no-cache on every blob: they all rotate in place and the download alias fronts them
|
||||
# with Front Door. Route caching is disabled there, but a stale script paired with a
|
||||
# fresh manifest would fail the bootstrap SHA check -- this keeps that class of bug
|
||||
# impossible even if caching is ever re-enabled.
|
||||
$headers = @{ 'x-ms-blob-type' = 'BlockBlob'; 'Content-Type' = $ScriptContentType; 'x-ms-blob-cache-control' = 'no-cache' }
|
||||
Invoke-RestMethod -Uri "$ContainerUrl/$($item.BlobName)?$writeSas" -Method Put -Headers $headers -Body $bytes | Out-Null
|
||||
|
||||
# Read straight back from the blob (authoritative, no CDN in the way) before touching the
|
||||
# mirror or the next file. The alias verification at the end would catch a bad body too,
|
||||
# but only after every blob was already overwritten -- these are live customer downloads.
|
||||
$written = Get-PublishedBytes "$ContainerUrl/$($item.BlobName)"
|
||||
if ($null -eq $written -or (Get-Sha256 $written) -ne $item.Sha256) {
|
||||
$writtenLength = if ($null -eq $written) { 'missing' } else { "$($written.Length) bytes" }
|
||||
throw ("Upload of $($item.BlobName) did not round-trip: sent $($bytes.Length) bytes, " +
|
||||
"blob now holds $writtenLength. Publishing stopped before the mirror and the " +
|
||||
'remaining files, so no further blob was touched.')
|
||||
}
|
||||
Write-Host " uploaded $($item.BlobName) ($($bytes.Length) bytes, round-trip verified)"
|
||||
|
||||
if ($legacySas) {
|
||||
try {
|
||||
Invoke-RestMethod -Uri "$LegacyContainerUrl/$($item.BlobName)?$legacySas" -Method Put -Headers $headers -Body $bytes | Out-Null
|
||||
Write-Host " mirrored $($item.BlobName) to legacy storage"
|
||||
} catch {
|
||||
Write-Host "::warning::legacy mirror failed for $($item.BlobName) -- $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 5. version oracles
|
||||
# version.json is the agent manifest; only bootstrapVersion belongs to us. Read from the
|
||||
# blob (not the alias) for an authoritative ETag, write back under If-Match so a concurrent
|
||||
# agent release cannot lose its version/sha256 fields.
|
||||
if (-not $DryRun) {
|
||||
try {
|
||||
$manifestResp = Invoke-WebRequest -Uri "$ContainerUrl/version.json" -Method Get -UseBasicParsing
|
||||
$manifestRaw = if ($manifestResp.Content -is [byte[]]) {
|
||||
[System.Text.Encoding]::UTF8.GetString($manifestResp.Content)
|
||||
} else { [string]$manifestResp.Content }
|
||||
$manifest = $manifestRaw | ConvertFrom-Json
|
||||
|
||||
if ($manifest.bootstrapVersion -eq $scriptVersion) {
|
||||
Write-Host "version.json already reports bootstrapVersion $scriptVersion"
|
||||
} else {
|
||||
$etag = '"' + ([string]($manifestResp.Headers['ETag'] | Select-Object -First 1)).Trim('"') + '"'
|
||||
$manifest | Add-Member -NotePropertyName 'bootstrapVersion' -NotePropertyValue $scriptVersion -Force
|
||||
$body = [System.Text.Encoding]::UTF8.GetBytes(($manifest | ConvertTo-Json -Compress))
|
||||
$manifestHeaders = @{
|
||||
'x-ms-blob-type' = 'BlockBlob'
|
||||
'Content-Type' = 'application/json'
|
||||
'x-ms-blob-cache-control' = 'no-cache'
|
||||
'If-Match' = $etag
|
||||
}
|
||||
Invoke-RestMethod -Uri "$ContainerUrl/version.json?$writeSas" -Method Put -Headers $manifestHeaders -Body $body | Out-Null
|
||||
Write-Host "version.json bootstrapVersion -> $scriptVersion (agent fields untouched)"
|
||||
|
||||
if ($legacySas) {
|
||||
try {
|
||||
$legacyHeaders = @{ 'x-ms-blob-type' = 'BlockBlob'; 'Content-Type' = 'application/json'; 'x-ms-blob-cache-control' = 'no-cache' }
|
||||
Invoke-RestMethod -Uri "$LegacyContainerUrl/version.json?$legacySas" -Method Put -Headers $legacyHeaders -Body $body | Out-Null
|
||||
} catch {
|
||||
Write-Host "::warning::legacy mirror failed for version.json -- $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
if ((Get-HttpStatus $_) -eq 412) {
|
||||
throw 'version.json changed while publishing (concurrent agent release). Re-run this workflow.'
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
$tableSas = if ($TableSasToken) { $TableSasToken.TrimStart('?') } else { '' }
|
||||
if ($DryRun) {
|
||||
Write-Host " [dry-run] would set AdminConfiguration.LatestBootstrapV2ScriptVersion = $scriptVersion"
|
||||
} elseif ([string]::IsNullOrEmpty($tableSas) -or $tableSas -eq 'XXX') {
|
||||
Write-Host 'SKIPPED: AdminConfiguration update -- set AZURE_TABLE_ADMIN_SAS_TOKEN to enable the portal version oracle'
|
||||
} else {
|
||||
$tableHeaders = @{
|
||||
'Content-Type' = 'application/json'
|
||||
'Accept' = 'application/json;odata=nometadata'
|
||||
'If-Match' = '*'
|
||||
}
|
||||
$tableBody = @{ LatestBootstrapV2ScriptVersion = $scriptVersion } | ConvertTo-Json
|
||||
Invoke-RestMethod -Uri "$TableUrl`?$tableSas" -Method Merge -Headers $tableHeaders -Body $tableBody | Out-Null
|
||||
Write-Host "AdminConfiguration.LatestBootstrapV2ScriptVersion = $scriptVersion"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 6. verify via alias
|
||||
if (-not $DryRun) {
|
||||
$pending = [System.Collections.ArrayList]::new()
|
||||
$publishSet | ForEach-Object { [void]$pending.Add($_) }
|
||||
|
||||
for ($attempt = 1; $attempt -le 6 -and $pending.Count -gt 0; $attempt++) {
|
||||
if ($attempt -gt 1) { Start-Sleep -Seconds 20 }
|
||||
foreach ($item in @($pending)) {
|
||||
$servedBytes = Get-PublishedBytes "$AliasUrl/$($item.BlobName)"
|
||||
if ($null -ne $servedBytes -and (Get-Sha256 $servedBytes) -eq $item.Sha256) {
|
||||
Write-Host " verified $($item.BlobName) via $AliasUrl"
|
||||
$pending.Remove($item)
|
||||
}
|
||||
}
|
||||
if ($pending.Count -gt 0) {
|
||||
Write-Host " attempt $attempt : $($pending.Count) blob(s) not current on the alias yet"
|
||||
}
|
||||
}
|
||||
|
||||
if ($pending.Count -gt 0) {
|
||||
$names = ($pending | ForEach-Object { $_.BlobName }) -join ', '
|
||||
throw ("Alias still serves stale content for: $names. The blobs were written, so this is a Front Door " +
|
||||
'cache issue -- purge with: az afd endpoint purge --resource-group rg-autopilotmonitor-prd-gwc ' +
|
||||
"--profile-name autopilotmonitor-fd --endpoint-name apm-download --content-paths '/agent/*'")
|
||||
}
|
||||
}
|
||||
|
||||
if ($env:GITHUB_STEP_SUMMARY) {
|
||||
$mode = if ($DryRun) { 'DRY RUN -- nothing uploaded' } else { 'published + verified via the download alias' }
|
||||
$summary = @(
|
||||
"### Bootstrap scripts: $mode",
|
||||
'',
|
||||
"Bootstrap script version: **$scriptVersion**",
|
||||
''
|
||||
) + ($publishSet | ForEach-Object { "- $($_.BlobName) -- sha256 $($_.Sha256)" })
|
||||
($summary -join [Environment]::NewLine) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8
|
||||
}
|
||||
|
||||
Write-Host 'Done.'
|
||||
@@ -0,0 +1,63 @@
|
||||
name: Bootstrap script gates
|
||||
|
||||
# Quality gate for the customer-facing PowerShell scripts, factored out so it exists
|
||||
# once and both callers run the identical checks:
|
||||
# * pester.yml -- pull requests (reports, does not block the merge button)
|
||||
# * publish-scripts.yml -- pushes to main, where it BLOCKS the upload to the
|
||||
# download alias. Devices execute these scripts as SYSTEM
|
||||
# during enrollment, so a red suite must never ship.
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
gates:
|
||||
name: Bootstrap guard tests (PS 5.1)
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Pester 5+
|
||||
shell: powershell
|
||||
run: |
|
||||
Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force -Scope CurrentUser | Out-Null
|
||||
Install-Module Pester -Scope CurrentUser -Force -SkipPublisherCheck -MinimumVersion 5.5.0
|
||||
|
||||
# Windows PowerShell 5.1 on purpose: that is the runtime IME uses to execute
|
||||
# platform scripts, so guard/relax logic is verified without resetting a device.
|
||||
- name: Run Pester
|
||||
shell: powershell
|
||||
run: |
|
||||
Import-Module Pester -MinimumVersion 5.5.0
|
||||
$c = New-PesterConfiguration
|
||||
$c.Run.Path = 'scripts/Bootstrap'
|
||||
$c.Run.Exit = $true
|
||||
$c.Output.Verbosity = 'Detailed'
|
||||
Invoke-Pester -Configuration $c
|
||||
|
||||
- name: Guard scripts stay pure ASCII
|
||||
shell: powershell
|
||||
run: |
|
||||
$failed = $false
|
||||
Get-ChildItem 'scripts/Bootstrap' -Filter '*.ps1' | ForEach-Object {
|
||||
$bytes = [System.IO.File]::ReadAllBytes($_.FullName)
|
||||
for ($i = 0; $i -lt $bytes.Length; $i++) {
|
||||
if ($bytes[$i] -gt 127) {
|
||||
Write-Host "::error file=scripts/Bootstrap/$($_.Name)::Non-ASCII byte 0x$('{0:X2}' -f $bytes[$i]) at offset $i. PS 5.1 (IME) reads BOM-less files as ANSI and corrupts multi-byte chars."
|
||||
$failed = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($failed) { exit 1 }
|
||||
Write-Host 'All bootstrap scripts are pure ASCII.'
|
||||
|
||||
# The publish chain renders the -Dev variant by literal substitution and refuses to
|
||||
# ship when an anchor moved. Running it dry here surfaces that (and any drift against
|
||||
# the alias) on the PR instead of on the push that would have published.
|
||||
- name: Publish dry run (dev-render anchors + version-bump guard + drift)
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Publish-BootstrapScripts.ps1 -DryRun
|
||||
@@ -33,6 +33,13 @@ permissions:
|
||||
id-token: write # keyless Sigstore signing (artifact attestation)
|
||||
attestations: write # store the provenance attestation
|
||||
|
||||
# Shared with publish-scripts.yml: both write version.json (this workflow the agent fields,
|
||||
# the script publisher bootstrapVersion under If-Match). Serialising them keeps a script push
|
||||
# from colliding with a release. Never cancel -- a half-published set is worse than a queue.
|
||||
concurrency:
|
||||
group: agent-blob-publish
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
LINE_MAJOR: 2
|
||||
AGENT_PROJECT: src/Agent/AutopilotMonitor.Agent.V2/AutopilotMonitor.Agent.V2.csproj
|
||||
@@ -46,8 +53,6 @@ env:
|
||||
VERSIONED_ZIP: AutopilotMonitor-Agent-v2.zip
|
||||
STABLE_ZIP: AutopilotMonitor-Agent.zip
|
||||
STABLE_MANIFEST: version.json
|
||||
STABLE_BOOTSTRAP: Install-AutopilotMonitor.ps1
|
||||
STABLE_BOOTSTRAP_DEV: Install-AutopilotMonitor-Dev.ps1
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@@ -181,6 +186,8 @@ jobs:
|
||||
# commit/buildUtc are the same two fields the backend and MCP manifests carry —
|
||||
# additive, and the bootstrap reads this file by field name, so older installers
|
||||
# ignore them. sha256/bootstrapVersion stay: those are the customer contract.
|
||||
# bootstrapVersion is seeded here so the published manifest is never briefly
|
||||
# missing it; Publish-BootstrapScripts.ps1 owns and re-asserts the field afterwards.
|
||||
$manifest | Add-Member -NotePropertyName "sha256" -NotePropertyValue $sha256Hash -Force
|
||||
$manifest | Add-Member -NotePropertyName "commit" -NotePropertyValue '${{ github.sha }}'.Substring(0, 7) -Force
|
||||
$manifest | Add-Member -NotePropertyName "buildUtc" -NotePropertyValue ([DateTime]::UtcNow.ToString('o')) -Force
|
||||
@@ -190,29 +197,12 @@ jobs:
|
||||
$manifest | ConvertTo-Json -Compress | Set-Content $env:MANIFEST_NAME -Encoding UTF8
|
||||
Write-Host "$env:MANIFEST_NAME content: $(Get-Content $env:MANIFEST_NAME)"
|
||||
|
||||
# Bootstrap = single generic source. Stable variant = source 1:1.
|
||||
# Dev variant = literal-string substitution of the two URL/manifest defaults.
|
||||
# Both are uploaded only on -PublishAsStable runs (versioned namespace doesn't carry bootstrap).
|
||||
if ('${{ inputs.publish_as_stable }}' -eq 'true') {
|
||||
$stableUrl = '$AgentDownloadUrl = "https://download.autopilotmonitor.com/agent/AutopilotMonitor-Agent.zip"'
|
||||
$devUrl = '$AgentDownloadUrl = "https://download.autopilotmonitor.com/agent/AutopilotMonitor-Agent-dev.zip"'
|
||||
$stableMan = '$VersionJsonName = "version.json"'
|
||||
$devMan = '$VersionJsonName = "version-dev.json"'
|
||||
|
||||
Copy-Item $env:BOOTSTRAP_SOURCE $env:STABLE_BOOTSTRAP -Force
|
||||
$content = Get-Content $env:BOOTSTRAP_SOURCE -Raw
|
||||
if ($content.IndexOf($stableUrl) -lt 0 -or $content.IndexOf($stableMan) -lt 0) {
|
||||
Write-Error "Bootstrap dev-render: anchor literal missing in $env:BOOTSTRAP_SOURCE"
|
||||
exit 1
|
||||
}
|
||||
$devContent = $content.Replace($stableUrl, $devUrl).Replace($stableMan, $devMan)
|
||||
[System.IO.File]::WriteAllText("$env:STABLE_BOOTSTRAP_DEV", $devContent, (New-Object System.Text.UTF8Encoding $false))
|
||||
}
|
||||
# The customer-facing scripts (stable + dev render, test script, grant script) are
|
||||
# published by .github/scripts/Publish-BootstrapScripts.ps1 -- see the upload step.
|
||||
|
||||
"versioned_zip_release=$versionedZip" >> $env:GITHUB_OUTPUT
|
||||
"sha256=$sha256Hash" >> $env:GITHUB_OUTPUT
|
||||
"exe_sha256=$exeSha256Hash" >> $env:GITHUB_OUTPUT
|
||||
"bootstrap_version=$bootstrapVersion" >> $env:GITHUB_OUTPUT
|
||||
|
||||
- name: Upload to Azure Blob Storage
|
||||
if: inputs.upload_blob
|
||||
@@ -264,22 +254,21 @@ jobs:
|
||||
Write-Host "PUBLISH AS STABLE -- writing V2 into stable namespace (customer-facing)" -ForegroundColor Magenta
|
||||
Send-Blob -Local $env:VERSIONED_ZIP -BlobName $env:STABLE_ZIP -ContentType "application/zip"
|
||||
Send-Blob -Local $env:MANIFEST_NAME -BlobName $env:STABLE_MANIFEST -ContentType "application/json"
|
||||
Send-Blob -Local $env:STABLE_BOOTSTRAP -BlobName $env:STABLE_BOOTSTRAP -ContentType "text/plain; charset=utf-8"
|
||||
Send-Blob -Local $env:STABLE_BOOTSTRAP_DEV -BlobName $env:STABLE_BOOTSTRAP_DEV -ContentType "text/plain; charset=utf-8"
|
||||
|
||||
if (Test-Path "scripts/Bootstrap/Test-ShouldBootstrapAgent.ps1") {
|
||||
Send-Blob -Local "scripts/Bootstrap/Test-ShouldBootstrapAgent.ps1" -BlobName "Test-ShouldBootstrapAgent.ps1" -ContentType "text/plain; charset=utf-8"
|
||||
}
|
||||
|
||||
# Customer-side Graph add-on grant script -- docs and admin UI link
|
||||
# https://download.autopilotmonitor.com/agent/Grant-AutopilotMonitorAddOn.ps1
|
||||
if (Test-Path "scripts/CustomerSetup/Grant-AutopilotMonitorAddOn.ps1") {
|
||||
Send-Blob -Local "scripts/CustomerSetup/Grant-AutopilotMonitorAddOn.ps1" -BlobName "Grant-AutopilotMonitorAddOn.ps1" -ContentType "text/plain; charset=utf-8"
|
||||
}
|
||||
# Customer-facing scripts: same publisher publish-scripts.yml runs on every push
|
||||
# to main, so a release lands a set that is consistent with what is already live
|
||||
# (dev render, version-bump guard, bootstrapVersion oracles, alias verification).
|
||||
# Runs AFTER version.json so its bootstrapVersion read sees this build's manifest.
|
||||
./.github/scripts/Publish-BootstrapScripts.ps1 `
|
||||
-SasToken '${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}' `
|
||||
-LegacySasToken '${{ secrets.AZURE_BLOB_AGENT_LEGACY_SAS_TOKEN }}' `
|
||||
-TableSasToken '${{ secrets.AZURE_TABLE_ADMIN_SAS_TOKEN }}'
|
||||
}
|
||||
|
||||
# Update AdminConfiguration LatestAgentV2* -- ONLY on stable cutover. The hash-oracle
|
||||
# must never point at a build that is not actually in the stable namespace.
|
||||
# LatestBootstrapV2ScriptVersion is NOT set here: it belongs to the script publisher
|
||||
# above, which also owns it on script-only pushes. One writer per field.
|
||||
$tableSasToken = "${{ secrets.AZURE_TABLE_ADMIN_SAS_TOKEN }}".TrimStart('?')
|
||||
if ('${{ inputs.publish_as_stable }}' -ne 'true') {
|
||||
Write-Host "SKIPPED: AdminConfiguration update (versioned-only run -- stable namespace unchanged)"
|
||||
@@ -289,22 +278,18 @@ jobs:
|
||||
$sha256 = "${{ steps.artifact.outputs.sha256 }}"
|
||||
$exeSha256 = "${{ steps.artifact.outputs.exe_sha256 }}"
|
||||
$agentVersion = "${{ steps.version.outputs.version }}"
|
||||
$bootstrapVersion = "${{ steps.artifact.outputs.bootstrap_version }}"
|
||||
$tableUrl = "https://autopilotmonitoreu.table.core.windows.net/AdminConfiguration(PartitionKey='GlobalConfig',RowKey='config')?$tableSasToken"
|
||||
$tableBody = @{
|
||||
LatestAgentV2Version = $agentVersion
|
||||
LatestAgentV2Sha256 = $sha256
|
||||
LatestAgentV2ExeSha256 = $exeSha256
|
||||
}
|
||||
if ($bootstrapVersion) {
|
||||
$tableBody["LatestBootstrapV2ScriptVersion"] = $bootstrapVersion
|
||||
}
|
||||
Invoke-RestMethod -Uri $tableUrl -Method Merge -Headers @{
|
||||
"Content-Type" = "application/json"
|
||||
"Accept" = "application/json;odata=nometadata"
|
||||
"If-Match" = "*"
|
||||
} -Body ($tableBody | ConvertTo-Json)
|
||||
Write-Host "AdminConfiguration updated: V2 version=$agentVersion, sha256=$sha256, exeSha256=$exeSha256, bootstrapVersion=$bootstrapVersion"
|
||||
Write-Host "AdminConfiguration updated: V2 version=$agentVersion, sha256=$sha256, exeSha256=$exeSha256"
|
||||
}
|
||||
|
||||
# Sigstore keyless provenance: binds the ZIP digest to this workflow run (repo, ref,
|
||||
|
||||
@@ -1,16 +1,20 @@
|
||||
name: Pester (Bootstrap scripts)
|
||||
|
||||
# Runs the Pester suite for the Intune bootstrap scripts under Windows PowerShell 5.1
|
||||
# (the runtime IME uses to execute platform scripts) so guard/relax logic changes are
|
||||
# verified without resetting a device. Like ci.yml it reports rather than blocks.
|
||||
# Pull-request view of the bootstrap script gates. Like ci.yml it reports rather than
|
||||
# blocks the merge button.
|
||||
#
|
||||
# There is deliberately no push trigger: publish-scripts.yml runs the very same
|
||||
# reusable gate workflow on every push to main and blocks the upload to the download
|
||||
# alias on it. A push trigger here would only duplicate that run, with no consequence.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['scripts/Bootstrap/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['scripts/Bootstrap/**']
|
||||
paths:
|
||||
- 'scripts/Bootstrap/**'
|
||||
- 'scripts/CustomerSetup/**'
|
||||
- '.github/scripts/Publish-BootstrapScripts.ps1'
|
||||
- '.github/workflows/bootstrap-script-gates.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -21,41 +25,5 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pester:
|
||||
name: Bootstrap guard tests (PS 5.1)
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Install Pester 5+
|
||||
shell: powershell
|
||||
run: |
|
||||
Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force -Scope CurrentUser | Out-Null
|
||||
Install-Module Pester -Scope CurrentUser -Force -SkipPublisherCheck -MinimumVersion 5.5.0
|
||||
|
||||
- name: Run Pester
|
||||
shell: powershell
|
||||
run: |
|
||||
Import-Module Pester -MinimumVersion 5.5.0
|
||||
$c = New-PesterConfiguration
|
||||
$c.Run.Path = 'scripts/Bootstrap'
|
||||
$c.Run.Exit = $true
|
||||
$c.Output.Verbosity = 'Detailed'
|
||||
Invoke-Pester -Configuration $c
|
||||
|
||||
- name: Guard scripts stay pure ASCII
|
||||
shell: powershell
|
||||
run: |
|
||||
$failed = $false
|
||||
Get-ChildItem 'scripts/Bootstrap' -Filter '*.ps1' | ForEach-Object {
|
||||
$bytes = [System.IO.File]::ReadAllBytes($_.FullName)
|
||||
for ($i = 0; $i -lt $bytes.Length; $i++) {
|
||||
if ($bytes[$i] -gt 127) {
|
||||
Write-Host "::error file=scripts/Bootstrap/$($_.Name)::Non-ASCII byte 0x$('{0:X2}' -f $bytes[$i]) at offset $i. PS 5.1 (IME) reads BOM-less files as ANSI and corrupts multi-byte chars."
|
||||
$failed = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($failed) { exit 1 }
|
||||
Write-Host 'All bootstrap scripts are pure ASCII.'
|
||||
gates:
|
||||
uses: ./.github/workflows/bootstrap-script-gates.yml
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
name: Publish customer scripts
|
||||
|
||||
# Keeps the scripts behind download.autopilotmonitor.com/agent/ current WITHOUT waiting
|
||||
# for an agent release. Before this workflow existed, the script blobs rotated only on a
|
||||
# publish_as_stable run of build-agent.yml, so a bootstrap fix sat unpublished until the
|
||||
# next agent cutover -- while the WDP bootstrap MSI downloads the published copy on every
|
||||
# enrolling device.
|
||||
#
|
||||
# Same publish code as the release path (.github/scripts/Publish-BootstrapScripts.ps1);
|
||||
# this workflow only supplies a second trigger for it.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'scripts/Bootstrap/**'
|
||||
- 'scripts/CustomerSetup/**'
|
||||
- '.github/scripts/Publish-BootstrapScripts.ps1'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Run every check and report drift, upload nothing'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Shared with build-agent.yml: a script publish and an agent release both touch
|
||||
# version.json, and the bootstrapVersion read-modify-write must not interleave with the
|
||||
# release writing its agent fields. Never cancel -- a half-published set is worse than a queue.
|
||||
concurrency:
|
||||
group: agent-blob-publish
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
gates:
|
||||
uses: ./.github/workflows/bootstrap-script-gates.yml
|
||||
|
||||
publish:
|
||||
name: Publish to the download alias
|
||||
needs: gates
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Publish bootstrap + customer scripts
|
||||
shell: pwsh
|
||||
run: |
|
||||
$dryRun = '${{ inputs.dry_run }}' -eq 'true'
|
||||
./.github/scripts/Publish-BootstrapScripts.ps1 `
|
||||
-SasToken '${{ secrets.AZURE_BLOB_AGENT_SAS_TOKEN }}' `
|
||||
-LegacySasToken '${{ secrets.AZURE_BLOB_AGENT_LEGACY_SAS_TOKEN }}' `
|
||||
-TableSasToken '${{ secrets.AZURE_TABLE_ADMIN_SAS_TOKEN }}' `
|
||||
-DryRun:$dryRun
|
||||
@@ -0,0 +1,127 @@
|
||||
---
|
||||
type: concept
|
||||
title: CMTrace Time Resolution — Per-Line Self-Anchoring
|
||||
description: How the agent converts bias-less CMTrace local times to UTC without trusting any process's timezone belief — each provably fresh line anchors its own offset against the agent clock on the 15-minute grid; everything not provably fresh falls back uniformly and says so.
|
||||
resource: src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime
|
||||
tags: [agent, ime, cmtrace, timestamps, timezone, calibration, provenance]
|
||||
timestamp: 2026-08-20
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
A CMTrace line carries local time and no offset — IME 1.104 writes `DateTime.Now.TimeOfDay`
|
||||
in both of its trace listeners. Converting such a line with the reader's own
|
||||
`TimeZoneInfo.Local` silently assumes the writing and reading processes agree on the zone.
|
||||
They do not: each caches its zone at process start and follows neither a later `tzutil` nor
|
||||
a Windows auto-timezone change. The error is `Offset_writer − Offset_reader` — zero only
|
||||
when both believe the same thing, right or wrong. Field measurement over 11,068 sessions
|
||||
found 26 sessions with a real error, from +1 h to −17 h.
|
||||
|
||||
Two designs failed before this one:
|
||||
|
||||
* **Fix the reader's belief** (`TimeZoneInfo.ClearCachedData`, live OS offset): makes the
|
||||
agent isolated-correct and thereby breaks the majority of sessions, where two equally
|
||||
stale beliefs cancel.
|
||||
* **One measured offset per file** (agent 2.0.1410, reverted in `04b1a7c6`): a single log
|
||||
file holds lines from multiple writer *eras*. `AgentExecutor.log` is written by
|
||||
short-lived child processes whose zone belief flips per process — interleaved in one
|
||||
file, and flipping *back* (fixture:
|
||||
`tests/fixtures/cmtrace-logs/agentexecutor-two-writer-eras-v1.cmtrace`). Any cross-line
|
||||
anchor — per file, or nearest-in-write-order — applies one era's offset to the other
|
||||
era's lines. In the field this shifted `script_started` events by −9 h against their
|
||||
correct completions.
|
||||
|
||||
# Schema
|
||||
|
||||
The resolution rule, in order:
|
||||
|
||||
1. **Writer-declared bias** (`+480` suffix): authoritative, used as-is. Never a calibration
|
||||
anchor — it needs no measurement and must not overwrite one.
|
||||
2. **Per-line self-anchoring** — only for lines read in a *provably fresh* pass:
|
||||
|
||||
```
|
||||
offset_line = round((local − agentUtcNow) / 15 min) · 15 min
|
||||
lineUtc = local − offset_line
|
||||
```
|
||||
|
||||
Every real UTC offset is a whole multiple of 15 minutes, so the rounding absorbs poll
|
||||
and flush latency. No cross-line state exists, so interleaved eras resolve line by
|
||||
line — the property the fixture test pins. The line's own timestamp still contributes
|
||||
sub-poll precision and ordering (two lines 6 ms apart stay 6 ms apart; a plain
|
||||
`occurredAt = now` could not do that). Guards: grid residual ≤ 2 min, offset within
|
||||
UTC−12 … UTC+14.
|
||||
3. **Uniform reader-zone fallback** for everything else: wrong in absolute terms whenever
|
||||
the writer held a different belief, but *self-consistent* — both ends of a duration are
|
||||
wrong by the same amount, so derived durations stay right. The revert's core lesson:
|
||||
partially corrected is strictly worse than uniformly wrong.
|
||||
|
||||
**Freshness** is what makes rule 2 sound, and it is strict:
|
||||
|
||||
* Every poll look at a file stamps `LastCheckedUtc` (`LogFilePositionTracker.MarkChecked`),
|
||||
including "no new data" and an empty first sight — so a file first seen empty
|
||||
(`AgentExecutor.log` appearing mid-enrollment) anchors from its very first content.
|
||||
* A pass's lines are fresh only if the file was previously observed *in this process* and
|
||||
the gap since the last look is ≤ `FreshLineMaxAge` (30 s — far above the 100 ms poll,
|
||||
far below the 13-minute zone where a line's *age* could round onto the offset grid).
|
||||
* A restart-restored position bookmark deliberately carries **no** freshness: the first
|
||||
pass after a restart reads downtime backlog and must never anchor.
|
||||
|
||||
**Provenance** (`sourceOffsetOrigin` in event DataJson): `bias` | `line-anchored` |
|
||||
`reader-zone-fallback` (plus retired `calibrated` from 2.0.1410 events). The applied
|
||||
offset is always recorded (`sourceOffsetMinutes`), the raw local time too
|
||||
(`sourceLocalTs`), so any event can be re-derived later. The per-*file* measurement still
|
||||
runs, purely observationally (`measuredWriterOffsetMinutes`, Info log when it disagrees
|
||||
with the process zone).
|
||||
|
||||
# Constraints
|
||||
|
||||
* An anchored line resolves to `now ± 2 min` *by construction*. The accepted edge: a
|
||||
freshly written line carrying a **replayed** old timestamp whose age is exactly
|
||||
N×15 min ± 2 min anchors to ≈now instead of its replayed past — bounded error, and
|
||||
sub-24 h replays are treated as current by the historic-replay guard anyway.
|
||||
* Snapshot readers (`StallProbeCollector`, `LogParserCollector`) cannot tail and therefore
|
||||
cannot prove freshness — they stay on the marked fallback. This is a documented limit,
|
||||
not an omission; sharing the per-file measurement with them would re-apply exactly what
|
||||
the revert removed.
|
||||
* Mixed pairs remain possible across a restart (start resolved from backlog fallback,
|
||||
completion line-anchored) and are recognizable via provenance.
|
||||
|
||||
# Backend regression tripwire
|
||||
|
||||
The backend watches for the failure mode this design eliminates. On every terminal ingest
|
||||
batch, the counter reconcile's existing event-partition scan additionally collects
|
||||
`Δ = ReceivedAt − OccurredUtc` per event, split by `Source == "ImeLogTracker"` versus the
|
||||
rest (`CmTraceSkewTripwire`, hooked in `EventIngestProcessor`). If
|
||||
`median(Δ_IME) − median(Δ_other)` is a clean 15-minute multiple (residual < 2 min — the
|
||||
same constants as the anchoring grid guard above) with ≥ 20 samples over ≥ 3 distinct
|
||||
upload batches per side, a `CmTraceTimeSkewRegression` ops event fires (category Agent,
|
||||
warning), carrying all numbers plus the session's `sourceOffsetOrigin` histogram.
|
||||
Bias-dominated sessions are suppressed (writer-declared offsets cannot be an anchoring
|
||||
regression); `measuredWriterOffsetMinutes` is never consulted (sticky after era
|
||||
flip-backs). Kill switch: app setting `CmTraceSkewTripwireDisabled`. Goal state: the event
|
||||
never fires — any hit is an anchoring case the per-line design misses, or a detector bug.
|
||||
|
||||
Device-clock problems (the customer-actionable cousin) are covered separately by the
|
||||
`clock_skew` analyze condition behind `ANALYZE-DEV-008`, which excludes IME-derived events
|
||||
precisely so an anchoring regression can never surface as a customer finding.
|
||||
|
||||
# Citations
|
||||
|
||||
* `CmTraceOffsetCalibrator.TryMeasureOffset` — the pure grid measurement, shared by
|
||||
anchoring and the observational per-file path.
|
||||
* `CmTraceSkewTripwire` + `CmTraceSkewTripwireTests` — the backend tripwire above.
|
||||
* `RuleEngine.ConditionEvaluators.EvaluateClockSkewCondition` + `RuleEngineClockSkewTests`
|
||||
— the customer-facing device-clock metrics (batch-median frames, spool-spread filter,
|
||||
plateau step detection, end-state persistence).
|
||||
* `ImeLogTracker.LogProcessing.cs` `ResolveEntryUtc` — the resolution order above.
|
||||
* `ImeLogTrackerLineAnchoringTests` — resolves-to-T across writer beliefs, interleaved
|
||||
eras in one chunk, freshness boundaries (first sight, empty first sight, poll gap,
|
||||
restart restore), the grid-replay edge, and the field fixture replayed at production
|
||||
poll cadence.
|
||||
* `CmTraceCalibratorFileIsolationTests` — pins per-file isolation of the observational
|
||||
measurement (session e9753578 logged a cross-file pairing the committed code could not
|
||||
be shown to produce; the mechanism remains open, the resolution path is immune by
|
||||
design, and `CalibrateFrom` logs the anchor timestamp as a tripwire).
|
||||
* `tasks/todo.md` (2026-08-20) — full evidence trail: field measurements, the revert,
|
||||
the two-writer-era fixture, and the disproven foreign-binary hypothesis (the SHA
|
||||
mismatch was the known release race against the 5-minute AdminConfiguration cache).
|
||||
@@ -143,11 +143,29 @@ advisory backstop. All routes converge on `CompleteThroughFinalizingOrDefer`:
|
||||
first-deployment window 100/101 for 200/210 without 110 — the RJ ESP was aborted,
|
||||
typically by an interactive logon reclassifying the run as secondary-user; emits
|
||||
the `realmjoin_first_deployment_incomplete` Warning, session 224b2087), and
|
||||
`Timeout` (hard 60-min deadline). The abort rule reads the *persisted*
|
||||
`Timeout` (60-min deadline from detection). The abort rule reads the *persisted*
|
||||
`RealmJoinFacts.LastDeploymentPhase` (updated by the typed `RealmJoinPhaseChanged`
|
||||
signal), so it survives agent restarts — the `RealmJoinDetected` replay path
|
||||
evaluates it too. A first observation of 210 does **not** release (RJ may have
|
||||
completed before agent boot and deploy afterwards).
|
||||
|
||||
**Activity-based timeout extension** (report 55e6afd61c9d — a large RJ package
|
||||
catalog outlived the 60-min window mid-install): when the timeout deadline fires
|
||||
while the first deployment is demonstrably active — `LastDeploymentPhase` 100/101
|
||||
*and* deployment activity (`RealmJoinPhaseChanged` or a package observation, tracked
|
||||
in `RealmJoinFacts.LastActivityUtc`; the phase captured at detection deliberately
|
||||
does not count) within the last 60 min — the handler re-arms the deadline to
|
||||
`lastActivity + 60 min` instead of timing out, emitting an Info
|
||||
`realmjoin_timeout_extended` event. Re-arms clamp to an absolute cap of
|
||||
**detection + 4 h**. Package activity is only observable at package *completion*
|
||||
(RJ writes the registry subkey when the install finishes), so a single install
|
||||
that is registry-silent for over 60 min still times out — the eventual
|
||||
`realmjoin_timeout` carries a `reason` payload (`hard_timeout` / `inactivity` /
|
||||
`absolute_cap`) telling the three shapes apart. A stale fire of an older deadline
|
||||
incarnation (armed due later than the fire time; `OccurredAtUtc = DueAtUtc` per
|
||||
scheduler contract) dead-ends as `realmjoin_timeout_stale_superseded_by_rearm`.
|
||||
The idle case (detected, no deployment activity ever) times out at 60 min
|
||||
unchanged, and phase 200/210 activity never extends.
|
||||
2. `Finalizing` stage + ~5 s `FinalizingGrace` deadline.
|
||||
3. Grace fires → `Completed`, `EnrollmentComplete` outcome, `enrollment_complete`
|
||||
timeline event with full audit payload.
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
---
|
||||
type: concept
|
||||
title: Diagnostics Package
|
||||
description: What the agent's diagnostics ZIP contains and why — the built-in section catalog shared with the portal, the RealmJoin and Device Preparation collection gates, the configured-path guards, caps, and the package manifest.
|
||||
resource: agent
|
||||
tags: [diagnostics, realmjoin, device-preparation, guardrails, portal]
|
||||
timestamp: 2026-08-23
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
The diagnostics package is the agent's last-resort support artifact: a ZIP built in memory
|
||||
by `DiagnosticsPackageService.BuildArchiveBytes` and uploaded through a short-lived SAS
|
||||
URL at the end of a session, at the WhiteGlove seal (`Always` mode) or on a server-requested
|
||||
on-demand collection ([server actions](../backend/server-actions-on-demand-diagnostics.md)).
|
||||
|
||||
## Archive layout
|
||||
|
||||
```
|
||||
sessioninfo.txt session/tenant/device/outcome, hardware via WMI
|
||||
<built-in sections> catalog order, see below
|
||||
AdditionalLogs/<last folder>/ admin-configured paths (global ∪ tenant)
|
||||
package-manifest.txt every packaging decision (always written)
|
||||
_TRUNCATED.txt only when a cap skipped something
|
||||
```
|
||||
|
||||
## The built-in catalog is the single source of truth
|
||||
|
||||
Everything the package collects **before** any configured path is data, not code paths:
|
||||
`DiagnosticsBuiltInSections.All` in Shared (`Models/Diagnostics/DiagnosticsBuiltInSections.cs`).
|
||||
The agent iterates it in order; the backend serves the same list to the portal
|
||||
(`GET /api/diagnostics/paths`, MemberRead) so what administrators see is exactly what the
|
||||
agent does. There is no second copy to drift.
|
||||
|
||||
| Id | ZIP folder | Source (unexpanded) | Patterns | Recursive | Condition |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| AgentLogs | `AgentLogs` | `%ProgramData%\AutopilotMonitor\Logs` | log patterns | no | Always |
|
||||
| ImeLogs | `ImeLogs` | `%ProgramData%\Microsoft\IntuneManagementExtension\Logs` | log patterns | no | Always |
|
||||
| ImeBootstrapperEventLog | `ImeLogs` | `C:\Windows\System32\winevt\Logs` | `BootstrapperAgentServiceLogProvider.evtx` | no | DevicePreparation |
|
||||
| AgentState | `AgentState` | `%ProgramData%\AutopilotMonitor\State` | log patterns + `*.complete`, `*.marker` | yes | Always |
|
||||
| AgentSpool | `AgentSpool` | `%ProgramData%\AutopilotMonitor\Spool` | `*.jsonl`, `*.json` | no | Always |
|
||||
| AgentMarkers | `AgentMarkers` | `%ProgramData%\AutopilotMonitor` | `*.complete`, `*.marker` | no | Always |
|
||||
| RealmJoinWindows | `RealmJoinLogs/Windows` | `C:\Windows\Logs` | `realmjoin*.log` | no | RealmJoinWatcher |
|
||||
| RealmJoinPackages | `RealmJoinLogs/Windows/RealmJoin` | `C:\Windows\Logs\RealmJoin` | `*.log` | yes | RealmJoinWatcher |
|
||||
| RealmJoinChoco | `RealmJoinLogs/Choco` | `%ProgramData%\RealmJoin\choco\logs` | `*.log` | yes | RealmJoinWatcher |
|
||||
| RealmJoinUserTray | `RealmJoinLogs/User` | `%LOGGED_ON_USER_PROFILE%\AppData\Local\RealmJoin` | `tray*.log` | no | RealmJoinWatcher |
|
||||
| RealmJoinUserLogs | `RealmJoinLogs/User/Logs` | `%LOGGED_ON_USER_PROFILE%\AppData\Local\RealmJoin\Logs` | `*.log` | yes | RealmJoinWatcher |
|
||||
|
||||
"Log patterns" = `*.log *.txt *.json *.jsonl *.etl *.evtx *.xml *.csv *.cab`. The five
|
||||
`Always` sections are the historical layout (PR1-B forensics) and are lock-tested in order;
|
||||
the RealmJoin ZIP layout mirrors the disk (`C:\Windows\Logs\{realmjoin*.log, RealmJoin\…}` →
|
||||
`RealmJoinLogs/Windows/…`, `%LOCALAPPDATA%\RealmJoin\{tray*.log, Logs\…}` →
|
||||
`RealmJoinLogs/User/…`), so a flat section and a tree section never produce the same entry
|
||||
name. Two sections may share a ZIP folder only when they read different source folders
|
||||
(`ImeLogs` + the bootstrapper evtx) — `(ZipFolder, SourceFolder)` pairs are unique.
|
||||
|
||||
### Conditions
|
||||
|
||||
`DiagnosticsSectionCondition` is evaluated by the agent (`IsSectionActive`, pure) against its
|
||||
effective configuration and the enrollment scenario; unknown values fail closed.
|
||||
|
||||
* **RealmJoinWatcher** — the tenant's RealmJoin Watcher toggle
|
||||
(`AnalyzerConfiguration.EnableRealmJoinWatcher`). The package service only sees
|
||||
`AgentConfiguration`, so `RemoteConfigMerger` mirrors that one Analyzers knob onto
|
||||
`AgentConfiguration.EnableRealmJoinWatcher` (the RealmJoin host keeps reading the remote
|
||||
block). Deliberately NOT additionally gated on the compile-time `RealmJoinTrackingEnabled`
|
||||
kill switch: if the live watcher is ever disabled, the logs become more useful, not less.
|
||||
No ConfigVersion bump — no new wire field.
|
||||
* **DevicePreparation** — `EnrollmentRegistryDetector.IsDeterministicDevicePreparation()`,
|
||||
the same deterministic `DevicePreparation\BootstrapperAgent\ExecutionContext` marker every
|
||||
other WDP gate keys on ([WDP scenario gates](wdp-scenario-gates.md)); CloudAssigned*
|
||||
fallback rules count as Classic. The probe is a registry read, evaluated once per build
|
||||
(`SCENARIO: devicePreparation=… realmJoinWatcher=…` in the manifest); a probe failure is
|
||||
treated as Classic and logged — the evtx export is skipped, never the package. The channel
|
||||
is exclusively locked while active, so the existing `wevtutil epl` export path
|
||||
(`ResolveChannelForEvtxFile`, field case 2026-08-17) handles it.
|
||||
|
||||
Inactive sections are never silent: `BUILT-IN SKIPPED (RealmJoin Watcher disabled): <Id>`,
|
||||
`BUILT-IN SKIPPED (not a Device Preparation enrollment): <Id>`,
|
||||
`BUILT-IN SKIPPED (no user session for token): <Id> '<source>'` (the
|
||||
`%LOGGED_ON_USER_PROFILE%` token resolves via `UserProfileResolver` only once an
|
||||
interactive user exists — identical to configured paths). Active ones log
|
||||
`BUILT-IN: <Id> -> folder='…' zip='…' recursive=…` followed by the per-pattern
|
||||
`ADDED / NO MATCH / FOLDER MISSING` lines.
|
||||
|
||||
## Built-ins bypass the configured-path guards — on purpose
|
||||
|
||||
`DiagnosticsPathGuards` ([gather-rule guardrails](../rules/gather-rule-guardrails.md))
|
||||
validates **admin-typed** paths: allowlisted prefixes, wildcard only in the last segment,
|
||||
`C:\Users` blocked except the token's `AppData\Local|Roaming`. The catalog is reviewed code,
|
||||
exactly like the five sections that never went through the guard before — and
|
||||
`%ProgramData%\RealmJoin` is not on the allowlist at all. What does apply to every section:
|
||||
the reparse-point skip (files and directories), the `.evtx` channel export, and the caps
|
||||
(100 MB per file, 500 MB total, 5000 files) tracked globally across sections by
|
||||
`BudgetTracker`, with every skip recorded in the manifest and `_TRUNCATED.txt`.
|
||||
|
||||
## Configured paths: global ∪ tenant
|
||||
|
||||
`GetAgentConfigFunction.MergeDiagnosticsLogPaths` sends global paths (Global Admin,
|
||||
`AdminConfiguration.DiagnosticsGlobalLogPathsJson`) first, then tenant paths
|
||||
(`TenantConfiguration.DiagnosticsLogPathsJson`), dropping blank entries and collapsing
|
||||
duplicates (trimmed, case-insensitive) onto the first occurrence — a path present in both
|
||||
lists used to produce duplicate ZIP entry names. The agent keys each entry's ZIP folder on
|
||||
the last directory name (`AdditionalLogs/<folder>`), so two configured paths ending in a
|
||||
same-named folder share one ZIP folder.
|
||||
|
||||
## Portal surfaces
|
||||
|
||||
* Global Admin → Settings → Diagnostics Log Paths: the built-in catalog (collapsed, read-only,
|
||||
neutral condition pills) above the editable global list; add-row first.
|
||||
* Tenant → Settings → Agent → Diagnostics Package: built-in catalog (header carries the
|
||||
tenant's persisted RealmJoin Watcher state, RealmJoin rows show on/off), the global paths
|
||||
read-only, then the tenant's own entries; every role with a tenant membership can read the
|
||||
catalog (the route is MemberRead, JWT-scoped). One line per entry (`DiagnosticsPathRow`),
|
||||
full values in tooltips.
|
||||
|
||||
## Test seams
|
||||
|
||||
* `DiagnosticsPackageService` internal ctor: the five historical folder overrides map onto
|
||||
catalog ids; `sectionFolderOverrides` covers any other section; `devicePreparationProbe`
|
||||
replaces the registry read (seam callers default to "not WDP", so existing rigs never touch
|
||||
the registry). `UserProfileResolver.SetForTesting(path|null)` / `Reset()` drive the token.
|
||||
* Catalog invariants live in the backend test project (`DiagnosticsBuiltInSectionsTests`),
|
||||
because Shared has no test project.
|
||||
|
||||
# Examples
|
||||
|
||||
A RealmJoin tenant (watcher on) with a signed-in user: `RealmJoinLogs/Windows/realmjoin.log`,
|
||||
`RealmJoinLogs/Windows/RealmJoin/Packages/<pkg>/<timestamp>_install.log`,
|
||||
`RealmJoinLogs/Choco/<pkg>/<timestamp>_install.log`, `RealmJoinLogs/User/tray.log`,
|
||||
`RealmJoinLogs/User/Logs/RjImeHost.log`. The same tenant's package collected before the
|
||||
desktop: the two `User` sections appear as `BUILT-IN SKIPPED (no user session for token)`.
|
||||
A Classic enrollment: `BUILT-IN SKIPPED (not a Device Preparation enrollment):
|
||||
ImeBootstrapperEventLog`. A tenant that had added
|
||||
`%LOGGED_ON_USER_PROFILE%\AppData\Local\RealmJoin\Logs\*.log` manually keeps working — the
|
||||
files simply appear twice (`AdditionalLogs/Logs/…` and `RealmJoinLogs/User/Logs/…`) until the
|
||||
entry is removed.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/DiagnosticsBuiltInSections.cs` — the catalog
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/DiagnosticsPackageService.cs` — `BuildArchiveBytes`, `AddBuiltInSection`, `IsSectionActive`, caps, evtx export
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigMerger.cs` — `EnableRealmJoinWatcher` mirror
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Diagnostics/GetDiagnosticsPathsFunction.cs` — `GET diagnostics/paths`
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs` — `MergeDiagnosticsLogPaths`
|
||||
* `src/Web/autopilot-monitor-web/components/diagnostics/` — `DiagnosticsPathRow`, `BuiltInSectionsList`, `builtInSectionDisplay`
|
||||
* Tests: `DiagnosticsPackageServiceTests` (RealmJoin on/off, no user session, bootstrapper evtx, probe failure), `DiagnosticsBuiltInSectionsTests`, `GetDiagnosticsPathsPayloadTests`, `GetAgentConfigFunctionTests.MergeDiagnosticsLogPaths_*`
|
||||
@@ -0,0 +1,113 @@
|
||||
---
|
||||
type: Concept
|
||||
title: DO GroupId Auto-Set — Delivery Optimization peering group from a network fingerprint
|
||||
description: EnableDoGroupIdAutoSet (ConfigVersion 40) makes the agent set the DOGroupId policy value to a deterministic GUID derived from the default gateway's IPv4 address and MAC (RealmJoin-compatible byte layout), so devices on the same local network form one DO peering group during enrollment; foreign DOGroupId/DOGroupIdSource policies always win.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/DoGroupIdService.cs
|
||||
tags:
|
||||
- agent
|
||||
- delivery-optimization
|
||||
- network
|
||||
- tenant-configuration
|
||||
timestamp: 2026-08-21T00:00:00+02:00
|
||||
---
|
||||
|
||||
# DO GroupId Auto-Set
|
||||
|
||||
## Schema
|
||||
|
||||
### The problem
|
||||
|
||||
With Delivery Optimization download mode Group (2), peering is scoped by GroupId. When no
|
||||
GroupId is configured, Windows derives it from AD site → authenticated domain SID → Entra
|
||||
tenant ID — for a cloud-native tenant that means **one tenant-wide group**, so "peers on
|
||||
the same network" degrades to "peers anywhere in the tenant" and DO's cloud service has to
|
||||
match them by public IP. A GroupId derived from the local network makes the group exactly
|
||||
"devices behind the same gateway" — including branch offices behind NAT. RealmJoin ships
|
||||
this as its network-fingerprint GroupId; our agent is on the device even earlier (during
|
||||
ESP, when the big app downloads actually happen), so it can seed the same value for
|
||||
Autopilot enrollments.
|
||||
|
||||
### The setting
|
||||
|
||||
`TenantConfiguration.EnableDoGroupIdAutoSet` (`bool?`, default null→false), served as
|
||||
`AgentConfigResponse.EnableDoGroupIdAutoSet` — **ConfigVersion 40**. Portal: Settings →
|
||||
Agent → Settings, toggle "Set Delivery Optimization Group ID", stored via the usual
|
||||
tenant-config roundtrip (`TableConfigRepository` Store+Map). Also shown in the admin
|
||||
tenant-config report. `RemoteConfigMerger` maps the bool 1:1; it survives the offline
|
||||
config cache like every plain feature flag.
|
||||
|
||||
The agent does NOT configure `DODownloadMode` — the fingerprint GroupId only has an
|
||||
effect when the tenant already deploys download mode Group (2) via Intune/GPO.
|
||||
|
||||
### The fingerprint
|
||||
|
||||
`DoGroupIdService.ComputeGroupId(ipv4Bytes, macBytes)` builds a 16-byte GUID:
|
||||
|
||||
```
|
||||
{101,48,6} + {0,0,0} + {ip[1], 0, ip[2], ip[3]} + mac[0..5] → new Guid(byte[16])
|
||||
```
|
||||
|
||||
Gateway discovery reuses the shared `NetworkInterfaceLocator` heuristic (Up, not
|
||||
Loopback/Tunnel, non-0.0.0.0 gateway; IPv4 gateway required), the gateway MAC comes from
|
||||
`ArpNativeMethods` (`iphlpapi!SendARP`). The byte layout — including the IP shuffle and
|
||||
the zero padding — is **RealmJoin-compatible and must never change**: devices managed by
|
||||
RealmJoin on the same network compute the identical GUID and peer with ours. The registry
|
||||
value is written as `Guid.ToString()` (braceless lowercase), identical to RealmJoin's
|
||||
`SetValue(name, Guid)` string form. Deliberate divergence from RealmJoin: **no
|
||||
random-GUID fallback** — a random per-device ID would isolate the device in a one-member
|
||||
group, which is worse than the OS default grouping, so no gateway/no MAC means skip.
|
||||
|
||||
### Ownership + conflict rules (never fight a foreign policy)
|
||||
|
||||
Target: `HKLM\SOFTWARE\Policies\Microsoft\Windows\DeliveryOptimization\DOGroupId`
|
||||
(64-bit registry view — the Policies store is WOW64-redirected; key paths reused from
|
||||
`DoThrottlePolicyReader`). Only this GPO-store value is written; the legacy pre-1607
|
||||
`GroupId` value name is intentionally not.
|
||||
|
||||
Skip conditions (Info event with `skipReason`, nothing written):
|
||||
|
||||
| skipReason | Meaning |
|
||||
|---|---|
|
||||
| `no_ipv4_gateway` | No active interface with an IPv4 default gateway |
|
||||
| `arp_failed` | Gateway MAC could not be resolved via SendARP |
|
||||
| `group_id_source_policy` | `DOGroupIdSource` configured in GPO or MDM store (dynamic group source wins) |
|
||||
| `mdm_policy_present` | `DOGroupId` managed via the Policy CSP store (Intune owns it) |
|
||||
| `foreign_value_present` | Existing GPO value neither matches the computed GUID nor the one we last wrote |
|
||||
|
||||
Ownership is tracked through the `StartupEventGate` fingerprint for the
|
||||
`do_group_id_auto_set` key: `written:<guid>` records the GUID we last wrote, so after a
|
||||
gateway change the agent recognizes its own stale value and updates it — while a value
|
||||
written by anyone else is never overwritten (`DecideAction`: empty→write, equal→no-op,
|
||||
ours→update, else→skip; unparseable existing values count as foreign). Every write is
|
||||
verified by readback.
|
||||
|
||||
### The event
|
||||
|
||||
One `do_group_id_auto_set` event per outcome change (Source `StartupEnvironmentProbes`,
|
||||
gated on the fingerprint — unchanged reruns across reboots stay silent; errors never
|
||||
latch, so each failing start keeps its retry trail). Severity: Info for
|
||||
success/skip, Warning for errors. Data: `groupId`, `previousValue`, `gatewayIp`,
|
||||
`gatewayMac`, `written`, `skipped`, `skipReason`, `success`, `error`.
|
||||
|
||||
## Examples
|
||||
|
||||
First boot on a network with gateway `192.168.178.1` / MAC `00-11-22-33-44-55`:
|
||||
|
||||
```
|
||||
do_group_id_auto_set (Info): Delivery Optimization GroupId set to
|
||||
00063065-0000-00a8-b201-001122334455 (gateway 192.168.178.1)
|
||||
```
|
||||
|
||||
Tenant already deploys a DO GroupId via Intune:
|
||||
|
||||
```
|
||||
do_group_id_auto_set (Info): Delivery Optimization GroupId not set: mdm_policy_present
|
||||
```
|
||||
|
||||
## Citations
|
||||
|
||||
* Service (fingerprint, ownership, write+readback): `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/DoGroupIdService.cs`
|
||||
* ARP interop: `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Interop/ArpNativeMethods.cs`
|
||||
* Probe wiring + event + fingerprint dedup: `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Runtime/StartupEnvironmentProbes.cs`
|
||||
* Backend serve + ConfigVersion: `../../src/Backend/AutopilotMonitor.Functions/Functions/Config/GetAgentConfigFunction.cs`
|
||||
* DO policy stores (key paths, Registry64 rationale): `../../src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/DoThrottlePolicyReader.cs`
|
||||
@@ -5,8 +5,12 @@ How the on-device agent works:
|
||||
* [Runtime Overview](overview.md) - lifecycle (install → boot guards → runtime → termination), collector hosts, the single-rail event pipeline to the backend, config & kill switch, on-disk layout.
|
||||
* [Decision Engine](decision-engine.md) - the pure-reducer state machine (DecisionCore): signals, state, effects, the A/B/C completion arms, invariants, threading.
|
||||
* [Logs, Persistence & Crash Recovery](logs-and-persistence.md) - every persisted file and why it exists; signal log vs spool; snapshot/journal; the replay-based recovery flow and quarantine.
|
||||
* [Diagnostics Package](diagnostics-package.md) - what the diagnostics ZIP contains: the built-in section catalog shared with the portal (Shared `DiagnosticsBuiltInSections`, served by GET diagnostics/paths), the RealmJoin Watcher and Device Preparation collection gates, why built-ins bypass the configured-path guards, global ∪ tenant path dedup, caps and the package manifest.
|
||||
* [Hello Wizard Un-Skip](hello-wizard-unskip.md) - why a single "WHfB disabled" policy read never decides completion: the HelloWizardStarted signal (prevention + cure) and the HelloTracker confirmation second read.
|
||||
* [Autopilot ZTD Diagnostics](autopilot-ztd-diagnostics.md) - Windows' own diagnostic surfaces for the profile-download flow: ModernDeployment event IDs (807/815/164/...), the Diagnostics\Autopilot registry key, deployment-service endpoints, and the known-issue error-code map.
|
||||
* [MDM Reboot Coalescing](mdm-reboot-coalescing.md) - attributing the mid-ESP coalesced reboot ("second sign-in") to the device-assigned policy URIs that forced it: aggregated 2800 observations (neutral, flush-time watermark) + ANALYZE-ESP-005 gated on an actually-observed reboot.
|
||||
* [Power-State Watcher](power-state-watcher.md) - live AC↔battery transitions and 50/30/15% downward threshold crossings via a WMI push subscription (no polling): power_state_change semantics (latch/debounce/cap, never armed without a battery) and the single-condition analyze rules ANALYZE-DEV-009/-010 on top.
|
||||
* [DO GroupId Auto-Set](do-group-id-auto-set.md) - the optional Delivery Optimization peering-group seed: deterministic GUID from default gateway IPv4 + MAC (RealmJoin-compatible), written to the DOGroupId policy value with readback, ownership-tracked so foreign Intune/GPO values are never overwritten; skip reasons and the do_group_id_auto_set event.
|
||||
* [System Timeline Watcher](system-timeline-watcher.md) - clock steps (system_clock_changed: oldTime/newTime/timeDeltaMs/process, micro-slew floor) and completed sleep episodes (system_sleep_episode: sleep/hibernate/Modern Standby with authoritative enteredAt/exitedAt) from one System-channel EventLogWatcher with pre-agent backfill; feeds the web clock-set jump cause, the standby duration note, time-attribution SleepSpans and ANALYZE-DEV-012.
|
||||
|
||||
Reading order for newcomers: overview → decision engine → persistence.
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
---
|
||||
type: Concept
|
||||
title: ESP Completion Starvation — the sits-d Cloud-PC complex
|
||||
description: Why five Windows 365 Cloud PCs finished enrolling but never completed in the product — a SkipUser=true flow produces none of the signals every completion-gate evaluation site hangs off — plus the two adjacent defects fixed alongside (edge-only user-apps-settled synthesis, orphaned Hello-wizard restart replay) and what a Shell-Core replay deliberately does not do.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals
|
||||
tags:
|
||||
- agent
|
||||
- esp
|
||||
- completion
|
||||
- skip-user
|
||||
- reboot
|
||||
- recovery
|
||||
timestamp: 2026-08-20T00:00:00+02:00
|
||||
---
|
||||
|
||||
# ESP Completion Starvation — the sits-d Cloud-PC complex
|
||||
|
||||
Seven Windows 365 Cloud PCs, same tenant, first sign-in the same morning. Two sessions
|
||||
succeeded within minutes; five sat at `Completion is waiting on: hello_resolution` from
|
||||
second five onward and ran into the six-hour max-lifetime watchdog — verdict `Incomplete`
|
||||
on devices whose users were working normally the whole time.
|
||||
|
||||
The first diagnosis blamed the mid-ESP reboot (DeviceGuard/DmaGuard, see
|
||||
[MDM Reboot Coalescing](mdm-reboot-coalescing.md)): the agent is killed in
|
||||
`Stage=EspAccountSetup` and, so the theory went, loses the Shell-Core ESP exit to the
|
||||
downtime. The evidence disproved it. `8110e262` ran **six hours uninterrupted** after its
|
||||
last reboot with the Shell-Core watcher armed — and Windows never wrote a 62407 in all
|
||||
that time, because on these machines **the user ESP page never existed**.
|
||||
|
||||
# Schema
|
||||
|
||||
## The actual defect — a satisfied gate nobody evaluates
|
||||
|
||||
The five stuck machines carried `SkipUserStatusPage=True` in the enrollment `FirstSync`
|
||||
key (read 10+ times across three agent processes, `True` on every read — not a stale
|
||||
single read). Windows honours it: no user ESP page, hence
|
||||
|
||||
* no Shell-Core 62407 (`esp_exiting`) — the page that would exit does not exist,
|
||||
* no `AccountSetupCategory` provisioning registry — nothing writes categories for a page
|
||||
that never renders,
|
||||
* desktop within seconds of sign-in.
|
||||
|
||||
The reducer is *built* for this flow: **arm B** of `ShouldTransitionToAwaitingHello`
|
||||
(`SkipUserEsp == true`) declares the gate open, and the missing-prerequisites bookkeeping
|
||||
already exempts the AccountSetup gate — which is why the sessions reported only
|
||||
`hello_resolution` as missing. But the gate is a predicate, not a process. Something has
|
||||
to evaluate it, and every evaluation site hangs off a signal a skip-user flow structurally
|
||||
cannot produce:
|
||||
|
||||
| Evaluation site | Carrier signal | On SkipUser=true |
|
||||
|---|---|---|
|
||||
| `HandleEspExitingV1` | Shell-Core 62407 | never fires |
|
||||
| `HandleEspPhaseChangedV1(FinalizingSetup)` | Shell-Core 62407 via coordinator | never fires |
|
||||
| `HandleDesktopArrivedV1` fast-path | requires Hello policy **disabled** | Hello was enabled |
|
||||
| `HandleImeUserSessionCompletedV1` arm-C attempt | requires the recorded final exit | unsatisfiable |
|
||||
|
||||
No knock, no promotion, no `HelloSafety` — and the `AdvisoryCompletion` backstop is armed
|
||||
exclusively from an `EspExiting` signal, so it never armed either. The session idles to
|
||||
the watchdog with an open gate.
|
||||
|
||||
## The fix — the observed skip is the exit evidence
|
||||
|
||||
`HandleImeUserSessionCompletedV1`'s completion attempt now accepts the observed skip in
|
||||
place of the final exit (`skipUserExitEquivalent`): on a flow where no page exists, no
|
||||
exit can ever say more than the `FirstSync` read already did. Everything else stays
|
||||
mandatory — AccountSetup anchor, genuine (at-or-after-anchor) IME user-session completion,
|
||||
DAD-validated real-user desktop — which makes this knock **stricter** than the existing
|
||||
Shell-Core-carried arm-B promotion (pinned in `ClassicAwaitingHelloGuardTests`, which
|
||||
requires no IME or desktop evidence at all). Hello semantics ride the existing rails:
|
||||
enabled-or-unknown promotes to `AwaitingHello` with `HelloSafety` armed (on Cloud PCs
|
||||
reached via RDP the Hello wizard never appears, so the synthetic timeout resolves the
|
||||
wait), disabled completes directly with the synthetic `Skipped` outcome.
|
||||
|
||||
The knock deliberately lives on the IME user-session edge, not on desktop arrival:
|
||||
|
||||
* **Ordering-robust against Fix 10.** An `EspPhaseChanged(AccountSetup)` arriving on
|
||||
`AwaitingHello` bounces the stage back and cancels `HelloSafety` (the premature-promotion
|
||||
guard). The IME AccountSetup phase line lands *after* the desktop on this flow, so a
|
||||
desktop-side knock would be undone two signals later. The IME phase line always precedes
|
||||
the user-session-complete line in log order, so a bounced promotion is re-knocked by the
|
||||
very next re-emission — including the restart re-parse.
|
||||
* **Right semantics.** The IME user session completes when the required-app processing is
|
||||
done; a desktop-side knock would mark the session `Succeeded` mid-installation.
|
||||
* Device Preparation is excluded — WDP keeps its Hello+Desktop conjunction and the
|
||||
`DevicePrepCompletion` backstop.
|
||||
|
||||
`ClassicSkipUserEspUserSessionCompletionTests` replays the real `8110e262` signal ordering
|
||||
end to end (bootstrap `EspConfigDetected` → Hello enabled → desktop → AccountSetup anchor →
|
||||
IME completion → HelloSafety → `enrollment_complete`), including the Fix-10
|
||||
bounce-then-re-knock cycle, and pins the negative space (no skip observed / no desktop /
|
||||
ghost pre-anchor IME completion / WDP).
|
||||
|
||||
Why five of seven identically administered Cloud PCs carried `SkipUserStatusPage=True` is
|
||||
a tenant-side question (ESP profile assignment state at each machine's provisioning
|
||||
moment); the value did not change during any observed session, and machine age does not
|
||||
cleanly explain the split. The agent's job is to complete the flow Windows actually ran —
|
||||
which it now does.
|
||||
|
||||
## Adjacent defect — the synthesis was edge-triggered
|
||||
|
||||
This one is real but belongs to the `SkipUser=false` flow (the two sessions that
|
||||
succeeded). The user-apps-settled synthesis
|
||||
(`MaybeSynthesizeAccountSetupCompleteFromSettledUserApps`, session `caa6cf50`) ran **only**
|
||||
from `OnEspExited`. On a tenant with ~138 required apps the ESP page exits while apps are
|
||||
still in flight: the single attempt misses and nothing ever re-checks, even when every
|
||||
tracked app reaches a terminal state minutes later.
|
||||
|
||||
The ESP exit is an **edge** (happens once); settled user-ESP apps are a **level** (reached
|
||||
later). `EspAndHelloTracker` records the edge in `_espExitObserved` and exposes
|
||||
`ReevaluateUserAppsSettledSynthesis()`, chained into `ImeLogTracker.OnAppStateChanged` and
|
||||
into the post-restore `onStateRestored` callback (a live exit observed before the IME
|
||||
state restore completes would otherwise never get a second look). Gate conditions are
|
||||
unchanged — no new completion path, only a second opportunity for the existing one. Only a
|
||||
**live, confirmed post-AccountSetup** exit may be remembered
|
||||
(`IsConfirmedPostAccountSetupExit`, stricter than the `IsIntermediateDeviceEspExit`
|
||||
forward guard: unknown counts as not confirmed).
|
||||
|
||||
## Adjacent defect — the restart replay was never wired up
|
||||
|
||||
`ShellCoreTracker`'s backfill had existed since session `772fe502` with **no caller**; the
|
||||
Shell-Core `EventLogWatcher` only delivers records written after `Start()`.
|
||||
`EspAndHelloHost.Start()` now calls it as `BackfillRecentHelloWizardStart` — which is
|
||||
exactly what it does and all it does:
|
||||
|
||||
| Record | Replayed | Why |
|
||||
|---|---|---|
|
||||
| 62404 Hello-wizard start | **yes** | Conservative: vetoes a premature "Hello is disabled" skip, can never complete a session by itself. The observation `772fe502` was about. |
|
||||
| 62407 ESP exit | **no** | Cannot be placed in time — see below. |
|
||||
| 62407 ESP failure | **no** | Re-injecting a historic failure as fresh can fail a session that recovered on retry (`ANALYZE-ESP-006`). |
|
||||
|
||||
A replayed exit is unusable, and every candidate ordering mechanism fails independently:
|
||||
|
||||
1. Windows writes the **identical** description `CommercialOOBE_ESPProgress_Page_Exiting`
|
||||
for the intermediate DeviceSetup→AccountSetup transition and the final exit.
|
||||
2. Every post-hoc ordering probe (AccountSetup registry, settled-apps) reads state as it
|
||||
is **now**, not as it was at the event's time.
|
||||
3. The reducer orders exits by **ingest ordinal**, not timestamp
|
||||
(`IsPostAccountSetupFinalExit`) — a historic exit replayed today gets a fresher ordinal
|
||||
than reality and reads as post-AccountSetup by construction.
|
||||
4. `HandleEspExitingV1` passes `espFinalExitInFlight: true` for every arriving exit; with
|
||||
restored state, arm C would open on a historic intermediate exit.
|
||||
|
||||
`ClassicEspExitingOnRestoredStateTests` pins that the reducer *does* open on any arriving
|
||||
exit; `ShellCoreTrackerReplayScopeTests` pins that the replay never produces one. Skipped
|
||||
62407 records are counted and reported once as an `agent_trace`
|
||||
(`reason: replayed_62407_not_orderable`) — silently dropping evidence is how this class of
|
||||
bug survives.
|
||||
|
||||
**Residual gap, named honestly:** a `SkipUser=false` session whose final exit falls
|
||||
exactly into an agent downtime *and* whose AccountSetup category never resolves
|
||||
(`1ec8f4c6` shape) still has no completion carrier after the restart. The normal case is
|
||||
covered — `ProvisioningStatusTracker` reads the current registry at startup, so a
|
||||
category that resolved during the downtime fires arm A. The residual shape is rare,
|
||||
surfaced by `ANALYZE-ESP-005` via the `session_stalled` interim trigger, and deliberately
|
||||
not patched with a replay that cannot be ordered.
|
||||
|
||||
### Sizing the replay window
|
||||
|
||||
A constant is provably wrong: the scheduled task has a `BootTrigger` only
|
||||
(`Program.InstallMode.BuildScheduledTaskXml`) — after an `exception_crash` the agent does
|
||||
not return until the next boot. `ResolveEspExitBackfillLookbackMinutes` reaches back to
|
||||
the last moment the previous run is known to have been alive (max of `snapshot.json`
|
||||
mtime and `LastBootUtc` for kill-type exits), floored at 5 minutes, capped at 360, off on
|
||||
`first_run`, and immune to cross-reboot clock skew (a future timestamp is ignored).
|
||||
|
||||
# Examples
|
||||
|
||||
Tenant `sits-d.cloud`, 2026-08-19, seven Windows 365 Cloud PCs, first sign-ins the same
|
||||
morning. The discriminator is a single registry value:
|
||||
|
||||
| Session | SkipUser | `esp_exiting` | AccountSetup registry | Reboots | Outcome |
|
||||
|---|---|---|---|---|---|
|
||||
| `08fc6bda` | False | yes | yes | 0 | Succeeded (10 min) |
|
||||
| `3b9291aa` | False | yes | yes | 0 | Succeeded (2.1 h) |
|
||||
| `8110e262` | True | — | — | 2 | Incomplete (6 h watchdog) |
|
||||
| `a89aac2d` | True | — | — | 1 | Incomplete |
|
||||
| `e7ba63c9` | True | — | — | 1 | Incomplete |
|
||||
| `cb4a485a` | True | — | — | 2 | Incomplete |
|
||||
| `3d6278fb` | True | — | — | 1 | Incomplete |
|
||||
|
||||
The reboot correlation was a red herring: `8110e262` ran six hours live after its final
|
||||
reboot and would have seen every signal. `8110e262` and `a89aac2d` both reached
|
||||
`138/138 completed, 0 failed` and a genuine IME user-session completion — with the fix,
|
||||
all five complete a few minutes after their IME user session settles.
|
||||
|
||||
`08fc6bda` exercises the edge-level fix on the same day: ESP page exited normally while
|
||||
`categorySucceeded` never confirmed — completed via the user-apps-settled synthesis.
|
||||
`3b9291aa` shows the independent starvation visibility gap: 99 `app_install_starved`
|
||||
events on a session recorded as `Succeeded` — surfaced from now on by `ANALYZE-APP-016`.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Classic.cs` — `HandleImeUserSessionCompletedV1`, `skipUserExitEquivalent`
|
||||
* `src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Shared.cs` — `ShouldTransitionToAwaitingHello` (arm B)
|
||||
* `src/Agent/AutopilotMonitor.DecisionCore.Tests/ClassicSkipUserEspUserSessionCompletionTests.cs` — end-to-end replay of the 8110e262 ordering
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/ShellCoreTracker.cs` — `BackfillRecentHelloWizardStart`, `ReplayBackfillRecords`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs` — `_espExitObserved`, `ReevaluateUserAppsSettledSynthesis`
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs` — lookback sizing, `OnAppStateChanged` chaining
|
||||
* [MDM Reboot Coalescing](mdm-reboot-coalescing.md) — the policy attribution for the reboots themselves
|
||||
* [Decision Engine](decision-engine.md) — the AccountSetup gate and completion arms
|
||||
@@ -0,0 +1,106 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Power-State Watcher — live AC/battery transitions and threshold events during enrollment
|
||||
description: How the agent observes AC↔battery switches and downward battery-threshold crossings mid-enrollment via a WMI push subscription (no polling), the power_state_change event semantics (transition field, 50/30/15 ladder, latch/debounce/cap), and the two analyze rules on top (ANALYZE-DEV-009/-010).
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/PowerStateWatcherHost.cs
|
||||
tags:
|
||||
- agent
|
||||
- power
|
||||
- battery
|
||||
- collector
|
||||
- wmi
|
||||
- analyze-rules
|
||||
timestamp: 2026-08-21T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Power-State Watcher
|
||||
|
||||
The startup probe (`power_state_check`, one-shot from `StartupEnvironmentProbes` via
|
||||
`PowerStateProbe`/`GetSystemPowerStatus`) only captures the power state at agent start. A
|
||||
device unplugged mid-enrollment — or draining its battery across a multi-hour ESP — was
|
||||
invisible. The watcher closes that gap **event-driven, with zero polling**.
|
||||
|
||||
# Mechanism
|
||||
|
||||
- `PowerStateWatcherHost` (`Orchestration/Hosts`) arms a WMI event subscription with a
|
||||
two-step chain:
|
||||
1. `SELECT * FROM Win32_PowerManagementEvent WHERE EventCode = 10` (PBT_APMPOWERSTATUSCHANGE:
|
||||
true kernel push, fires on AC↔DC switches and battery-percentage steps).
|
||||
2. **Fallback** `SELECT * FROM __InstanceModificationEvent WITHIN 30 WHERE TargetInstance
|
||||
ISA 'Win32_Battery'` — required because on Windows 11 24H2/25H2 (observed on builds
|
||||
26200 and 26220, session 161b838c) the `MS_Power_Management_Event_Provider` is still
|
||||
registered in the repository but its activation fails with `WBEM_E_NOT_FOUND`: the
|
||||
legacy push provider no longer loads on exactly the OS builds new devices enroll with.
|
||||
The WITHIN sampling runs inside WinMgmt (one Win32_Battery snapshot every 30 s), not in
|
||||
the agent; worst-case detection latency ≈ 35 s incl. debounce. `collector_degraded`
|
||||
(`watcher_arm_failed`) is emitted only when BOTH attempts fail; the agent log's arm
|
||||
line names the active mode.
|
||||
|
||||
Either way the arriving event's payload is ignored — each arrival re-probes via the
|
||||
existing `PowerStateProbe.Probe()` (cheap P/Invoke).
|
||||
- Snapshots are diffed by the pure `PowerStateTransitionTracker`
|
||||
(`Monitoring/Enrollment/SystemSignals`), which decides what to emit. The host only owns
|
||||
WMI, a 5 s trailing-edge debounce timer (dock-flap AC→battery→AC inside the window
|
||||
collapses to a no-diff) and the emit path (`InformationalEventPost`, single rail).
|
||||
- **No battery / probe error ⇒ WMI is never armed** — desktops and VMs pay nothing. WMI
|
||||
arming runs on a background thread (WinMgmt can hang in OOBE; `ConsoleBypassWatcher`
|
||||
precedent); an arm failure emits one `collector_degraded` (`watcher_arm_failed`).
|
||||
- Registered as a plain always-on peripheral host after `NetworkChangeHost`, deliberately
|
||||
NOT under `PeriodicCollectorLifecycleHost`: it is push-based (zero idle cost) and must be
|
||||
alive precisely when the device idles on battery. Kill-switch:
|
||||
`Collectors.EnablePowerStateWatcher` (default true, compile-time default like the other
|
||||
collector toggles; ConfigVersion 38).
|
||||
|
||||
# Event: power_state_change
|
||||
|
||||
`data.transition` discriminates three shapes (payload always carries `onAcPower`,
|
||||
`batteryPercent`, `isCharging`, `batteryLifeMinutes`; unknowns as `"unknown"`):
|
||||
|
||||
| transition | Severity | Immediate | Notes |
|
||||
|---|---|---|---|
|
||||
| `ac_to_battery` | Warning | yes | one per accepted edge |
|
||||
| `battery_to_ac` | Info | no | recovery |
|
||||
| `threshold_crossed` | Info (50) / Warning (30) / Error (15) | 30+15 only | extra field `thresholdPercent` |
|
||||
|
||||
Threshold semantics (tracker):
|
||||
|
||||
- Ladder 50/30/15, evaluated only while ON battery; **each level latches once per agent
|
||||
lifetime** (charge-up + re-drain does not re-emit — no plug-cycle alarm spam).
|
||||
- A multi-level jump emits only the LOWEST newly-crossed level (60%→10% ⇒ one `15` event).
|
||||
- Entering battery power already below a level counts as crossing it; the arm-time baseline
|
||||
does too — an enrollment STARTING at 12% battery emits its `thresholdPercent: 15` event
|
||||
immediately (the analyze rule would otherwise never see sessions that began low).
|
||||
- Lifetime emission cap 20 (storm backstop against a flapping dock / dying battery
|
||||
controller); the host logs one warning when it engages.
|
||||
|
||||
Backend/agent hygiene: `power_state_change` is classified as non-activity on both sides —
|
||||
`SignalActivityClassifier.NonActivityEventTypes` (agent: must not reset the
|
||||
PeriodicCollectorLifecycleHost/StallProbeHost idle clocks) and
|
||||
`EventIngestProcessor.IsPeriodicOrStallEvent` (backend: must not heal a Stalled session —
|
||||
battery drain is not enrollment progress).
|
||||
|
||||
# Analyze Rules
|
||||
|
||||
- **ANALYZE-DEV-009** (high): battery crossed 15% while on battery — matches the
|
||||
agent-stamped `thresholdPercent equals 15`.
|
||||
- **ANALYZE-DEV-010** (warning): AC→battery switch — matches `transition equals
|
||||
ac_to_battery`.
|
||||
- Both: `evaluateOn: ["enrollment_end", "whiteglove_sealed", "on_event:power_state_change"]`
|
||||
(live interim finding + terminal finalization), `markSessionAsFailedDefault: false`.
|
||||
- Both are deliberately **single-required-condition**: `event_data` conditions scan events
|
||||
independently (no same-instance join), so a two-condition rule (e.g. `transition ==
|
||||
threshold_crossed AND batteryPercent < 15`) could false-positive across two different
|
||||
power events. `thresholdPercent` exists only on threshold events and its value pins the
|
||||
ladder level, so one condition fully identifies the instance.
|
||||
- `{{token}}` caveat: `AddDataFieldsToEvidence` whitelists only generic identifiers
|
||||
(appId/errorCode/…), so `{{batteryPercent}}` would render literally. DEV-009 interpolates
|
||||
`{{thresholdPercent}}` (the matched condition's own dataField); the exact percentage lives
|
||||
on the linked event in the timeline.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/PowerStateWatcherHost.cs`
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/PowerStateTransitionTracker.cs`
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Runtime/PowerStateProbe.cs` (probe, unchanged)
|
||||
- `rules/analyze/ANALYZE-DEV-009.json`, `rules/analyze/ANALYZE-DEV-010.json`
|
||||
- [analyze-rule-triggers](../rules/analyze-rule-triggers.md) (interim-trigger design; the WG battery rule was its motivating case)
|
||||
@@ -0,0 +1,110 @@
|
||||
---
|
||||
type: Concept
|
||||
title: System Timeline Watcher — clock steps and sleep episodes from the System event log
|
||||
description: How the agent turns two enrollment blind spots into timeline ground truth via one EventLogWatcher on the System channel — system_clock_changed (Kernel-General 1, oldTime/newTime/timeDeltaMs/process) and system_sleep_episode (Power-Troubleshooter 1 classic sleep/hibernate, Kernel-Power 507 Modern Standby) — including backfill of pre-agent records, the payload-times-are-authoritative rule, SleepSpans in time attribution, the web clock-set jump cause, and ANALYZE-DEV-012.
|
||||
resource: /src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/SystemTimelineTracker.cs
|
||||
tags:
|
||||
- agent
|
||||
- eventlog
|
||||
- time
|
||||
- clock-skew
|
||||
- standby
|
||||
- collector
|
||||
- time-attribution
|
||||
- analyze-rules
|
||||
timestamp: 2026-08-21T00:00:00+02:00
|
||||
---
|
||||
|
||||
# System Timeline Watcher
|
||||
|
||||
Two things silently distort every session timeline and its wall-clock duration:
|
||||
|
||||
1. **The OS clock gets stepped** — classically w32time correcting a wrong BIOS clock early
|
||||
in OOBE, but also tzutil/MDM actions. Before this watcher the jump was only *inferred*
|
||||
(TimeJumpBadge, `clock_skew`/DEV-008); Windows records the ground truth.
|
||||
2. **The device sleeps mid-enrollment** — the timeline shows an unexplained gap, the
|
||||
duration looks inflated, and nothing says "the device was simply asleep for 56 minutes".
|
||||
|
||||
`SystemTimelineTracker` (hosted by `SystemTimelineWatcherHost`, config flag
|
||||
`Collectors.EnableSystemTimelineWatcher`, ConfigVersion 39) subscribes to the **System**
|
||||
channel with one provider-qualified XPath and forwards both facts as events.
|
||||
|
||||
# Sources and events
|
||||
|
||||
| Source (System channel) | Emitted event | Key payload |
|
||||
|---|---|---|
|
||||
| `Microsoft-Windows-Kernel-General` EventID 1 | `system_clock_changed` | `oldTime`, `newTime` (ISO-8601), signed `timeDeltaMs`, `reason`/`reasonText` (1=`application_set`, 2=`hardware_clock_sync`), `processName`/`processId` |
|
||||
| `Microsoft-Windows-Power-Troubleshooter` EventID 1 | `system_sleep_episode` | `kind`=`sleep`/`hibernate` (EffectiveState 5 ⇒ hibernate), `enteredAt`=SleepTime, `exitedAt`=WakeTime, `durationSeconds`, `wakeSourceType`/`wakeSourceText` |
|
||||
| `Microsoft-Windows-Kernel-Power` EventID 507 | `system_sleep_episode` | `kind`=`modern_standby`, `exitedAt`=TimeCreated, `enteredAt`=exit − DurationInUs, `durationSeconds` (scenario), `sleepDurationSeconds` (real sleep), `reason`, `onAcPower`, `batteryRemainingCapacityOnExit` |
|
||||
|
||||
**506 is deliberately not consumed.** 507 carries the full episode; pairing 506/507 would
|
||||
need an orphan-tolerant state machine across agent restarts for zero information gain — an
|
||||
episode without a 507 never completed and would not be emitted under any design.
|
||||
|
||||
# Thresholds (constants, not knobs)
|
||||
|
||||
- `MinClockDeltaMs = 2000` — w32time logs Kernel-General 1 for routine **1 ms micro-slews**
|
||||
(verified live, several per hour); only genuine steps are emitted. Suppressed records are
|
||||
still claimed by the dedup watermark.
|
||||
- `ClockDeltaWarningMs = 5 min` — larger steps are Warning + immediate upload (can break
|
||||
token/cert validation windows; matches the web TimeJumpBadge threshold).
|
||||
- `MinSleepDurationSeconds = 60` — **Modern Standby only**: every screen-off scenario logs a
|
||||
506/507 pair, including seconds-long ones with `SleepDurationInUs 0` (`SleepEntered=false`);
|
||||
the single duration gate suppresses both. Classic Power-Troubleshooter episodes have no
|
||||
floor — they only exist for genuinely completed S3/S4 transitions.
|
||||
|
||||
# Payload times are authoritative
|
||||
|
||||
The decisive clock correction happens exactly when the device clock is untrustworthy, and
|
||||
the backend clamps implausible event timestamps (±168 h/24 h) to server time. Every semantic
|
||||
instant therefore travels as an explicit ISO-8601 payload field (`oldTime`/`newTime`,
|
||||
`enteredAt`/`exitedAt`); consumers must never read the event's own `Timestamp` for these.
|
||||
The event `Timestamp` is set to the record's TimeCreated (WindowsUpdateTracker backfill
|
||||
convention) so rows land where the wake/step happened.
|
||||
|
||||
# Backfill and dedup
|
||||
|
||||
Template: `WindowsUpdateTracker`. `LoadWatermark → arm watcher → backfill` with lookback
|
||||
`SystemEventBackfillLookbackMinutes` (default 1440 = 24 h; 0 disables). The System .evtx
|
||||
persists across OOBE reboots, so the pre-agent w32time correction and pre-agent standby
|
||||
episodes are recovered with `backfilled=true`. Cross-restart dedup via RecordId watermark
|
||||
(`system-timeline-watermark.json` in the state directory) plus an intra-run `HashSet` —
|
||||
deliberately **not** a high-water mark, which would drop exactly the older, never-emitted
|
||||
pre-agent records (the live watcher arms before the backfill scan).
|
||||
|
||||
# Non-activity classification (both sides)
|
||||
|
||||
Both event types are environment observation, not enrollment progress:
|
||||
`SignalActivityClassifier.NonActivityEventTypes` (agent — must not reset idle/stall clocks)
|
||||
and `EventIngestProcessor.IsPeriodicOrStallEvent` (backend — a device that slept through its
|
||||
stall must stay Stalled).
|
||||
|
||||
# Consumers
|
||||
|
||||
- **Web timeline** (`EventTimeline.tsx`): badge blocks for both events; the TimeJumpBadge
|
||||
gains the ground-truth cause `clock-set` — `classifyTimeJump` receives the session's
|
||||
`system_clock_changed` deltas and matches a backward display step against a backward clock
|
||||
set within max(60 s, 20 %) (`lib/timeProvenance.ts`).
|
||||
- **Duration story** (`SessionInfoCard` via `useSessionDerivedData.standbySeconds`): the
|
||||
wall-clock duration deliberately keeps the pause (the enrollment really took that long);
|
||||
a `· 🌙 56m standby` note explains it. Episodes are dedup'd on `enteredAt`.
|
||||
- **Time attribution** (`TimeAttributionCalculator.BuildSleepSpans`, AttributionVersion 3):
|
||||
`SleepSpan`s mirror `RebootSpan`s — cross-cutting annotations clipped to the observation
|
||||
windows (a WhiteGlove-pause episode contributes only its in-window flanks), never a slice
|
||||
of the wall-clock partition. Persisted as `SleepSeconds`/`SleepSpansJson`; rendered as the
|
||||
standby chip in `TimeAttributionLane`.
|
||||
- **ANALYZE-DEV-012** (info, `trigger: single`, `on_event:system_sleep_episode`): fires once
|
||||
per session for episodes ≥ 5 min. Deliberately the ONLY new rule: clock issues already
|
||||
have DEV-008 (`clock_skew`) — a second clock rule would double-report the same action item
|
||||
(user decision 2026-08-21); clock steps stay timeline ground truth with badges instead.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/SystemTimelineTracker.cs`
|
||||
- `src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/SystemTimelineWatcherHost.cs`
|
||||
- `src/Backend/AutopilotMonitor.Functions/Helpers/TimeAttributionCalculator.cs` (`BuildSleepSpans`)
|
||||
- `src/Web/autopilot-monitor-web/lib/timeProvenance.ts` (`clock-set`, `findExplainingClockChange`)
|
||||
- `rules/analyze/ANALYZE-DEV-012.json`
|
||||
- Live payload evidence (2026-08-21, Win11 26220): Kernel-General 1 with `TimeDeltaInMs=1`
|
||||
micro-slews; Kernel-Power 507 with `SleepDurationInUs=2324095807`; Power-Troubleshooter 1
|
||||
with `SleepTime`/`WakeTime` across a hibernate.
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
type: concept
|
||||
title: Attempt-Scoped App Install Durations
|
||||
description: AppInstallSummary.DurationSeconds measures the status-defining (last) install attempt via a LastAttemptStartedAt max-fold anchor — replacing the first-observation-to-last-terminal span that billed multi-pass IME processing gaps to trivial apps; time attribution counts the union of active attempt segments.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.AppInstall.cs
|
||||
tags: [backend, agent, ingest, app-installs, metrics, time-attribution, durations]
|
||||
timestamp: 2026-08-21
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
The IME processes the ESP app list in multiple passes: a device-ESP evaluation pass can
|
||||
touch an app (often ending `Skipped`) long before the real install runs in a later pass.
|
||||
`AppInstallSummary` is keyed by `{SessionId}_{AppName}` and folded from ALL of the app's
|
||||
events, so `DurationSeconds = last terminal − first observation` spanned those passes.
|
||||
Field case (session `44862cd3`, "Set TimeZone"): evaluation pass 09:16, real install
|
||||
10:22:58 → 10:23:21 — reported 4002 s for a 23-second install. Fleet averages, the
|
||||
Slowest-Apps ranking, p95s, and the version-regression radar all inherited the inflation;
|
||||
the rule engine's `app_install_duration` (already attempt-scoped) contradicted the
|
||||
dashboard for the same install.
|
||||
|
||||
# Schema
|
||||
|
||||
Three durable anchors on the row — each with a distinct, non-interchangeable job:
|
||||
|
||||
| Column | Fold | Role |
|
||||
|---|---|---|
|
||||
| `StartedAt` | min (earliest wins) | First observation. Server-side window/bucket filter of every AppInstallSummaries scan and the radar's version-ordering key. Never re-anchored. |
|
||||
| `LastAttemptStartedAt` | **max** (in-batch and in `ReconcileAppInstallSummaryWithExisting`) | Anchor of `DurationSeconds`. Null = sentinel (row predates 2026-08, or no start observed) → duration falls back to the historical `StartedAt` span. Replay of an older batch can never regress it. |
|
||||
| `InstallPassCount` | additive, guarded by `LastAttemptStartedAt` monotonicity (a batch whose newest start is not newer than the stored anchor adds 0) | Explains "finished long after first seen" (web ×N chip); 2+ is normal. |
|
||||
|
||||
`DurationSeconds = CompletedAt − LastAttemptStartedAt`, computed at the terminal event and
|
||||
re-derived in one central reconcile site covering in-order and Q4 out-of-order batches.
|
||||
A `LastAttemptStartedAt` newer than `CompletedAt` means a fresh pass is in flight — the
|
||||
completed attempt's stored duration stands (the 0 value omits the column; Merge preserves).
|
||||
`DownloadDurationSeconds` pairs the latest `app_download_started` with the NEXT
|
||||
`app_install_started` (attempt-scoped, in-batch). The full span stays derivable as
|
||||
`CompletedAt − StartedAt` — deliberately not a column.
|
||||
|
||||
Weaker-terminal guard (both in-batch and cross-batch): `app_install_completed` with
|
||||
state `Skipped`/`Postponed` never overrides an `Installed`/`Error` terminal, nor its
|
||||
`CompletedAt`/`DurationSeconds` — mirroring the long-pinned `app_install_skipped` rule.
|
||||
|
||||
# Consumers
|
||||
|
||||
* **Fleet metrics / apps dashboard / PDF** (`MetricsMath`, `AppsAnalyticsHelper`): values
|
||||
drop to attempt scale; labels say "install time (final attempt)". The 6 h
|
||||
`MaxPlausibleInstallDurationSeconds` gate remains (an attempt longer than the agent's
|
||||
lifetime is still unobservable).
|
||||
* **AppVersionRegressionRadar**: cutover gate in `MeasuredAppGroups` — only rows with
|
||||
`LastAttemptStartedAt` enter the duration population, on BOTH comparison sides. A mixed
|
||||
population would mask real regressions and falsely re-arm active episodes for up to the
|
||||
35-day horizon. `MinMeasuredInstalls` keeps it silent until enough post-cutover rows
|
||||
exist. Thresholds (2.0× / 300 s) were tuned on span data — re-evaluate ~4 weeks after
|
||||
rollout.
|
||||
* **Time attribution** (`TimeAttributionCalculator.BuildBlockingAppIntervals`):
|
||||
sequence-ordered start→terminal pairing yields one segment per pass/attempt. Per-app
|
||||
seconds = SUM of clamped segments (failed attempts and evaluation passes count — they
|
||||
occupied the path); occupancy merges the segments, never the per-app hulls; hull
|
||||
endpoints remain first-start→last-terminal for display and what-if. Deliberately
|
||||
BROADER than the metrics duration: active time across all passes vs. final attempt.
|
||||
* **Agent payloads** (`AppInstallTiming` / `ImeLogTrackerAdapter.UpdateAppTiming`): a
|
||||
terminal→active transition re-arms BOTH endpoints, so event `durationSeconds` also
|
||||
describes the attempt (needs the agent release carrying this change).
|
||||
* **Rule engine** `app_install_duration`: was already attempt-scoped — now consistent.
|
||||
|
||||
# Rollout
|
||||
|
||||
No backfill (user decision, analog to the CMTrace legacy): pre-cutover rows keep span
|
||||
values, so time-series charts step down visibly at the changeover (documented in the
|
||||
public platform changelog). Sentinel-gated columns follow the standard chain:
|
||||
`BuildAppInstallSummaryEntity` + `MapToAppInstallSummary` + `AppsDashboardProjection`
|
||||
(the only projection whose consumers read the new columns).
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.AppInstall.cs` — fold + weaker-terminal guard + attempt pairing
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Metrics.cs` — reconcile max-fold, pass-count guard, central duration recompute
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/AppVersionRegressionRadar.cs` — cutover gate
|
||||
* `src/Backend/AutopilotMonitor.Functions/Helpers/TimeAttributionCalculator.cs` — per-pass segments + occupancy
|
||||
* `src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs` — payload timing re-arm
|
||||
* Tests: `AppInstallTerminalStateAndReconcileTests`, `EventTimestampValidationTests`, `AppVersionRegressionRadarTests`, `TimeAttributionCalculatorTests`
|
||||
@@ -0,0 +1,93 @@
|
||||
---
|
||||
type: concept
|
||||
title: Plan Downgrade Policy & the Retention Grace Period
|
||||
description: What happens when a tenant loses Pro (explicit downgrade or trial expiry) — every entitlement gates read-time and reversibly, except retention, which gets a 30-day grace before the Community cap starts hard-deleting; the anchor model, the ops events, and the deliberate non-grace of everything else.
|
||||
resource: backend
|
||||
tags: [plan, edition, downgrade, retention, grace-period, trial, entitlements]
|
||||
timestamp: 2026-08-22
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
## The downgrade model: gate, don't destroy
|
||||
|
||||
Edition is resolved at READ time (`FeatureEntitlementCatalog.ResolveEdition`): Pro ⇔
|
||||
permanent tier `pro`/legacy `enterprise` OR an active trial. A plan mutation is a single
|
||||
field write (`PATCH config/{tenantId}/plan`, GA-only) — there is no downgrade event
|
||||
handler, no cascade. On Pro → Community every entitlement simply gates on the next read
|
||||
(≤ ~5 min config cache):
|
||||
|
||||
* **Unrestricted Mode** — effective value flips false; the GA on-request flag and the
|
||||
tenant toggle survive untouched (inert). Re-upgrade restores it verbatim.
|
||||
* **Delegated (MSP) admin** — scope suppressed at the home-tenant gate
|
||||
(`DelegatedAdminService.ApplyHomeTenantGateAsync`); grant rows / tenant groups /
|
||||
assignments are never touched. Resurrects on re-upgrade.
|
||||
* **OOBE bootstrap** — endpoints 403/404 unless the additive GA per-tenant flag
|
||||
(`BootstrapTokenEnabled`) is set; `BootstrapSessions` rows are kept.
|
||||
* **Rate limits / MCP quota** — pure computed-at-read floors/plan names.
|
||||
|
||||
**Retention is the one exception**: the retention sweep (12-hourly
|
||||
`SessionDeletionMaintenance` → `SessionRetentionFanoutService`) hard-deletes sessions
|
||||
older than `GetEffectiveRetentionDays` = `min(stored DataRetentionDays, edition cap)`.
|
||||
Without protection, a downgrade (or trial expiry) with stored retention 365 would start
|
||||
irreversibly cascade-deleting the 90–365-day band on the next tick.
|
||||
|
||||
## The retention grace period (30 days)
|
||||
|
||||
`FeatureEntitlementCatalog.RetentionDowngradeGraceDays = 30`. For 30 days after losing
|
||||
Pro, `GetEffectiveRetentionDays` keeps clamping to the **Pro** cap (365); only after the
|
||||
grace does the Community cap (90) bite. The stored `DataRetentionDays` is never rewritten.
|
||||
`days <= 0` (GA-only "infinite") short-circuits before any cap — those tenants are never
|
||||
swept, grace or not.
|
||||
|
||||
**Anchor** — "when was Pro lost" (`TenantEntitlementService.GetRetentionGraceEndUtc`,
|
||||
returns null while effectively Pro; grace end = latest anchor + 30d):
|
||||
|
||||
* **Explicit downgrade**: `TenantConfiguration.ProDowngradedUtc` (backend-only field, no
|
||||
agent ConfigVersion impact), maintained exclusively by the plan endpoint's pure mutation
|
||||
core `PlanManagementFunction.ApplyPlanChanges`. Stamped when the EFFECTIVE edition
|
||||
(before vs. after, same `nowUtc`) transitions Pro → Community — this covers both a
|
||||
planTier downgrade and a GA explicitly ending a trial (`trialExpiresUtc: null`, where
|
||||
the trial timestamp disappears and could not anchor). Cleared on any transition back to
|
||||
effectively Pro (PATCH pro, trial grant, self-service trial start), so a re-upgrade also
|
||||
resets the grace clock.
|
||||
* **Trial expiry**: no write happens — the stored `TrialExpiresUtc` itself is the anchor
|
||||
(read-time, like edition resolution). A planTier downgrade under a still-active trial
|
||||
deliberately does NOT stamp: the edition is still Pro, and the later trial expiry
|
||||
anchors the grace then.
|
||||
|
||||
The field is on the patch-endpoint `BaseDeniedFields` and `RevertProtectedFields` lists
|
||||
(a config revert must not time-travel the anchor) and roundtrips the table-serialization
|
||||
contract (Store+Map + roundtrip tests). Legacy rows read null → no anchor → no grace,
|
||||
which is correct: no tenant had been downgraded before the feature existed.
|
||||
|
||||
## Ops visibility (dual-registered event types)
|
||||
|
||||
* `TenantPlanDowngraded` (Warning) — emitted by the PATCH handler on an effective
|
||||
Pro→Community transition; carries `retentionGraceEndsUtc` + stored retention days. The
|
||||
response body also returns `retentionGraceEndsUtc`.
|
||||
* `TenantRetentionGraceExpiring` / `TenantRetentionGraceEnded` (Warning) — emitted by the
|
||||
daily `TrialExpirySweepFunction` (03:30 UTC) with the same stateless window mechanics as
|
||||
the trial events (≤3-day heads-up re-fired daily; ended once via the 24h look-back), and
|
||||
ONLY when data is actually at risk: stored `DataRetentionDays` above the Community cap
|
||||
(0 = infinite ⇒ never). Re-upgraded tenants go silent automatically (grace resolves null).
|
||||
|
||||
All three are registered in the web `OPS_EVENT_TYPES` catalog (OpsAlertRulesSection.tsx)
|
||||
so Telegram alert rules can route them.
|
||||
|
||||
## Deliberate non-goals
|
||||
|
||||
* No grace for Unrestricted/MSP/bootstrap/limits — reversible gates need none.
|
||||
* The retention WRITE cap (`TenantConfigValidation`) does not honor the grace: during the
|
||||
grace a tenant admin cannot SET a value above the Community cap. The grace protects
|
||||
existing data; new writes follow the new plan.
|
||||
* The feature-flags endpoint keeps publishing the plain edition cap
|
||||
(`retentionCapDays`) — it feeds the settings input maximum, which is the write cap.
|
||||
|
||||
# Citations
|
||||
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/TenantEntitlementService.cs` — grace + effective retention
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Config/PlanManagementFunction.cs` — `ApplyPlanChanges` anchor lifecycle
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Maintenance/TrialExpirySweepFunction.cs` — grace event windows
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/Deletion/SessionRetentionFanoutService.cs` — enforcement point
|
||||
* Tests: `TenantEntitlementServiceTests`, `PlanManagementTransitionTests`, `TrialExpirySweepFunctionTests`, `SessionRetentionFanoutServiceTests`, `TenantConfigTableSerializationTests`
|
||||
@@ -0,0 +1,84 @@
|
||||
---
|
||||
type: concept
|
||||
title: Send-Time Frame Separation (X-Send-Time-Utc)
|
||||
description: One device-clock send timestamp per upload batch splits ReceivedAt − OccurredUtc into pure spool delay and pure device-vs-server clock offset — turning clock diagnostics from median statistics into direct per-request measurement.
|
||||
resource: src/Backend/AutopilotMonitor.Functions/Functions/Ingest/IngestTelemetryFunction.cs
|
||||
tags: [backend, agent, ingest, timestamps, clock-skew, telemetry, provenance]
|
||||
timestamp: 2026-08-21
|
||||
---
|
||||
|
||||
# Problem
|
||||
|
||||
Events carry two time anchors: `OccurredUtc` (device clock at emission) and `ReceivedAt`
|
||||
(server clock at ingest). Their difference conflates two physically distinct phenomena:
|
||||
|
||||
* **Spool delay** — the event waited offline or in retry backoff before upload. Expected,
|
||||
harmless, minutes to hours.
|
||||
* **Device-vs-server clock offset** — the actual diagnostic signal.
|
||||
|
||||
A uniform spool backlog (events emitted close together, uploaded after an outage) is
|
||||
indistinguishable from a clock offset in that delta; a draining backlog fakes a clock jump
|
||||
ramp. Every consumer had to filter this statistically (spread caps, plateau guards,
|
||||
persistence tests) — and the first field clock-jump measurement failed exactly on this
|
||||
ambiguity.
|
||||
|
||||
# Schema
|
||||
|
||||
The agent stamps one header per upload attempt (`BackendTelemetryUploader`):
|
||||
|
||||
```
|
||||
X-Send-Time-Utc: <DateTime.UtcNow, ISO-8601 round-trip>
|
||||
```
|
||||
|
||||
Per **attempt**, not per batch content: a retry after backoff is a new send moment. The
|
||||
ingest function parses it (`ParseSendTimeHeader`: round-trip parse, UTC-normalized, values
|
||||
before year 2000 rejected as garbage — deliberately **no upper clamp**, a future-dated
|
||||
send time IS the clock-error measurement), and `StampServerFields` writes it as `SentAt`
|
||||
onto every event of the request — server-stamped, request-level, never trusted per-event
|
||||
from the wire. Stored as the nullable `SentAt` column on Events (absent for pre-header
|
||||
agents), mapped back everywhere events are read, exposed as `sentAt` in the `fields=`
|
||||
projection and verbatim on the raw endpoints.
|
||||
|
||||
The separation is then exact:
|
||||
|
||||
```
|
||||
SentAt − OccurredUtc = pure spool delay (both device frame; offset cancels)
|
||||
ReceivedAt − SentAt = network latency + clock offset (the only frame crossing)
|
||||
```
|
||||
|
||||
Network latency is seconds; anything beyond is device clock error, per request, no
|
||||
statistics required.
|
||||
|
||||
# Consumers
|
||||
|
||||
* **`clock_skew` analyze condition** (ANALYZE-DEV-008): batches whose events carry
|
||||
`SentAt` measure the batch offset directly as `SentAt − ReceivedAt` — spool-immune by
|
||||
construction, so the spread cap and the IME/clamped exclusions are unnecessary there
|
||||
(no event timestamps are read at all; IME-only sessions become measurable). Batches
|
||||
without `SentAt` keep the legacy per-event median + spread-cap path. Downstream logic
|
||||
(stability check, plateau step detection, end-state persistence) is shared; evidence
|
||||
carries `sentAtBatchCount`.
|
||||
* **`CmTraceSkewTripwire` deliberately unchanged**: `median(Δ_IME) − median(Δ_other)`
|
||||
cancels the device clock offset as common mode — send time adds nothing to that
|
||||
differential measurement.
|
||||
|
||||
# Constraints
|
||||
|
||||
* `SentAt` comes from the same device wall clock as the event timestamps — a clock jump
|
||||
between emission and send lands in the spool-delay term. That is the intended place for
|
||||
it to become visible, not a defect.
|
||||
* Signals and DecisionTransitions do not carry `SentAt` (no consumer); the header is
|
||||
request-level, so extending them later is a storage-only change.
|
||||
* Fleet coverage grows with agent rollout; all analyses must treat `SentAt == null`
|
||||
(legacy rows, old agents) as "legacy path", never as zero.
|
||||
|
||||
# Citations
|
||||
|
||||
* `BackendTelemetryUploader.UploadBatchAsync` — header stamping per attempt.
|
||||
* `IngestTelemetryFunction.ParseSendTimeHeader` + `IngestTelemetryFunctionTests` — parse
|
||||
contract incl. the no-upper-clamp decision.
|
||||
* `EventIngestProcessor.StampServerFields` + `IngestCriticalPathTests` — server-side
|
||||
stamping, forged per-event values overridden.
|
||||
* `RuleEngine.ConditionEvaluators.EvaluateClockSkewCondition` + `RuleEngineClockSkewTests`
|
||||
— the two measurement paths (the backlog-alias test pair pins why SentAt exists).
|
||||
* `tasks/todo.md` P14 (2026-08-21) — decision trail incl. why the tripwire stays as is.
|
||||
@@ -4,7 +4,7 @@ title: Tenant Activation & the Once-Only Welcome Mail
|
||||
description: How a signup becomes an activated tenant (manual approve or auto-approve queue), and how the welcome mail is sent exactly once even though activation and the user-provided address race each other.
|
||||
resource: backend
|
||||
tags: [activation, auto-approve, welcome-email, preview-whitelist, race]
|
||||
timestamp: 2026-08-02
|
||||
timestamp: 2026-08-23
|
||||
---
|
||||
|
||||
# Schema
|
||||
@@ -58,6 +58,46 @@ The Global-Admin resend endpoint (`POST /api/preview/send-welcome-email/{tenantI
|
||||
always sends (explicit intent) and consumes the marker best-effort so the automatic
|
||||
paths never duplicate afterwards.
|
||||
|
||||
## Mail transport
|
||||
|
||||
Both transactional mails (welcome, post-offboarding farewell) go through the
|
||||
provider-neutral `EmailService` (`IEmailService` for the welcome path,
|
||||
`IOffboardFarewellEmailSender` for the farewell path). The provider is an implementation
|
||||
detail confined to one private transport method plus the `Email:*` configuration
|
||||
section — a provider swap must not rename classes, DI registrations or callers.
|
||||
|
||||
* Current provider: **Mailchimp Transactional (Mandrill)**, `POST {Email:Endpoint}`
|
||||
(`messages/send`), API key in the JSON body, typed `HttpClient` with the shared
|
||||
`ResiliencePolicies.Notification` retry policy (15 s timeout).
|
||||
* Settings (App Settings use `Email__Key`): `Email:ApiKey` (required — empty means every
|
||||
send is a logged no-op, never an error), `Email:Endpoint`, `Email:FromAddress`
|
||||
(default `noreply@autopilotmonitor.com`), `Email:FromName` (default `Autopilot Monitor`).
|
||||
* Result handling: per-recipient status `sent`/`queued`/`scheduled` = success;
|
||||
`rejected`/`invalid` (with `reject_reason`), non-2xx or malformed bodies = warning.
|
||||
Never throws — both callers rely on that fail-soft contract.
|
||||
* `track_opens`/`track_clicks` are explicitly `false` and `auto_text` is `true`: the provider
|
||||
receives only the recipient address and the tenant domain, which is exactly the claim on
|
||||
the privacy page and the trust data-flows page. Changing tracking = changing those pages.
|
||||
|
||||
## Template overrides (operator-editable)
|
||||
|
||||
Global Admins can replace either HTML body without a deployment (portal: Settings →
|
||||
Email Templates). `EmailTemplateService` resolves the body for every send: the stored
|
||||
override wins, otherwise the built-in `EmailTemplates.*`. Overrides live in the
|
||||
`PreviewConfig` table (partition `EmailTemplates`, row key `welcome` | `farewell`,
|
||||
properties `Html`, `UpdatedBy`, `UpdatedUtc`), cached 5 minutes, invalidated on save/reset.
|
||||
|
||||
* The only placeholder is `{{domainName}}`; an empty domain renders as "your organization",
|
||||
matching the built-ins. Subjects are not editable (pinned constants).
|
||||
* Limit 30 000 characters (Table string property cap is 32K UTF-16 chars).
|
||||
* The send path is fail-soft: a storage error falls back to the built-in body. The admin
|
||||
read path is not — a broken store surfaces as an error instead of silently showing the
|
||||
built-in.
|
||||
* Endpoints (all GlobalAdminOnly): `GET|PUT|DELETE /api/global/email-templates/{kind}` and
|
||||
`POST /api/global/email-templates/{kind}/test`. The test send goes to the caller's tenant
|
||||
contact address (activation notification email → `ContactEmail` → caller UPN) with the
|
||||
caller's tenant domain, optionally with an unsaved draft body; provider tag `test`.
|
||||
|
||||
# Examples
|
||||
|
||||
* Auto-approve wins the race: approve at T+64 s finds no address → logs "deferred to the
|
||||
@@ -73,3 +113,8 @@ paths never duplicate afterwards.
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Rules/PreviewWhitelistFunction.cs` — save-path trigger, GA resend
|
||||
* `src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableConfigRepository.cs` — conditional inserts (approved / welcome-email-sent)
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/TenantApprovalServiceTests.cs` — ordering + dedup pins
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EmailService.cs` — provider-neutral transport (Mandrill wire format, result handling, tracking off)
|
||||
* `src/Backend/AutopilotMonitor.Functions.Tests/EmailServiceTests.cs` — wire-format, fail-soft and gate-ordering pins
|
||||
* `src/Backend/AutopilotMonitor.Functions/Services/EmailTemplateService.cs` — override resolution, placeholder rendering, cache
|
||||
* `src/Backend/AutopilotMonitor.Functions/Functions/Admin/EmailTemplatesFunction.cs` — GA endpoints (read / override / reset / test send)
|
||||
* `src/Web/autopilot-monitor-web/app/admin/settings/sections/SectionEmailTemplates.tsx` — portal editor, preview frame, test send
|
||||
|
||||
+11
-1
@@ -13,20 +13,26 @@ https://docs.autopilotmonitor.com (separate repository).
|
||||
|
||||
* [Architecture Guide](architecture.md) - High-level architecture overview and solution layout for contributors.
|
||||
* [Version Contract](versioning.md) - One version shape for agent, backend and MCP: the ETag-CAS counter blob that mints build numbers, and the manifests published only after a deploy is verified live.
|
||||
* [Customer Script Publishing](script-publishing.md) - How the customer-facing PowerShell scripts reach download.autopilotmonitor.com/agent/, why publishing is decoupled from agent releases, and the guards keeping the published copy, the version badge and the portal's version oracle in agreement.
|
||||
* [URL Registry](url-registry.md) - Every well-known host lives in one registry file per component (Constants.cs / utils/config.ts / src/config.ts), enforced by guard tests; includes the agent-download migration to the download alias and its deploy-order constraint.
|
||||
* [Drift Guards](drift-guards.md) - The 2026-08 fragility-audit follow-up contracts: SignalR message-name catalog (typed on both ends), the inverted-tick RowKey codec, tombstone table tagging, the shrink-only OkAsync baseline, per-section config PATCH with the section field map, and the web's scoped-endpoint builders + sidebar visibility matrix.
|
||||
|
||||
# Agent
|
||||
|
||||
* [V2 Agent](agent/index.md) - How the on-device agent works: [runtime overview](agent/overview.md), [decision engine](agent/decision-engine.md) (reducer, signals, completion arms), [logs & persistence](agent/logs-and-persistence.md) (signal log, snapshot, crash recovery), [Hello wizard un-skip](agent/hello-wizard-unskip.md) (why a single "WHfB disabled" read never decides completion), [Autopilot ZTD diagnostics](agent/autopilot-ztd-diagnostics.md) (event IDs, diagnostic registry, endpoints, error-code map).
|
||||
* [V2 Agent](agent/index.md) - How the on-device agent works: [runtime overview](agent/overview.md), [decision engine](agent/decision-engine.md) (reducer, signals, completion arms), [logs & persistence](agent/logs-and-persistence.md) (signal log, snapshot, crash recovery), [diagnostics package](agent/diagnostics-package.md) (built-in section catalog shared with the portal, RealmJoin/Device Preparation collection gates, caps, manifest), [Hello wizard un-skip](agent/hello-wizard-unskip.md) (why a single "WHfB disabled" read never decides completion), [Autopilot ZTD diagnostics](agent/autopilot-ztd-diagnostics.md) (event IDs, diagnostic registry, endpoints, error-code map).
|
||||
* [Build Counter Blob](agent/build-counter-blob.md) - How agent build numbers are minted: shared counter blob with ETag-CAS so local and CI builds never collide; CI additionally attests build provenance (Sigstore).
|
||||
* [Agent Endpoint Migration](agent/endpoint-migration.md) - How the backend re-homes agents to a new API base URL via the config channel (backend move or per-tenant region move): live-fetch-only, allowlist-validated on both sides, one hop, kill wins.
|
||||
* [Gather Rule Debug Log](agent/gather-rule-debug-log.md) - EnableGatherRuleDebugLog (ConfigVersion 36): the per-tenant local trace file that explains every gather-rule evaluation decision — scope skips, on_change suppression, empty collector results, logparser position/parse-mode outcomes — for rules that produce no timeline events.
|
||||
* [MDM Reboot Coalescing](agent/mdm-reboot-coalescing.md) - Attributing the mid-ESP coalesced reboot ("second sign-in") to the device-assigned policy URIs that forced it: aggregated DM-Enterprise 2800 observations (neutral, flush-time watermark) + the ANALYZE-ESP-005 advisory gated on an actually-observed reboot.
|
||||
* [ESP Completion Starvation — the sits-d Cloud-PC complex](agent/post-reboot-completion-recovery.md) - Why a SkipUser=true flow (no user ESP page → no Shell-Core 62407, no AccountSetup registry) left the completion gate satisfied but never evaluated — five Cloud PCs idled six hours at "waiting on: hello_resolution" — and the fix: the observed skip stands in for the final-exit evidence in the IME user-session completion attempt. Plus the two adjacent defects fixed alongside: the edge-only user-apps-settled synthesis (now level-triggered) and the never-wired Shell-Core restart replay (now Hello-wizard-only, with the reasoning why a replayed 62407 must never reach the reducer).
|
||||
* [Secure Boot CA-2023 Detection](agent/secureboot-ca2023-detection.md) - Why ANALYZE-SEC-001 v3 reads the UEFI db/KEK variables directly instead of trusting the SecureBoot\Servicing registry, and how the one-sided uefiCa2023FirmwareConfirmed marker plus a not_exists precondition give absence-tolerant suppression across old and new agents.
|
||||
* [Continue-Anyway Observation Mode](agent/continue-anyway-observation.md) - EnableEspContinueAnywayObservation (ConfigVersion 37, operator-set): Device-phase ESP terminal failures on Continue-Anyway profiles get a bounded 60-min observation instead of an immediate hard fail; the DAD-validated real-user desktop + Hello gate detect the true end, the session completes Succeeded + EspSoftFailure (amber "with issues" badge), expiry un-defangs to the original esp_terminal_failure.
|
||||
* [CMTrace Time Resolution](agent/cmtrace-time-resolution.md) - Converting bias-less CMTrace local times to UTC without trusting any process's timezone belief: per-line self-anchoring on the 15-minute offset grid for provably fresh lines, uniform marked reader-zone fallback for backlog, and the provenance vocabulary (bias / line-anchored / reader-zone-fallback) that keeps every event re-derivable.
|
||||
* [Registry App-State Second Pillar](agent/registry-app-state-pillar.md) - IME's authoritative per-app state read straight from the registry (Win32Apps EnforcementStateMessage, EspTrackingWin32Apps, StatusServiceReports) via a snapshot-and-diff watcher; registry/log reconciliation doubles as the built-in IME log-pattern drift alarm.
|
||||
* [WDP Scenario Gates & Completion Backstop](agent/wdp-scenario-gates.md) - How the agent gates Autopilot/ESP-only machinery off on Windows Device Preparation sessions (deterministic ExecutionContext marker, fail-toward-Classic), which noise it suppresses (ZTD probe, esp_config unknown, ModernDeployment 100/417/418/1005, starved sweep), and the WDP completion model: High-confidence seed, desktop-based arm D, and the 30-min device_prep_completion backstop.
|
||||
* [Power-State Watcher](agent/power-state-watcher.md) - Live AC↔battery transitions and 50/30/15% downward battery-threshold crossings during enrollment via a WMI Win32_PowerManagementEvent push subscription (zero polling, never armed on battery-less devices): the power_state_change event (transition field, per-level lifetime latch, 5s debounce, emission cap) and the analyze rules ANALYZE-DEV-009 (battery <15%, high) / ANALYZE-DEV-010 (AC→battery, warning) with on_event interim triggers.
|
||||
* [System Timeline Watcher](agent/system-timeline-watcher.md) - Ground truth for two timeline blind spots from one System-channel EventLogWatcher with pre-agent backfill: system_clock_changed (Kernel-General 1 — oldTime/newTime/signed timeDeltaMs/reason/setting process; 2 s micro-slew floor, ≥5 min Warning+immediate) and system_sleep_episode (Power-Troubleshooter 1 sleep/hibernate + Kernel-Power 507 Modern Standby; authoritative enteredAt/exitedAt payload instants, 60 s Modern-Standby floor). Consumed by the TimeJumpBadge clock-set cause, the SessionInfoCard standby note, time-attribution SleepSpans (AttributionVersion 3) and ANALYZE-DEV-012.
|
||||
* [DO GroupId Auto-Set](agent/do-group-id-auto-set.md) - EnableDoGroupIdAutoSet (ConfigVersion 40): the agent seeds the Delivery Optimization DOGroupId policy with a deterministic network-fingerprint GUID (default gateway IPv4 + SendARP MAC, RealmJoin-compatible byte layout) so same-network devices form one DO peering group during ESP; ownership-tracked via the StartupEventGate fingerprint, foreign DOGroupId/DOGroupIdSource policies always win, no random-GUID fallback.
|
||||
* [ESP Policy-Provider Stall Detection](agent/esp-policy-provider-stall.md) - The EnrollmentStatusTracking CSP's no-timeout Setup/Apps wait is keyed to the Intune-registered "Sidecar" provider by NAME: co-management's "ConfigMgr" parks the user ESP at "Apps (Identifying)" for days even with TrackingPoliciesCreated=1 (issue #106); the always-on two-arm esp_policy_provider_stalled tripwire (provider_incomplete | sidecar_provider_missing) and the SOFTWARE\Microsoft\Windows\Autopilot gather-rule allowlist entry that surface it.
|
||||
|
||||
# Rules
|
||||
@@ -34,16 +40,20 @@ https://docs.autopilotmonitor.com (separate repository).
|
||||
* [Gather Rule Phase Scoping, One-Shot Triggers & Emit-on-Change](rules/gather-rule-phase-scoping.md) - Restricting gather rules to enrollment phases (activePhases / activeFromPhase sticky latch), one-shot collection at a phase's start/end (phase_change / phase_exit), and emitMode "on_change" result-dedup.
|
||||
* [Gather Rule Guardrails](rules/gather-rule-guardrails.md) - Why the allowlists in rules/guardrails.json are enforced on the agent and nowhere else, which hard blocks survive unrestricted mode, and the rule that every collector must call a guard and emit security_warning on refusal.
|
||||
* [Rule Authoring Surface](rules/rule-authoring-surface.md) - How customer AIs author rules against the real contract: the baked MCP resources (guide/schemas/guardrails via combine.js), the MCP-local validate_rule pre-flight, and the side-effect-free POST rules/analyze/dryrun whose trace twin shares the production evaluators (parity + side-effect-freedom pinned by tests).
|
||||
* [Rule ID Namespace & Collision Guards](rules/rule-id-namespace.md) - Why rule identity is the human-readable ruleId (no GUIDs) with tenant isolation via Table Storage partitioning, the reserved built-in namespace (RuleIdPolicy), which collisions are possible (custom-vs-custom across tenants: allowed and harmless), and the guards keeping the global RuleStats aggregate catalog-only and the global partition inside the reserved namespace (combine.js, reseed, cross-tenant RuleState GC).
|
||||
* [Analyze Rule Evaluation Triggers](rules/analyze-rule-triggers.md) - Opt-in `evaluateOn` interim triggers (`whiteglove_sealed`, `on_event:<type>`) so rules can fire before a session is terminal, with RuleResults update-semantics (interim → finalize/resolve, NotifiedAt notification dedupe) replacing the permanent dedupe freeze, KO/stat suppression on interim runs, the ingest-side cached trigger registry, and the authoring lints that keep interim rules monotonic.
|
||||
|
||||
# Backend
|
||||
|
||||
* [Business Timestamps](backend/business-timestamps.md) - Why the Azure Tables system Timestamp is never authoritative (migrations reset it), and the OccurredUtc + RowKey-decode compensation for AuditLogs/OpsEvents/Events after the 2026-07-18 migration.
|
||||
* [Send-Time Frame Separation](backend/send-time-frame-separation.md) - One device-clock send timestamp per upload batch (X-Send-Time-Utc → SentAt on Events) splits ReceivedAt − OccurredUtc into pure spool delay and pure device-vs-server clock offset; the clock_skew condition measures SentAt batches directly (spool-immune), the CMTrace tripwire deliberately stays on its differential math.
|
||||
* [OOBE Bootstrap go-Code Flow](backend/oobe-bootstrap-go-flow.md) - The bootstrap short-code lifecycle (portal-created session → `irm go.autopilotmonitor.com/CODE | iex` at Shift+F10 → GetBootstrapScriptFunction), the always-HTTP-200 text/plain contract, the two-line injection defense (GUID/URL/expiry validator + single-quoted PS literals, golden-pinned against the legacy TS template), and the per-IP rate-limit story behind Front Door.
|
||||
* [Device Journeys & First-Time-Right](backend/device-journeys.md) - How a device's terminal enrollment attempts are grouped into journeys (trim+casefold device key with junk-serial exclusion, first-success end, 30-day gap, WhiteGlove = one attempt), the DeviceHistories chain with its tombstone-driven deleted-session pruning, the additive daily FTR aggregates, and the banner/fleet/MCP surfaces.
|
||||
* [Enrollment Time Attribution](backend/time-attribution.md) - How a terminal session's authoritative wall clock (DurationSeconds, WhiteGlove pause excluded) is partitioned into named segments with an explicit unattributed remainder, ESP-blocking apps as positive evidence with "up to" what-if bounds, reboot spans from event gaps, and the breakdown/aggregate tables the sweep maintains.
|
||||
* [Attempt-Scoped App Install Durations](backend/app-install-duration-attempt.md) - AppInstallSummary.DurationSeconds = status-defining (last) install attempt via the LastAttemptStartedAt max-fold anchor instead of the first-observation→last-terminal span that billed multi-pass IME gaps to trivial apps; InstallPassCount, the weaker-terminal guard, the radar's cutover gate, and time attribution's union-of-attempt-segments occupancy.
|
||||
* [Rule Regression Radar](backend/rule-regression-radar.md) - How analyze-rule hit-rate regressions are detected (7-day window ≥2× the 28-day baseline with disjoint Wilson intervals), which suppressions keep the radar quiet (grace period, edit-in-window, kill switch), how alert episodes live in the ruleregression tracker keyspace with re-arm semantics, and the on-fire dimension correlation with its "correlated, not necessarily causal" wording contract.
|
||||
* [App Version Duration Regression Radar](backend/app-version-regression-radar.md) - How app versions whose median install duration regressed vs. the previous version are detected (first-seen-ordered versions, median lift ≥2× plus a 5-minute absolute floor over measured installs only), how alert episodes live in the appversionregression tracker keyspace with drain/median re-arm semantics, and the bell/ops-event/analytics surfaces.
|
||||
* [Plan Downgrade Policy & the Retention Grace Period](backend/plan-downgrade-retention-grace.md) - What happens when a tenant loses Pro: every entitlement gates read-time and reversibly ("gate, don't destroy"), except retention — a 30-day grace keeps the Pro cap before the Community cap starts hard-deleting; the ProDowngradedUtc/trial-expiry anchor model, the TenantPlanDowngraded/RetentionGrace* ops events, and the deliberate non-grace of everything else.
|
||||
* [Tenant Activation & the Once-Only Welcome Mail](backend/tenant-activation-welcome-mail.md) - How a signup becomes an activated tenant (manual approve or the auto-approve queue), and the write-then-read + conditional-marker arbitration that sends the welcome mail exactly once even though activation and the user-typed notification address race each other.
|
||||
* [W365 Cloud PC Device Validation](backend/cloudpc-device-validation.md) - Why Cloud PCs can never pass the Autopilot serial lookup, and the cert-CN-bound CloudPc fallback validator: Subject CN (= Intune device id) matched against the tenant's Graph virtualEndpoint/cloudPCs inventory — admits exactly the service-provisioned Cloud PCs, nothing else.
|
||||
* [ServerActions & On-Demand Diagnostics](backend/server-actions-on-demand-diagnostics.md) - The queue-on-session, deliver-on-ingest action channel, the TenantAdminOrOperator tier with in-function per-type re-gating, the portal's Collect Logs flow (quick-config orders rotate_config before request_diagnostics), and the hosted-prefix sweep in the offboarding cascade.
|
||||
|
||||
+46
@@ -1,5 +1,51 @@
|
||||
# Log
|
||||
|
||||
## 2026-08-23
|
||||
|
||||
* **New**: `agent/diagnostics-package.md` — the diagnostics ZIP's built-in collection is now ONE catalog in Shared (`DiagnosticsBuiltInSections.All`): the agent iterates it in order and the backend serves the same list to the portal (`GET diagnostics/paths`, MemberRead), so what administrators see is exactly what the agent collects. New conditional sections: RealmJoin client/package/Chocolatey/per-user logs behind the tenant's RealmJoin Watcher toggle (mirrored onto `AgentConfiguration.EnableRealmJoinWatcher` by the merger) and the Device Preparation bootstrapper event log behind the deterministic WDP marker (probe once per build, fail toward Classic); every skip is a manifest line. Built-ins bypass `DiagnosticsPathGuards` by design (reviewed code, `%ProgramData%\RealmJoin` is not allowlisted); caps/reparse/evtx export still apply. `MergeDiagnosticsLogPaths` dedups global ∪ tenant paths. Portal: collapsed read-only catalog + one-line rows in both cards, tenant context dropped the GA-only global fetch.
|
||||
* **Update**: `rules/gather-rule-guardrails.md` — `DiagnosticsPathGuards` scope clarified: configured diagnostics paths only, never the built-in catalog.
|
||||
* **Update**: `backend/tenant-activation-welcome-mail.md` — new "Template overrides" section: Global Admins can replace the welcome/farewell HTML in the portal (Settings → Email Templates); `EmailTemplateService` resolves override-or-built-in per send (PreviewConfig table, partition `EmailTemplates`, 5-min cache, `{{domainName}}` placeholder, 30k limit, fail-soft send path), GA endpoints `global/email-templates/{kind}` (+`/test` to the caller's tenant contact address). Also: the mail header is image-free (CSS-drawn brand mark, gradient stripes and clip-path edge as progressive enhancement).
|
||||
|
||||
## 2026-08-22
|
||||
|
||||
* **Update**: `backend/tenant-activation-welcome-mail.md` — new "Mail transport" section: transactional mail now goes through the provider-neutral `EmailService` (`IEmailService` + `IOffboardFarewellEmailSender`) with the provider confined to one private transport method and the `Email:*` settings (`ApiKey`, `Endpoint`, `FromAddress`, `FromName`); current provider Mailchimp Transactional (Mandrill) replacing Resend (NuGet dropped, typed `HttpClient` + `ResiliencePolicies.Notification`). Result-array semantics (`sent`/`queued`/`scheduled` ok, `rejected`/`invalid` warn), open/click tracking explicitly off because the privacy/trust claim depends on it.
|
||||
|
||||
* **New**: `backend/plan-downgrade-retention-grace.md` — plan downgrade policy: every Pro entitlement gates read-time and reversibly on downgrade ("gate, don't destroy"), except retention, which now gets a 30-day grace (`FeatureEntitlementCatalog.RetentionDowngradeGraceDays`) before the Community cap starts hard-deleting the 90–365-day band. Anchor model: `ProDowngradedUtc` stamped/cleared by the plan endpoint's pure `ApplyPlanChanges` core on EFFECTIVE-edition transitions (covers explicitly ended trials; planTier downgrade under an active trial deliberately doesn't stamp), trial expiry anchors read-time via `TrialExpiresUtc`. New Warning ops events `TenantPlanDowngraded` (PATCH handler) and `TenantRetentionGraceExpiring`/`TenantRetentionGraceEnded` (TrialExpirySweep window mechanics, only when stored retention exceeds the Community cap). Field is patch-denied + revert-protected; retention WRITE cap deliberately ignores the grace.
|
||||
|
||||
## 2026-08-21
|
||||
|
||||
* **New**: `agent/do-group-id-auto-set.md` — optional DO peering-group seed (`EnableDoGroupIdAutoSet`, default off, ConfigVersion 40): `DoGroupIdService` derives a deterministic GUID from the default gateway's IPv4 + SendARP MAC in RealmJoin's exact byte layout (cross-product peering) and writes `DOGroupId` in the 64-bit GPO Policies store with readback verification. Conservative by design: no random-GUID fallback (skip on missing gateway/MAC), `DOGroupIdSource`/MDM-managed values/foreign GPO values always win (skip reasons in the `do_group_id_auto_set` event), own stale values are recognized via the `written:<guid>` StartupEventGate fingerprint and updated after a gateway change. Gateway heuristic extracted into the shared `NetworkInterfaceLocator` (dedup of NetworkChangeDetector + DeviceInfoCollector copies).
|
||||
|
||||
* **New**: `agent/system-timeline-watcher.md` — clock-step and sleep-episode ground truth from the System event log: one `EventLogWatcher` (Kernel-General 1, Power-Troubleshooter 1, Kernel-Power 507; 506 deliberately unconsumed) emits `system_clock_changed` (oldTime/newTime/signed timeDeltaMs/reason/process; 2 s micro-slew floor, ≥5 min = Warning+immediate) and `system_sleep_episode` (kind sleep/hibernate/modern_standby, authoritative `enteredAt`/`exitedAt` payload instants, 60 s Modern-Standby floor). WindowsUpdateTracker-style backfill (24 h default, `system-timeline-watermark.json`) recovers pre-agent records; ConfigVersion 39 (`Collectors.EnableSystemTimelineWatcher`). Consumers: TimeJumpBadge cause `clock-set` (ground truth beats inference), SessionInfoCard standby note next to the unchanged wall-clock duration, `SleepSpan`s in time attribution (AttributionVersion 3, RebootSpan clone), and ANALYZE-DEV-012 (info, single, ≥5 min episode; deliberately the only new rule — clock issues stay with DEV-008).
|
||||
|
||||
* **New**: `agent/power-state-watcher.md` — live AC/battery observation during enrollment: `PowerStateWatcherHost` subscribes to the WMI push `Win32_PowerManagementEvent EventCode 10` (no polling; re-probes `GetSystemPowerStatus` per push, 5 s trailing-edge debounce, never armed on battery-less devices, `Collectors.EnablePowerStateWatcher` kill-switch, ConfigVersion 38). The pure `PowerStateTransitionTracker` emits `power_state_change` with `transition` = ac_to_battery (Warning, immediate) / battery_to_ac (Info) / threshold_crossed at 50/30/15% (each level latched once per lifetime, lowest-crossed-only on jumps, baseline-below-level covered, emission cap 20). Classified non-activity on both sides (SignalActivityClassifier + IsPeriodicOrStallEvent). New analyze rules ANALYZE-DEV-009 (thresholdPercent=15, high) and ANALYZE-DEV-010 (transition=ac_to_battery, warning), both with `on_event:power_state_change` interim triggers and deliberately single-required-condition (event_data conditions have no same-instance join).
|
||||
|
||||
* **Update**: `rules/rule-id-namespace.md` — the export→paste round-trip is no longer lossy: the JSON editors now map rule-shaped exports into the form (`gatherRuleToForm` unflattens the nested gather `parameters` object, analyze `jsonToForm` maps `evaluateOn` onto the eval checkboxes; `tags` and the effective `markSessionAsFailed` carry into the created rule). Both editors still accept the serialized-form shape; edit-mode pastes merge, with an explicit `parameters`/`evaluateOn` winning over stale flat fields.
|
||||
|
||||
* **New**: `rules/rule-id-namespace.md` — rule identity model: human-readable `ruleId` strings (no GUIDs), tenant isolation via Table Storage partitioning (`global` = built-in/community, tenantId = custom), the reserved built-in namespace from `RuleIdPolicy`, and the four collision guards: catalog-only global `RuleStats` aggregate (same-ID custom rules of unrelated tenants no longer sum into one `global_{ruleId}` row), combine.js duplicate/namespace fail, GitHub-reseed rejection of non-reserved IDs into the global partition, and the namespace gate on the cross-tenant RuleState GC.
|
||||
|
||||
* **New**: `backend/app-install-duration-attempt.md` — attempt-scoped app install durations: `DurationSeconds` = status-defining (last) attempt anchored on the new max-folded `LastAttemptStartedAt` (StartedAt stays the earliest-wins window/version-ordering key; legacy rows without the anchor keep the span fallback); `InstallPassCount` with replay-guarded additive fold; weaker-terminal guard (completed-with-Skipped never overrides Installed/Error, in-batch and cross-batch); attempt-scoped `DownloadDurationSeconds` pairing; the version-regression radar's both-sides cutover gate; time attribution counts the union of per-pass active segments (occupancy merges segments, hulls stay for display/what-if); agent payload timing re-arms both endpoints on terminal→active. No backfill — pre-cutover rows keep span semantics.
|
||||
|
||||
* **Update**: `agent/decision-engine.md` — activity-based RealmJoin timeout extension (report 55e6afd61c9d: a large RJ package catalog outlived the 60-min window mid-install — the timer arms at RJ-agent-MSI detection, so the first deployment had effectively ~44 min). When the `RealmJoinTimeout` deadline fires with `LastDeploymentPhase` 100/101 and deployment activity (phase change or package observation, persisted in `RealmJoinFacts.LastActivityUtc`; the detection-time phase capture deliberately excluded) inside the last 60 min, the deadline re-arms to `lastActivity + 60 min` (Info event `realmjoin_timeout_extended`), clamped at detection + 4 h. `realmjoin_timeout` now carries `reason` = `hard_timeout` / `inactivity` / `absolute_cap`; a fire of a superseded deadline incarnation dead-ends. Idle detection and phase-200/210 sessions keep the original 60-min behavior.
|
||||
|
||||
* **New**: `backend/send-time-frame-separation.md` — P14: the agent stamps one device-clock send timestamp per upload attempt (`X-Send-Time-Utc`), the ingest function server-stamps it as `SentAt` on every event of the request. `SentAt − OccurredUtc` is pure spool delay, `ReceivedAt − SentAt` is network + device-vs-server clock offset — the two were indistinguishable before and had to be separated statistically. The `clock_skew` condition measures SentAt-carrying batches directly (spool-immune, no spread cap, IME-only sessions become measurable; evidence carries `sentAtBatchCount`), legacy batches keep the median path; the CMTrace tripwire deliberately stays on its differential math (common-mode clock offset cancels there).
|
||||
|
||||
## 2026-08-20
|
||||
|
||||
* **Update**: `agent/cmtrace-time-resolution.md` — new "Backend regression tripwire" section: the P6 split. Operator side: `CmTraceSkewTripwire` piggybacks on the terminal counter reconcile's partition scan (`Δ = ReceivedAt − OccurredUtc` per source, 15-min grid + 2-min residual, ≥20 samples / ≥3 distinct batches per side, bias-dominated sessions suppressed, `measuredWriterOffsetMinutes` never consulted) and emits the `CmTraceTimeSkewRegression` ops event — goal state: never fires; kill switch `CmTraceSkewTripwireDisabled`. Customer side: new `clock_skew` analyze condition source (`skewMetric: clock_jump | sustained_offset`, threshold in seconds, batch-median frames with spool-spread filter, plateau step detection, end-state persistence, IME-derived events excluded) behind the new built-in `ANALYZE-DEV-008` (300 s = the Kerberos/Entra 5-minute skew boundary) — the during-enrollment complement of ANALYZE-DEV-007's one-shot NTP check. P5 (agent-side observability event) is subsumed by the tripwire.
|
||||
|
||||
* **New**: `agent/cmtrace-time-resolution.md` — the era-aware answer to the CMTrace timezone skew: a bias-less line read in a provably fresh pass anchors its own UTC offset against the agent clock (15-minute grid, 2-minute residual), so interleaved writer eras in one file resolve line by line; everything not provably fresh keeps the uniform reader-zone fallback and says so via `sourceOffsetOrigin`. Freshness is a per-file `LastCheckedUtc` stamped on every poll look (empty first sights included) with a 30-second window, never restored across restarts. Replaces the reverted per-file offset application (04b1a7c6); the per-file measurement stays observational, now with the anchor timestamp logged as a tripwire for the still-unexplained e9753578 label/anchor crossing.
|
||||
|
||||
* **Update**: `agent/post-reboot-completion-recovery.md` — retitled to "ESP Completion Starvation — the sits-d Cloud-PC complex" and reframed on the evidence: the reboot was a red herring. The five stuck Cloud PCs carried `SkipUserStatusPage=True` (read 10+ times across three agent processes, `True` every time) and Windows honoured it — no user ESP page, hence no Shell-Core 62407 and no AccountSetup provisioning registry, verified over a six-hour uninterrupted post-reboot run with the watcher armed. Arm B of `ShouldTransitionToAwaitingHello` was open from the bootstrap onward, but every evaluation site hangs off a signal a skip-user flow cannot produce, and the `AdvisoryCompletion` backstop arms exclusively from an `EspExiting` signal — an open gate nobody knocks on. Fix: `HandleImeUserSessionCompletedV1` accepts the observed skip in place of the final-exit evidence (`skipUserExitEquivalent`; anchor + genuine IME completion + real-user desktop stay mandatory, WDP excluded). The knock deliberately lives on the IME edge, not desktop arrival — ordering-robust against Fix 10's AccountSetup bounce-back and semantically "user-phase work finished" rather than mid-install. `ClassicSkipUserEspUserSessionCompletionTests` replays the real 8110e262 ordering end to end. Docs and comments no longer claim an ESP-exit recovery that does not exist (stray duplicate summary in `ShellCoreTracker`, `ImeLogHost` restore-callback comment); the residual `SkipUser=false` exit-in-downtime gap is named honestly and surfaced by `ANALYZE-ESP-005` via `session_stalled` instead of being patched with an unorderable replay.
|
||||
|
||||
## 2026-08-19
|
||||
|
||||
* **New**: `agent/post-reboot-completion-recovery.md` — a forced mid-ESP reboot (DeviceGuard/DmaGuard) used to cost the session its AccountSetup completion permanently. Two defects: `ShellCoreTracker.BackfillRecentEspExitEvents()` existed but had no caller, so every agent restart dropped an ESP exit (62407) written while it was down; and the user-apps-settled synthesis was edge-triggered on `EspExited` only, so apps settling afterwards never re-opened the gate. `EspAndHelloHost.Start()` now backfills with a lookback sized to the downtime (`previousExit=reboot_kill` → since `LastBootUtc`, clamped to 360 min), and `ReevaluateUserAppsSettledSynthesis()` is chained into `ImeLogTracker.OnAppStateChanged`. No new completion path — the gate conditions are unchanged. Evidence: sits-d Cloud PCs, five `Incomplete` sessions on healthy devices, two of them at `138/138 apps, 0 failed`. Hardened over two review rounds: the remembered edge must be a **live** exit that is **confirmed** post-AccountSetup (a replay carries no evidence of its own position — Shell-Core writes the same `CommercialOOBE_ESPProgress_Page_Exiting` for the intermediate and the final exit, and the registry check reads *now*, not the event's time); the replay hands over the **newest** exit rather than the oldest; the replay is narrowed to the Hello-wizard start (62404) alone — a conservative fact that can only make the agent wait longer. A replayed ESP exit is never re-raised at all: Windows writes the identical description for the intermediate and the final exit, the reducer orders exits by ingest ordinal (fresher than reality for a replay) and `HandleEspExitingV1` passes `espFinalExitInFlight: true` for every arriving exit, so arm C of `ShouldTransitionToAwaitingHello` would open on a historic handoff once the restored state carries AccountSetupEntered + IME user session + desktop. A replayed ESP failure is likewise dropped (it could fail a session that recovered). Skipped records are reported as an `agent_trace`. The window is **measured** from the previous run's last known liveness (snapshot mtime, widened by `LastBootUtc`) instead of a constant — the scheduled task has a BootTrigger only and no restart-on-failure, so a crashed agent returns hours later and five minutes would lose the exit every time. On `first_run` the replay stays off, leaving the happy path untouched.
|
||||
|
||||
* **Update**: `rules/analyze-rule-triggers.md` — `on_event:session_stalled` documented as an established interim trigger for the never-terminal stuck session (one emission per session, no throttle needed). `ANALYZE-ESP-005` carries it: both its conditions are monotonic and it has no terminal precondition, and it fires at confidence 95 on the sits-d stalled stream. `ANALYZE-APP-015` deliberately does NOT — it is the worked example of the terminal-precondition trap (lint 1) — and `ANALYZE-ESP-001` does not either, because its `phase_duration` gate on a still-open phase is unverified mid-run. New `ANALYZE-APP-016` ("Required Apps Never Started — ESP Apps Gate Starved") turns the existing `app_install_starved` events into a customer-facing finding and, unlike `ANALYZE-APP-015`, fires even when the session succeeded — session `3b9291aa` was recorded `Succeeded` with 99 required apps that never started installing.
|
||||
|
||||
* **New**: `script-publishing.md` — the customer script publishing chain. `.github/scripts/Publish-BootstrapScripts.ps1` becomes the single owner (dev render, version-bump guard, uploads, version oracles, alias verification), called both by the new `publish-scripts.yml` (push to `main` on `scripts/Bootstrap/**` and `scripts/CustomerSetup/**`) and by the `publish_as_stable` branch of `build-agent.yml`. Script publishing no longer waits for an agent release. Blobs are published in a deterministic CRLF/UTF-8-no-BOM form so byte comparisons do not depend on the checkout; the Pester + ASCII gates move into the reusable `bootstrap-script-gates.yml` and block the upload; `AdminConfiguration.LatestBootstrapV2ScriptVersion` now has exactly one writer. The version-bump guard compares a code fingerprint (PowerShell token stream without comments, newlines and line continuations), so only behaviour changes require a `$ScriptVersion` bump — comment and formatting changes publish with a warning.
|
||||
|
||||
## 2026-08-18
|
||||
|
||||
* **New**: `agent/wdp-scenario-gates.md` — WDP scenario gates (afee7ae0 audit): deterministic ExecutionContext marker as the only gate key (fail-toward-Classic), suppressed Autopilot/ESP-only emissions and hosts (autopilot_profile*/ZTD probe, esp_config unknown null-gate, EspPolicyProviderStallHost, ProvisioningStatusTracker, ModernDeployment noise IDs 100/417/418/1005 + error-backfill skip, starved sweep), and the WDP completion model (High-confidence v2 seed via `enrollmentTypeDeterministic`, desktop-based arm D, 30-min `device_prep_completion` backstop, parked-tripwire WDP branch).
|
||||
|
||||
@@ -98,9 +98,13 @@ Interim evaluation changes what a rule's conditions mean mid-flight. Three new l
|
||||
|
||||
# Application to the Motivating Cases
|
||||
|
||||
- WG battery rule (sits-d): `evaluateOn: ["enrollment_end", "whiteglove_sealed"]` — technician gets the channel alert at the bench; Part-2 terminal pass finalizes or resolves the finding.
|
||||
- WG battery rule (sits-d): shipped 2026-08-21 as ANALYZE-DEV-009 (battery below 15% while on battery) and ANALYZE-DEV-010 (AC→battery switch), both `evaluateOn: ["enrollment_end", "whiteglove_sealed", "on_event:power_state_change"]` on the live power watcher's `power_state_change` events (see `../agent/power-state-watcher.md`) — technician gets the channel alert at the bench; the terminal pass finalizes the finding. Both are deliberately single-required-condition rules: `event_data` conditions scan events independently (no same-instance join), so DEV-009 matches the agent-stamped `thresholdPercent` and DEV-010 matches `transition`.
|
||||
- Hybrid-affinity stuck (esa): ANALYZE-ID-004 gets `evaluateOn: ["enrollment_end", "on_event:hybrid_login_pending"]` — the second overdue-login event (threshold 65) produces a live finding on the still-InProgress session. ANALYZE-APP-015 stays `enrollment_end`-only until its conditions are hardened (lint 1).
|
||||
|
||||
- Stuck after a forced mid-ESP reboot (sits-d Cloud PCs, 2026-08-19): `on_event:session_stalled` is the established trigger for the never-terminal stuck session. The agent emits `session_stalled` exactly once per session (60-minute stall probe), so the trigger is rare by construction and needs no throttle. Five Cloud PCs sat at `InProgress` with **zero findings** for hours — a mid-ESP reboot had killed the agent in `Stage=EspAccountSetup` — before the max-lifetime watchdog eventually terminalised them as `Incomplete`. ANALYZE-ESP-005 carries the trigger and fires on that stream at confidence 95 (verified against session `8110e262`): both its conditions (`mdm_policy_reboot_required.firstRebootUri` exists, `system_reboot_detected` count ≥ 1) are **monotonic** and it has no terminal preconditions, so lint 1 does not apply. ANALYZE-APP-016 is interim-safe for the same reason (`app_install_starved` count ≥ 1 only ever grows).
|
||||
|
||||
Two rules were considered and deliberately **not** given the trigger: ANALYZE-APP-015 is the worked example of lint 1 above and stays `enrollment_end`-only; ANALYZE-ESP-001 gates on `phase_duration` of a phase that is still open mid-run, and that evaluation is not pinned by a test — an unverified trigger is not worth a possible false positive on every slow DeviceSetup.
|
||||
|
||||
# Product Decisions (settled 2026-08-14)
|
||||
|
||||
1. Notify-on-resolve (a "finding healed" message)? **No** (v1) — resolution is visible in UI/MCP.
|
||||
|
||||
@@ -31,6 +31,12 @@ item in `AutopilotMonitor.Agent.V2.Core.csproj`) and regenerated into the portal
|
||||
| `IsCommandAllowed` | `allowedCommands` | **exact** match, trimmed — never prefix |
|
||||
| `IsEventLogChannelAllowed` | `eventLogChannels` | prefix, boundary `/` or end |
|
||||
|
||||
`DiagnosticsPathGuards.IsDiagnosticsPathAllowed` (allowlist `diagnosticsPathPrefixes`, a C#
|
||||
literal mirrored in the JSON) applies the same hard blocks and the same user-profile
|
||||
exception to **admin-configured diagnostics paths** only. The diagnostics package's built-in
|
||||
sections are reviewed code and never pass through it — see
|
||||
[Diagnostics Package](../agent/diagnostics-package.md).
|
||||
|
||||
**Hard blocks live in code, not in the JSON**, so a parse error cannot lift them — on a
|
||||
parse failure the allowlists fall back to empty and everything is blocked (fail-closed),
|
||||
but the hard blocks must hold regardless of configuration:
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
type: concept
|
||||
title: Rule ID Namespace & Collision Guards
|
||||
description: How gather/analyze rule identity is scoped (tenant partition vs global partition), why there are no GUIDs, which collisions are possible, and the guards that keep the reserved built-in namespace and the global RuleStats aggregate clean.
|
||||
resource: backend, rules
|
||||
tags: [rules, ruleid, namespace, collision, rulestats, tenant-isolation]
|
||||
timestamp: 2026-08-21
|
||||
---
|
||||
|
||||
# Schema
|
||||
|
||||
Rule identity is the human-readable `ruleId` string — there are no GUIDs, deliberately: the ID appears in logs, mails, notifications, RuleResults, the agent config payload, and docs, and tenant isolation is provided by Table Storage partitioning instead.
|
||||
|
||||
| Store | PartitionKey | RowKey | Content |
|
||||
|---|---|---|---|
|
||||
| `GatherRules` / `AnalyzeRules` | `"global"` | ruleId | Built-in + community rules (shared, all tenants) |
|
||||
| `GatherRules` / `AnalyzeRules` | tenantId | ruleId | Custom rules (visible only to that tenant) |
|
||||
| `RuleStates` | tenantId | ruleId | Per-tenant enable/notify overrides — built-in/community rules ONLY (custom rules fold state into their own row) |
|
||||
| `RuleStats` | date | `{tenantId}_{ruleId}` and `global_{ruleId}` | Daily fire/eval telemetry; the global row is catalog-only (see guards) |
|
||||
|
||||
Namespace split (`RuleIdPolicy.cs`, shared model):
|
||||
|
||||
- **Reserved built-in namespace**: `(ANALYZE|GATHER)-<CATEGORY>-<NUMBER>` with `CATEGORY != CUSTOM`, case-insensitive, including currently unused numbers (gaps = retired rules that may return).
|
||||
- **Tenant namespace**: everything else — `ANALYZE-CUSTOM-NNN` (portal suggestion), `-CUSTOM` suffixes, org prefixes (`CONTOSO-WIFI-001`), freeform.
|
||||
|
||||
# Collision model
|
||||
|
||||
- **Custom vs built-in**: impossible via API — `CreateRuleAsync`/`UpdateRuleAsync` reject reserved IDs (409) and check existence against `global` ∪ own tenant partition. Legacy debris is neutralized at merge time: global wins, tenant copy dropped with a warning (pinned by `RuleMergeCollisionGuardTests`, motivated by the 5ca2b350 duplicate-key incident).
|
||||
- **Custom vs custom across tenants**: allowed and harmless — separate partitions, separate evaluation, separate RuleResults/tenant stats. Two tenants both using `ANALYZE-CUSTOM-101` never interact.
|
||||
- **JSON copy between tenants** (export → paste into create editor): goes through the same create path, so an ID already taken in the receiving tenant surfaces as 409 — the user renames. There is no bulk import endpoint. The JSON editors accept both shapes — a rule export and the serialized form — via `gatherRuleToForm` / `jsonToForm` (web `types.ts` of each rules page): nested gather `parameters` are unflattened, analyze `evaluateOn` maps onto the eval checkboxes, and `tags` plus the effective `markSessionAsFailed` survive the import.
|
||||
|
||||
# Guards (all enforced, 2026-08-21)
|
||||
|
||||
1. **Global RuleStats aggregate is catalog-only.** The `global_{ruleId}` row would otherwise sum same-ID custom rules of unrelated tenants (title/severity last-writer-wins). Analyze gates on `rule.IsBuiltIn || rule.IsCommunity` (`AnalyzeOnEnrollmentEndHandler`), gather gates on `RuleIdPolicy.IsReservedBuiltInId` (`EventIngestProcessor.RuleStats`) because agent events carry no flag — valid because custom rules can never occupy the reserved pattern.
|
||||
2. **Catalog build fails on namespace violations.** `rules/scripts/combine.js` rejects duplicate IDs (case-insensitive) and any gather/analyze built-in outside the reserved pattern; `BuiltInRuleCatalogPolicyTests` pins the same invariants on the embedded resource actually deployed.
|
||||
3. **GitHub reseed rejects non-reserved IDs** before writing to the `global` partition (`ReseedFromGitHubFunction`) — a seeded `*-CUSTOM-*` rule would silently shadow every tenant's same-ID custom rule at merge time.
|
||||
4. **Cross-tenant RuleState GC is namespace-gated.** `DeleteRuleStatesForRuleIdAcrossTenantsAsync` deletes by bare RowKey across all partitions; it now refuses non-reserved IDs with `failed=-1` (callers then skip the global catalog delete, same contract as enumeration failure).
|
||||
|
||||
Invariant chain: schema pattern + combine.js + reseed guard ⇒ the `global` partition only ever contains reserved-pattern IDs ⇒ the pattern is a safe built-in/custom discriminator everywhere no `IsBuiltIn` flag is available.
|
||||
|
||||
# Citations
|
||||
|
||||
- `src/Shared/AutopilotMonitor.Shared/Models/Rules/RuleIdPolicy.cs` — reserved pattern + rationale
|
||||
- `src/Backend/AutopilotMonitor.Functions/Services/AnalyzeRuleService.cs`, `GatherRuleService.cs` — create/update guards, merge collision guard
|
||||
- `src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Rules.cs` — key schemes, cross-tenant GC guard
|
||||
- `src/Backend/AutopilotMonitor.Functions.Tests/BuiltInRuleCatalogPolicyTests.cs`, `RuleIdReservationTests.cs`, `RuleMergeCollisionGuardTests.cs`
|
||||
- `rules/scripts/combine.js` — build-time namespace/duplicate guard
|
||||
@@ -0,0 +1,137 @@
|
||||
---
|
||||
type: Concept
|
||||
title: Customer Script Publishing — download.autopilotmonitor.com/agent/
|
||||
description: How the customer-facing PowerShell scripts reach the download alias, why publishing is decoupled from agent releases, and the guards that keep the published copy, the version badge and the portal's version oracle in agreement.
|
||||
resource: /.github/scripts/Publish-BootstrapScripts.ps1
|
||||
tags:
|
||||
- build
|
||||
- ci
|
||||
- release
|
||||
- bootstrap
|
||||
- download-alias
|
||||
timestamp: 2026-08-19T00:00:00+02:00
|
||||
---
|
||||
|
||||
# Overview
|
||||
|
||||
Four PowerShell scripts are served to customers from `https://download.autopilotmonitor.com/agent/`:
|
||||
|
||||
| Blob | Source | Consumed by |
|
||||
| --- | --- | --- |
|
||||
| `Install-AutopilotMonitor.ps1` | `scripts/Bootstrap/Install-AutopilotMonitor.ps1` | Intune platform script (customer uploads a copy); downloaded live on every device by the WDP bootstrap MSI |
|
||||
| `Install-AutopilotMonitor-Dev.ps1` | rendered from the same source | dev fleet |
|
||||
| `Test-ShouldBootstrapAgent.ps1` | `scripts/Bootstrap/Test-ShouldBootstrapAgent.ps1` | customer dry run via `irm ... \| iex` |
|
||||
| `Grant-AutopilotMonitorAddOn.ps1` | `scripts/CustomerSetup/Grant-AutopilotMonitorAddOn.ps1` | optional Graph permission grant |
|
||||
|
||||
`Uninstall-AutopilotMonitor.ps1` is deliberately not published — nothing links it, and an
|
||||
unauthenticated uninstall script on the public download host is not a feature.
|
||||
|
||||
The alias is a Front Door route in front of the `agent` container; see
|
||||
[url-registry.md](url-registry.md) for the host registry and the migration to the alias.
|
||||
|
||||
# Publishing chain
|
||||
|
||||
`.github/scripts/Publish-BootstrapScripts.ps1` is the single owner. Two workflows call it
|
||||
with identical arguments:
|
||||
|
||||
* **`publish-scripts.yml`** — on every push to `main` touching `scripts/Bootstrap/**`,
|
||||
`scripts/CustomerSetup/**` or the publisher itself, plus `workflow_dispatch` with a
|
||||
`dry_run` input.
|
||||
* **`build-agent.yml`** — inside the `publish_as_stable` branch, after `version.json` is
|
||||
uploaded, so a release lands a set consistent with what is already live.
|
||||
|
||||
Before this split, the script blobs rotated **only** on a stable agent release. A bootstrap
|
||||
fix therefore sat unpublished until the next agent cutover — while the WDP MSI downloads the
|
||||
published copy on every enrolling device. Script publishing is now decoupled from the agent
|
||||
release cadence; the release path still publishes, so it can never fall behind either.
|
||||
|
||||
Both workflows share the concurrency group `agent-blob-publish` (`cancel-in-progress: false`).
|
||||
Both write `version.json`, and the publisher's `bootstrapVersion` read-modify-write must not
|
||||
interleave with a release writing the agent fields.
|
||||
|
||||
# Publisher steps
|
||||
|
||||
1. **Parse** `$ScriptVersion` from the bootstrap source.
|
||||
2. **Dev render** — literal substitution of the two `$AgentDownloadUrl` / `$VersionJsonName`
|
||||
defaults. A missing anchor literal is a hard failure: a silently un-substituted dev script
|
||||
would point the dev fleet at the stable agent.
|
||||
3. **Version-bump guard** — if the published bootstrap changed *behaviour* under an unchanged
|
||||
`$ScriptVersion`, the run aborts. Comment and formatting changes publish with a warning.
|
||||
See [Version oracles](#version-oracles).
|
||||
4. **Upload** every blob with `Cache-Control: no-cache` (they all rotate in place), mirrored
|
||||
fail-soft to the legacy storage account.
|
||||
5. **Reconcile the version oracles.**
|
||||
6. **Verify through the alias** — re-download each blob and compare SHA-256, six attempts
|
||||
20 s apart. A persistent mismatch means blobs were written but Front Door serves stale
|
||||
content, and the error names the `az afd endpoint purge` command.
|
||||
|
||||
## Deterministic publish form
|
||||
|
||||
Every blob is published as **CRLF, UTF-8 without BOM**, normalized by the publisher rather
|
||||
than taken from the working copy. Git normalizes line endings on checkout — a Windows runner
|
||||
yields CRLF, a `core.autocrlf=false` clone yields LF — so publishing raw working-copy bytes
|
||||
would emit different blobs for identical source and make every byte comparison, guard and
|
||||
verification alike, depend on where the workflow ran. CRLF is also what PS 5.1 expects on the
|
||||
device and what has always been published.
|
||||
|
||||
The ASCII-only rule for `scripts/Bootstrap/*.ps1` (enforced by the gates below) is unrelated
|
||||
but adjacent: PS 5.1 reads BOM-less files as ANSI and corrupts multi-byte characters.
|
||||
|
||||
## Version oracles
|
||||
|
||||
Three places state the bootstrap script version, and all three are derived from
|
||||
`$ScriptVersion` in the source:
|
||||
|
||||
* `version.json.bootstrapVersion` — read by the docs badge. Written by the publisher as a
|
||||
read-modify-write under `If-Match`, so a concurrent agent release keeps its own fields.
|
||||
`build-agent.yml` seeds the field when it builds the manifest so the published manifest is
|
||||
never briefly missing it; the publisher owns and re-asserts it afterwards.
|
||||
* `AdminConfiguration.LatestBootstrapV2ScriptVersion` — read by the portal to tell a customer
|
||||
their uploaded copy is outdated. **Only** the publisher writes it; `build-agent.yml` writes
|
||||
the `LatestAgentV2*` fields and nothing else. One writer per field.
|
||||
* The `Bootstrap script version: v<x>` marker the script logs, parsed out of captured platform
|
||||
script stdout by `utils/bootstrapVersion.ts`.
|
||||
|
||||
The version-bump guard exists because those three would otherwise keep asserting a version
|
||||
that no longer matches what customers download: a behaviour change under an unchanged version
|
||||
is invisible to every consumer, and customers decide from that version whether to re-upload
|
||||
their Intune copy.
|
||||
|
||||
"Changed" means changed **behaviour**, matching what the source file's own comment on
|
||||
`$ScriptVersion` already asks for ("bump on meaningful changes"). The guard compares a code
|
||||
fingerprint — the PowerShell token stream with comments, newlines and line continuations
|
||||
removed — so comment fixes, typo corrections in comments and reflow publish with a
|
||||
`::warning::` instead of failing. Tokenising rather than regex-stripping keeps a `#` inside a
|
||||
string literal from being mistaken for a comment.
|
||||
|
||||
# Gates
|
||||
|
||||
`.github/workflows/bootstrap-script-gates.yml` is a `workflow_call` workflow holding the
|
||||
checks once:
|
||||
|
||||
* Pester suite under **Windows PowerShell 5.1** — the runtime IME uses for platform scripts.
|
||||
* ASCII-only scan over `scripts/Bootstrap/*.ps1`.
|
||||
* `Publish-BootstrapScripts.ps1 -DryRun` — exercises the dev-render anchors and the
|
||||
version-bump guard, and reports per blob whether the alias is current, stale or missing.
|
||||
|
||||
Two callers: `pester.yml` on pull requests (reports, does not block the merge button), and
|
||||
`publish-scripts.yml` on pushes to `main`, where the gates job is a `needs:` dependency of the
|
||||
publish job and therefore **blocks** the upload. `pester.yml` deliberately has no push trigger
|
||||
— it would duplicate the gate run with no consequence.
|
||||
|
||||
# Operating it
|
||||
|
||||
* Dry run and drift check, no credentials needed for the read paths:
|
||||
`pwsh .github/scripts/Publish-BootstrapScripts.ps1 -DryRun`
|
||||
* Republish without a code change: `workflow_dispatch` on **Publish customer scripts** with
|
||||
`dry_run=false`.
|
||||
* Front Door serving stale content after a successful upload:
|
||||
`az afd endpoint purge --resource-group rg-autopilotmonitor-prd-gwc --profile-name autopilotmonitor-fd --endpoint-name apm-download --content-paths '/agent/*'`
|
||||
|
||||
# Citations
|
||||
|
||||
* `.github/scripts/Publish-BootstrapScripts.ps1`
|
||||
* `.github/workflows/publish-scripts.yml`, `.github/workflows/bootstrap-script-gates.yml`, `.github/workflows/pester.yml`
|
||||
* `.github/workflows/build-agent.yml` (the `publish_as_stable` branch of the upload step)
|
||||
* `src/Agent/AutopilotMonitor.BootstrapMsi/Invoke-BootstrapDownload.ps1`
|
||||
* `src/Web/autopilot-monitor-web/utils/bootstrapVersion.ts`
|
||||
@@ -0,0 +1,113 @@
|
||||
{
|
||||
"$schema": "../schema/analyze-rule.schema.json",
|
||||
"ruleId": "ANALYZE-APP-016",
|
||||
"title": "Required Apps Never Started — ESP Apps Gate Starved",
|
||||
"description": "One or more required user-ESP apps never started installing while the ESP AccountSetup apps gate waited on them — the agent observed the ESP page exit with those apps still in a non-terminal, never-started state. Unlike ANALYZE-APP-015 (which reports that completion was DEFERRED because of a pending app, and stays silent once the session reaches a terminal verdict) this rule reports the delivery gap itself and fires regardless of the final outcome: an enrollment that Windows called done while required apps never ran is a successful enrollment with missing software, and the operator must see it.",
|
||||
"severity": "warning",
|
||||
"category": "apps",
|
||||
"version": "1.0.0",
|
||||
"author": "Autopilot Monitor",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed",
|
||||
"on_event:session_stalled"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 75,
|
||||
"confidenceThreshold": 70,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "starved_apps_present",
|
||||
"source": "event_count",
|
||||
"eventType": "app_install_starved",
|
||||
"operator": "count_gte",
|
||||
"value": "1",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "many_starved_apps",
|
||||
"source": "event_count",
|
||||
"eventType": "app_install_starved",
|
||||
"operator": "count_gte",
|
||||
"value": "5",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "appName",
|
||||
"source": "event_type",
|
||||
"eventType": "app_install_starved",
|
||||
"dataField": "appName",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "state",
|
||||
"source": "event_type",
|
||||
"eventType": "app_install_starved",
|
||||
"dataField": "state",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [
|
||||
{
|
||||
"signal": "many_starved_apps",
|
||||
"condition": "exists",
|
||||
"weight": 15
|
||||
},
|
||||
{
|
||||
"signal": "appName",
|
||||
"condition": "exists",
|
||||
"weight": 10
|
||||
}
|
||||
],
|
||||
"explanation": "At least one **required** app assigned to the user ESP never started installing. The Intune Management Extension never moved it out of its initial state (`{{state}}`) — it did not download, did not install, and did not fail — while the ESP AccountSetup apps gate was waiting on it. First app observed: `{{appName}}`.\n\nEvery starved app has its own `app_install_starved` timeline event carrying `appId`, `appName` and `state` — open the timeline and filter on that event type for the complete list.\n\nThis is a **delivery** problem, not an install error, and it is invisible in the Intune console: the app simply never appears. Typical causes:\n\n- far more required apps targeted at the ESP than the gate can service in the available window,\n- requirement rules that are never met on this device (OS / architecture / registry / script checks),\n- an unsatisfied dependency or supersedence chain,\n- an app that needs a context unavailable during provisioning (e.g. a VPN client waiting on a portal sign-in),\n- a mid-ESP reboot that cut the gate short — check whether the mid-ESP reboot advisory also fired for this session.\n\nNote that this finding is independent of the session verdict. A session can be marked **Succeeded** and still be missing every app listed here.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Shrink the ESP blocking set",
|
||||
"steps": [
|
||||
"Open the Enrollment Status Page profile and review the blocking app list — every required app there must finish before the user reaches the desktop.",
|
||||
"Move apps that do not have to be present at first logon from *required* to *available*, or remove them from the ESP's blocking apps selection.",
|
||||
"Prefer a small, deliberately chosen blocking set (security agent, VPN, management tooling) over the full application catalogue."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Unblock the individual apps",
|
||||
"steps": [
|
||||
"Collect the app names from the `app_install_starved` timeline events in this session.",
|
||||
"For each app, review its requirement rules and dependencies in Intune — IntuneManagementExtension.log records every requirement evaluation.",
|
||||
"Verify the assignment actually targets the signed-in user (user-targeted apps need an established Entra user affinity; check the hybrid user-affinity finding if it also fired).",
|
||||
"Re-run an enrollment and confirm the apps now reach a terminal state before the ESP exits."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Rule out a truncated gate",
|
||||
"steps": [
|
||||
"Check whether a mid-ESP reboot was observed for this session — a forced restart during AccountSetup cuts the apps gate short and can starve everything still queued.",
|
||||
"If so, reassign the reboot-forcing device policies to user groups so the restart lands after AccountSetup."
|
||||
]
|
||||
}
|
||||
],
|
||||
"relatedDocs": [
|
||||
{
|
||||
"title": "Microsoft Learn: Enrollment Status Page — blocking apps",
|
||||
"url": "https://learn.microsoft.com/en-us/intune/intune-service/enrollment/windows-enrollment-status"
|
||||
},
|
||||
{
|
||||
"title": "Microsoft Learn: Win32 app requirement rules",
|
||||
"url": "https://learn.microsoft.com/en-us/intune/intune-service/apps/apps-win32-add"
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"apps",
|
||||
"esp",
|
||||
"starvation",
|
||||
"assignment",
|
||||
"blocking",
|
||||
"delivery-gap"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"$schema": "../schema/analyze-rule.schema.json",
|
||||
"ruleId": "ANALYZE-DEV-008",
|
||||
"title": "Device Clock Wrong During Enrollment — Jump or Sustained Offset",
|
||||
"description": "The device's clock changed by five or more minutes mid-session (clock jump) or ran five or more minutes off server time for the whole session (sustained offset), measured against the server receive frame of every uploaded event. Complements ANALYZE-DEV-007, which is a one-shot NTP check at agent startup and misses devices where NTP was unreachable or the clock changed later. IME-log-derived events are excluded from the measurement.",
|
||||
"severity": "warning",
|
||||
"category": "device",
|
||||
"version": "1.0.0",
|
||||
"author": "Autopilot Monitor",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 60,
|
||||
"confidenceThreshold": 50,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "clock_jump",
|
||||
"source": "clock_skew",
|
||||
"skewMetric": "clock_jump",
|
||||
"operator": "gte",
|
||||
"value": "300",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "sustained_offset",
|
||||
"source": "clock_skew",
|
||||
"skewMetric": "sustained_offset",
|
||||
"operator": "gte",
|
||||
"value": "300",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [],
|
||||
"explanation": "This device's {{skewSummary}} (five-minute threshold). A clock that is off by five or more minutes breaks time-sensitive enrollment mechanisms: Kerberos tolerates at most 5 minutes of skew (Hybrid Join), Entra ID token validation allows about 5 minutes on token lifetimes, and freshly issued SCEP/PKCS certificates can be rejected as \"not yet valid\" when the clock is more than ~10 minutes behind the CA.\n\nUnlike ANALYZE-DEV-007 (a one-shot NTP comparison at agent startup), this finding is measured across the whole session against the server's receive time — it also catches devices where the NTP check could not run (UDP 123 blocked) or where the clock changed after startup (time sync correcting a wrong RTC, Windows auto-timezone, manual change).",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Fix the device clock and time source",
|
||||
"steps": [
|
||||
"Check the BIOS/RTC clock and the CMOS battery if the wrong time recurs on the same hardware.",
|
||||
"At OOBE, run 'w32tm /resync /force' (Shift+F10 command prompt) to sync with time.windows.com.",
|
||||
"Ensure UDP port 123 to time.windows.com is open in the deployment network.",
|
||||
"Review 'Set time automatically' and 'Set time zone automatically' behavior on the deployment image — automatic timezone changes mid-enrollment show up as clock jumps."
|
||||
]
|
||||
}
|
||||
],
|
||||
"relatedDocs": [
|
||||
{
|
||||
"title": "Microsoft Learn: Windows Autopilot known issues — device provisioning can fail (clock skew)",
|
||||
"url": "https://learn.microsoft.com/en-us/autopilot/known-issues"
|
||||
},
|
||||
{
|
||||
"title": "Microsoft Learn: Windows Autopilot requirements — Networking (NTP)",
|
||||
"url": "https://learn.microsoft.com/en-us/autopilot/requirements?tabs=networking"
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"device",
|
||||
"clock-skew",
|
||||
"clock-jump",
|
||||
"time-sync",
|
||||
"kerberos",
|
||||
"certificates",
|
||||
"known-issue"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"$schema": "../schema/analyze-rule.schema.json",
|
||||
"ruleId": "ANALYZE-DEV-009",
|
||||
"title": "Battery Critically Low During Enrollment",
|
||||
"description": "Detects that the device's battery dropped to 15% or less while enrolling on battery power (power_state_change threshold event from the agent's live power watcher). A device dying mid-enrollment leaves a half-provisioned state that typically needs a full reset; this rule surfaces the risk while there is still time to plug the device in. Uses a single required condition on the agent-stamped thresholdPercent field (the data field 'transition' on the event is unrelated to this rule's 'trigger' property).",
|
||||
"severity": "high",
|
||||
"category": "device",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed",
|
||||
"on_event:power_state_change"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 90,
|
||||
"confidenceThreshold": 60,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "battery_threshold_15",
|
||||
"source": "event_data",
|
||||
"eventType": "power_state_change",
|
||||
"dataField": "thresholdPercent",
|
||||
"operator": "equals",
|
||||
"value": "15",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"explanation": "The device's battery fell below **{{thresholdPercent}}%** while the enrollment was running on battery power. If the battery drains completely mid-enrollment, the device is left in a half-provisioned state — apps, policies, or the user profile may be incomplete, and in most cases the device has to be reset and re-enrolled. The agent detected the threshold crossing live via a Windows power-status notification; the exact percentage at that moment is recorded on the matched power_state_change event in the session timeline.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Secure power for the running enrollment",
|
||||
"steps": [
|
||||
"Connect the device to AC power immediately — the enrollment continues normally once power is stable.",
|
||||
"If the device shut down before completing, check whether the session reached the desktop; if not, reset the device (Autopilot reset or reimage) and re-enroll it plugged in.",
|
||||
"For technician/White Glove flows, make AC power part of the bench checklist — pre-provisioning on battery is the most common source of this alert.",
|
||||
"If this fires across many devices of one model, review that model's battery health (the hardware_spec event carries design vs. full-charge capacity)."
|
||||
]
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"device",
|
||||
"power",
|
||||
"battery",
|
||||
"environment"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"$schema": "../schema/analyze-rule.schema.json",
|
||||
"ruleId": "ANALYZE-DEV-010",
|
||||
"title": "Enrollment Switched From AC to Battery Power",
|
||||
"description": "Detects that the device lost AC power and switched to battery while the enrollment was running (power_state_change event with transition ac_to_battery from the agent's live power watcher). Uses a single required condition so the match is pinned to one event instance; the event payload carries the battery percentage at the moment of the switch. The agent debounces power flapping (5s) so a brief unplug/replug does not produce this event.",
|
||||
"severity": "warning",
|
||||
"category": "device",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed",
|
||||
"on_event:power_state_change"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 85,
|
||||
"confidenceThreshold": 60,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "switched_to_battery",
|
||||
"source": "event_data",
|
||||
"eventType": "power_state_change",
|
||||
"dataField": "transition",
|
||||
"operator": "equals",
|
||||
"value": "ac_to_battery",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"explanation": "The device switched from AC to battery power mid-enrollment. Enrollments are power-hungry — app downloads, Windows Update and disk encryption drain a battery quickly, and Windows may also throttle CPU and network on battery, slowing the enrollment down. The battery percentage at the moment of the switch is recorded on the matched power_state_change event in the session timeline; if the device stayed on battery, look there for subsequent threshold events (50/30/15%) — a later battery_to_ac event means power was restored.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Restore AC power",
|
||||
"steps": [
|
||||
"Reconnect the device to AC power; the enrollment continues normally.",
|
||||
"Check the session timeline for a subsequent battery_to_ac event (power restored) or threshold_crossed events (battery draining) to judge urgency.",
|
||||
"If this alert is frequent in a deployment area, check docking stations/USB-C hubs for loose power delivery — a flapping dock is a common cause.",
|
||||
"Make 'plugged in during setup' part of the end-user/technician enrollment instructions."
|
||||
]
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"device",
|
||||
"power",
|
||||
"battery",
|
||||
"environment"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"$schema": "../schema/analyze-rule.schema.json",
|
||||
"ruleId": "ANALYZE-DEV-012",
|
||||
"title": "Device Entered Standby During Enrollment",
|
||||
"description": "Detects that the device completed a sleep, hibernate or Modern Standby episode of at least 5 minutes while the enrollment was running (system_sleep_episode event from the agent's system timeline watcher, recorded retroactively at wake from the Windows event log — pre-agent episodes are backfilled). Uses a single required condition on the episode's durationSeconds so the match is pinned to one event instance and the interpolated duration is the matched value. Short Modern Standby screen-off scenarios (under 60 s of real sleep) never emit the event, and this rule adds a 5-minute floor so brief lid-close moments do not fire it.",
|
||||
"severity": "info",
|
||||
"category": "device",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed",
|
||||
"on_event:system_sleep_episode"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 90,
|
||||
"confidenceThreshold": 60,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "slept_during_enrollment",
|
||||
"source": "event_data",
|
||||
"eventType": "system_sleep_episode",
|
||||
"dataField": "durationSeconds",
|
||||
"operator": "gte",
|
||||
"value": "300",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"explanation": "The device was asleep for {{durationSeconds}} seconds during the enrollment window. Nothing progresses while the device sleeps — downloads pause, the IME stops processing and the ESP simply waits — so the wall-clock enrollment duration includes this pause and can look much slower than the enrollment actually was. The matched system_sleep_episode event in the session timeline carries the exact enter/exit times, the episode kind (sleep, hibernate or Modern Standby) and, for Modern Standby, whether the device was on AC power. A long unexplained gap in the timeline right before that event is this pause, not missing telemetry.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Keep the device awake during enrollment",
|
||||
"steps": [
|
||||
"Check the matched system_sleep_episode event in the timeline for the episode bounds and kind — the enrollment duration minus the episode duration is the real active time.",
|
||||
"If the device slept on battery, connect AC power during setup; Windows sleeps far more aggressively on battery.",
|
||||
"Instruct users/technicians to keep the lid open and the device powered until the enrollment finishes.",
|
||||
"For recurring cases, review the power plan applied during OOBE/ESP (e.g. via provisioning or Intune policy) — consider disabling sleep on AC for the enrollment window."
|
||||
]
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"device",
|
||||
"power",
|
||||
"standby",
|
||||
"sleep",
|
||||
"environment"
|
||||
]
|
||||
}
|
||||
@@ -5,12 +5,13 @@
|
||||
"description": "Attributes an observed enrollment reboot to device-assigned MDM policies that Windows flagged as reboot-required. Fires ONLY when both facts are present in the session: at least one actually-observed reboot (system_reboot_detected) AND reboot-required policy flags (mdm_policy_reboot_required). Policy flags alone — e.g. from a user-policy sync during AccountSetup — force no reboot and stay advisory-silent.",
|
||||
"severity": "warning",
|
||||
"category": "esp",
|
||||
"version": "1.2.0",
|
||||
"version": "1.3.0",
|
||||
"author": "Autopilot Monitor",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed"
|
||||
"whiteglove_sealed",
|
||||
"on_event:session_stalled"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 70,
|
||||
|
||||
+325
-2
@@ -831,6 +831,118 @@
|
||||
"pending"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-APP-016",
|
||||
"title": "Required Apps Never Started — ESP Apps Gate Starved",
|
||||
"description": "One or more required user-ESP apps never started installing while the ESP AccountSetup apps gate waited on them — the agent observed the ESP page exit with those apps still in a non-terminal, never-started state. Unlike ANALYZE-APP-015 (which reports that completion was DEFERRED because of a pending app, and stays silent once the session reaches a terminal verdict) this rule reports the delivery gap itself and fires regardless of the final outcome: an enrollment that Windows called done while required apps never ran is a successful enrollment with missing software, and the operator must see it.",
|
||||
"severity": "warning",
|
||||
"category": "apps",
|
||||
"version": "1.0.0",
|
||||
"author": "Autopilot Monitor",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed",
|
||||
"on_event:session_stalled"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 75,
|
||||
"confidenceThreshold": 70,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "starved_apps_present",
|
||||
"source": "event_count",
|
||||
"eventType": "app_install_starved",
|
||||
"operator": "count_gte",
|
||||
"value": "1",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"signal": "many_starved_apps",
|
||||
"source": "event_count",
|
||||
"eventType": "app_install_starved",
|
||||
"operator": "count_gte",
|
||||
"value": "5",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "appName",
|
||||
"source": "event_type",
|
||||
"eventType": "app_install_starved",
|
||||
"dataField": "appName",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "state",
|
||||
"source": "event_type",
|
||||
"eventType": "app_install_starved",
|
||||
"dataField": "state",
|
||||
"operator": "exists",
|
||||
"value": "",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [
|
||||
{
|
||||
"signal": "many_starved_apps",
|
||||
"condition": "exists",
|
||||
"weight": 15
|
||||
},
|
||||
{
|
||||
"signal": "appName",
|
||||
"condition": "exists",
|
||||
"weight": 10
|
||||
}
|
||||
],
|
||||
"explanation": "At least one **required** app assigned to the user ESP never started installing. The Intune Management Extension never moved it out of its initial state (`{{state}}`) — it did not download, did not install, and did not fail — while the ESP AccountSetup apps gate was waiting on it. First app observed: `{{appName}}`.\n\nEvery starved app has its own `app_install_starved` timeline event carrying `appId`, `appName` and `state` — open the timeline and filter on that event type for the complete list.\n\nThis is a **delivery** problem, not an install error, and it is invisible in the Intune console: the app simply never appears. Typical causes:\n\n- far more required apps targeted at the ESP than the gate can service in the available window,\n- requirement rules that are never met on this device (OS / architecture / registry / script checks),\n- an unsatisfied dependency or supersedence chain,\n- an app that needs a context unavailable during provisioning (e.g. a VPN client waiting on a portal sign-in),\n- a mid-ESP reboot that cut the gate short — check whether the mid-ESP reboot advisory also fired for this session.\n\nNote that this finding is independent of the session verdict. A session can be marked **Succeeded** and still be missing every app listed here.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Shrink the ESP blocking set",
|
||||
"steps": [
|
||||
"Open the Enrollment Status Page profile and review the blocking app list — every required app there must finish before the user reaches the desktop.",
|
||||
"Move apps that do not have to be present at first logon from *required* to *available*, or remove them from the ESP's blocking apps selection.",
|
||||
"Prefer a small, deliberately chosen blocking set (security agent, VPN, management tooling) over the full application catalogue."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Unblock the individual apps",
|
||||
"steps": [
|
||||
"Collect the app names from the `app_install_starved` timeline events in this session.",
|
||||
"For each app, review its requirement rules and dependencies in Intune — IntuneManagementExtension.log records every requirement evaluation.",
|
||||
"Verify the assignment actually targets the signed-in user (user-targeted apps need an established Entra user affinity; check the hybrid user-affinity finding if it also fired).",
|
||||
"Re-run an enrollment and confirm the apps now reach a terminal state before the ESP exits."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Rule out a truncated gate",
|
||||
"steps": [
|
||||
"Check whether a mid-ESP reboot was observed for this session — a forced restart during AccountSetup cuts the apps gate short and can starve everything still queued.",
|
||||
"If so, reassign the reboot-forcing device policies to user groups so the restart lands after AccountSetup."
|
||||
]
|
||||
}
|
||||
],
|
||||
"relatedDocs": [
|
||||
{
|
||||
"title": "Microsoft Learn: Enrollment Status Page — blocking apps",
|
||||
"url": "https://learn.microsoft.com/en-us/intune/intune-service/enrollment/windows-enrollment-status"
|
||||
},
|
||||
{
|
||||
"title": "Microsoft Learn: Win32 app requirement rules",
|
||||
"url": "https://learn.microsoft.com/en-us/intune/intune-service/apps/apps-win32-add"
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"apps",
|
||||
"esp",
|
||||
"starvation",
|
||||
"assignment",
|
||||
"blocking",
|
||||
"delivery-gap"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-CORR-003",
|
||||
"title": "Proxy Configuration Causing Download Failure",
|
||||
@@ -1404,6 +1516,216 @@
|
||||
"known-issue"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-DEV-008",
|
||||
"title": "Device Clock Wrong During Enrollment — Jump or Sustained Offset",
|
||||
"description": "The device's clock changed by five or more minutes mid-session (clock jump) or ran five or more minutes off server time for the whole session (sustained offset), measured against the server receive frame of every uploaded event. Complements ANALYZE-DEV-007, which is a one-shot NTP check at agent startup and misses devices where NTP was unreachable or the clock changed later. IME-log-derived events are excluded from the measurement.",
|
||||
"severity": "warning",
|
||||
"category": "device",
|
||||
"version": "1.0.0",
|
||||
"author": "Autopilot Monitor",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 60,
|
||||
"confidenceThreshold": 50,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "clock_jump",
|
||||
"source": "clock_skew",
|
||||
"skewMetric": "clock_jump",
|
||||
"operator": "gte",
|
||||
"value": "300",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"signal": "sustained_offset",
|
||||
"source": "clock_skew",
|
||||
"skewMetric": "sustained_offset",
|
||||
"operator": "gte",
|
||||
"value": "300",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"confidenceFactors": [],
|
||||
"explanation": "This device's {{skewSummary}} (five-minute threshold). A clock that is off by five or more minutes breaks time-sensitive enrollment mechanisms: Kerberos tolerates at most 5 minutes of skew (Hybrid Join), Entra ID token validation allows about 5 minutes on token lifetimes, and freshly issued SCEP/PKCS certificates can be rejected as \"not yet valid\" when the clock is more than ~10 minutes behind the CA.\n\nUnlike ANALYZE-DEV-007 (a one-shot NTP comparison at agent startup), this finding is measured across the whole session against the server's receive time — it also catches devices where the NTP check could not run (UDP 123 blocked) or where the clock changed after startup (time sync correcting a wrong RTC, Windows auto-timezone, manual change).",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Fix the device clock and time source",
|
||||
"steps": [
|
||||
"Check the BIOS/RTC clock and the CMOS battery if the wrong time recurs on the same hardware.",
|
||||
"At OOBE, run 'w32tm /resync /force' (Shift+F10 command prompt) to sync with time.windows.com.",
|
||||
"Ensure UDP port 123 to time.windows.com is open in the deployment network.",
|
||||
"Review 'Set time automatically' and 'Set time zone automatically' behavior on the deployment image — automatic timezone changes mid-enrollment show up as clock jumps."
|
||||
]
|
||||
}
|
||||
],
|
||||
"relatedDocs": [
|
||||
{
|
||||
"title": "Microsoft Learn: Windows Autopilot known issues — device provisioning can fail (clock skew)",
|
||||
"url": "https://learn.microsoft.com/en-us/autopilot/known-issues"
|
||||
},
|
||||
{
|
||||
"title": "Microsoft Learn: Windows Autopilot requirements — Networking (NTP)",
|
||||
"url": "https://learn.microsoft.com/en-us/autopilot/requirements?tabs=networking"
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"device",
|
||||
"clock-skew",
|
||||
"clock-jump",
|
||||
"time-sync",
|
||||
"kerberos",
|
||||
"certificates",
|
||||
"known-issue"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-DEV-009",
|
||||
"title": "Battery Critically Low During Enrollment",
|
||||
"description": "Detects that the device's battery dropped to 15% or less while enrolling on battery power (power_state_change threshold event from the agent's live power watcher). A device dying mid-enrollment leaves a half-provisioned state that typically needs a full reset; this rule surfaces the risk while there is still time to plug the device in. Uses a single required condition on the agent-stamped thresholdPercent field (the data field 'transition' on the event is unrelated to this rule's 'trigger' property).",
|
||||
"severity": "high",
|
||||
"category": "device",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed",
|
||||
"on_event:power_state_change"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 90,
|
||||
"confidenceThreshold": 60,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "battery_threshold_15",
|
||||
"source": "event_data",
|
||||
"eventType": "power_state_change",
|
||||
"dataField": "thresholdPercent",
|
||||
"operator": "equals",
|
||||
"value": "15",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"explanation": "The device's battery fell below **{{thresholdPercent}}%** while the enrollment was running on battery power. If the battery drains completely mid-enrollment, the device is left in a half-provisioned state — apps, policies, or the user profile may be incomplete, and in most cases the device has to be reset and re-enrolled. The agent detected the threshold crossing live via a Windows power-status notification; the exact percentage at that moment is recorded on the matched power_state_change event in the session timeline.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Secure power for the running enrollment",
|
||||
"steps": [
|
||||
"Connect the device to AC power immediately — the enrollment continues normally once power is stable.",
|
||||
"If the device shut down before completing, check whether the session reached the desktop; if not, reset the device (Autopilot reset or reimage) and re-enroll it plugged in.",
|
||||
"For technician/White Glove flows, make AC power part of the bench checklist — pre-provisioning on battery is the most common source of this alert.",
|
||||
"If this fires across many devices of one model, review that model's battery health (the hardware_spec event carries design vs. full-charge capacity)."
|
||||
]
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"device",
|
||||
"power",
|
||||
"battery",
|
||||
"environment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-DEV-010",
|
||||
"title": "Enrollment Switched From AC to Battery Power",
|
||||
"description": "Detects that the device lost AC power and switched to battery while the enrollment was running (power_state_change event with transition ac_to_battery from the agent's live power watcher). Uses a single required condition so the match is pinned to one event instance; the event payload carries the battery percentage at the moment of the switch. The agent debounces power flapping (5s) so a brief unplug/replug does not produce this event.",
|
||||
"severity": "warning",
|
||||
"category": "device",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed",
|
||||
"on_event:power_state_change"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 85,
|
||||
"confidenceThreshold": 60,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "switched_to_battery",
|
||||
"source": "event_data",
|
||||
"eventType": "power_state_change",
|
||||
"dataField": "transition",
|
||||
"operator": "equals",
|
||||
"value": "ac_to_battery",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"explanation": "The device switched from AC to battery power mid-enrollment. Enrollments are power-hungry — app downloads, Windows Update and disk encryption drain a battery quickly, and Windows may also throttle CPU and network on battery, slowing the enrollment down. The battery percentage at the moment of the switch is recorded on the matched power_state_change event in the session timeline; if the device stayed on battery, look there for subsequent threshold events (50/30/15%) — a later battery_to_ac event means power was restored.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Restore AC power",
|
||||
"steps": [
|
||||
"Reconnect the device to AC power; the enrollment continues normally.",
|
||||
"Check the session timeline for a subsequent battery_to_ac event (power restored) or threshold_crossed events (battery draining) to judge urgency.",
|
||||
"If this alert is frequent in a deployment area, check docking stations/USB-C hubs for loose power delivery — a flapping dock is a common cause.",
|
||||
"Make 'plugged in during setup' part of the end-user/technician enrollment instructions."
|
||||
]
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"device",
|
||||
"power",
|
||||
"battery",
|
||||
"environment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-DEV-012",
|
||||
"title": "Device Entered Standby During Enrollment",
|
||||
"description": "Detects that the device completed a sleep, hibernate or Modern Standby episode of at least 5 minutes while the enrollment was running (system_sleep_episode event from the agent's system timeline watcher, recorded retroactively at wake from the Windows event log — pre-agent episodes are backfilled). Uses a single required condition on the episode's durationSeconds so the match is pinned to one event instance and the interpolated duration is the matched value. Short Modern Standby screen-off scenarios (under 60 s of real sleep) never emit the event, and this rule adds a 5-minute floor so brief lid-close moments do not fire it.",
|
||||
"severity": "info",
|
||||
"category": "device",
|
||||
"version": "1.0.0",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed",
|
||||
"on_event:system_sleep_episode"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 90,
|
||||
"confidenceThreshold": 60,
|
||||
"conditions": [
|
||||
{
|
||||
"signal": "slept_during_enrollment",
|
||||
"source": "event_data",
|
||||
"eventType": "system_sleep_episode",
|
||||
"dataField": "durationSeconds",
|
||||
"operator": "gte",
|
||||
"value": "300",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"explanation": "The device was asleep for {{durationSeconds}} seconds during the enrollment window. Nothing progresses while the device sleeps — downloads pause, the IME stops processing and the ESP simply waits — so the wall-clock enrollment duration includes this pause and can look much slower than the enrollment actually was. The matched system_sleep_episode event in the session timeline carries the exact enter/exit times, the episode kind (sleep, hibernate or Modern Standby) and, for Modern Standby, whether the device was on AC power. A long unexplained gap in the timeline right before that event is this pause, not missing telemetry.",
|
||||
"remediation": [
|
||||
{
|
||||
"title": "Keep the device awake during enrollment",
|
||||
"steps": [
|
||||
"Check the matched system_sleep_episode event in the timeline for the episode bounds and kind — the enrollment duration minus the episode duration is the real active time.",
|
||||
"If the device slept on battery, connect AC power during setup; Windows sleeps far more aggressively on battery.",
|
||||
"Instruct users/technicians to keep the lid open and the device powered until the enrollment finishes.",
|
||||
"For recurring cases, review the power plan applied during OOBE/ESP (e.g. via provisioning or Intune policy) — consider disabling sleep on AC for the enrollment window."
|
||||
]
|
||||
}
|
||||
],
|
||||
"markSessionAsFailedDefault": false,
|
||||
"tags": [
|
||||
"device",
|
||||
"power",
|
||||
"standby",
|
||||
"sleep",
|
||||
"environment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ruleId": "ANALYZE-ENRL-001",
|
||||
"title": "Enrollment Failed",
|
||||
@@ -2069,12 +2391,13 @@
|
||||
"description": "Attributes an observed enrollment reboot to device-assigned MDM policies that Windows flagged as reboot-required. Fires ONLY when both facts are present in the session: at least one actually-observed reboot (system_reboot_detected) AND reboot-required policy flags (mdm_policy_reboot_required). Policy flags alone — e.g. from a user-policy sync during AccountSetup — force no reboot and stay advisory-silent.",
|
||||
"severity": "warning",
|
||||
"category": "esp",
|
||||
"version": "1.2.0",
|
||||
"version": "1.3.0",
|
||||
"author": "Autopilot Monitor",
|
||||
"enabled": true,
|
||||
"evaluateOn": [
|
||||
"enrollment_end",
|
||||
"whiteglove_sealed"
|
||||
"whiteglove_sealed",
|
||||
"on_event:session_stalled"
|
||||
],
|
||||
"trigger": "single",
|
||||
"baseConfidence": 70,
|
||||
|
||||
@@ -169,7 +169,12 @@
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "Source of the signal.",
|
||||
"enum": ["event_type", "event_data", "event_data_array", "phase_duration", "event_count", "app_install_duration", "event_correlation"]
|
||||
"enum": ["event_type", "event_data", "event_data_array", "phase_duration", "event_count", "app_install_duration", "event_correlation", "clock_skew"]
|
||||
},
|
||||
"skewMetric": {
|
||||
"type": "string",
|
||||
"description": "For clock_skew only: which device-clock metric to evaluate. clock_jump = persistent step in the device's clock frame mid-session; sustained_offset = whole session ran on a clock off by at least the threshold. value is the threshold in seconds; operator is limited to gt/gte on the magnitude. IME-log-derived events are excluded from the measurement.",
|
||||
"enum": ["clock_jump", "sustained_offset"]
|
||||
},
|
||||
"eventType": {
|
||||
"type": "string",
|
||||
|
||||
@@ -13,18 +13,26 @@ const rulesRoot = path.resolve(__dirname, '..');
|
||||
|
||||
// ── Rule combination ────────────────────────────────────────────────────────
|
||||
|
||||
// Shipped gather/analyze rules must stay inside the reserved built-in namespace:
|
||||
// (ANALYZE|GATHER)-<CATEGORY>-<NNN> with CATEGORY !== CUSTOM. The CUSTOM category is
|
||||
// the sanctioned tenant namespace (RuleIdPolicy.cs) — a built-in shipped there would
|
||||
// silently shadow every tenant's same-ID custom rule at merge time (global wins).
|
||||
const RESERVED_BUILTIN_ID = /^(ANALYZE|GATHER)-(?!CUSTOM-)[A-Z]+-\d{3}$/;
|
||||
|
||||
const configs = [
|
||||
{
|
||||
dir: path.join(rulesRoot, 'gather'),
|
||||
output: path.join(rulesRoot, 'dist', 'gather-rules.json'),
|
||||
schema: '../schema/gather-rule.schema.json',
|
||||
idField: 'ruleId'
|
||||
idField: 'ruleId',
|
||||
reservedNamespace: RESERVED_BUILTIN_ID
|
||||
},
|
||||
{
|
||||
dir: path.join(rulesRoot, 'analyze'),
|
||||
output: path.join(rulesRoot, 'dist', 'analyze-rules.json'),
|
||||
schema: '../schema/analyze-rule.schema.json',
|
||||
idField: 'ruleId'
|
||||
idField: 'ruleId',
|
||||
reservedNamespace: RESERVED_BUILTIN_ID
|
||||
},
|
||||
{
|
||||
dir: path.join(rulesRoot, 'ime-log-patterns'),
|
||||
@@ -37,12 +45,27 @@ const configs = [
|
||||
for (const config of configs) {
|
||||
const files = fs.readdirSync(config.dir).filter(f => f.endsWith('.json')).sort();
|
||||
const rules = [];
|
||||
const seenIds = new Map(); // lowercased id -> filename (case variants collide for humans and RuleIdPolicy)
|
||||
|
||||
for (const file of files) {
|
||||
const content = fs.readFileSync(path.join(config.dir, file), 'utf8');
|
||||
const rule = JSON.parse(content);
|
||||
// Remove $schema from individual entries (it's on the wrapper)
|
||||
delete rule['$schema'];
|
||||
|
||||
const id = rule[config.idField] || '';
|
||||
const idKey = id.toLowerCase();
|
||||
if (seenIds.has(idKey)) {
|
||||
console.error(`ERROR: duplicate ${config.idField} "${id}" in ${file} (already defined in ${seenIds.get(idKey)})`);
|
||||
process.exit(1);
|
||||
}
|
||||
seenIds.set(idKey, file);
|
||||
|
||||
if (config.reservedNamespace && !config.reservedNamespace.test(id)) {
|
||||
console.error(`ERROR: ${file}: ${config.idField} "${id}" is outside the reserved built-in namespace ${config.reservedNamespace} — built-ins must never use the tenant CUSTOM namespace`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
rules.push(rule);
|
||||
}
|
||||
|
||||
|
||||
@@ -28,10 +28,13 @@
|
||||
(validates Windows 365 Cloud PCs against the tenant's Cloud PC inventory --
|
||||
Cloud PCs are never Autopilot-registered, so this enables agent monitoring
|
||||
of Cloud PC first-connect enrollment)
|
||||
- All -> every optional permission above, in one run
|
||||
(also works with -Revoke to remove all optional grants at once)
|
||||
|
||||
.PARAMETER Permissions
|
||||
Microsoft Graph application permission names to grant directly. Use when the admin UI
|
||||
rendered a copy-paste command for you, or when you have a specific list in mind.
|
||||
Microsoft Graph application permission names to grant directly. Use when you copied the
|
||||
permission list from the admin UI ("Copy permissions"), or when you have a specific list
|
||||
in mind.
|
||||
|
||||
.PARAMETER TenantId
|
||||
Tenant ID (GUID) or verified domain. Optional -- defaults to the signed-in context.
|
||||
@@ -47,12 +50,16 @@
|
||||
.\Grant-AutopilotMonitorAddOn.ps1 -ClientId "<your-client-id>" -Features ScriptDisplayNames
|
||||
|
||||
.EXAMPLE
|
||||
# Same effect, using the explicit-permission form (what the admin UI generates):
|
||||
# Same effect, using the explicit-permission form:
|
||||
.\Grant-AutopilotMonitorAddOn.ps1 `
|
||||
-ClientId "<your-client-id>" `
|
||||
-Permissions "DeviceManagementScripts.Read.All" `
|
||||
-TenantId "contoso.onmicrosoft.com"
|
||||
|
||||
.EXAMPLE
|
||||
# Grant every optional capability in one run:
|
||||
.\Grant-AutopilotMonitorAddOn.ps1 -ClientId "<your-client-id>" -Features All
|
||||
|
||||
.EXAMPLE
|
||||
# Verify current state only, no changes:
|
||||
.\Grant-AutopilotMonitorAddOn.ps1 -ClientId "<your-client-id>" -Features ScriptDisplayNames -VerifyOnly
|
||||
@@ -65,6 +72,12 @@
|
||||
Requires:
|
||||
- Microsoft.Graph.Authentication PowerShell module (auto-installs if missing).
|
||||
- Sign-in as Global Administrator OR Privileged Role Administrator OR Cloud Application Administrator.
|
||||
|
||||
Sign-in behaviour:
|
||||
In Azure Cloud Shell the script signs in via the ambient identity endpoint (a delegated
|
||||
token for the already-signed-in user) - no device-code prompt, which Conditional Access
|
||||
policies commonly block. Outside Cloud Shell it falls back to a normal interactive
|
||||
Microsoft Graph sign-in. -TenantId only applies to the interactive fallback.
|
||||
- The Autopilot Monitor app must be admin-consented in your tenant first
|
||||
(its service principal must exist before grants can be added).
|
||||
|
||||
@@ -80,7 +93,7 @@ param(
|
||||
[string]$ClientId,
|
||||
|
||||
[Parameter(Mandatory = $true, ParameterSetName = 'ByFeatures')]
|
||||
[ValidateSet('ScriptDisplayNames', 'W365CloudPcValidation')]
|
||||
[ValidateSet('All', 'ScriptDisplayNames', 'W365CloudPcValidation')]
|
||||
[string[]]$Features,
|
||||
|
||||
[Parameter(Mandatory = $true, ParameterSetName = 'ByPermissions')]
|
||||
@@ -106,7 +119,11 @@ $FeatureCatalog = @{
|
||||
}
|
||||
|
||||
# Translate Features -> Permissions when caller used the high-level form.
|
||||
# 'All' is a script-side meta-value: it expands to every catalog feature.
|
||||
if ($PSCmdlet.ParameterSetName -eq 'ByFeatures') {
|
||||
if ($Features -contains 'All') {
|
||||
$Features = @($FeatureCatalog.Keys)
|
||||
}
|
||||
$Permissions = @()
|
||||
foreach ($f in $Features) {
|
||||
$Permissions += $FeatureCatalog[$f]
|
||||
@@ -133,7 +150,25 @@ if ($TenantId) { $connectParams['TenantId'] = $TenantId }
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "[Auth] Connecting to Microsoft Graph..." -ForegroundColor Cyan
|
||||
Connect-MgGraph @connectParams | Out-Null
|
||||
|
||||
# Preferred path: ambient identity. In Azure Cloud Shell the local identity endpoint issues a
|
||||
# delegated Graph token for the signed-in user, which avoids the device-code flow that
|
||||
# Conditional Access policies commonly block. Outside Azure this throws and we fall back to
|
||||
# an interactive sign-in with explicit scopes.
|
||||
$identityConnected = $false
|
||||
try {
|
||||
Connect-MgGraph -Identity -NoWelcome -ErrorAction Stop | Out-Null
|
||||
# An ambient identity can exist yet hold no Graph permissions (e.g. a plain VM managed
|
||||
# identity) - probe with a read before committing to this session.
|
||||
Invoke-MgGraphRequest -Method GET -Uri "https://graph.microsoft.com/v1.0/servicePrincipals?`$select=id&`$top=1" -ErrorAction Stop | Out-Null
|
||||
$identityConnected = $true
|
||||
Write-Host "[Auth] Connected via ambient identity (Azure Cloud Shell / managed identity)." -ForegroundColor Green
|
||||
}
|
||||
catch {
|
||||
if (Get-MgContext) { Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null }
|
||||
Write-Host "[Auth] No usable ambient identity - falling back to interactive sign-in..." -ForegroundColor Yellow
|
||||
Connect-MgGraph @connectParams | Out-Null
|
||||
}
|
||||
|
||||
$ctx = Get-MgContext
|
||||
if (-not $ctx) {
|
||||
@@ -144,7 +179,10 @@ Write-Host "[Auth] Signed in as : $($ctx.Account)"
|
||||
Write-Host "[Auth] Tenant : $($ctx.TenantId)"
|
||||
Write-Host "[Auth] Granted scopes: $($ctx.Scopes -join ', ')"
|
||||
|
||||
if (-not ($ctx.Scopes -contains 'AppRoleAssignment.ReadWrite.All')) {
|
||||
# Scope check only applies to the interactive path: an ambient-identity (Cloud Shell) token
|
||||
# carries a fixed scope set (incl. Directory.AccessAsUser.All) instead of the requested scopes,
|
||||
# and its effective rights follow the signed-in user's directory role.
|
||||
if (-not $identityConnected -and -not ($ctx.Scopes -contains 'AppRoleAssignment.ReadWrite.All')) {
|
||||
Write-Warning "Scope 'AppRoleAssignment.ReadWrite.All' was NOT granted. The signed-in user likely lacks the required role (Global Admin / Privileged Role Admin / Cloud App Admin)."
|
||||
if (-not $VerifyOnly) { return }
|
||||
}
|
||||
|
||||
+4
-2
@@ -37,6 +37,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
nameof(AgentConfigResponse.EnrollmentSummaryLaunchRetrySeconds),
|
||||
nameof(AgentConfigResponse.NtpServer),
|
||||
nameof(AgentConfigResponse.EnableTimezoneAutoSet),
|
||||
nameof(AgentConfigResponse.EnableDoGroupIdAutoSet),
|
||||
nameof(AgentConfigResponse.DiagnosticsUploadEnabled),
|
||||
nameof(AgentConfigResponse.DiagnosticsUploadMode),
|
||||
nameof(AgentConfigResponse.DiagnosticsLogPaths),
|
||||
@@ -52,6 +53,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
// Merge reads two Collectors knobs (AgentMaxLifetimeMinutes, HelloWaitTimeoutSeconds);
|
||||
// the remaining CollectorConfiguration flags flow untouched to DefaultComponentFactory.
|
||||
nameof(AgentConfigResponse.Collectors),
|
||||
// Merge reads one Analyzers knob (EnableRealmJoinWatcher → diagnostics RealmJoin sections);
|
||||
// the remaining flags flow untouched to DefaultComponentFactory + AgentAnalyzerManager.
|
||||
nameof(AgentConfigResponse.Analyzers),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
@@ -69,8 +73,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
nameof(AgentConfigResponse.ImeLogPatterns),
|
||||
// Passed by AgentRuntimeHost into IComponentFactory.CreateCollectorHosts → ImeLogHost sealing re-emit.
|
||||
nameof(AgentConfigResponse.WhiteGloveSealingPatternIds),
|
||||
// Consumed by DefaultComponentFactory (RealmJoin/KeepAwake/ConsoleBypass host gates) + AgentAnalyzerManager.
|
||||
nameof(AgentConfigResponse.Analyzers),
|
||||
// Self-update ZIP integrity second trust channel — BinaryIntegrityVerifier / Program update path.
|
||||
nameof(AgentConfigResponse.LatestAgentSha256),
|
||||
// Running-EXE integrity verification — BinaryIntegrityVerifier post-config check.
|
||||
|
||||
+29
@@ -43,6 +43,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
EnrollmentSummaryLaunchRetrySeconds = 300,
|
||||
NtpServer = "pool.ntp.org",
|
||||
EnableTimezoneAutoSet = true,
|
||||
EnableDoGroupIdAutoSet = true,
|
||||
DiagnosticsUploadEnabled = true,
|
||||
DiagnosticsUploadMode = "OnFailure",
|
||||
DiagnosticsLogPaths = new List<DiagnosticsLogPath>
|
||||
@@ -83,6 +84,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(300, agent.EnrollmentSummaryLaunchRetrySeconds);
|
||||
Assert.Equal("pool.ntp.org", agent.NtpServer);
|
||||
Assert.True(agent.EnableTimezoneAutoSet);
|
||||
Assert.True(agent.EnableDoGroupIdAutoSet);
|
||||
Assert.True(agent.DiagnosticsUploadEnabled);
|
||||
Assert.Equal("OnFailure", agent.DiagnosticsUploadMode);
|
||||
Assert.Single(agent.DiagnosticsLogPaths);
|
||||
@@ -366,6 +368,33 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(30, agent.HelloWaitTimeoutSeconds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Merge_mirrors_analyzers_realmjoin_watcher_onto_agent_config()
|
||||
{
|
||||
// The diagnostics package gates its RealmJoin sections on this flag but never
|
||||
// sees the remote response — the merger carries the one Analyzers knob over.
|
||||
var agent = NewAgentConfig();
|
||||
var remote = FullRemote();
|
||||
remote.Analyzers = new AnalyzerConfiguration { EnableRealmJoinWatcher = true };
|
||||
|
||||
RemoteConfigMerger.Merge(agent, remote);
|
||||
|
||||
Assert.True(agent.EnableRealmJoinWatcher);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Merge_clears_realmjoin_watcher_when_remote_analyzers_missing()
|
||||
{
|
||||
var agent = NewAgentConfig();
|
||||
agent.EnableRealmJoinWatcher = true; // stale cached true must not survive
|
||||
var remote = FullRemote();
|
||||
remote.Analyzers = null!;
|
||||
|
||||
RemoteConfigMerger.Merge(agent, remote);
|
||||
|
||||
Assert.False(agent.EnableRealmJoinWatcher);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Merge_replaces_diagnostics_log_paths_with_empty_list_when_remote_null()
|
||||
{
|
||||
|
||||
+25
-12
@@ -17,14 +17,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Integration
|
||||
{
|
||||
/// <summary>
|
||||
/// Plan §EspExiting Adapter — wire-through verification of the production EspExiting path:
|
||||
/// <c>ShellCoreTracker.HandleBackfillRecord</c> → <c>EspExited</c> event → inner re-raise on
|
||||
/// <c>EspAndHelloTracker.EspExited</c> → <c>EspAndHelloTrackerAdapter</c> → ingress → engine
|
||||
/// reducer. This is the path the live V2-Host uses; without the coordinator forward the
|
||||
/// EspExiting signal would never reach the engine in production.
|
||||
/// <c>ShellCoreTracker.ProcessEvent</c> (live Shell-Core 62407) → <c>EspExited</c> event →
|
||||
/// inner re-raise on <c>EspAndHelloTracker.EspExited</c> → <c>EspAndHelloTrackerAdapter</c> →
|
||||
/// ingress → engine reducer. Without the coordinator forward the EspExiting signal would
|
||||
/// never reach the engine in production.
|
||||
/// <para>
|
||||
/// The first test isolates the tracker's <c>EspExited</c> contract (source-event timestamp
|
||||
/// preserved through backfill); the coordinator tests drive the full production chain through
|
||||
/// to the <c>HelloSafety</c> deadline when AccountSetup is already entered.
|
||||
/// preserved rather than collapsed to wall-clock-now); the coordinator tests drive the full
|
||||
/// production chain through to the <c>HelloSafety</c> deadline when AccountSetup is already
|
||||
/// entered.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// It used to drive that first test through <c>HandleBackfillRecord</c> because the replay is
|
||||
/// easier to call than the event-log watcher — but the replay no longer raises
|
||||
/// <c>EspExited</c> at all (a replayed 62407 cannot be placed in time; see
|
||||
/// <c>ShellCoreTrackerReplayScopeTests</c>), and it never was the production path: the replay
|
||||
/// had no caller when this test was written.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ShellCoreToEngineEspExitingTests
|
||||
@@ -34,12 +42,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Integration
|
||||
"CommercialOOBE_ESPProgress_Page_Exiting"; // matches EspExitingPattern in ShellCoreTracker
|
||||
|
||||
[Fact]
|
||||
public void Tracker_backfill_record_raises_EspExited_with_source_timestamp()
|
||||
public void Tracker_live_record_raises_EspExited_with_source_timestamp()
|
||||
{
|
||||
// Tracker-level contract: HandleBackfillRecord raises EspExited carrying the source
|
||||
// event time (not Clock.UtcNow). The EspExited→EspExiting signal mapping and SourceOrigin
|
||||
// are covered by the coordinator tests below; here we isolate the backfill→event link
|
||||
// that the production coordinator depends on.
|
||||
// Tracker-level contract: the live 62407 path raises EspExited carrying the source
|
||||
// event time (record.TimeCreated, not Clock.UtcNow). The EspExited→EspExiting signal
|
||||
// mapping and SourceOrigin are covered by the coordinator tests below; here we isolate
|
||||
// the event-log→event link that the production coordinator depends on.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
var clock = new VirtualClock(ClockNow);
|
||||
@@ -57,7 +65,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Integration
|
||||
// Source time IS in the past relative to the clock — verify the tracker carries the
|
||||
// historical timestamp on the event rather than collapsing to Clock.UtcNow.
|
||||
var sourceTime = ClockNow.AddMinutes(-2);
|
||||
tracker.HandleBackfillRecord(ShellCoreTracker.EventId_ShellCore_WebAppEvent, EspExitingDescription, sourceTime);
|
||||
tracker.ProcessEvent(
|
||||
eventId: ShellCoreTracker.EventId_ShellCore_WebAppEvent,
|
||||
description: EspExitingDescription,
|
||||
timestamp: sourceTime,
|
||||
providerName: "Microsoft-Windows-Shell-Core",
|
||||
isBackfill: false);
|
||||
|
||||
Assert.NotNull(captured);
|
||||
Assert.Equal(sourceTime, captured!.OccurredAtUtc);
|
||||
|
||||
+48
@@ -170,5 +170,53 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
Assert.Empty(f.StarvedPosts());
|
||||
Assert.Empty(coordinator.StarvedAppsReported);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reevaluate_does_not_re_emit_starved_warnings()
|
||||
{
|
||||
// sits-d Cloud-PC fix (2026-08-19): the synthesis re-check now runs on EVERY IME
|
||||
// app-state transition. It must not turn the one-shot starvation warning into a
|
||||
// per-transition stream — the re-check path suppresses the emission entirely
|
||||
// (the per-appId dedupe would swallow the duplicates, but walking the probe
|
||||
// hundreds of times on the IME hot path is waste, not safety).
|
||||
using var f = new Fixture();
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => false,
|
||||
() => new[] { StarvedApp("app-1", "Contoso Backgrounds") });
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
var afterExit = f.StarvedPosts().Count;
|
||||
Assert.Equal(1, afterExit);
|
||||
|
||||
for (var i = 0; i < 10; i++) coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.Equal(afterExit, f.StarvedPosts().Count);
|
||||
Assert.Single(coordinator.StarvedAppsReported);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Starved_apps_that_settle_later_still_complete_the_enrollment()
|
||||
{
|
||||
// The real sits-d shape: the ESP exits with apps still starved (warning emitted),
|
||||
// the apps then all settle, and the re-check must open the gate — the warning
|
||||
// already on the timeline is history, not a permanent veto.
|
||||
using var f = new Fixture();
|
||||
var settled = false;
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => settled,
|
||||
() => new[] { StarvedApp("app-1", "Contoso Backgrounds") });
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
Assert.Single(f.StarvedPosts());
|
||||
Assert.DoesNotContain(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete);
|
||||
|
||||
settled = true;
|
||||
coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.Equal(1, f.Ingress.Posted.Count(p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete));
|
||||
Assert.Single(f.StarvedPosts());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+182
-2
@@ -35,14 +35,25 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
public Fixture() { Logger = new AgentLogger(Tmp.Path, AgentLogLevel.Debug); }
|
||||
|
||||
public EspAndHelloTracker BuildCoordinator(Func<bool> settledProbe)
|
||||
=> BuildCoordinator(settledProbe, accountSetupActivityProbe: () => true, skipUserEsp: false);
|
||||
|
||||
/// <summary>
|
||||
/// Codex review P1 (2026-08-19): the remembered exit edge is gated on positive evidence
|
||||
/// that the exit is post-AccountSetup, so these two probes decide whether the deferred
|
||||
/// re-check may fire at all.
|
||||
/// </summary>
|
||||
public EspAndHelloTracker BuildCoordinator(
|
||||
Func<bool> settledProbe,
|
||||
Func<bool> accountSetupActivityProbe,
|
||||
bool? skipUserEsp)
|
||||
{
|
||||
return new EspAndHelloTracker(
|
||||
sessionId: "S1",
|
||||
tenantId: "T1",
|
||||
post: new InformationalEventPost(TrackerPostSink, Clock),
|
||||
logger: Logger,
|
||||
skipConfigProbe: () => ((bool?)false, (bool?)false),
|
||||
accountSetupActivityProbe: () => true,
|
||||
skipConfigProbe: () => (skipUserEsp, (bool?)false),
|
||||
accountSetupActivityProbe: accountSetupActivityProbe,
|
||||
userEspAppsSettledProbe: settledProbe);
|
||||
}
|
||||
|
||||
@@ -159,5 +170,174 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.Equal(DecisionSignalKind.EspExiting, posted.Kind);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
// sits-d Cloud-PC fix (2026-08-19) — the synthesis is no longer edge-only.
|
||||
// Sessions 8110e262 / a89aac2d: the ESP page exited while 138 required user-ESP apps
|
||||
// were still in flight, they all reached a terminal state (0 failed) minutes later,
|
||||
// and because nothing re-checked, the enrollment never completed.
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Reevaluate_after_apps_settle_synthesizes_without_a_second_exit()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
var settled = false;
|
||||
using var coordinator = f.BuildCoordinator(() => settled);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
Assert.DoesNotContain(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete);
|
||||
|
||||
// The apps settle later; only the IME app-state callback fires — there is NO second
|
||||
// ESP exit on a real device, which is exactly why the edge-only version starved.
|
||||
settled = true;
|
||||
coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.Equal(1, f.Ingress.Posted.Count(p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete));
|
||||
Assert.True(coordinator.UserAppsSettledSynthesisFiredForTest);
|
||||
|
||||
var info = Assert.Single(f.TrackerPostSink.Posted);
|
||||
var data = Assert.IsAssignableFrom<System.Collections.Generic.IReadOnlyDictionary<string, object>>(info.TypedPayload);
|
||||
Assert.Equal("esp_exited_user_apps_settled_category_unresolved", data["fallbackReason"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reevaluate_without_an_observed_esp_exit_never_synthesizes()
|
||||
{
|
||||
// The gate needs BOTH facts. Settled apps alone must never open it — otherwise every
|
||||
// quiet moment during AccountSetup would look like completion.
|
||||
using var f = new Fixture();
|
||||
using var coordinator = f.BuildCoordinator(() => true);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.Empty(f.Ingress.Posted);
|
||||
Assert.Empty(f.TrackerPostSink.Posted);
|
||||
Assert.False(coordinator.UserAppsSettledSynthesisFiredForTest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reevaluate_is_idempotent_after_the_synthesis_fired()
|
||||
{
|
||||
// The IME callback fires per app-state transition — on these tenants that is 138+
|
||||
// times. The synthesis must stay fire-once.
|
||||
using var f = new Fixture();
|
||||
using var coordinator = f.BuildCoordinator(() => true);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
for (var i = 0; i < 20; i++) coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.Equal(1, f.Ingress.Posted.Count(p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete));
|
||||
Assert.Single(f.TrackerPostSink.Posted);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
// Codex review P1 (2026-08-19): Shell-Core raises 62407 at EVERY ESP phase transition.
|
||||
// Remembering the intermediate DeviceSetup→AccountSetup exit would let the deferred
|
||||
// re-check open the strong AccountSetup gate as soon as the last user app settles —
|
||||
// while the AccountSetup page is still up and its other subcategories still running.
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Intermediate_device_exit_is_not_remembered_as_the_edge()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
var settled = false;
|
||||
// Classic user-driven enrollment (SkipUser=false) that has NOT reached AccountSetup:
|
||||
// this 62407 is the DeviceSetup→AccountSetup transition, not the final exit.
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => settled, accountSetupActivityProbe: () => false, skipUserEsp: false);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
|
||||
// Apps settle afterwards — the level is reached, but the edge was never a valid one.
|
||||
settled = true;
|
||||
for (var i = 0; i < 10; i++) coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.DoesNotContain(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete);
|
||||
Assert.False(coordinator.UserAppsSettledSynthesisFiredForTest);
|
||||
Assert.Empty(f.TrackerPostSink.Posted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_skip_user_is_treated_as_unconfirmed()
|
||||
{
|
||||
// Erring strict costs at worst today's stall; erring loose costs a premature Succeeded.
|
||||
using var f = new Fixture();
|
||||
var settled = false;
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => settled, accountSetupActivityProbe: () => false, skipUserEsp: null);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
settled = true;
|
||||
coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.False(coordinator.UserAppsSettledSynthesisFiredForTest);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Skip_user_profile_confirms_the_device_exit_as_final()
|
||||
{
|
||||
// SkipUser=true means there is no user ESP at all — the Device-ESP exit IS the final
|
||||
// one and no second exit is coming, so the edge is legitimately remembered.
|
||||
using var f = new Fixture();
|
||||
var settled = false;
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => settled, accountSetupActivityProbe: () => false, skipUserEsp: true);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
settled = true;
|
||||
coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.Equal(1, f.Ingress.Posted.Count(p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Account_setup_activity_confirms_the_edge_for_the_deferred_recheck()
|
||||
{
|
||||
// The real reboot shape: AccountSetup activity is visible in the registry, the final
|
||||
// exit lands, the apps settle a moment later.
|
||||
using var f = new Fixture();
|
||||
var settled = false;
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => settled, accountSetupActivityProbe: () => true, skipUserEsp: false);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
Assert.DoesNotContain(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete);
|
||||
|
||||
settled = true;
|
||||
coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
Assert.Equal(1, f.Ingress.Posted.Count(p => p.Kind == DecisionSignalKind.AccountSetupProvisioningComplete));
|
||||
}
|
||||
|
||||
// NOTE: the "replayed exit" cases that used to live here are gone by construction —
|
||||
// ShellCoreTracker never re-raises a 62407 from the replay any more, so no backfilled exit
|
||||
// can reach this handler at all. That contract is pinned one layer up, in
|
||||
// ShellCoreTrackerReplayScopeTests, together with the reducer-side proof of why it matters
|
||||
// (ClassicEspExitingOnRestoredStateTests).
|
||||
|
||||
[Fact]
|
||||
public void Reevaluate_stays_silent_while_apps_are_still_unsettled()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var coordinator = f.BuildCoordinator(() => false);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
for (var i = 0; i < 5; i++) coordinator.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.Equal(DecisionSignalKind.EspExiting, posted.Kind);
|
||||
Assert.False(coordinator.UserAppsSettledSynthesisFiredForTest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
{
|
||||
/// <summary>
|
||||
/// Pins the per-FILE isolation of the offset measurement across a full read cycle.
|
||||
///
|
||||
/// <para>
|
||||
/// Background (2026-08-20, session e9753578): the field showed "+02:00 measured for
|
||||
/// IntuneManagementExtension.log" although that file held no +2-era line — the anchor had
|
||||
/// crossed file boundaries. The committed code was proven innocent (this very scenario is
|
||||
/// green here); the leak lived in a binary built from an uncommitted tree. This test keeps
|
||||
/// the property pinned so a real regression of the file scoping can never ship silently
|
||||
/// again — it replays the exact field pass structure: a NEW file appearing mid-run with a
|
||||
/// different writer era than the file already being tailed.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class CmTraceCalibratorFileIsolationTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 8, 20, 12, 13, 0, DateTimeKind.Utc);
|
||||
|
||||
private static List<ImeLogPattern> Patterns() => new List<ImeLogPattern>
|
||||
{
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-MARK", Category = "always", Enabled = true,
|
||||
Pattern = @"marker (?<n>\d+)",
|
||||
Action = "noop",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
};
|
||||
|
||||
private static string Line(string message, DateTime utcInstant, TimeSpan writerOffset)
|
||||
{
|
||||
var local = utcInstant + writerOffset;
|
||||
return $"<![LOG[{message}]LOG]!><time=\"{local:HH:mm:ss.fffffff}\" date=\"{local:M-d-yyyy}\" " +
|
||||
"component=\"X\" context=\"\" type=\"1\" thread=\"1\" file=\"\">";
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NewFileWithOtherEra_MustNotRecalibrateTheOldFile()
|
||||
{
|
||||
var pdt = TimeSpan.FromHours(-7); // IME service era
|
||||
var cest = TimeSpan.FromHours(2); // fresh child-process era
|
||||
|
||||
using var tmp = new TempDirectory();
|
||||
var now = T0;
|
||||
var tracker = new ImeLogTracker(
|
||||
logFolder: tmp.Path,
|
||||
patterns: Patterns(),
|
||||
logger: new AgentLogger(tmp.Path, AgentLogLevel.Info));
|
||||
try
|
||||
{
|
||||
tracker.UtcNowProvider = () => now;
|
||||
var imeLog = Path.Combine(tmp.Path, "IntuneManagementExtension.log");
|
||||
var execLog = Path.Combine(tmp.Path, "AgentExecutor.log");
|
||||
|
||||
// Pass 1: IME.log first sight (no calibration).
|
||||
File.AppendAllText(imeLog, Line("marker 1", now, pdt) + Environment.NewLine);
|
||||
await tracker.CheckLogFilesAsync(CancellationToken.None);
|
||||
|
||||
// Pass 2: IME.log grew -> measures -7.
|
||||
now = T0.AddSeconds(10);
|
||||
File.AppendAllText(imeLog, Line("marker 2", now, pdt) + Environment.NewLine);
|
||||
await tracker.CheckLogFilesAsync(CancellationToken.None);
|
||||
|
||||
TimeSpan imeMeasured;
|
||||
Assert.True(tracker.OffsetCalibrator.TryGetOffset("IntuneManagementExtension.log", out imeMeasured));
|
||||
Assert.Equal(pdt, imeMeasured);
|
||||
|
||||
// Pass 3: AgentExecutor.log appears for the FIRST time with fresh +2-era lines
|
||||
// while IME.log grows too — the exact field pass structure. The +2 anchor must
|
||||
// never land under IME.log's key.
|
||||
now = T0.AddSeconds(20);
|
||||
File.AppendAllText(execLog, Line("marker 3", now, cest) + Environment.NewLine);
|
||||
File.AppendAllText(imeLog, Line("marker 4", now, pdt) + Environment.NewLine);
|
||||
await tracker.CheckLogFilesAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(tracker.OffsetCalibrator.TryGetOffset("IntuneManagementExtension.log", out imeMeasured));
|
||||
Assert.Equal(pdt, imeMeasured);
|
||||
|
||||
// Pass 4: both grow — each file measures its own writer's era.
|
||||
now = T0.AddSeconds(30);
|
||||
File.AppendAllText(execLog, Line("marker 5", now, cest) + Environment.NewLine);
|
||||
File.AppendAllText(imeLog, Line("marker 6", now, pdt) + Environment.NewLine);
|
||||
await tracker.CheckLogFilesAsync(CancellationToken.None);
|
||||
|
||||
TimeSpan execMeasured;
|
||||
Assert.True(tracker.OffsetCalibrator.TryGetOffset("AgentExecutor.log", out execMeasured));
|
||||
Assert.Equal(cest, execMeasured);
|
||||
Assert.True(tracker.OffsetCalibrator.TryGetOffset("IntuneManagementExtension.log", out imeMeasured));
|
||||
Assert.Equal(pdt, imeMeasured);
|
||||
}
|
||||
finally
|
||||
{
|
||||
tracker.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
-12
@@ -67,29 +67,34 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
Assert.Equal("EnforcementState: Installing app X", entry.Message);
|
||||
Assert.Equal("AppEnforce", entry.Component);
|
||||
Assert.Equal(1, entry.Type);
|
||||
Assert.NotEqual(default, entry.Timestamp);
|
||||
Assert.True(entry.HasTimestamp);
|
||||
Assert.NotEqual(default, entry.LocalTimestamp);
|
||||
// Bias-less line: the parser refuses to guess the writer's zone, so no UTC value.
|
||||
Assert.Null(entry.BiasMinutes);
|
||||
Assert.Null(entry.TimestampUtc);
|
||||
Assert.Equal(DateTimeKind.Unspecified, entry.LocalTimestamp.Kind);
|
||||
}
|
||||
|
||||
// ── Structurally valid but unparseable timestamp → UtcNow fallback ────
|
||||
// ── Structurally valid but unparseable timestamp → flagged, not invented ──
|
||||
|
||||
[Fact]
|
||||
public void TryParseLine_falls_back_to_utcnow_for_unparseable_timestamp()
|
||||
public void TryParseLine_flags_unparseable_timestamp_instead_of_inventing_one()
|
||||
{
|
||||
// time/date satisfy the regex character classes ([\d:.]+ / [\d-]+) so the line MATCHES,
|
||||
// but "13-45-2026" / "25:99:99" fail every DateTime format → the parser falls back to
|
||||
// DateTime.UtcNow (CmTraceLogParser.cs:57) instead of throwing.
|
||||
var before = DateTime.UtcNow;
|
||||
// but "13-45-2026" / "25:99:99" fail every DateTime format. The parser used to stamp
|
||||
// DateTime.UtcNow here, which made a fabricated value indistinguishable from a real
|
||||
// one downstream. It now reports HasTimestamp=false and leaves the choice of fallback
|
||||
// to the caller, which is the only party that knows what a safe default is in context.
|
||||
var line = CmTraceLine("some message", "25:99:99.0000000", "13-45-2026");
|
||||
|
||||
var ok = CmTraceLogParser.TryParseLine(line, out var entry);
|
||||
var after = DateTime.UtcNow;
|
||||
|
||||
Assert.True(ok);
|
||||
Assert.NotNull(entry);
|
||||
Assert.Equal("some message", entry.Message);
|
||||
// Don't assert an exact instant — only that the fallback stamped a fresh "now"
|
||||
// (a successfully-parsed date would land in 2026, far outside this window).
|
||||
Assert.InRange(entry.Timestamp, before.AddSeconds(-5), after.AddSeconds(5));
|
||||
Assert.False(entry.HasTimestamp);
|
||||
Assert.Equal(default, entry.LocalTimestamp);
|
||||
Assert.Null(entry.TimestampUtc);
|
||||
}
|
||||
|
||||
// ── UTC-bias suffix on the time field → honored, not dropped ──────────
|
||||
@@ -108,10 +113,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
|
||||
Assert.True(ok);
|
||||
Assert.NotNull(entry);
|
||||
// A writer-declared bias IS the fact we otherwise have to measure, so it is honored
|
||||
// and resolved to UTC right here — unlike the bias-less form.
|
||||
Assert.Equal(expectedOffsetMinutes, entry.BiasMinutes);
|
||||
Assert.NotNull(entry.TimestampUtc);
|
||||
Assert.Equal(
|
||||
new DateTime(2026, 7, 29, 11, 46, 19, DateTimeKind.Utc).AddTicks(226610).AddMinutes(expectedOffsetMinutes),
|
||||
entry.Timestamp);
|
||||
Assert.Equal(DateTimeKind.Utc, entry.Timestamp.Kind);
|
||||
entry.TimestampUtc!.Value);
|
||||
Assert.Equal(DateTimeKind.Utc, entry.TimestampUtc!.Value.Kind);
|
||||
}
|
||||
|
||||
// ── Oversized line → parses, no throw, no truncation of the message ───
|
||||
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
{
|
||||
/// <summary>
|
||||
/// Pins the measurement contract of <see cref="CmTraceOffsetCalibrator"/>.
|
||||
///
|
||||
/// <para>
|
||||
/// The cases are taken from real sessions: +1 h (GMT to W. Europe, session bfff9b1c) and
|
||||
/// -17 h (a writer left on the Pacific OOBE default while the agent had moved to
|
||||
/// E. Australia). The backlog case is the one the residual guard cannot catch on its own and
|
||||
/// is the reason the freshness rule exists.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class CmTraceOffsetCalibratorTests
|
||||
{
|
||||
private const string Source = "IntuneManagementExtension.log";
|
||||
private static readonly DateTime UtcNow = new DateTime(2026, 08, 20, 07, 31, 46, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>A line written "now" by a writer that believes it is at <paramref name="offset"/>.</summary>
|
||||
private static DateTime LineWrittenNow(TimeSpan offset, DateTime utcNow)
|
||||
=> DateTime.SpecifyKind(utcNow + offset, DateTimeKind.Unspecified);
|
||||
|
||||
[Theory]
|
||||
[InlineData(2)] // CEST — the bfff9b1c writer
|
||||
[InlineData(1)] // BST — what the agent wrongly assumed there
|
||||
[InlineData(-7)] // PDT — the OOBE default seen in the -17 h sessions
|
||||
[InlineData(10)] // E. Australia
|
||||
[InlineData(14)] // Line Islands, the upper bound
|
||||
[InlineData(-12)] // Dateline, the lower bound
|
||||
public void MeasuresWholeHourOffsets(int hours)
|
||||
{
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
var offset = TimeSpan.FromHours(hours);
|
||||
|
||||
Assert.True(sut.TryCalibrate(Source, LineWrittenNow(offset, UtcNow), UtcNow));
|
||||
|
||||
TimeSpan measured;
|
||||
Assert.True(sut.TryGetOffset(Source, out measured));
|
||||
Assert.Equal(offset, measured);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(330)] // India, UTC+5:30
|
||||
[InlineData(345)] // Nepal, UTC+5:45
|
||||
[InlineData(570)] // Adelaide, UTC+9:30 — the +30 min sessions in the field data
|
||||
public void MeasuresQuarterHourOffsets(int minutes)
|
||||
{
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
var offset = TimeSpan.FromMinutes(minutes);
|
||||
|
||||
Assert.True(sut.TryCalibrate(Source, LineWrittenNow(offset, UtcNow), UtcNow));
|
||||
|
||||
TimeSpan measured;
|
||||
Assert.True(sut.TryGetOffset(Source, out measured));
|
||||
Assert.Equal(offset, measured);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AbsorbsPollAndFlushLatency()
|
||||
{
|
||||
// The line was written a moment before we parsed it. That lag must vanish into the
|
||||
// grid rather than shift the measured offset.
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
var lineLocal = LineWrittenNow(TimeSpan.FromHours(2), UtcNow.AddMilliseconds(-450));
|
||||
|
||||
Assert.True(sut.TryCalibrate(Source, lineLocal, UtcNow));
|
||||
|
||||
TimeSpan measured;
|
||||
Assert.True(sut.TryGetOffset(Source, out measured));
|
||||
Assert.Equal(TimeSpan.FromHours(2), measured);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RejectsBacklogLine_EvenWhenItRoundsCleanly()
|
||||
{
|
||||
// THE case the residual guard cannot catch alone: a line exactly 30 minutes old is a
|
||||
// clean multiple of the grid, so it rounds with zero residual — and would calibrate
|
||||
// the offset half an hour wrong. Only the caller's freshness rule prevents this from
|
||||
// ever being offered; this test pins that a 30-minute-old line does not silently
|
||||
// produce the true offset.
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
var thirtyMinutesOld = LineWrittenNow(TimeSpan.FromHours(2), UtcNow.AddMinutes(-30));
|
||||
|
||||
Assert.True(sut.TryCalibrate(Source, thirtyMinutesOld, UtcNow));
|
||||
|
||||
TimeSpan measured;
|
||||
Assert.True(sut.TryGetOffset(Source, out measured));
|
||||
Assert.Equal(TimeSpan.FromMinutes(90), measured);
|
||||
Assert.NotEqual(TimeSpan.FromHours(2), measured);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RejectsOffGridCandidate()
|
||||
{
|
||||
// 7 minutes off the grid: not fresh, and not a timezone either.
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
var offGrid = LineWrittenNow(TimeSpan.FromHours(2), UtcNow.AddMinutes(-7));
|
||||
|
||||
Assert.False(sut.TryCalibrate(Source, offGrid, UtcNow));
|
||||
|
||||
TimeSpan ignored;
|
||||
Assert.False(sut.TryGetOffset(Source, out ignored));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RejectsOffsetOutsideAnyRealTimezone()
|
||||
{
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
var absurd = LineWrittenNow(TimeSpan.FromHours(20), UtcNow);
|
||||
|
||||
Assert.False(sut.TryCalibrate(Source, absurd, UtcNow));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelfHeals_WhenTheWritingProcessPicksUpANewZone()
|
||||
{
|
||||
// The writer restarts mid-session and now believes CEST instead of the stale BST.
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
Assert.True(sut.TryCalibrate(Source, LineWrittenNow(TimeSpan.FromHours(1), UtcNow), UtcNow));
|
||||
|
||||
var later = UtcNow.AddMinutes(5);
|
||||
Assert.True(sut.TryCalibrate(Source, LineWrittenNow(TimeSpan.FromHours(2), later), later));
|
||||
|
||||
TimeSpan measured;
|
||||
Assert.True(sut.TryGetOffset(Source, out measured));
|
||||
Assert.Equal(TimeSpan.FromHours(2), measured);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IgnoresAnchorOlderThanTheCurrentOne()
|
||||
{
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
Assert.True(sut.TryCalibrate(Source, LineWrittenNow(TimeSpan.FromHours(2), UtcNow), UtcNow));
|
||||
|
||||
// An out-of-order line from before the current anchor must not move the offset back.
|
||||
var earlier = UtcNow.AddMinutes(-10);
|
||||
Assert.False(sut.TryCalibrate(Source, LineWrittenNow(TimeSpan.FromHours(1), earlier), earlier));
|
||||
|
||||
TimeSpan measured;
|
||||
Assert.True(sut.TryGetOffset(Source, out measured));
|
||||
Assert.Equal(TimeSpan.FromHours(2), measured);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void KeepsSourcesIndependent()
|
||||
{
|
||||
// IntuneManagementExtension.log and AppWorkload.log are written by different
|
||||
// processes and may hold different beliefs.
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
Assert.True(sut.TryCalibrate("IntuneManagementExtension.log", LineWrittenNow(TimeSpan.FromHours(2), UtcNow), UtcNow));
|
||||
Assert.True(sut.TryCalibrate("AppWorkload.log", LineWrittenNow(TimeSpan.FromHours(-7), UtcNow), UtcNow));
|
||||
|
||||
TimeSpan ime, appWorkload;
|
||||
Assert.True(sut.TryGetOffset("IntuneManagementExtension.log", out ime));
|
||||
Assert.True(sut.TryGetOffset("AppWorkload.log", out appWorkload));
|
||||
Assert.Equal(TimeSpan.FromHours(2), ime);
|
||||
Assert.Equal(TimeSpan.FromHours(-7), appWorkload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvesTheRealFieldLine_FromSessionBfff9b1c()
|
||||
{
|
||||
// Raw line: <![LOG[Agent version is: 1.104.102.0]LOG]!><time="09:31:46.5192284" ...>
|
||||
// Server-side ReceivedAt puts the true UTC at 07:31:46.4. The old code produced
|
||||
// 08:31:46.5192284 — local minus the agent's stale +1 h belief.
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
var trueUtc = new DateTime(2026, 08, 20, 07, 31, 46, 400, DateTimeKind.Utc);
|
||||
var lineLocal = new DateTime(2026, 08, 20, 09, 31, 46, 519, DateTimeKind.Unspecified);
|
||||
|
||||
Assert.True(sut.TryCalibrate(Source, lineLocal, trueUtc));
|
||||
|
||||
DateTime resolved;
|
||||
Assert.True(sut.TryResolveUtc(Source, lineLocal, out resolved));
|
||||
Assert.Equal(DateTimeKind.Utc, resolved.Kind);
|
||||
Assert.Equal(new DateTime(2026, 08, 20, 07, 31, 46, 519, DateTimeKind.Utc), resolved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReportsUncalibratedSource_SoTheCallerCanFlagIt()
|
||||
{
|
||||
var sut = new CmTraceOffsetCalibrator();
|
||||
|
||||
DateTime resolved;
|
||||
Assert.False(sut.TryResolveUtc(Source, UtcNow, out resolved));
|
||||
}
|
||||
}
|
||||
}
|
||||
+441
@@ -0,0 +1,441 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Logging;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-line self-anchoring — the era-aware successor to the reverted per-file offset
|
||||
/// application (04b1a7c6).
|
||||
///
|
||||
/// <para>
|
||||
/// The property under test: a line read in a provably FRESH pass resolves to the instant it
|
||||
/// was written, whatever timezone its writer believed — and neighbouring lines from writers
|
||||
/// with DIFFERENT beliefs each resolve to their own era, because no cross-line state is
|
||||
/// involved. That interleaving (fixture: AgentExecutor.log with PDT and CEST children in one
|
||||
/// file, flipping both ways) is exactly what broke every cross-line anchor design.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// The freshness boundary is equally load-bearing: backlog (first sight, restart catch-up,
|
||||
/// stalled polls) must NEVER anchor, because an old line's age can round onto the 15-minute
|
||||
/// offset grid. Those paths keep the uniform reader-zone fallback.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerLineAnchoringTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 8, 20, 12, 13, 0, DateTimeKind.Utc);
|
||||
|
||||
private static List<ImeLogPattern> Patterns() => new List<ImeLogPattern>
|
||||
{
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-ALL", Category = "always", Enabled = true,
|
||||
Pattern = @"marker (?<n>\d+)",
|
||||
Action = "noop",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>A CMTrace line as a writer holding <paramref name="writerOffset"/> renders instant <paramref name="utcInstant"/>.</summary>
|
||||
private static string Line(string message, DateTime utcInstant, TimeSpan writerOffset)
|
||||
{
|
||||
var local = utcInstant + writerOffset;
|
||||
return $"<![LOG[{message}]LOG]!><time=\"{local:HH:mm:ss.fffffff}\" date=\"{local:M-d-yyyy}\" " +
|
||||
"component=\"X\" context=\"\" type=\"1\" thread=\"1\" file=\"\">";
|
||||
}
|
||||
|
||||
private sealed class Harness : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
public ImeLogTracker Tracker { get; }
|
||||
public DateTime Now { get; set; } = T0;
|
||||
|
||||
public Harness(string? stateDirectory = null)
|
||||
{
|
||||
Tracker = new ImeLogTracker(
|
||||
logFolder: _tmp.Path,
|
||||
patterns: Patterns(),
|
||||
logger: new AgentLogger(_tmp.Path, AgentLogLevel.Info),
|
||||
stateDirectory: stateDirectory);
|
||||
Tracker.UtcNowProvider = () => Now;
|
||||
}
|
||||
|
||||
public string PathOf(string fileName) => Path.Combine(_tmp.Path, fileName);
|
||||
|
||||
public void Append(string fileName, string line)
|
||||
=> File.AppendAllText(PathOf(fileName), line + Environment.NewLine);
|
||||
|
||||
public Task Pass() => Tracker.CheckLogFilesAsync(CancellationToken.None);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Tracker.Dispose();
|
||||
_tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// The writer's belief. A fresh line must resolve to the instant it was written,
|
||||
// WHATEVER that belief is — including agreement with this process (the majority case,
|
||||
// which a fix to the agent's own zone cache would have broken).
|
||||
[InlineData(2)] // CEST
|
||||
[InlineData(1)] // BST
|
||||
[InlineData(0)] // UTC
|
||||
[InlineData(-7)] // PDT — the OOBE default behind the -9 h and -17 h field cases
|
||||
[InlineData(10)] // E. Australia
|
||||
public async Task FreshLine_ResolvesToItsTrueInstant_ForAnyWriterBelief(int writerOffsetHours)
|
||||
{
|
||||
var writerOffset = TimeSpan.FromHours(writerOffsetHours);
|
||||
using var h = new Harness();
|
||||
|
||||
// First sight: backlog, must not anchor.
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 1", h.Now, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
var writtenAt = h.Now.AddMilliseconds(-80); // written just before the poll sees it
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 2", writtenAt, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
Assert.Equal(CmTraceOffsetOrigin.LineAnchored, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
Assert.Equal(writtenAt, h.Tracker.LastMatchedLogTimestamp);
|
||||
Assert.Equal(writerOffsetHours * 60, h.Tracker.LastMatchedSourceOffsetMinutes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InterleavedEras_InOneGrowingPass_EachLineResolvesToItsOwnEra()
|
||||
{
|
||||
// The e9753578 file shape: two writer processes with different zone beliefs
|
||||
// interleaved in ONE file. Any per-file offset gets one of them wrong by 9 h;
|
||||
// per-line anchoring must get both right within the same chunk.
|
||||
var pdt = TimeSpan.FromHours(-7);
|
||||
var cest = TimeSpan.FromHours(2);
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append("AgentExecutor.log", Line("marker 1", h.Now, pdt));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
var wrotePdt = h.Now.AddMilliseconds(-90);
|
||||
var wroteCest = h.Now.AddMilliseconds(-50);
|
||||
h.Append("AgentExecutor.log", Line("marker 2", wrotePdt, pdt));
|
||||
h.Append("AgentExecutor.log", Line("marker 3", wroteCest, cest));
|
||||
|
||||
var resolved = new List<(DateTime? Ts, CmTraceOffsetOrigin Origin, int? Offset)>();
|
||||
Action<string> capture = _ => resolved.Add((
|
||||
h.Tracker.LastMatchedLogTimestamp,
|
||||
h.Tracker.LastMatchedSourceOffsetOrigin,
|
||||
h.Tracker.LastMatchedSourceOffsetMinutes));
|
||||
h.Tracker.OnPatternMatched += capture;
|
||||
try
|
||||
{
|
||||
await h.Pass();
|
||||
}
|
||||
finally
|
||||
{
|
||||
h.Tracker.OnPatternMatched -= capture;
|
||||
}
|
||||
|
||||
Assert.Equal(2, resolved.Count);
|
||||
Assert.Equal(wrotePdt, resolved[0].Ts);
|
||||
Assert.Equal(-420, resolved[0].Offset);
|
||||
Assert.Equal(wroteCest, resolved[1].Ts);
|
||||
Assert.Equal(120, resolved[1].Offset);
|
||||
Assert.All(resolved, r => Assert.Equal(CmTraceOffsetOrigin.LineAnchored, r.Origin));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FirstSightBacklog_FallsBackToReaderZone()
|
||||
{
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 1", h.Now.AddHours(-3), TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
Assert.Equal(CmTraceOffsetOrigin.None, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FileFirstSeenEmpty_ItsFirstContentIsFresh()
|
||||
{
|
||||
// AgentExecutor.log in the field: the file APPEARS mid-enrollment. Seeing it empty
|
||||
// (or not existing content-wise) counts as an observation, so its very first lines
|
||||
// are provably written between passes and may anchor. This closes the warm-up gap
|
||||
// for exactly the file whose eras flip the most.
|
||||
using var h = new Harness();
|
||||
|
||||
File.WriteAllText(h.PathOf("AgentExecutor.log"), string.Empty);
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(5);
|
||||
var writtenAt = h.Now.AddMilliseconds(-60);
|
||||
h.Append("AgentExecutor.log", Line("marker 1", writtenAt, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
Assert.Equal(CmTraceOffsetOrigin.LineAnchored, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
Assert.Equal(writtenAt, h.Tracker.LastMatchedLogTimestamp);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PollGapBeyondFreshWindow_FallsBack()
|
||||
{
|
||||
// A stalled agent (GC, CPU starvation) re-polls after a long gap: the new bytes span
|
||||
// the whole gap and their age is unbounded, so the pass must not anchor — even
|
||||
// though most of its lines may in fact be recent. Conservative by design.
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 1", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
h.Now = T0.AddSeconds(10);
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 2", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
Assert.Equal(CmTraceOffsetOrigin.LineAnchored, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
|
||||
// Gap far beyond FreshLineMaxAge.
|
||||
h.Now = h.Now + ImeLogTracker.FreshLineMaxAge + TimeSpan.FromSeconds(10);
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 3", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
Assert.Equal(CmTraceOffsetOrigin.None, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
|
||||
// And the very next tight poll is fresh again.
|
||||
h.Now = h.Now.AddSeconds(1);
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 4", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
Assert.Equal(CmTraceOffsetOrigin.LineAnchored, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RestartWithRestoredBookmark_FirstPassIsBacklog_SecondPassAnchors()
|
||||
{
|
||||
// The persisted position bookmark survives a restart; the freshness guarantee must
|
||||
// not — the first pass after a restart reads the downtime backlog, whose lines can
|
||||
// be arbitrarily old.
|
||||
using var stateDir = new TempDirectory();
|
||||
using var h = new Harness(stateDirectory: stateDir.Path);
|
||||
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 1", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
h.Tracker.SaveStateForTest();
|
||||
|
||||
// "Restart": a second tracker over the same folder restores the bookmark.
|
||||
using var h2 = new HarnessOver(h, stateDir.Path);
|
||||
h2.Tracker.LoadStateForTest();
|
||||
|
||||
// Backlog written during the downtime — 30 min old, exactly on the offset grid,
|
||||
// the nastiest possible backlog shape. Must NOT anchor.
|
||||
h2.Now = T0.AddMinutes(30);
|
||||
h2.Append(Line("marker 2", T0.AddSeconds(20), TimeSpan.FromHours(2)));
|
||||
await h2.Pass();
|
||||
Assert.Equal(CmTraceOffsetOrigin.None, h2.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
|
||||
// The next tight poll is fresh again.
|
||||
h2.Now = h2.Now.AddSeconds(1);
|
||||
var writtenAt = h2.Now.AddMilliseconds(-70);
|
||||
h2.Append(Line("marker 3", writtenAt, TimeSpan.FromHours(2)));
|
||||
await h2.Pass();
|
||||
Assert.Equal(CmTraceOffsetOrigin.LineAnchored, h2.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
Assert.Equal(writtenAt, h2.Tracker.LastMatchedLogTimestamp);
|
||||
}
|
||||
|
||||
/// <summary>Second tracker over the FIRST harness's log folder — the "restarted agent".</summary>
|
||||
private sealed class HarnessOver : IDisposable
|
||||
{
|
||||
private readonly string _logFolder;
|
||||
public ImeLogTracker Tracker { get; }
|
||||
public DateTime Now { get; set; }
|
||||
|
||||
public HarnessOver(Harness original, string stateDirectory)
|
||||
{
|
||||
_logFolder = Path.GetDirectoryName(original.PathOf("x"));
|
||||
Tracker = new ImeLogTracker(
|
||||
logFolder: _logFolder,
|
||||
patterns: Patterns(),
|
||||
logger: new AgentLogger(_logFolder, AgentLogLevel.Info),
|
||||
stateDirectory: stateDirectory);
|
||||
Tracker.UtcNowProvider = () => Now;
|
||||
}
|
||||
|
||||
public void Append(string line)
|
||||
=> File.AppendAllText(Path.Combine(_logFolder, "IntuneManagementExtension.log"), line + Environment.NewLine);
|
||||
|
||||
public Task Pass() => Tracker.CheckLogFilesAsync(CancellationToken.None);
|
||||
|
||||
public void Dispose() => Tracker.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FreshlyWrittenReplayLine_OnExactGridAge_ResolvesToRoughlyNow()
|
||||
{
|
||||
// Characterization of the one accepted edge: a line freshly WRITTEN but carrying a
|
||||
// REPLAYED timestamp whose age is exactly a grid multiple (here 30 min) measures
|
||||
// that age as an "offset" and resolves to ~now instead of its replayed past. The
|
||||
// error is bounded by construction (an anchored line always lands at now ± 2 min),
|
||||
// and sub-24h replays are treated as current by the historic-replay guard anyway.
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 1", h.Now, TimeSpan.Zero));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
var replayedInstant = h.Now.AddMinutes(-30);
|
||||
h.Append("IntuneManagementExtension.log", Line("marker 2", replayedInstant, TimeSpan.Zero));
|
||||
await h.Pass();
|
||||
|
||||
Assert.Equal(CmTraceOffsetOrigin.LineAnchored, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
Assert.NotNull(h.Tracker.LastMatchedLogTimestamp);
|
||||
var error = (h.Tracker.LastMatchedLogTimestamp!.Value - h.Now).Duration();
|
||||
Assert.True(error <= CmTraceOffsetCalibrator.MaxGridResidual,
|
||||
$"anchored line must resolve to now ± residual, was off by {error}");
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
// The committed field fixture — session e9753578, ImeLogs/AgentExecutor.log.
|
||||
// One file, two writer eras (PDT hour-05 / CEST hour-14), interleaved, flipping
|
||||
// BOTH ways. Replayed here as the live tail the field agent saw: entry by entry,
|
||||
// with the clock following the true write time.
|
||||
// =====================================================================
|
||||
|
||||
private static string FixturePath()
|
||||
{
|
||||
var dir = AppContext.BaseDirectory;
|
||||
for (int i = 0; i < 12; i++)
|
||||
{
|
||||
if (File.Exists(Path.Combine(dir, "AutopilotMonitor.sln")))
|
||||
return Path.Combine(dir, "tests", "fixtures", "cmtrace-logs",
|
||||
"agentexecutor-two-writer-eras-v1.cmtrace");
|
||||
var parent = Directory.GetParent(dir)?.FullName;
|
||||
if (parent == null || parent == dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
throw new DirectoryNotFoundException(
|
||||
"Could not locate repo root (AutopilotMonitor.sln) walking up from " + AppContext.BaseDirectory);
|
||||
}
|
||||
|
||||
/// <summary>Group the fixture's physical lines into complete CMTrace entries (multiline-safe).</summary>
|
||||
private static List<string> FixtureEntries()
|
||||
{
|
||||
var entries = new List<string>();
|
||||
string? pending = null;
|
||||
foreach (var rawWithBom in File.ReadAllLines(FixturePath()))
|
||||
{
|
||||
// The fixture keeps the original file's BOM verbatim — it sat at the START of
|
||||
// the real AgentExecutor.log, which is mid-file in this excerpt.
|
||||
var raw = rawWithBom.TrimStart('');
|
||||
if (pending == null)
|
||||
{
|
||||
if (raw.Length == 0 || raw[0] == '#') continue;
|
||||
if (!raw.StartsWith("<![LOG[", StringComparison.Ordinal)) continue;
|
||||
if (raw.Contains("]LOG]!>")) { entries.Add(raw); continue; }
|
||||
pending = raw;
|
||||
}
|
||||
else
|
||||
{
|
||||
pending += "\n" + raw;
|
||||
if (raw.Contains("]LOG]!>")) { entries.Add(pending!); pending = null; }
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TwoWriterEraFixture_EveryLineOfBothErasResolvesToItsTrueInstant()
|
||||
{
|
||||
// The committed fixture is a curated, verbatim excerpt of the transition window
|
||||
// (commit policy: anonymized, versioned) — 20 entries covering both eras and the flip.
|
||||
var entries = FixtureEntries();
|
||||
Assert.True(entries.Count >= 15, $"fixture unexpectedly small: {entries.Count} entries");
|
||||
|
||||
using var h = new Harness();
|
||||
|
||||
// Match every fixture entry.
|
||||
h.Tracker.CompilePatterns(new List<ImeLogPattern>
|
||||
{
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-ANY", Category = "always", Enabled = true,
|
||||
Pattern = ".+", Action = "noop",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
});
|
||||
|
||||
// The file exists (empty) before its first content — first sight is the observation
|
||||
// baseline, everything after is a fresh tail, exactly like the field.
|
||||
File.WriteAllText(h.PathOf("AgentExecutor.log"), string.Empty);
|
||||
h.Now = new DateTime(2026, 8, 20, 12, 13, 30, DateTimeKind.Utc);
|
||||
await h.Pass();
|
||||
|
||||
var anchored = 0;
|
||||
var resolvedList = new List<(DateTime Resolved, DateTime Expected)>();
|
||||
Action<string> capture = _ =>
|
||||
{
|
||||
if (h.Tracker.LastMatchedSourceOffsetOrigin == CmTraceOffsetOrigin.LineAnchored)
|
||||
anchored++;
|
||||
if (h.Tracker.LastMatchedLogTimestamp.HasValue && _expectedForCurrentEntry.HasValue)
|
||||
resolvedList.Add((h.Tracker.LastMatchedLogTimestamp.Value, _expectedForCurrentEntry.Value));
|
||||
};
|
||||
h.Tracker.OnPatternMatched += capture;
|
||||
try
|
||||
{
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
CmTraceLogEntry? parsed;
|
||||
Assert.True(CmTraceLogParser.TryParseLine(entry, out parsed), "fixture entry did not parse");
|
||||
|
||||
// Era by local hour: the fixture holds exactly two eras — PDT (UTC-7) lines
|
||||
// render at local hour 05, CEST (UTC+2) lines at hour 14 (fixture header).
|
||||
var eraOffset = parsed.LocalTimestamp.Hour == 14
|
||||
? TimeSpan.FromHours(2)
|
||||
: TimeSpan.FromHours(-7);
|
||||
var trueUtc = DateTime.SpecifyKind(parsed.LocalTimestamp - eraOffset, DateTimeKind.Utc);
|
||||
|
||||
// The clock follows the write; never move backwards (interleaved eras are
|
||||
// written in true-UTC order, but guard against sub-ms jitter).
|
||||
var readAt = trueUtc.AddMilliseconds(120);
|
||||
if (readAt > h.Now) h.Now = readAt;
|
||||
|
||||
// In production the 100 ms poll keeps running through write silences — every
|
||||
// empty look refreshes the freshness window. The fixture's transition window
|
||||
// spans multi-minute write gaps, so replay that cadence with one empty poll
|
||||
// just before each append; without it the NEXT pass would (correctly!)
|
||||
// classify the gap as a stall and refuse to anchor.
|
||||
h.Now = h.Now.AddMilliseconds(-100);
|
||||
await h.Pass();
|
||||
h.Now = readAt;
|
||||
|
||||
_expectedForCurrentEntry = trueUtc;
|
||||
File.AppendAllText(h.PathOf("AgentExecutor.log"), entry + Environment.NewLine);
|
||||
await h.Pass();
|
||||
_expectedForCurrentEntry = null;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
h.Tracker.OnPatternMatched -= capture;
|
||||
}
|
||||
|
||||
// Every matched entry resolved to the instant its writer rendered — across BOTH
|
||||
// eras, including the flips. This is the exact property whose absence produced the
|
||||
// -9 h script durations.
|
||||
Assert.Equal(entries.Count, resolvedList.Count);
|
||||
Assert.All(resolvedList, r => Assert.Equal(r.Expected, r.Resolved));
|
||||
Assert.True(anchored == resolvedList.Count,
|
||||
$"expected every entry line-anchored, got {anchored}/{resolvedList.Count}");
|
||||
|
||||
// Both eras genuinely exercised: PDT entries render at local hour 05, CEST at 14.
|
||||
Assert.Contains(resolvedList, r => (r.Expected + TimeSpan.FromHours(-7)).Hour == 5);
|
||||
Assert.Contains(resolvedList, r => (r.Expected + TimeSpan.FromHours(2)).Hour == 14);
|
||||
}
|
||||
|
||||
private DateTime? _expectedForCurrentEntry;
|
||||
}
|
||||
}
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
{
|
||||
/// <summary>
|
||||
/// End-to-end offset calibration through a real read cycle.
|
||||
///
|
||||
/// <para>
|
||||
/// A CMTrace line states local time and nothing else, so its UTC value depends on which zone
|
||||
/// the WRITING process believed it was in. The tracker used to substitute its own belief,
|
||||
/// which is right only when the two happen to agree.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// These tests are deliberately independent of the machine they run on: the assertion is
|
||||
/// always "a line written at instant T resolves to T", for every writer belief. That single
|
||||
/// property covers both field cases at once — the writer disagreeing with us (the bug) and
|
||||
/// the writer agreeing with us (the majority, which a naive fix would have broken).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerOffsetCalibrationTests
|
||||
{
|
||||
private const string LogFileName = "IntuneManagementExtension.log";
|
||||
private static readonly DateTime T0 = new DateTime(2026, 8, 20, 7, 30, 0, DateTimeKind.Utc);
|
||||
|
||||
private static List<ImeLogPattern> Patterns() => new List<ImeLogPattern>
|
||||
{
|
||||
new ImeLogPattern
|
||||
{
|
||||
PatternId = "T-MARK", Category = "always", Enabled = true,
|
||||
Pattern = @"marker (?<n>\d+)",
|
||||
Action = "noop",
|
||||
Parameters = new Dictionary<string, string>(),
|
||||
},
|
||||
};
|
||||
|
||||
/// <summary>A CMTrace line as a writer holding <paramref name="writerOffset"/> renders instant <paramref name="utcInstant"/>.</summary>
|
||||
private static string Line(string message, DateTime utcInstant, TimeSpan writerOffset)
|
||||
{
|
||||
var local = utcInstant + writerOffset;
|
||||
return $"<![LOG[{message}]LOG]!><time=\"{local:HH:mm:ss.fffffff}\" date=\"{local:M-d-yyyy}\" " +
|
||||
"component=\"IntuneManagementExtension\" context=\"\" type=\"1\" thread=\"1\" file=\"\">";
|
||||
}
|
||||
|
||||
private sealed class Harness : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
public ImeLogTracker Tracker { get; }
|
||||
public DateTime Now { get; set; } = T0;
|
||||
|
||||
public Harness()
|
||||
{
|
||||
Tracker = new ImeLogTracker(
|
||||
logFolder: _tmp.Path,
|
||||
patterns: Patterns(),
|
||||
logger: new AgentLogger(_tmp.Path, AgentLogLevel.Info));
|
||||
Tracker.UtcNowProvider = () => Now;
|
||||
}
|
||||
|
||||
public string LogPath => Path.Combine(_tmp.Path, LogFileName);
|
||||
|
||||
public void Append(string line) => File.AppendAllText(LogPath, line + Environment.NewLine);
|
||||
|
||||
public Task Pass() => Tracker.CheckLogFilesAsync(CancellationToken.None);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Tracker.Dispose();
|
||||
_tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// The writer's belief. The MEASUREMENT must recover it exactly, whatever it is.
|
||||
[InlineData(2)] // CEST
|
||||
[InlineData(1)] // BST
|
||||
[InlineData(-7)] // PDT — the OOBE default behind the -17 h and -9 h field cases
|
||||
[InlineData(10)] // E. Australia
|
||||
[InlineData(0)] // UTC
|
||||
public async Task MeasuresTheWritersOffsetExactly_ForAnyBelief(int writerOffsetHours)
|
||||
{
|
||||
// NOTE: the per-FILE measurement asserted here is OBSERVATIONAL — its application was
|
||||
// reverted on 2026-08-20 (04b1a7c6) because one offset per file is wrong when a file
|
||||
// holds interleaved writer eras. Resolution now anchors per LINE instead (see
|
||||
// ImeLogTrackerLineAnchoringTests, which restores the "a line written at T resolves
|
||||
// to T" property era-safely). The per-file measurement stays for the Info log and
|
||||
// measuredWriterOffsetMinutes provenance.
|
||||
var writerOffset = TimeSpan.FromHours(writerOffsetHours);
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append(Line("marker 1", h.Now, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
h.Append(Line("marker 2", h.Now, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
TimeSpan measured;
|
||||
Assert.True(h.Tracker.OffsetCalibrator.TryGetOffset(LogFileName, out measured));
|
||||
Assert.Equal(writerOffset, measured);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MeasuresTheWritersOffset_NotItsOwn()
|
||||
{
|
||||
using var h = new Harness();
|
||||
var writerOffset = TimeSpan.FromHours(2);
|
||||
|
||||
h.Append(Line("marker 1", h.Now, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
h.Append(Line("marker 2", h.Now, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
TimeSpan measured;
|
||||
Assert.True(h.Tracker.OffsetCalibrator.TryGetOffset(LogFileName, out measured));
|
||||
Assert.Equal(writerOffset, measured);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DoesNotCalibrateOnFirstSightOfAFile()
|
||||
{
|
||||
// The first pass reads from position 0, so the newest line in the file may be hours
|
||||
// old. Anchoring on it would measure the line's age instead of the writer's offset.
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append(Line("marker 1", T0.AddHours(-3), TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
TimeSpan ignored;
|
||||
Assert.False(h.Tracker.OffsetCalibrator.TryGetOffset(LogFileName, out ignored));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FollowsTheWriterAcrossARestartIntoANewZone()
|
||||
{
|
||||
// IME restarts mid-enrollment and picks up the timezone the OS moved to. The
|
||||
// measurement has to follow, otherwise every later line is off by the difference.
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append(Line("marker 1", h.Now, TimeSpan.FromHours(1)));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
h.Append(Line("marker 2", h.Now, TimeSpan.FromHours(1)));
|
||||
await h.Pass();
|
||||
|
||||
TimeSpan before;
|
||||
Assert.True(h.Tracker.OffsetCalibrator.TryGetOffset(LogFileName, out before));
|
||||
Assert.Equal(TimeSpan.FromHours(1), before);
|
||||
|
||||
// Writer restarts, now on +2.
|
||||
h.Now = T0.AddSeconds(20);
|
||||
h.Append(Line("marker 3", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(30);
|
||||
var writtenAt = h.Now;
|
||||
h.Append(Line("marker 4", writtenAt, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
TimeSpan after;
|
||||
Assert.True(h.Tracker.OffsetCalibrator.TryGetOffset(LogFileName, out after));
|
||||
Assert.Equal(TimeSpan.FromHours(2), after);
|
||||
// The measurement follows the restart. Applying it is what the 2026-08-20 revert
|
||||
// removed — that is exactly the case a file with two writer eras breaks.
|
||||
Assert.Equal(120, h.Tracker.LastMatchedMeasuredWriterOffsetMinutes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HonorsAWriterDeclaredBiasWithoutMeasuring()
|
||||
{
|
||||
// A line carrying its own bias states the offset outright; it must be used as-is and
|
||||
// must never become a calibration anchor.
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append(Line("marker 1", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
var writtenAt = h.Now;
|
||||
var local = writtenAt + TimeSpan.FromHours(-8); // a PST writer
|
||||
h.Append($"<![LOG[marker 2]LOG]!><time=\"{local:HH:mm:ss.fffffff}+480\" date=\"{local:M-d-yyyy}\" " +
|
||||
"component=\"IntuneManagementExtension\" context=\"\" type=\"1\" thread=\"1\" file=\"\">");
|
||||
await h.Pass();
|
||||
|
||||
Assert.True(h.Tracker.LastMatchedLogTimestamp.HasValue);
|
||||
Assert.Equal(writtenAt, h.Tracker.LastMatchedLogTimestamp.Value);
|
||||
|
||||
TimeSpan ignored;
|
||||
Assert.False(h.Tracker.OffsetCalibrator.TryGetOffset(LogFileName, out ignored));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordsMeasuredOffsetSeparatelyFromTheAppliedOne()
|
||||
{
|
||||
// The regression that forced the revert shipped an event tagged "calibrated" while
|
||||
// carrying an offset that did not hold for that line. Measured and applied are now
|
||||
// separate fields so a future reader cannot mistake one for the other.
|
||||
using var h = new Harness();
|
||||
var writerOffset = TimeSpan.FromHours(-7); // PDT, the OOBE default
|
||||
|
||||
h.Append(Line("marker 1", h.Now, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
h.Append(Line("marker 2", h.Now, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(20);
|
||||
h.Append(Line("marker 3", h.Now, writerOffset));
|
||||
await h.Pass();
|
||||
|
||||
// Measured: the writer's actual belief, per file.
|
||||
Assert.Equal(-420, h.Tracker.LastMatchedMeasuredWriterOffsetMinutes);
|
||||
|
||||
// Applied: the line's OWN anchor (fresh pass), never the per-file "calibrated" value —
|
||||
// the two must stay distinguishable even when they agree numerically.
|
||||
Assert.Equal(CmTraceOffsetOrigin.LineAnchored, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
Assert.NotEqual(CmTraceOffsetOrigin.Calibrated, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordsProvenance_AsFallback_BeforeAnyCalibration()
|
||||
{
|
||||
// The warm-up pass: the offset is not measured yet, so the reader zone is assumed and
|
||||
// the event has to say so rather than look source-grounded.
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append(Line("marker 1", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
Assert.Equal(CmTraceOffsetOrigin.None, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
Assert.True(h.Tracker.LastMatchedSourceLocalTimestamp.HasValue);
|
||||
|
||||
// The APPLIED offset is reported even on the fallback path — an event that states no
|
||||
// offset at all cannot be recomputed later, which defeats the purpose of recording
|
||||
// provenance. Asserted against the running machine's zone so this stays portable.
|
||||
var readerZone = (int)TimeZoneInfo.Local
|
||||
.GetUtcOffset(h.Tracker.LastMatchedSourceLocalTimestamp!.Value).TotalMinutes;
|
||||
Assert.Equal(readerZone, h.Tracker.LastMatchedSourceOffsetMinutes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordsProvenance_ForADeclaredBias()
|
||||
{
|
||||
using var h = new Harness();
|
||||
|
||||
h.Append(Line("marker 1", h.Now, TimeSpan.FromHours(2)));
|
||||
await h.Pass();
|
||||
|
||||
h.Now = T0.AddSeconds(10);
|
||||
var local = h.Now + TimeSpan.FromHours(-8);
|
||||
h.Append($"<![LOG[marker 2]LOG]!><time=\"{local:HH:mm:ss.fffffff}+480\" date=\"{local:M-d-yyyy}\" " +
|
||||
"component=\"IntuneManagementExtension\" context=\"\" type=\"1\" thread=\"1\" file=\"\">");
|
||||
await h.Pass();
|
||||
|
||||
Assert.Equal(CmTraceOffsetOrigin.Bias, h.Tracker.LastMatchedSourceOffsetOrigin);
|
||||
// Reported in the same sense as a measured offset (local = UTC + offset), so the two
|
||||
// origins are directly comparable; the wire bias uses the opposite convention.
|
||||
Assert.Equal(-480, h.Tracker.LastMatchedSourceOffsetMinutes);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+188
-2
@@ -1,9 +1,11 @@
|
||||
#nullable enable
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
@@ -46,7 +48,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
Directory.CreateDirectory(SpoolFolder);
|
||||
}
|
||||
|
||||
public DiagnosticsPackageService Build(System.Action<AgentConfiguration>? mutateConfig = null)
|
||||
public DiagnosticsPackageService Build(
|
||||
System.Action<AgentConfiguration>? mutateConfig = null,
|
||||
IReadOnlyDictionary<string, string>? sectionFolderOverrides = null,
|
||||
System.Func<bool>? devicePreparationProbe = null)
|
||||
{
|
||||
// BackendApiClient is required by the public ctor but BuildArchiveBytes
|
||||
// never touches it — construct with a throwaway HttpClient. Tests only
|
||||
@@ -71,7 +76,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
imeLogFolderOverride: ImeTmp.Path,
|
||||
agentStateFolderOverride: StateFolder,
|
||||
agentSpoolFolderOverride: SpoolFolder,
|
||||
agentDataFolderOverride: DataFolder);
|
||||
agentDataFolderOverride: DataFolder,
|
||||
sectionFolderOverrides: sectionFolderOverrides,
|
||||
devicePreparationProbe: devicePreparationProbe);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
@@ -287,6 +294,185 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Runtime
|
||||
Assert.DoesNotContain(entries, e => e.StartsWith("AgentSpool/"));
|
||||
}
|
||||
|
||||
// ===================================================== built-in catalog gates ====
|
||||
// The RealmJoin sections ride behind the tenant's RealmJoin Watcher toggle, the
|
||||
// Device Preparation bootstrapper event log behind the deterministic WDP marker.
|
||||
// Every conditional section is skipped WITH a manifest line, never silently.
|
||||
|
||||
/// <summary>RealmJoin system-side folders redirected to temp (user-side goes via UserProfileResolver).</summary>
|
||||
private sealed class RealmJoinRig : System.IDisposable
|
||||
{
|
||||
public TempDirectory WindowsLogs { get; } = new TempDirectory();
|
||||
public TempDirectory Choco { get; } = new TempDirectory();
|
||||
public TempDirectory UserProfile { get; } = new TempDirectory();
|
||||
public string PackagesFolder => Path.Combine(WindowsLogs.Path, "RealmJoin");
|
||||
|
||||
public RealmJoinRig()
|
||||
{
|
||||
File.WriteAllText(Path.Combine(WindowsLogs.Path, "realmjoin.log"), "rj");
|
||||
File.WriteAllText(Path.Combine(WindowsLogs.Path, "realmjoin2.log"), "rj2");
|
||||
var pkg = Path.Combine(PackagesFolder, "Packages", "generic-7zip");
|
||||
Directory.CreateDirectory(pkg);
|
||||
File.WriteAllText(Path.Combine(pkg, "install.log"), "pkg");
|
||||
var choco = Path.Combine(Choco.Path, "generic-7zip");
|
||||
Directory.CreateDirectory(choco);
|
||||
File.WriteAllText(Path.Combine(choco, "2026-06-30_install.log"), "choco");
|
||||
var userRj = Path.Combine(UserProfile.Path, "AppData", "Local", "RealmJoin");
|
||||
Directory.CreateDirectory(Path.Combine(userRj, "Logs", "generic-office"));
|
||||
File.WriteAllText(Path.Combine(userRj, "tray.log"), "tray");
|
||||
File.WriteAllText(Path.Combine(userRj, "config.pjson"), "not a log");
|
||||
File.WriteAllText(Path.Combine(userRj, "Logs", "RjImeHost.log"), "host");
|
||||
File.WriteAllText(Path.Combine(userRj, "Logs", "generic-office", "usersettings.log"), "pkg-user");
|
||||
}
|
||||
|
||||
public IReadOnlyDictionary<string, string> Overrides => new Dictionary<string, string>
|
||||
{
|
||||
["RealmJoinWindows"] = WindowsLogs.Path,
|
||||
["RealmJoinPackages"] = PackagesFolder,
|
||||
["RealmJoinChoco"] = Choco.Path,
|
||||
};
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
WindowsLogs.Dispose();
|
||||
Choco.Dispose();
|
||||
UserProfile.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveBytes_skips_realmjoin_sections_when_watcher_disabled()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
using var rj = new RealmJoinRig();
|
||||
UserProfileResolver.SetForTesting(rj.UserProfile.Path);
|
||||
try
|
||||
{
|
||||
var bytes = rig.Build(sectionFolderOverrides: rj.Overrides).BuildArchiveBytes(enrollmentSucceeded: true);
|
||||
|
||||
var entries = ZipEntryNames(bytes);
|
||||
Assert.DoesNotContain(entries, e => e.StartsWith("RealmJoinLogs/"));
|
||||
var manifest = ReadZipEntryText(bytes, "package-manifest.txt")!;
|
||||
Assert.Contains("SCENARIO: devicePreparation=False realmJoinWatcher=False", manifest);
|
||||
Assert.Contains("BUILT-IN SKIPPED (RealmJoin Watcher disabled): RealmJoinWindows", manifest);
|
||||
Assert.Contains("BUILT-IN SKIPPED (RealmJoin Watcher disabled): RealmJoinUserLogs", manifest);
|
||||
}
|
||||
finally
|
||||
{
|
||||
UserProfileResolver.Reset();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveBytes_includes_realmjoin_sections_when_watcher_enabled()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
using var rj = new RealmJoinRig();
|
||||
UserProfileResolver.SetForTesting(rj.UserProfile.Path);
|
||||
try
|
||||
{
|
||||
var bytes = rig.Build(cfg => cfg.EnableRealmJoinWatcher = true, rj.Overrides)
|
||||
.BuildArchiveBytes(enrollmentSucceeded: false);
|
||||
|
||||
var entries = ZipEntryNames(bytes);
|
||||
// ZIP layout mirrors the disk layout: flat client logs + package tree under
|
||||
// Windows/, tray logs + per-user Logs tree under User/.
|
||||
Assert.Contains("RealmJoinLogs/Windows/realmjoin.log", entries);
|
||||
Assert.Contains("RealmJoinLogs/Windows/realmjoin2.log", entries);
|
||||
Assert.Contains("RealmJoinLogs/Windows/RealmJoin/Packages/generic-7zip/install.log", entries);
|
||||
Assert.Contains("RealmJoinLogs/Choco/generic-7zip/2026-06-30_install.log", entries);
|
||||
Assert.Contains("RealmJoinLogs/User/tray.log", entries);
|
||||
Assert.Contains("RealmJoinLogs/User/Logs/RjImeHost.log", entries);
|
||||
Assert.Contains("RealmJoinLogs/User/Logs/generic-office/usersettings.log", entries);
|
||||
// Only tray*.log is collected from the user RealmJoin root — config stays out.
|
||||
Assert.DoesNotContain(entries, e => e.EndsWith("config.pjson"));
|
||||
var manifest = ReadZipEntryText(bytes, "package-manifest.txt")!;
|
||||
Assert.Contains("BUILT-IN: RealmJoinUserLogs -> folder=", manifest);
|
||||
Assert.DoesNotContain("BUILT-IN SKIPPED (RealmJoin Watcher disabled)", manifest);
|
||||
}
|
||||
finally
|
||||
{
|
||||
UserProfileResolver.Reset();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveBytes_skips_realmjoin_user_sections_without_user_session()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
using var rj = new RealmJoinRig();
|
||||
// Token present, no interactive user detected → user-profile sections skip with a
|
||||
// manifest line; the system-side sections are unaffected.
|
||||
UserProfileResolver.SetForTesting(null!);
|
||||
try
|
||||
{
|
||||
var bytes = rig.Build(cfg => cfg.EnableRealmJoinWatcher = true, rj.Overrides)
|
||||
.BuildArchiveBytes(enrollmentSucceeded: true);
|
||||
|
||||
var entries = ZipEntryNames(bytes);
|
||||
Assert.Contains("RealmJoinLogs/Windows/realmjoin.log", entries);
|
||||
Assert.DoesNotContain(entries, e => e.StartsWith("RealmJoinLogs/User"));
|
||||
var manifest = ReadZipEntryText(bytes, "package-manifest.txt")!;
|
||||
Assert.Contains("BUILT-IN SKIPPED (no user session for token): RealmJoinUserTray", manifest);
|
||||
Assert.Contains("BUILT-IN SKIPPED (no user session for token): RealmJoinUserLogs", manifest);
|
||||
}
|
||||
finally
|
||||
{
|
||||
UserProfileResolver.Reset();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveBytes_collects_bootstrapper_event_log_only_on_device_preparation()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
using var winevt = new TempDirectory();
|
||||
// Synthetic evtx: no registered channel claims the file, so the export falls
|
||||
// back to a raw copy — the point here is the scenario gate, not wevtutil.
|
||||
File.WriteAllBytes(Path.Combine(winevt.Path, "BootstrapperAgentServiceLogProvider.evtx"), new byte[] { 1, 2, 3 });
|
||||
var overrides = new Dictionary<string, string> { ["ImeBootstrapperEventLog"] = winevt.Path };
|
||||
|
||||
var classic = rig.Build(sectionFolderOverrides: overrides, devicePreparationProbe: () => false)
|
||||
.BuildArchiveBytes(enrollmentSucceeded: true);
|
||||
Assert.DoesNotContain("ImeLogs/BootstrapperAgentServiceLogProvider.evtx", ZipEntryNames(classic));
|
||||
Assert.Contains("BUILT-IN SKIPPED (not a Device Preparation enrollment): ImeBootstrapperEventLog",
|
||||
ReadZipEntryText(classic, "package-manifest.txt"));
|
||||
|
||||
var wdp = rig.Build(sectionFolderOverrides: overrides, devicePreparationProbe: () => true)
|
||||
.BuildArchiveBytes(enrollmentSucceeded: true);
|
||||
Assert.Contains("ImeLogs/BootstrapperAgentServiceLogProvider.evtx", ZipEntryNames(wdp));
|
||||
Assert.Contains("SCENARIO: devicePreparation=True", ReadZipEntryText(wdp, "package-manifest.txt"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildArchiveBytes_treats_failing_device_preparation_probe_as_classic()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
var bytes = rig.Build(devicePreparationProbe: () => throw new System.InvalidOperationException("registry boom"))
|
||||
.BuildArchiveBytes(enrollmentSucceeded: true);
|
||||
|
||||
var manifest = ReadZipEntryText(bytes, "package-manifest.txt")!;
|
||||
Assert.Contains("SCENARIO PROBE FAILED (treated as Classic): registry boom", manifest);
|
||||
Assert.Contains("BUILT-IN SKIPPED (not a Device Preparation enrollment): ImeBootstrapperEventLog", manifest);
|
||||
Assert.Contains("sessioninfo.txt", ZipEntryNames(bytes)); // archive still built
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DiagnosticsSectionCondition.Always, false, false, true)]
|
||||
[InlineData(DiagnosticsSectionCondition.RealmJoinWatcher, false, true, false)]
|
||||
[InlineData(DiagnosticsSectionCondition.RealmJoinWatcher, true, false, true)]
|
||||
[InlineData(DiagnosticsSectionCondition.DevicePreparation, true, false, false)]
|
||||
[InlineData(DiagnosticsSectionCondition.DevicePreparation, false, true, true)]
|
||||
[InlineData((DiagnosticsSectionCondition)999, true, true, false)]
|
||||
public void IsSectionActive_gates_on_condition(DiagnosticsSectionCondition condition, bool watcher, bool wdp, bool expected)
|
||||
{
|
||||
var section = new DiagnosticsBuiltInSection("X", "X", @"C:\X", new[] { "*.log" }, false, "x", condition);
|
||||
var cfg = Cfg();
|
||||
cfg.EnableRealmJoinWatcher = watcher;
|
||||
|
||||
Assert.Equal(expected, DiagnosticsPackageService.IsSectionActive(section, cfg, wdp));
|
||||
}
|
||||
|
||||
private static string[] ZipEntryNames(byte[] zipBytes)
|
||||
{
|
||||
using var ms = new MemoryStream(zipBytes);
|
||||
|
||||
+322
@@ -0,0 +1,322 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Scope of the Shell-Core replay: it recovers the Hello-wizard start and NOTHING else.
|
||||
/// <para>
|
||||
/// The replay had existed since session 772fe502 with no caller. Wiring it up (2026-08-19)
|
||||
/// activated three rails at once, and only one of them is safe to replay:
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item><b>62404 Hello-wizard start — replayed.</b> A conservative fact: it vetoes a
|
||||
/// premature "Hello is disabled" skip and can never by itself complete a session, so
|
||||
/// replaying it can only make the agent wait longer, never finish early.</item>
|
||||
/// <item><b>62407 ESP exit — never replayed.</b> Windows writes the identical description
|
||||
/// for the intermediate DeviceSetup-to-AccountSetup transition and the final exit, so the
|
||||
/// record carries no evidence of its own position; the reducer orders exits by ingest
|
||||
/// ordinal, which for a replay is fresher than reality; and arm C of
|
||||
/// ShouldTransitionToAwaitingHello opens on any arriving exit once the restored state
|
||||
/// carries AccountSetupEntered + a genuine IME user session + desktop arrival. See
|
||||
/// ClassicEspExitingOnRestoredStateTests for the reducer half of that proof.</item>
|
||||
/// <item><b>62407 ESP failure — never replayed.</b> Re-injecting a historic failure as
|
||||
/// fresh can fail a session that recovered (ANALYZE-ESP-006).</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class ShellCoreTrackerReplayScopeTests
|
||||
{
|
||||
private static readonly DateTime ClockNow = new DateTime(2026, 8, 19, 8, 43, 16, DateTimeKind.Utc);
|
||||
|
||||
// The REAL Shell-Core wording. Byte-identical for the intermediate DeviceSetup-to-
|
||||
// AccountSetup transition and for the final post-AccountSetup exit — that identity is the
|
||||
// whole reason a replayed exit cannot be classified.
|
||||
private const string EspExitDescription = "CommercialOOBE_ESPProgress_Page_Exiting";
|
||||
private const string EspFailureDescription = "CommercialOOBE_ESPProgress_Failure";
|
||||
private const string AadHelloDescription = "CloudExperienceHost web app activity: AADHello";
|
||||
private const string OtherWebAppDescription = "CloudExperienceHost web app activity: SomethingElse";
|
||||
|
||||
private sealed class Fixture : IDisposable
|
||||
{
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public FakeSignalIngressSink PostSink { get; } = new FakeSignalIngressSink();
|
||||
public VirtualClock Clock { get; } = new VirtualClock(ClockNow);
|
||||
|
||||
public ShellCoreTracker Build() => new ShellCoreTracker(
|
||||
sessionId: "S1",
|
||||
tenantId: "T1",
|
||||
post: new InformationalEventPost(PostSink, Clock),
|
||||
logger: new AgentLogger(Tmp.Path, AgentLogLevel.Info),
|
||||
helloTracker: null);
|
||||
|
||||
public List<FakeSignalIngressSink.PostedSignal> AgentTraces() =>
|
||||
PostSink.Posted.Where(p =>
|
||||
p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == "agent_trace").ToList();
|
||||
|
||||
public void Dispose() => Tmp.Dispose();
|
||||
}
|
||||
|
||||
private static (int, string, DateTime) Record(int id, string description, DateTime at)
|
||||
=> (id, description, at);
|
||||
|
||||
// ------------------------------------------------------------------ the wizard rail
|
||||
|
||||
[Fact]
|
||||
public void The_hello_wizard_start_is_replayed_with_its_original_source_time()
|
||||
{
|
||||
// Session 772fe502: an agent restarted while the user sits inside the Hello wizard
|
||||
// would otherwise never learn the wizard ran. This is the observation the replay
|
||||
// exists for.
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var order = new List<string>();
|
||||
HelloWizardStartedEventArgs? captured = null;
|
||||
tracker.HelloWizardStarted += (_, args) => { order.Add("wizard"); captured = args; };
|
||||
tracker.FinalizingSetupPhaseTriggered += (_, reason) => order.Add($"finalizing:{reason}");
|
||||
|
||||
var duringDowntime = ClockNow.AddMinutes(-37);
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>
|
||||
{
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppStarted, AadHelloDescription, duringDowntime),
|
||||
});
|
||||
|
||||
Assert.Equal(new[] { "wizard", "finalizing:hello_wizard_started" }, order);
|
||||
Assert.NotNull(captured);
|
||||
// The historical time must survive the replay — collapsing to now would date the
|
||||
// wizard after the restart and skew every duration derived from it.
|
||||
Assert.Equal(duringDowntime, captured!.OccurredAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_wizard_replay_is_single_shot_across_repeated_records()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var starts = 0;
|
||||
tracker.HelloWizardStarted += (_, __) => starts++;
|
||||
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>
|
||||
{
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppStarted, AadHelloDescription, ClockNow.AddMinutes(-30)),
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppStarted, AadHelloDescription, ClockNow.AddMinutes(-20)),
|
||||
});
|
||||
|
||||
Assert.Equal(1, starts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_non_hello_web_app_start_is_ignored()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var starts = 0;
|
||||
tracker.HelloWizardStarted += (_, __) => starts++;
|
||||
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>
|
||||
{
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppStarted, OtherWebAppDescription, ClockNow.AddMinutes(-5)),
|
||||
});
|
||||
|
||||
Assert.Equal(0, starts);
|
||||
Assert.Empty(f.AgentTraces());
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ the exit rail
|
||||
|
||||
[Fact]
|
||||
public void An_intermediate_only_window_raises_nothing_that_can_reach_the_reducer()
|
||||
{
|
||||
// THE regression this whole review chain is about: restart, and the only 62407 in the
|
||||
// window is the historic DeviceSetup-to-AccountSetup handoff. Nothing may leave the
|
||||
// tracker on a rail the decision engine listens to.
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var espExits = 0;
|
||||
var finalizing = new List<string>();
|
||||
var failures = 0;
|
||||
tracker.EspExited += (_, __) => espExits++;
|
||||
tracker.FinalizingSetupPhaseTriggered += (_, reason) => finalizing.Add(reason);
|
||||
tracker.EspFailureDetected += (_, __) => failures++;
|
||||
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>
|
||||
{
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppEvent, EspExitDescription, ClockNow.AddMinutes(-52)),
|
||||
});
|
||||
|
||||
Assert.Equal(0, espExits);
|
||||
Assert.Empty(finalizing);
|
||||
Assert.Equal(0, failures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Even_several_exits_in_the_window_stay_out_of_the_decision_stream()
|
||||
{
|
||||
// "Newest exit wins" was an earlier attempt at this problem. It does not help: if the
|
||||
// final exit has not happened yet, the newest record in the window is still the
|
||||
// intermediate one. Picking a record is the wrong lever — none of them is usable.
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var espExits = 0;
|
||||
tracker.EspExited += (_, __) => espExits++;
|
||||
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>
|
||||
{
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppEvent, EspExitDescription, ClockNow.AddMinutes(-52)),
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppEvent, EspExitDescription, ClockNow.AddMinutes(-9)),
|
||||
});
|
||||
|
||||
Assert.Equal(0, espExits);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_historic_esp_failure_is_never_re_injected()
|
||||
{
|
||||
// A session that failed and then recovered on retry (ANALYZE-ESP-006) must not be
|
||||
// failed again by its own history.
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var failures = new List<string>();
|
||||
tracker.EspFailureDetected += (_, failureType) => failures.Add(failureType);
|
||||
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>
|
||||
{
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppEvent, EspFailureDescription, ClockNow.AddMinutes(-40)),
|
||||
});
|
||||
|
||||
Assert.Empty(failures);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ visibility
|
||||
|
||||
[Fact]
|
||||
public void Skipped_records_are_reported_once_as_an_agent_trace()
|
||||
{
|
||||
// Dropping evidence silently is how this class of bug survives. The gap has to stay
|
||||
// visible to whoever debugs the session later — as an informational event, which is
|
||||
// decision-neutral by construction.
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>
|
||||
{
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppEvent, EspExitDescription, ClockNow.AddMinutes(-52)),
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppEvent, EspFailureDescription, ClockNow.AddMinutes(-40)),
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppEvent, EspExitDescription, ClockNow.AddMinutes(-9)),
|
||||
});
|
||||
|
||||
var trace = Assert.Single(f.AgentTraces());
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(trace.TypedPayload);
|
||||
Assert.Equal(2, data["skippedEspExits"]);
|
||||
Assert.Equal(1, data["skippedEspFailures"]);
|
||||
Assert.Equal("replayed_62407_not_orderable", data["reason"]);
|
||||
Assert.Equal(ClockNow.AddMinutes(-52).ToString("o"), data["oldestSkippedUtc"]);
|
||||
Assert.Equal(ClockNow.AddMinutes(-9).ToString("o"), data["newestSkippedUtc"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_wizard_only_window_reports_nothing_as_skipped()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>
|
||||
{
|
||||
Record(ShellCoreTracker.EventId_ShellCore_WebAppStarted, AadHelloDescription, ClockNow.AddMinutes(-30)),
|
||||
});
|
||||
|
||||
Assert.Empty(f.AgentTraces());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_empty_batch_is_a_no_op()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var raised = 0;
|
||||
tracker.EspExited += (_, __) => raised++;
|
||||
tracker.HelloWizardStarted += (_, __) => raised++;
|
||||
|
||||
tracker.ReplayBackfillRecords(new List<(int, string, DateTime)>());
|
||||
|
||||
Assert.Equal(0, raised);
|
||||
Assert.Empty(f.AgentTraces());
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ live path untouched
|
||||
|
||||
[Fact]
|
||||
public void The_live_path_still_raises_the_exit_normally()
|
||||
{
|
||||
// Narrowing the REPLAY must not narrow live observation — that is the agent's primary
|
||||
// signal source and the one the completion recovery actually runs on.
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var exits = new List<DateTime>();
|
||||
tracker.EspExited += (_, args) => exits.Add(args.OccurredAtUtc);
|
||||
|
||||
tracker.ProcessEvent(
|
||||
eventId: ShellCoreTracker.EventId_ShellCore_WebAppEvent,
|
||||
description: EspExitDescription,
|
||||
timestamp: ClockNow,
|
||||
providerName: "Microsoft-Windows-Shell-Core",
|
||||
isBackfill: false);
|
||||
|
||||
Assert.Equal(ClockNow, Assert.Single(exits));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_live_path_still_raises_esp_failures()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var tracker = f.Build();
|
||||
|
||||
var failures = new List<string>();
|
||||
tracker.EspFailureDetected += (_, failureType) => failures.Add(failureType);
|
||||
|
||||
tracker.ProcessEvent(
|
||||
eventId: ShellCoreTracker.EventId_ShellCore_WebAppEvent,
|
||||
description: EspFailureDescription,
|
||||
timestamp: ClockNow,
|
||||
providerName: "Microsoft-Windows-Shell-Core",
|
||||
isBackfill: false);
|
||||
|
||||
Assert.Single(failures);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ window clamp
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 1)]
|
||||
[InlineData(-42, 1)]
|
||||
[InlineData(5, 5)]
|
||||
[InlineData(ShellCoreTracker.BackfillLookbackMaxMinutes, ShellCoreTracker.BackfillLookbackMaxMinutes)]
|
||||
[InlineData(ShellCoreTracker.BackfillLookbackMaxMinutes + 1, ShellCoreTracker.BackfillLookbackMaxMinutes)]
|
||||
[InlineData(int.MaxValue, ShellCoreTracker.BackfillLookbackMaxMinutes)]
|
||||
public void Lookback_is_clamped_to_a_sane_window(int requested, int expected)
|
||||
{
|
||||
// int.MaxValue matters: the query multiplies minutes by 60_000, so an unclamped value
|
||||
// would overflow into a negative timediff and silently match nothing.
|
||||
Assert.Equal(expected, ShellCoreTracker.ClampLookbackMinutes(requested));
|
||||
}
|
||||
}
|
||||
}
|
||||
+258
@@ -0,0 +1,258 @@
|
||||
#nullable enable
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Runtime;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Pure state machine behind the power watcher: AC↔battery edges, the 50/30/15 threshold
|
||||
/// ladder (latch once per lifetime, lowest-crossed-only on multi-level jumps, baseline
|
||||
/// coverage for sessions starting low), null-percent tolerance and the storm cap.
|
||||
/// </summary>
|
||||
public sealed class PowerStateTransitionTrackerTests
|
||||
{
|
||||
private static PowerStateResult Ac(int? percent = 80, bool charging = true) => new PowerStateResult
|
||||
{
|
||||
OnAcPower = true,
|
||||
HasBattery = true,
|
||||
BatteryPercent = percent,
|
||||
IsCharging = charging,
|
||||
BatteryLifeMinutes = null,
|
||||
};
|
||||
|
||||
private static PowerStateResult Battery(int? percent, int? lifeMinutes = 90) => new PowerStateResult
|
||||
{
|
||||
OnAcPower = false,
|
||||
HasBattery = true,
|
||||
BatteryPercent = percent,
|
||||
IsCharging = false,
|
||||
BatteryLifeMinutes = lifeMinutes,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Baseline_on_ac_is_silent()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
Assert.Empty(tracker.Baseline(Ac()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Baseline_without_battery_or_with_probe_error_is_silent()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
Assert.Empty(tracker.Baseline(new PowerStateResult { OnAcPower = true, HasBattery = false }));
|
||||
Assert.Empty(tracker.Baseline(new PowerStateResult { ProbeError = "boom" }));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Baseline_on_battery_above_ladder_is_silent()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
Assert.Empty(tracker.Baseline(Battery(80)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Baseline_at_12_percent_emits_exactly_the_15_threshold()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
|
||||
var emissions = tracker.Baseline(Battery(12));
|
||||
|
||||
var e = Assert.Single(emissions);
|
||||
Assert.Equal(PowerStateTransitionTracker.TransitionThresholdCrossed, e.Transition);
|
||||
Assert.Equal(15, e.ThresholdPercent);
|
||||
Assert.Equal(12, e.BatteryPercent);
|
||||
Assert.Equal(EventSeverity.Error, e.Severity);
|
||||
Assert.True(e.ImmediateUpload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Baseline_at_25_percent_emits_30_and_latches_50()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
|
||||
var baseline = tracker.Baseline(Battery(25));
|
||||
var e = Assert.Single(baseline);
|
||||
Assert.Equal(30, e.ThresholdPercent);
|
||||
Assert.Equal(EventSeverity.Warning, e.Severity);
|
||||
Assert.True(e.ImmediateUpload);
|
||||
|
||||
// 50 and 30 are latched by the baseline; draining to 40% stays silent, 14% emits 15.
|
||||
Assert.Empty(tracker.Evaluate(Battery(40)));
|
||||
var next = Assert.Single(tracker.Evaluate(Battery(14)));
|
||||
Assert.Equal(15, next.ThresholdPercent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ac_to_battery_emits_warning_with_immediate_upload()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Ac(63));
|
||||
|
||||
var e = Assert.Single(tracker.Evaluate(Battery(63)));
|
||||
|
||||
Assert.Equal(PowerStateTransitionTracker.TransitionAcToBattery, e.Transition);
|
||||
Assert.Null(e.ThresholdPercent);
|
||||
Assert.Equal(EventSeverity.Warning, e.Severity);
|
||||
Assert.True(e.ImmediateUpload);
|
||||
Assert.False(e.OnAcPower);
|
||||
Assert.Equal(63, e.BatteryPercent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Battery_to_ac_emits_info_without_immediate_upload()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Battery(70));
|
||||
|
||||
var e = Assert.Single(tracker.Evaluate(Ac(70)));
|
||||
|
||||
Assert.Equal(PowerStateTransitionTracker.TransitionBatteryToAc, e.Transition);
|
||||
Assert.Equal(EventSeverity.Info, e.Severity);
|
||||
Assert.False(e.ImmediateUpload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_diff_stays_silent()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Ac());
|
||||
|
||||
Assert.Empty(tracker.Evaluate(Ac()));
|
||||
Assert.Empty(tracker.Evaluate(Ac(79)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unplug_below_threshold_emits_transition_and_threshold_together()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Ac(10));
|
||||
|
||||
var emissions = tracker.Evaluate(Battery(10));
|
||||
|
||||
Assert.Equal(2, emissions.Count);
|
||||
Assert.Equal(PowerStateTransitionTracker.TransitionAcToBattery, emissions[0].Transition);
|
||||
Assert.Equal(PowerStateTransitionTracker.TransitionThresholdCrossed, emissions[1].Transition);
|
||||
Assert.Equal(15, emissions[1].ThresholdPercent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Multi_level_drop_emits_lowest_crossed_level_only()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Battery(60));
|
||||
|
||||
var e = Assert.Single(tracker.Evaluate(Battery(10)));
|
||||
|
||||
Assert.Equal(15, e.ThresholdPercent);
|
||||
// All levels are latched — recovering and re-draining emits nothing more.
|
||||
tracker.Evaluate(Ac(90));
|
||||
Assert.Single(tracker.Evaluate(Battery(45)), x => x.Transition == PowerStateTransitionTracker.TransitionAcToBattery);
|
||||
Assert.Empty(tracker.Evaluate(Battery(13)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Each_level_latches_once_per_lifetime()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Battery(70));
|
||||
|
||||
Assert.Single(tracker.Evaluate(Battery(48))); // 50
|
||||
Assert.Empty(tracker.Evaluate(Battery(47))); // still inside 50, no re-emit
|
||||
Assert.Single(tracker.Evaluate(Battery(29))); // 30
|
||||
Assert.Single(tracker.Evaluate(Battery(15))); // 15 (boundary: <= level)
|
||||
Assert.Empty(tracker.Evaluate(Battery(5))); // ladder exhausted
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Threshold_severity_ladder_is_info_warning_error()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Battery(70));
|
||||
|
||||
var fifty = Assert.Single(tracker.Evaluate(Battery(50)));
|
||||
Assert.Equal(EventSeverity.Info, fifty.Severity);
|
||||
Assert.False(fifty.ImmediateUpload);
|
||||
|
||||
var thirty = Assert.Single(tracker.Evaluate(Battery(30)));
|
||||
Assert.Equal(EventSeverity.Warning, thirty.Severity);
|
||||
Assert.True(thirty.ImmediateUpload);
|
||||
|
||||
var fifteen = Assert.Single(tracker.Evaluate(Battery(15)));
|
||||
Assert.Equal(EventSeverity.Error, fifteen.Severity);
|
||||
Assert.True(fifteen.ImmediateUpload);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Null_battery_percent_skips_thresholds_but_ac_diff_still_works()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Ac(null));
|
||||
|
||||
var e = Assert.Single(tracker.Evaluate(Battery(null, lifeMinutes: null)));
|
||||
|
||||
Assert.Equal(PowerStateTransitionTracker.TransitionAcToBattery, e.Transition);
|
||||
Assert.Null(e.BatteryPercent);
|
||||
Assert.Contains("battery level unknown", e.Message);
|
||||
Assert.Empty(tracker.Evaluate(Battery(null, lifeMinutes: null)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Probe_error_mid_run_preserves_state()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Ac());
|
||||
|
||||
Assert.Empty(tracker.Evaluate(new PowerStateResult { ProbeError = "transient" }));
|
||||
// State was not corrupted by the error — the real transition still diffs cleanly.
|
||||
Assert.Single(tracker.Evaluate(Battery(60)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_without_baseline_latches_like_a_baseline()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
|
||||
var e = Assert.Single(tracker.Evaluate(Battery(12)));
|
||||
|
||||
// No phantom ac_to_battery from the implicit default state — only the threshold.
|
||||
Assert.Equal(PowerStateTransitionTracker.TransitionThresholdCrossed, e.Transition);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Emission_cap_suppresses_and_flips_flag_once()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker(maxEmissions: 4);
|
||||
tracker.Baseline(Ac());
|
||||
|
||||
// Flapping power source: each edge is one emission.
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
Assert.Single(tracker.Evaluate(Battery(80)));
|
||||
Assert.Single(tracker.Evaluate(Ac(80)));
|
||||
}
|
||||
Assert.False(tracker.EmissionCapReached);
|
||||
|
||||
Assert.Empty(tracker.Evaluate(Battery(80)));
|
||||
Assert.True(tracker.EmissionCapReached);
|
||||
Assert.Empty(tracker.Evaluate(Ac(80)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Message_carries_percent_and_transition_context()
|
||||
{
|
||||
var tracker = new PowerStateTransitionTracker();
|
||||
tracker.Baseline(Ac(62));
|
||||
|
||||
var toBattery = Assert.Single(tracker.Evaluate(Battery(62)));
|
||||
Assert.Contains("AC to battery", toBattery.Message);
|
||||
Assert.Contains("battery 62%", toBattery.Message);
|
||||
|
||||
var backToAc = Assert.Single(tracker.Evaluate(Ac(63)));
|
||||
Assert.Contains("back to AC", backToAc.Message);
|
||||
Assert.Contains("charging", backToAc.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
+429
@@ -0,0 +1,429 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// SystemTimelineTracker — clock steps (Kernel-General 1) and completed sleep episodes
|
||||
/// (Power-Troubleshooter 1 / Kernel-Power 507) from the System channel. Drives the primitive
|
||||
/// <see cref="SystemTimelineTracker.ProcessEvent"/> test seam (no real EventRecord, which is
|
||||
/// abstract + Windows-only), mirroring the WindowsUpdateTracker test pattern.
|
||||
/// </summary>
|
||||
public sealed class SystemTimelineTrackerTests : IDisposable
|
||||
{
|
||||
private static readonly DateTime At = new DateTime(2026, 8, 21, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
private readonly FakeSignalIngressSink _sink;
|
||||
private readonly SystemTimelineTracker _tracker;
|
||||
private long _nextRecordId = 1;
|
||||
|
||||
public SystemTimelineTrackerTests()
|
||||
{
|
||||
_sink = new FakeSignalIngressSink();
|
||||
var post = new InformationalEventPost(_sink, new VirtualClock(At));
|
||||
var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
|
||||
_tracker = new SystemTimelineTracker(
|
||||
sessionId: "sess-stl",
|
||||
tenantId: "tenant-stl",
|
||||
post: post,
|
||||
logger: logger,
|
||||
backfillEnabled: false,
|
||||
stateDirectory: null); // in-memory dedup for most tests
|
||||
}
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// Emission helpers
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
private void EmitClockChange(
|
||||
DateTime? oldTime, DateTime? newTime, string? rawDeltaMs = null,
|
||||
int reason = 1, long? recordId = null, bool isBackfill = false, DateTime? timeCreatedUtc = null)
|
||||
{
|
||||
var data = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (oldTime.HasValue) data["OldTime"] = oldTime.Value.ToString("o");
|
||||
if (newTime.HasValue) data["NewTime"] = newTime.Value.ToString("o");
|
||||
if (rawDeltaMs != null) data["TimeDeltaInMs"] = rawDeltaMs;
|
||||
data["Reason"] = reason.ToString();
|
||||
data["ProcessName"] = @"\Device\HarddiskVolume3\Windows\System32\svchost.exe";
|
||||
data["ProcessID"] = "6892";
|
||||
|
||||
_tracker.ProcessEvent(
|
||||
providerName: SystemTimelineTracker.ProviderKernelGeneral,
|
||||
eventId: SystemTimelineTracker.EventId_ClockChange,
|
||||
recordId: recordId ?? _nextRecordId++,
|
||||
timeCreatedUtc: timeCreatedUtc ?? At,
|
||||
eventData: data,
|
||||
isBackfill: isBackfill);
|
||||
}
|
||||
|
||||
private void EmitClassicResume(
|
||||
DateTime sleepTime, DateTime wakeTime, string? effectiveState = "1",
|
||||
string? wakeSourceText = null, long? recordId = null, bool isBackfill = false, DateTime? timeCreatedUtc = null)
|
||||
{
|
||||
var data = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["SleepTime"] = sleepTime.ToString("o"),
|
||||
["WakeTime"] = wakeTime.ToString("o"),
|
||||
["WakeSourceType"] = "0",
|
||||
};
|
||||
if (effectiveState != null) data["EffectiveState"] = effectiveState;
|
||||
if (wakeSourceText != null) data["WakeSourceText"] = wakeSourceText;
|
||||
|
||||
_tracker.ProcessEvent(
|
||||
providerName: SystemTimelineTracker.ProviderPowerTroubleshooter,
|
||||
eventId: SystemTimelineTracker.EventId_ClassicResume,
|
||||
recordId: recordId ?? _nextRecordId++,
|
||||
timeCreatedUtc: timeCreatedUtc ?? wakeTime,
|
||||
eventData: data,
|
||||
isBackfill: isBackfill);
|
||||
}
|
||||
|
||||
private void EmitModernStandbyExit(
|
||||
long sleepDurationUs, long durationUs, string powerStateAc = "true",
|
||||
long? recordId = null, bool isBackfill = false, DateTime? timeCreatedUtc = null)
|
||||
{
|
||||
var data = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["SleepDurationInUs"] = sleepDurationUs.ToString(),
|
||||
["DurationInUs"] = durationUs.ToString(),
|
||||
["Reason"] = "32",
|
||||
["PowerStateAc"] = powerStateAc,
|
||||
["BatteryRemainingCapacityOnExit"] = "34180",
|
||||
};
|
||||
|
||||
_tracker.ProcessEvent(
|
||||
providerName: SystemTimelineTracker.ProviderKernelPower,
|
||||
eventId: SystemTimelineTracker.EventId_ModernStandbyExit,
|
||||
recordId: recordId ?? _nextRecordId++,
|
||||
timeCreatedUtc: timeCreatedUtc ?? At,
|
||||
eventData: data,
|
||||
isBackfill: isBackfill);
|
||||
}
|
||||
|
||||
private IReadOnlyList<FakeSignalIngressSink.PostedSignal> ByType(string eventType) =>
|
||||
_sink.Posted.Where(p =>
|
||||
p.Kind == DecisionSignalKind.InformationalEvent
|
||||
&& p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == eventType).ToList();
|
||||
|
||||
private static IReadOnlyDictionary<string, object> Data(FakeSignalIngressSink.PostedSignal s) =>
|
||||
(IReadOnlyDictionary<string, object>)s.TypedPayload!;
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// system_clock_changed
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ClockStep_AboveThreshold_EmitsInfo_WithSignedDeltaAndNormalizedTimes()
|
||||
{
|
||||
var oldTime = At;
|
||||
var newTime = At.AddSeconds(10);
|
||||
EmitClockChange(oldTime, newTime, reason: 1);
|
||||
|
||||
var s = Assert.Single(ByType(Constants.EventTypes.SystemClockChanged));
|
||||
Assert.Equal("Info", s.Payload![SignalPayloadKeys.Severity]);
|
||||
Assert.Equal("false", s.Payload![SignalPayloadKeys.ImmediateUpload]);
|
||||
|
||||
var data = Data(s);
|
||||
Assert.Equal(10_000L, data["timeDeltaMs"]);
|
||||
Assert.Equal("application_set", data["reasonText"]);
|
||||
Assert.Equal(1, data["reason"]);
|
||||
Assert.Equal(oldTime.ToString("o"), data["oldTime"]);
|
||||
Assert.Equal(newTime.ToString("o"), data["newTime"]);
|
||||
Assert.Equal(6892, data["processId"]);
|
||||
Assert.Contains("forward", (string)s.Payload![SignalPayloadKeys.Message]);
|
||||
Assert.Contains("svchost.exe", (string)s.Payload![SignalPayloadKeys.Message]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1)]
|
||||
[InlineData(1999)]
|
||||
public void ClockMicroStep_BelowThreshold_Suppressed(int deltaMs)
|
||||
{
|
||||
EmitClockChange(At, At.AddMilliseconds(deltaMs));
|
||||
Assert.Empty(ByType(Constants.EventTypes.SystemClockChanged));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClockStep_Backward_EmitsNegativeDelta()
|
||||
{
|
||||
EmitClockChange(oldTime: At, newTime: At.AddSeconds(-3));
|
||||
|
||||
var s = Assert.Single(ByType(Constants.EventTypes.SystemClockChanged));
|
||||
Assert.Equal(-3000L, Data(s)["timeDeltaMs"]);
|
||||
Assert.Contains("backward", (string)s.Payload![SignalPayloadKeys.Message]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClockStep_AtLeastFiveMinutes_WarningAndImmediateUpload()
|
||||
{
|
||||
EmitClockChange(At, At.AddMinutes(5), reason: 2);
|
||||
|
||||
var s = Assert.Single(ByType(Constants.EventTypes.SystemClockChanged));
|
||||
Assert.Equal("Warning", s.Payload![SignalPayloadKeys.Severity]);
|
||||
Assert.Equal("true", s.Payload![SignalPayloadKeys.ImmediateUpload]);
|
||||
Assert.Equal("hardware_clock_sync", Data(s)["reasonText"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClockStep_NoUsableTimesOrDelta_Skipped()
|
||||
{
|
||||
EmitClockChange(oldTime: null, newTime: null, rawDeltaMs: "not-a-number");
|
||||
Assert.Empty(ByType(Constants.EventTypes.SystemClockChanged));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClockStep_FallsBackToTimeDeltaInMs_WhenTimesUnparseable()
|
||||
{
|
||||
// Only the raw delta parses — the direction information of NewTime/OldTime is gone,
|
||||
// but the magnitude still gates and emits.
|
||||
EmitClockChange(oldTime: null, newTime: null, rawDeltaMs: "7200000");
|
||||
|
||||
var s = Assert.Single(ByType(Constants.EventTypes.SystemClockChanged));
|
||||
Assert.Equal(7_200_000L, Data(s)["timeDeltaMs"]);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// system_sleep_episode — classic (Power-Troubleshooter 1)
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ClassicResume_EffectiveState5_EmitsHibernateEpisode()
|
||||
{
|
||||
var sleepAt = At.AddHours(-2);
|
||||
EmitClassicResume(sleepAt, At, effectiveState: "5", wakeSourceText: "Power Button");
|
||||
|
||||
var s = Assert.Single(ByType(Constants.EventTypes.SystemSleepEpisode));
|
||||
Assert.Equal("Info", s.Payload![SignalPayloadKeys.Severity]);
|
||||
|
||||
var data = Data(s);
|
||||
Assert.Equal("hibernate", data["kind"]);
|
||||
Assert.Equal(7200L, data["durationSeconds"]);
|
||||
Assert.Equal(sleepAt.ToString("o"), data["enteredAt"]);
|
||||
Assert.Equal(At.ToString("o"), data["exitedAt"]);
|
||||
Assert.Equal("Power Button", data["wakeSourceText"]);
|
||||
Assert.Contains("hibernate", (string)s.Payload![SignalPayloadKeys.Message]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClassicResume_NonHibernateState_EmitsSleepKind()
|
||||
{
|
||||
EmitClassicResume(At.AddMinutes(-30), At, effectiveState: "4");
|
||||
Assert.Equal("sleep", Data(Assert.Single(ByType(Constants.EventTypes.SystemSleepEpisode)))["kind"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ClassicResume_ShortEpisode_StillEmits()
|
||||
{
|
||||
// No duration floor for classic sleep: the event only exists for genuinely completed
|
||||
// S3/S4 transitions, so a 30 s real sleep is still signal.
|
||||
EmitClassicResume(At.AddSeconds(-30), At);
|
||||
|
||||
var s = Assert.Single(ByType(Constants.EventTypes.SystemSleepEpisode));
|
||||
Assert.Equal(30L, Data(s)["durationSeconds"]);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// system_sleep_episode — Modern Standby (Kernel-Power 507)
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ModernStandbyExit_EmitsEpisode_WithDerivedEnterTime()
|
||||
{
|
||||
// Values from a live 507 on this dev machine: 38 min scenario, ~38 min actual sleep.
|
||||
EmitModernStandbyExit(sleepDurationUs: 2_324_095_807, durationUs: 2_330_201_652);
|
||||
|
||||
var s = Assert.Single(ByType(Constants.EventTypes.SystemSleepEpisode));
|
||||
var data = Data(s);
|
||||
Assert.Equal("modern_standby", data["kind"]);
|
||||
Assert.Equal(2330L, data["durationSeconds"]);
|
||||
Assert.Equal(2324L, data["sleepDurationSeconds"]);
|
||||
Assert.Equal(true, data["onAcPower"]);
|
||||
Assert.Equal(34180L, data["batteryRemainingCapacityOnExit"]);
|
||||
Assert.Equal(At.ToString("o"), data["exitedAt"]);
|
||||
Assert.Equal(At.AddMilliseconds(-2_330_201.652).ToString("o"), data["enteredAt"]);
|
||||
Assert.Contains("Modern Standby", (string)s.Payload![SignalPayloadKeys.Message]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModernStandbyExit_SleepBelowFloor_Suppressed()
|
||||
{
|
||||
EmitModernStandbyExit(sleepDurationUs: 30_000_000, durationUs: 90_000_000); // 30 s sleep
|
||||
Assert.Empty(ByType(Constants.EventTypes.SystemSleepEpisode));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModernStandbyExit_ScreenOffWithoutSleep_Suppressed()
|
||||
{
|
||||
// SleepEntered=false scenarios carry SleepDurationInUs 0 — the single duration gate
|
||||
// suppresses them without a separate branch.
|
||||
EmitModernStandbyExit(sleepDurationUs: 0, durationUs: 6_106_696);
|
||||
Assert.Empty(ByType(Constants.EventTypes.SystemSleepEpisode));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ModernStandbyExit_ScenarioShorterThanSleep_ClampsToSleepDuration()
|
||||
{
|
||||
EmitModernStandbyExit(sleepDurationUs: 120_000_000, durationUs: 0);
|
||||
|
||||
var data = Data(Assert.Single(ByType(Constants.EventTypes.SystemSleepEpisode)));
|
||||
Assert.Equal(120L, data["durationSeconds"]);
|
||||
Assert.Equal(120L, data["sleepDurationSeconds"]);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// Backfill semantics
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void BackfilledEvent_UsesEventTimeForTimelineTimestamp_AndFlagsData()
|
||||
{
|
||||
// A pre-agent standby episode (record created before the agent started) must land on
|
||||
// the timeline where the device actually woke, not at agent start.
|
||||
var wakeTime = At.AddHours(-3);
|
||||
EmitClassicResume(wakeTime.AddHours(-1), wakeTime, isBackfill: true, timeCreatedUtc: wakeTime);
|
||||
|
||||
var s = Assert.Single(ByType(Constants.EventTypes.SystemSleepEpisode));
|
||||
Assert.Equal(wakeTime, s.OccurredAtUtc);
|
||||
Assert.Equal(true, Data(s)["backfilled"]);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// Dedup / watermark
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void SameRecordId_ProcessedTwice_EmitsOnce()
|
||||
{
|
||||
EmitClockChange(At, At.AddMinutes(1), recordId: 42);
|
||||
EmitClockChange(At, At.AddMinutes(1), recordId: 42);
|
||||
|
||||
Assert.Single(ByType(Constants.EventTypes.SystemClockChanged));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BackfillEventWithLowerRecordId_AfterLiveEvent_IsStillEmitted()
|
||||
{
|
||||
// The live watcher is armed BEFORE backfill runs, so a live record with a higher
|
||||
// RecordId can be processed first. An older, never-emitted backfill record must NOT be
|
||||
// suppressed — pre-agent episodes are the whole point (high-water-mark trap).
|
||||
EmitClockChange(At, At.AddMinutes(1), recordId: 100); // live, high
|
||||
EmitClockChange(At, At.AddMinutes(2), recordId: 50, isBackfill: true); // backfill, older
|
||||
|
||||
Assert.Equal(2, ByType(Constants.EventTypes.SystemClockChanged).Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Watermark_PersistsAcrossTrackerInstances()
|
||||
{
|
||||
var post = new InformationalEventPost(_sink, new VirtualClock(At));
|
||||
var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
|
||||
var clockData = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["OldTime"] = At.ToString("o"),
|
||||
["NewTime"] = At.AddMinutes(10).ToString("o"),
|
||||
["Reason"] = "1",
|
||||
};
|
||||
|
||||
var first = new SystemTimelineTracker("s", "t", post, logger, backfillEnabled: false, stateDirectory: _tmp.Path);
|
||||
first.ProcessEvent(SystemTimelineTracker.ProviderKernelGeneral, SystemTimelineTracker.EventId_ClockChange,
|
||||
recordId: 500, timeCreatedUtc: At, eventData: clockData, isBackfill: false);
|
||||
|
||||
// A fresh tracker (agent restart) must load the watermark and skip the already-emitted
|
||||
// record when the backfill re-reads it.
|
||||
var second = new SystemTimelineTracker("s", "t", post, logger, backfillEnabled: false, stateDirectory: _tmp.Path);
|
||||
second.LoadWatermark();
|
||||
second.ProcessEvent(SystemTimelineTracker.ProviderKernelGeneral, SystemTimelineTracker.EventId_ClockChange,
|
||||
recordId: 500, timeCreatedUtc: At, eventData: clockData, isBackfill: true);
|
||||
|
||||
Assert.Single(ByType(Constants.EventTypes.SystemClockChanged));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnknownProviderOrEventId_Ignored()
|
||||
{
|
||||
_tracker.ProcessEvent("Microsoft-Windows-Kernel-Power", eventId: 566, recordId: _nextRecordId++,
|
||||
timeCreatedUtc: At, eventData: new Dictionary<string, string>(), isBackfill: false);
|
||||
_tracker.ProcessEvent("Some-Other-Provider", eventId: 1, recordId: _nextRecordId++,
|
||||
timeCreatedUtc: At, eventData: new Dictionary<string, string>(), isBackfill: false);
|
||||
|
||||
Assert.Empty(_sink.Posted);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// XPath + parsing
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void BuildXPath_ContainsAllThreeProviderIdPairs_AndNo506()
|
||||
{
|
||||
var xpath = SystemTimelineTracker.BuildXPath();
|
||||
Assert.Contains("Provider[@Name='Microsoft-Windows-Kernel-General'] and EventID=1", xpath);
|
||||
Assert.Contains("Provider[@Name='Microsoft-Windows-Power-Troubleshooter'] and EventID=1", xpath);
|
||||
Assert.Contains("Provider[@Name='Microsoft-Windows-Kernel-Power'] and EventID=507", xpath);
|
||||
Assert.DoesNotContain("506", xpath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildBackfillXPath_AppendsTimediffClause()
|
||||
{
|
||||
var xpath = SystemTimelineTracker.BuildBackfillXPath(86_400_000);
|
||||
Assert.Contains("TimeCreated[timediff(@SystemTime) <= 86400000]", xpath);
|
||||
Assert.Contains("EventID=507", xpath);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ParseEventData_RealKernelGeneralPayload_YieldsAuthoritativeFields()
|
||||
{
|
||||
// Verbatim (values shortened) from a live Kernel-General 1 record on a 26220 dev machine.
|
||||
const string xml =
|
||||
"<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'>" +
|
||||
"<System><Provider Name='Microsoft-Windows-Kernel-General'/><EventID>1</EventID></System>" +
|
||||
"<EventData>" +
|
||||
"<Data Name='NewTime'>2026-08-21T09:40:02.1987028Z</Data>" +
|
||||
"<Data Name='OldTime'>2026-08-21T09:40:02.1973028Z</Data>" +
|
||||
"<Data Name='TimeDeltaInMs'>1</Data>" +
|
||||
"<Data Name='Reason'>1</Data>" +
|
||||
"<Data Name='ProcessName'>\\Device\\HarddiskVolume3\\Windows\\System32\\svchost.exe</Data>" +
|
||||
"<Data Name='ProcessID'>6892</Data>" +
|
||||
"</EventData></Event>";
|
||||
|
||||
var data = SystemTimelineTracker.ParseEventData(xml);
|
||||
Assert.Equal("1", data["TimeDeltaInMs"]);
|
||||
Assert.Equal("2026-08-21T09:40:02.1987028Z", data["newtime"]); // case-insensitive keys
|
||||
Assert.NotNull(SystemTimelineTracker.TryParseUtc(data["OldTime"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FormatDuration_CompactForms()
|
||||
{
|
||||
Assert.Equal("2h 05m", SystemTimelineTracker.FormatDuration(TimeSpan.FromMinutes(125)));
|
||||
Assert.Equal("56m 12s", SystemTimelineTracker.FormatDuration(TimeSpan.FromSeconds(56 * 60 + 12)));
|
||||
Assert.Equal("45s", SystemTimelineTracker.FormatDuration(TimeSpan.FromSeconds(45)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProcessLeafName_StripsDevicePath()
|
||||
{
|
||||
Assert.Equal("svchost.exe",
|
||||
SystemTimelineTracker.ProcessLeafName(@"\Device\HarddiskVolume3\Windows\System32\svchost.exe"));
|
||||
Assert.Null(SystemTimelineTracker.ProcessLeafName(null));
|
||||
Assert.Null(SystemTimelineTracker.ProcessLeafName(@"\Device\HarddiskVolume3\"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
{
|
||||
public class DoGroupIdServiceTests
|
||||
{
|
||||
// -------------------------------------------------------------------
|
||||
// ComputeGroupId — RealmJoin byte-layout compatibility
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ComputeGroupId_MatchesHandComputedVector()
|
||||
{
|
||||
// Layout: {101,48,6} + {0,0,0} + {ip[1],0,ip[2],ip[3]} + mac — through new Guid(byte[])
|
||||
// the first three groups print little-endian, the last eight bytes verbatim.
|
||||
var guid = DoGroupIdService.ComputeGroupId(
|
||||
new byte[] { 192, 168, 178, 1 },
|
||||
new byte[] { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55 });
|
||||
|
||||
Assert.Equal("00063065-0000-00a8-b201-001122334455", guid.ToString());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(new byte[] { 10, 0, 0, 1 }, new byte[] { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF })]
|
||||
[InlineData(new byte[] { 192, 168, 1, 254 }, new byte[] { 0x00, 0x50, 0x56, 0x01, 0x02, 0x03 })]
|
||||
[InlineData(new byte[] { 172, 16, 254, 3 }, new byte[] { 0xF0, 0x9F, 0xC2, 0x00, 0x00, 0x01 })]
|
||||
public void ComputeGroupId_MatchesRealmJoinReferenceImplementation(byte[] ip, byte[] mac)
|
||||
{
|
||||
Assert.Equal(RealmJoinReferenceGuid(ip, mac), DoGroupIdService.ComputeGroupId(ip, mac));
|
||||
}
|
||||
|
||||
/// <summary>Verbatim port of RealmJoin's ConvertToGuid — the compatibility contract.</summary>
|
||||
private static Guid RealmJoinReferenceGuid(byte[] ipBytes, byte[] address)
|
||||
{
|
||||
byte[] prefix = { 101, 48, 6 };
|
||||
var stuff = new byte[4];
|
||||
stuff[0] = ipBytes[1];
|
||||
stuff[2] = ipBytes[2];
|
||||
stuff[3] = ipBytes[3];
|
||||
|
||||
return new Guid(prefix
|
||||
.Concat(new byte[16 - prefix.Length - stuff.Length - address.Length])
|
||||
.Concat(stuff)
|
||||
.Concat(address)
|
||||
.ToArray());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeGroupId_RejectsNonIpv4AndBadMacLengths()
|
||||
{
|
||||
var ip4 = new byte[] { 10, 0, 0, 1 };
|
||||
var mac = new byte[] { 1, 2, 3, 4, 5, 6 };
|
||||
|
||||
Assert.Throws<ArgumentException>(() => DoGroupIdService.ComputeGroupId(new byte[16], mac));
|
||||
Assert.Throws<ArgumentException>(() => DoGroupIdService.ComputeGroupId(null!, mac));
|
||||
Assert.Throws<ArgumentException>(() => DoGroupIdService.ComputeGroupId(ip4, new byte[8]));
|
||||
Assert.Throws<ArgumentException>(() => DoGroupIdService.ComputeGroupId(ip4, null!));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeGroupId_RegistryFormat_IsBracelessLowercase()
|
||||
{
|
||||
// RealmJoin writes SetValue(name, Guid) which stringifies via Guid.ToString() ("d"
|
||||
// format) — our written value must be the identical string form.
|
||||
var text = DoGroupIdService.ComputeGroupId(
|
||||
new byte[] { 10, 1, 2, 3 }, new byte[] { 0xAB, 0xCD, 0xEF, 0x01, 0x02, 0x03 }).ToString();
|
||||
|
||||
Assert.DoesNotContain("{", text);
|
||||
Assert.Equal(text.ToLowerInvariant(), text);
|
||||
Assert.Equal(36, text.Length);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// DecideAction — ownership matrix
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
private const string Computed = "00063065-0000-00a8-b201-001122334455";
|
||||
private const string Other = "11111111-2222-3333-4444-555555555555";
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
public void DecideAction_NoExistingValue_Writes(string? existing)
|
||||
{
|
||||
Assert.Equal(DoGroupIdAction.Write, DoGroupIdService.DecideAction(existing, Computed, null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecideAction_ExistingEqualsComputed_IsNoOp()
|
||||
{
|
||||
Assert.Equal(DoGroupIdAction.NoOp, DoGroupIdService.DecideAction(Computed, Computed, null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecideAction_ExistingEqualsComputed_ToleratesBracesAndCase()
|
||||
{
|
||||
// A value written by another tool in "{...}"/uppercase form is still semantically ours.
|
||||
var braced = "{" + Computed.ToUpperInvariant() + "}";
|
||||
Assert.Equal(DoGroupIdAction.NoOp, DoGroupIdService.DecideAction(braced, Computed, null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecideAction_ExistingIsOurStaleValue_WritesUpdate()
|
||||
{
|
||||
// Gateway changed: the registry still holds the GUID we wrote for the old network.
|
||||
Assert.Equal(DoGroupIdAction.Write, DoGroupIdService.DecideAction(Other, Computed, Other));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecideAction_ForeignValue_Skips()
|
||||
{
|
||||
Assert.Equal(DoGroupIdAction.SkipForeign, DoGroupIdService.DecideAction(Other, Computed, null));
|
||||
Assert.Equal(DoGroupIdAction.SkipForeign, DoGroupIdService.DecideAction(Other, Computed, Computed));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DecideAction_NonGuidGarbageExisting_SkipsAsForeign()
|
||||
{
|
||||
// Never overwrite a value we can't even parse — conservative by design.
|
||||
Assert.Equal(DoGroupIdAction.SkipForeign, DoGroupIdService.DecideAction("not-a-guid", Computed, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -117,6 +118,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
hosts,
|
||||
"AadJoinHost wires its onRealUserJoined callback to the already-constructed DesktopArrivalHost.");
|
||||
|
||||
// Codex review P2 (2026-08-19): EspAndHelloHost.Start() replays the Shell-Core
|
||||
// ESP exit, but ImeLogTracker only restores its persisted app states in its own
|
||||
// Start() — which happens later. Pin the order that makes ImeLogHost's
|
||||
// onStateRestored nudge necessary, so a future reshuffle either keeps the nudge
|
||||
// meaningful or fails here loudly.
|
||||
AssertBefore<EspAndHelloHost, ImeLogHost>(
|
||||
hosts,
|
||||
"the ESP-exit replay runs before the IME app-state restore, which is why " +
|
||||
"ImeLogHost re-triggers the user-apps-settled synthesis after LoadState.");
|
||||
|
||||
// Closure-latch sanity (factory lines ~139/297): the EspAndHelloHost probes close
|
||||
// over the ImeLogHost reference, which is assigned during the same factory call —
|
||||
// the host must therefore exist in the returned list.
|
||||
@@ -130,5 +141,40 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ime_app_state_callback_is_chained_into_the_user_apps_settled_recheck()
|
||||
{
|
||||
// sits-d Cloud-PC fix (2026-08-19): the AccountSetup synthesis is re-evaluated on every
|
||||
// terminal IME app transition, which only works while the factory keeps this chain
|
||||
// wired. It is five lines that a refactor can drop silently and whose absence costs a
|
||||
// whole class of sessions their completion — so pin it here.
|
||||
var surfaces = CreateSurfaces();
|
||||
var hosts = surfaces.Hosts;
|
||||
try
|
||||
{
|
||||
var imeHost = hosts.OfType<ImeLogHost>().Single();
|
||||
var handler = imeHost.Tracker.OnAppStateChanged;
|
||||
|
||||
Assert.NotNull(handler);
|
||||
|
||||
// Invoking it must reach the re-check without throwing. The re-check is a no-op
|
||||
// here (no ESP exit observed), which is the point — it must stay cheap and silent
|
||||
// on the IME hot path until both facts are in.
|
||||
var pkg = new AppPackageState("app-1", listPos: 0);
|
||||
var ex = Record.Exception(() => handler!(
|
||||
pkg,
|
||||
AppInstallationState.Installing,
|
||||
AppInstallationState.Installed));
|
||||
Assert.Null(ex);
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var host in hosts)
|
||||
{
|
||||
try { host.Dispose(); } catch { /* construction-only test — best-effort cleanup */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// The Shell-Core ESP-exit replay is RESTART RECOVERY: it re-reads observations the agent could
|
||||
/// not make because no agent process was running. The window is therefore measured against the
|
||||
/// previous run's last known liveness, not guessed from a constant.
|
||||
/// <para>
|
||||
/// Why a constant is not enough (Codex review P2): the scheduled task carries a BootTrigger
|
||||
/// only, with no restart-on-failure. A crashed agent does not come back until the NEXT BOOT —
|
||||
/// possibly hours later, long after Windows wrote the final ESP exit. Five minutes would lose
|
||||
/// it every time.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class EspExitBackfillLookbackPolicyTests
|
||||
{
|
||||
private static readonly DateTime Now = new DateTime(2026, 8, 19, 8, 43, 16, DateTimeKind.Utc);
|
||||
private const int Default = ShellCoreTracker.BackfillLookbackMinutes;
|
||||
|
||||
private static int Resolve(string? exitType, DateTime? lastAlive = null, DateTime? bootUtc = null)
|
||||
=> DefaultComponentFactory.ResolveEspExitBackfillLookbackMinutes(exitType, bootUtc, lastAlive, Now);
|
||||
|
||||
// ---- no previous run => nothing to recover ---------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData("first_run")]
|
||||
[InlineData("FIRST_RUN")]
|
||||
[InlineData("")]
|
||||
[InlineData(null)]
|
||||
public void No_previous_run_means_no_replay(string? exitType)
|
||||
{
|
||||
Assert.Equal(0, Resolve(exitType));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void First_run_stays_off_even_with_timestamps_available()
|
||||
{
|
||||
// The exit type decides, not the presence of a timestamp.
|
||||
Assert.Equal(0, Resolve("first_run", lastAlive: Now.AddHours(-3), bootUtc: Now.AddHours(-3)));
|
||||
}
|
||||
|
||||
// ---- the default is a floor, never a ceiling --------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData("clean")]
|
||||
[InlineData("hard_kill")]
|
||||
[InlineData("exception_crash")]
|
||||
[InlineData("reboot_kill")]
|
||||
public void Without_any_timestamp_the_default_applies(string exitType)
|
||||
{
|
||||
Assert.Equal(Default, Resolve(exitType));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_restart_that_returned_within_seconds_keeps_the_default()
|
||||
{
|
||||
Assert.Equal(Default, Resolve("clean", lastAlive: Now.AddSeconds(-20)));
|
||||
}
|
||||
|
||||
// ---- the crash case P2 is about ---------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void A_crash_recovered_only_at_the_next_boot_gets_the_whole_gap()
|
||||
{
|
||||
// BootTrigger-only task: the agent was gone for three hours while the ESP finished.
|
||||
Assert.Equal(181, Resolve("exception_crash", lastAlive: Now.AddHours(-3)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Snapshot_mtime_carries_the_window_when_no_boot_time_exists()
|
||||
{
|
||||
// LastBootUtc is only populated for hard_kill / reboot_kill — exception_crash and
|
||||
// clean have nothing else to go on.
|
||||
Assert.Equal(46, Resolve("clean", lastAlive: Now.AddMinutes(-45)));
|
||||
}
|
||||
|
||||
// ---- reboot: both inputs available, wider wins ------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void The_wider_of_the_two_inputs_wins()
|
||||
{
|
||||
// Snapshot last written 40 min ago, boot 12 min ago — the gap started at the snapshot.
|
||||
Assert.Equal(41, Resolve("reboot_kill", lastAlive: Now.AddMinutes(-40), bootUtc: Now.AddMinutes(-12)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Boot_time_alone_still_covers_a_reboot_without_a_snapshot()
|
||||
{
|
||||
Assert.Equal(41, Resolve("reboot_kill", bootUtc: Now.AddMinutes(-40)));
|
||||
}
|
||||
|
||||
// ---- robustness -------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void A_timestamp_in_the_future_is_ignored()
|
||||
{
|
||||
// Clock skew across a reboot must never produce a negative window.
|
||||
Assert.Equal(Default, Resolve("reboot_kill", lastAlive: Now.AddMinutes(5), bootUtc: Now.AddMinutes(9)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_future_timestamp_does_not_suppress_a_valid_one()
|
||||
{
|
||||
Assert.Equal(31, Resolve("reboot_kill", lastAlive: Now.AddMinutes(9), bootUtc: Now.AddMinutes(-30)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_very_long_gap_is_clamped_by_the_tracker()
|
||||
{
|
||||
// The policy may propose more than the cap; ClampLookbackMinutes is the backstop, so
|
||||
// the two together can never produce an unbounded event-log query.
|
||||
var proposed = Resolve("exception_crash", lastAlive: Now.AddHours(-20));
|
||||
Assert.True(proposed > ShellCoreTracker.BackfillLookbackMaxMinutes);
|
||||
Assert.Equal(ShellCoreTracker.BackfillLookbackMaxMinutes,
|
||||
ShellCoreTracker.ClampLookbackMinutes(proposed));
|
||||
}
|
||||
|
||||
// ---- the snapshot probe itself ----------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Missing_snapshot_reads_as_null_not_as_an_exception()
|
||||
{
|
||||
using var tmp = new Harness.TempDirectory();
|
||||
Assert.Null(DefaultComponentFactory.ReadPreviousRunLastAliveUtc(tmp.Path));
|
||||
Assert.Null(DefaultComponentFactory.ReadPreviousRunLastAliveUtc(string.Empty));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_existing_snapshot_reports_its_write_time()
|
||||
{
|
||||
using var tmp = new Harness.TempDirectory();
|
||||
var path = System.IO.Path.Combine(tmp.Path, "snapshot.json");
|
||||
System.IO.File.WriteAllText(path, "{}");
|
||||
|
||||
var read = DefaultComponentFactory.ReadPreviousRunLastAliveUtc(tmp.Path);
|
||||
|
||||
Assert.NotNull(read);
|
||||
Assert.Equal(DateTimeKind.Utc, read!.Value.Kind);
|
||||
Assert.True((DateTime.UtcNow - read.Value).Duration() < TimeSpan.FromMinutes(5));
|
||||
}
|
||||
}
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// Codex review P2 (2026-08-19) — <see cref="ImeLogHost"/> starts AFTER <c>EspAndHelloHost</c>,
|
||||
/// and <c>ImeLogTracker.Start()</c> restores the persisted package states silently (LoadState
|
||||
/// raises no <c>OnAppStateChanged</c>). On an agent restart the Shell-Core ESP-exit replay
|
||||
/// therefore runs against an empty app list. Without the post-restore nudge the exact reboot
|
||||
/// case the completion recovery exists for — apps already terminal BEFORE the reboot, only the
|
||||
/// final exit lost to the downtime — would still never complete.
|
||||
/// </summary>
|
||||
public sealed class ImeLogHostStateRestoredCallbackTests : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
private ImeLogHost CreateHost(Action? onStateRestored)
|
||||
=> new ImeLogHost(
|
||||
sessionId: "s",
|
||||
tenantId: "t",
|
||||
logger: new AgentLogger(_tmp.Path, AgentLogLevel.Info),
|
||||
ingress: new FakeSignalIngressSink(),
|
||||
clock: new VirtualClock(new DateTime(2026, 8, 19, 8, 43, 16, DateTimeKind.Utc)),
|
||||
imeLogPathOverride: _tmp.Path,
|
||||
imeMatchLogPath: null,
|
||||
imePatterns: null,
|
||||
stateDirectory: _tmp.Path,
|
||||
whiteGloveSealingPatternIds: null,
|
||||
onStateRestored: onStateRestored);
|
||||
|
||||
[Fact]
|
||||
public void Start_invokes_the_state_restored_callback_exactly_once()
|
||||
{
|
||||
var calls = 0;
|
||||
using var host = CreateHost(() => calls++);
|
||||
|
||||
host.Start();
|
||||
try { Assert.Equal(1, calls); }
|
||||
finally { host.Stop(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_throwing_callback_does_not_break_host_start()
|
||||
{
|
||||
// Fail-soft like every other co-wired callback in the orchestration layer: a broken
|
||||
// re-check must never take the IME log host — the session's primary signal source —
|
||||
// down with it.
|
||||
using var host = CreateHost(() => throw new InvalidOperationException("recheck boom"));
|
||||
|
||||
var ex = Record.Exception(() => host.Start());
|
||||
try { Assert.Null(ex); }
|
||||
finally { host.Stop(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_callback_is_a_valid_configuration()
|
||||
{
|
||||
using var host = CreateHost(null);
|
||||
|
||||
var ex = Record.Exception(() => host.Start());
|
||||
try { Assert.Null(ex); }
|
||||
finally { host.Stop(); }
|
||||
}
|
||||
}
|
||||
}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// PowerStateWatcherHost — no-battery/probe-error devices never arm, a low-battery baseline
|
||||
/// emits its threshold event onto the single rail, and debounced ticks translate tracker
|
||||
/// emissions into <c>power_state_change</c> events. WMI arming itself stays untested
|
||||
/// (ConsoleBypassWatcher precedent); <c>StartCore(armWmi: false)</c> exercises everything else.
|
||||
/// </summary>
|
||||
public sealed class PowerStateWatcherHostTests : IDisposable
|
||||
{
|
||||
private static readonly DateTime At = new DateTime(2026, 8, 21, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
private readonly FakeSignalIngressSink _sink = new FakeSignalIngressSink();
|
||||
private readonly AgentLogger _logger;
|
||||
private PowerStateResult _probeResult = new PowerStateResult { OnAcPower = true, HasBattery = true, BatteryPercent = 80 };
|
||||
|
||||
public PowerStateWatcherHostTests()
|
||||
{
|
||||
_logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
|
||||
}
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
private PowerStateWatcherHost Build() => new PowerStateWatcherHost(
|
||||
sessionId: "sess-power",
|
||||
tenantId: "tenant-power",
|
||||
ingress: _sink,
|
||||
clock: new VirtualClock(At),
|
||||
logger: _logger,
|
||||
probe: () => _probeResult);
|
||||
|
||||
private IReadOnlyList<FakeSignalIngressSink.PostedSignal> PowerEvents() =>
|
||||
_sink.Posted.Where(p =>
|
||||
p.Kind == DecisionSignalKind.InformationalEvent
|
||||
&& p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == Constants.EventTypes.PowerStateChange).ToList();
|
||||
|
||||
[Fact]
|
||||
public void No_battery_emits_nothing()
|
||||
{
|
||||
_probeResult = new PowerStateResult { OnAcPower = true, HasBattery = false };
|
||||
|
||||
Build().StartCore(armWmi: false);
|
||||
|
||||
Assert.Empty(_sink.Posted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Probe_error_emits_nothing()
|
||||
{
|
||||
_probeResult = new PowerStateResult { ProbeError = "GetSystemPowerStatus returned false" };
|
||||
|
||||
Build().StartCore(armWmi: false);
|
||||
|
||||
Assert.Empty(_sink.Posted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Baseline_below_threshold_emits_threshold_event_with_immediate_upload()
|
||||
{
|
||||
_probeResult = new PowerStateResult { OnAcPower = false, HasBattery = true, BatteryPercent = 12, BatteryLifeMinutes = 34 };
|
||||
|
||||
Build().StartCore(armWmi: false);
|
||||
|
||||
var s = Assert.Single(PowerEvents());
|
||||
Assert.Equal("Error", s.Payload![SignalPayloadKeys.Severity]);
|
||||
Assert.Equal("true", s.Payload![SignalPayloadKeys.ImmediateUpload]);
|
||||
Assert.Contains("15%", s.Payload![SignalPayloadKeys.Message]);
|
||||
|
||||
var data = (IReadOnlyDictionary<string, object>)s.TypedPayload!;
|
||||
Assert.Equal("threshold_crossed", data["transition"]);
|
||||
Assert.Equal(15, data["thresholdPercent"]);
|
||||
Assert.Equal(12, data["batteryPercent"]);
|
||||
Assert.Equal(false, data["onAcPower"]);
|
||||
Assert.Equal(34, data["batteryLifeMinutes"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tick_after_unplug_emits_ac_to_battery_warning()
|
||||
{
|
||||
var host = Build();
|
||||
host.StartCore(armWmi: false);
|
||||
Assert.Empty(PowerEvents());
|
||||
|
||||
_probeResult = new PowerStateResult { OnAcPower = false, HasBattery = true, BatteryPercent = 79 };
|
||||
host.TickForTest();
|
||||
|
||||
var s = Assert.Single(PowerEvents());
|
||||
Assert.Equal("Warning", s.Payload![SignalPayloadKeys.Severity]);
|
||||
Assert.Equal("true", s.Payload![SignalPayloadKeys.ImmediateUpload]);
|
||||
|
||||
var data = (IReadOnlyDictionary<string, object>)s.TypedPayload!;
|
||||
Assert.Equal("ac_to_battery", data["transition"]);
|
||||
Assert.Equal(79, data["batteryPercent"]);
|
||||
Assert.False(data.ContainsKey("thresholdPercent"));
|
||||
|
||||
// Replug: Info recovery event without immediate upload.
|
||||
_probeResult = new PowerStateResult { OnAcPower = true, HasBattery = true, BatteryPercent = 79, IsCharging = true };
|
||||
host.TickForTest();
|
||||
|
||||
Assert.Equal(2, PowerEvents().Count);
|
||||
var back = PowerEvents()[1];
|
||||
Assert.Equal("Info", back.Payload![SignalPayloadKeys.Severity]);
|
||||
Assert.Equal("false", back.Payload![SignalPayloadKeys.ImmediateUpload]);
|
||||
Assert.Equal("battery_to_ac", ((IReadOnlyDictionary<string, object>)back.TypedPayload!)["transition"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tick_with_unchanged_state_emits_nothing()
|
||||
{
|
||||
var host = Build();
|
||||
host.StartCore(armWmi: false);
|
||||
|
||||
host.TickForTest();
|
||||
host.TickForTest();
|
||||
|
||||
Assert.Empty(PowerEvents());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stopped_host_ignores_ticks()
|
||||
{
|
||||
var host = Build();
|
||||
host.StartCore(armWmi: false);
|
||||
host.Stop();
|
||||
|
||||
_probeResult = new PowerStateResult { OnAcPower = false, HasBattery = true, BatteryPercent = 50 };
|
||||
host.TickForTest();
|
||||
|
||||
Assert.Empty(PowerEvents());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_battery_percent_serializes_as_unknown()
|
||||
{
|
||||
var host = Build();
|
||||
host.StartCore(armWmi: false);
|
||||
|
||||
_probeResult = new PowerStateResult { OnAcPower = false, HasBattery = true, BatteryPercent = null, BatteryLifeMinutes = null };
|
||||
host.TickForTest();
|
||||
|
||||
var data = (IReadOnlyDictionary<string, object>)Assert.Single(PowerEvents()).TypedPayload!;
|
||||
Assert.Equal("unknown", data["batteryPercent"]);
|
||||
Assert.Equal("unknown", data["batteryLifeMinutes"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
@@ -35,6 +35,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
// OS-eventlog forwarding — observability, not enrollment progress (session 8bc1180f:
|
||||
// an EventID-100 burst must not keep the periodic collectors alive).
|
||||
[InlineData("modern_deployment_log")]
|
||||
// Environment observation — battery drain on an idle device must not reset idle clocks.
|
||||
[InlineData("power_state_change")]
|
||||
// Agent health / control / transport — not device/enrollment progress (P2).
|
||||
[InlineData("collector_degraded")]
|
||||
[InlineData("telemetry_upload_poisoned")]
|
||||
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// SystemTimelineWatcherHost — thin ICollectorHost wrapper over SystemTimelineTracker.
|
||||
/// Real event-log arming stays untested by convention (ConsoleBypassWatcher precedent);
|
||||
/// these tests pin the lifecycle contract only.
|
||||
/// </summary>
|
||||
public sealed class SystemTimelineWatcherHostTests : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
private SystemTimelineWatcherHost CreateHost() => new SystemTimelineWatcherHost(
|
||||
sessionId: "sess-stl",
|
||||
tenantId: "tenant-stl",
|
||||
logger: new AgentLogger(_tmp.Path, AgentLogLevel.Info),
|
||||
ingress: new FakeSignalIngressSink(),
|
||||
clock: new VirtualClock(new DateTime(2026, 8, 21, 10, 0, 0, DateTimeKind.Utc)),
|
||||
backfillLookbackMinutes: 0,
|
||||
stateDirectory: _tmp.Path);
|
||||
|
||||
[Fact]
|
||||
public void Stop_BeforeStart_DoesNotThrow()
|
||||
{
|
||||
using (var host = CreateHost())
|
||||
{
|
||||
host.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dispose_Twice_DoesNotThrow()
|
||||
{
|
||||
var host = CreateHost();
|
||||
host.Dispose();
|
||||
host.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Name_IsStable()
|
||||
{
|
||||
using (var host = CreateHost())
|
||||
{
|
||||
Assert.Equal("SystemTimelineTracker", host.Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+108
@@ -339,6 +339,114 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Runtime
|
||||
Assert.False(StartupEnvironmentProbes.IsNtpWithinTolerance(null!));
|
||||
}
|
||||
|
||||
// ================================================================= DO GroupId auto-set
|
||||
// Builder/fingerprint tests only — TrySetGroupId itself does live registry + ARP I/O and
|
||||
// must never run in a unit test (it would write HKLM on the build machine).
|
||||
|
||||
[Fact]
|
||||
public void BuildDoGroupIdEvent_written_success_is_info_with_full_data()
|
||||
{
|
||||
var result = new DoGroupIdSetResult
|
||||
{
|
||||
Success = true,
|
||||
Written = true,
|
||||
GroupId = "00063065-0000-00a8-b201-001122334455",
|
||||
PreviousValue = null,
|
||||
GatewayIp = "192.168.178.1",
|
||||
GatewayMac = "00-11-22-33-44-55",
|
||||
};
|
||||
|
||||
var evt = StartupEnvironmentProbes.BuildDoGroupIdEvent(Config(), result);
|
||||
|
||||
Assert.Equal("do_group_id_auto_set", evt.EventType);
|
||||
Assert.Equal(EventSeverity.Info, evt.Severity);
|
||||
Assert.Equal("StartupEnvironmentProbes", evt.Source);
|
||||
Assert.Equal("S1", evt.SessionId);
|
||||
Assert.Equal("T1", evt.TenantId);
|
||||
Assert.Contains("00063065-0000-00a8-b201-001122334455", evt.Message);
|
||||
Assert.Equal("192.168.178.1", evt.Data["gatewayIp"]);
|
||||
Assert.Equal("00-11-22-33-44-55", evt.Data["gatewayMac"]);
|
||||
Assert.Equal(true, evt.Data["written"]);
|
||||
Assert.Equal(true, evt.Data["success"]);
|
||||
Assert.Equal(false, evt.Data["skipped"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildDoGroupIdEvent_noop_success_says_already_set()
|
||||
{
|
||||
var result = new DoGroupIdSetResult
|
||||
{
|
||||
Success = true,
|
||||
Written = false,
|
||||
GroupId = "00063065-0000-00a8-b201-001122334455",
|
||||
};
|
||||
|
||||
var evt = StartupEnvironmentProbes.BuildDoGroupIdEvent(Config(), result);
|
||||
|
||||
Assert.Equal(EventSeverity.Info, evt.Severity);
|
||||
Assert.Contains("already set", evt.Message);
|
||||
Assert.Equal(false, evt.Data["written"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildDoGroupIdEvent_skip_is_info_with_reason()
|
||||
{
|
||||
var result = new DoGroupIdSetResult
|
||||
{
|
||||
Skipped = true,
|
||||
SkipReason = "foreign_value_present",
|
||||
PreviousValue = "11111111-2222-3333-4444-555555555555",
|
||||
};
|
||||
|
||||
var evt = StartupEnvironmentProbes.BuildDoGroupIdEvent(Config(), result);
|
||||
|
||||
Assert.Equal(EventSeverity.Info, evt.Severity);
|
||||
Assert.Contains("foreign_value_present", evt.Message);
|
||||
Assert.Equal("foreign_value_present", evt.Data["skipReason"]);
|
||||
Assert.Equal("11111111-2222-3333-4444-555555555555", evt.Data["previousValue"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildDoGroupIdEvent_error_is_warning()
|
||||
{
|
||||
var result = new DoGroupIdSetResult { Error = "readback mismatch" };
|
||||
|
||||
var evt = StartupEnvironmentProbes.BuildDoGroupIdEvent(Config(), result);
|
||||
|
||||
Assert.Equal(EventSeverity.Warning, evt.Severity);
|
||||
Assert.Contains("readback mismatch", evt.Message);
|
||||
Assert.Equal("readback mismatch", evt.Data["error"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoGroupIdFingerprint_roundtrips_ownership_marker_for_success()
|
||||
{
|
||||
var fp = StartupEnvironmentProbes.ComputeDoGroupIdFingerprint(new DoGroupIdSetResult
|
||||
{
|
||||
Success = true,
|
||||
GroupId = "00063065-0000-00a8-b201-001122334455",
|
||||
});
|
||||
|
||||
Assert.Equal("written:00063065-0000-00a8-b201-001122334455", fp);
|
||||
Assert.Equal("00063065-0000-00a8-b201-001122334455", StartupEnvironmentProbes.ExtractLastWrittenGroupId(fp));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoGroupIdFingerprint_skip_carries_no_ownership_and_errors_never_dedupe()
|
||||
{
|
||||
var skipFp = StartupEnvironmentProbes.ComputeDoGroupIdFingerprint(new DoGroupIdSetResult
|
||||
{
|
||||
Skipped = true,
|
||||
SkipReason = "mdm_policy_present",
|
||||
});
|
||||
Assert.Equal("skipped:mdm_policy_present", skipFp);
|
||||
Assert.Null(StartupEnvironmentProbes.ExtractLastWrittenGroupId(skipFp));
|
||||
|
||||
// Errors: no fingerprint → the probe re-runs AND re-emits each failing start.
|
||||
Assert.Null(StartupEnvironmentProbes.ComputeDoGroupIdFingerprint(new DoGroupIdSetResult { Error = "boom" }));
|
||||
Assert.Null(StartupEnvironmentProbes.ExtractLastWrittenGroupId(null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunAsync_skips_geo_timezone_and_ntp_once_all_succeeded_only_power_state_emits()
|
||||
{
|
||||
|
||||
+11
-9
@@ -245,11 +245,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Terminal_to_active_retry_rearms_CompletedAt_and_final_terminal_covers_total_occupancy()
|
||||
public void Terminal_to_active_retry_rearms_BothEndpoints_and_final_terminal_covers_last_attempt()
|
||||
{
|
||||
// Audit Q1: the IME retry path (Error -> Downloading -> Installed is permitted by
|
||||
// AppPackageState.UpdateState). Pre-fix, CompletedAt froze at the first Error, so
|
||||
// the final Installed payload carried the first failure's completedAt/duration.
|
||||
// Audit Q1 + 2026-08 attempt-duration change: the IME retry path
|
||||
// (Error -> Downloading -> Installed is permitted by AppPackageState.UpdateState).
|
||||
// A terminal->active transition re-arms BOTH endpoints, so the final Installed
|
||||
// payload describes the retry ATTEMPT (matching the backend's AppInstallSummaries
|
||||
// semantics) — not the first-start -> last-terminal span across attempts.
|
||||
using var f = new ImeLogTrackerAdapterFixture(T0);
|
||||
using var adapter = new ImeLogTrackerAdapter(f.Tracker, f.Ingress, f.Clock);
|
||||
var app = new AppPackageState("app-R", 0);
|
||||
@@ -261,23 +263,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Error, AppInstallationState.Downloading);
|
||||
|
||||
// Re-armed while active again: the retry's download event carries no completedAt,
|
||||
// and the FIRST start stamp is kept.
|
||||
// and the start stamp is the RETRY's re-entry time (fresh attempt window).
|
||||
Assert.Null(adapter.AppTimings["app-R"].CompletedAtUtc);
|
||||
var retryDownload = f.InfoEvent(SharedEventTypes.AppDownloadStarted);
|
||||
Assert.False(retryDownload.Payload!.ContainsKey("completedAt"));
|
||||
Assert.False(retryDownload.Payload.ContainsKey("durationSeconds"));
|
||||
Assert.Equal(T0.ToString("o"), retryDownload.Payload["startedAt"]);
|
||||
Assert.Equal(T0.AddSeconds(15).ToString("o"), retryDownload.Payload["startedAt"]);
|
||||
|
||||
f.Clock.Advance(TimeSpan.FromSeconds(20));
|
||||
adapter.TriggerAppStateFromTest(app, AppInstallationState.Downloading, AppInstallationState.Installed);
|
||||
|
||||
var timing = adapter.AppTimings["app-R"];
|
||||
Assert.Equal(T0, timing.StartedAtUtc);
|
||||
Assert.Equal(T0.AddSeconds(15), timing.StartedAtUtc);
|
||||
Assert.Equal(T0.AddSeconds(35), timing.CompletedAtUtc);
|
||||
Assert.Equal(35.0, timing.DurationSeconds);
|
||||
Assert.Equal(20.0, timing.DurationSeconds);
|
||||
|
||||
var completed = Assert.Single(f.InfoEvents(SharedEventTypes.AppInstallComplete));
|
||||
Assert.Equal("35.00", completed.Payload!["durationSeconds"]);
|
||||
Assert.Equal("20.00", completed.Payload!["durationSeconds"]);
|
||||
Assert.Equal(T0.AddSeconds(35).ToString("o"), completed.Payload["completedAt"]);
|
||||
}
|
||||
|
||||
|
||||
+25
@@ -107,6 +107,31 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.False(req.Headers.ContainsKey("X-Bootstrap-Token"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UploadBatchAsync_stamps_send_time_header_per_attempt()
|
||||
{
|
||||
// P14: X-Send-Time-Utc carries the device clock at the SEND moment, round-trip
|
||||
// ISO-8601, one value per request. A retry is a new send moment — the header is
|
||||
// stamped inside UploadBatchAsync, so each attempt gets a fresh value.
|
||||
using var rig = new Rig();
|
||||
rig.Handler.QueueStatus(HttpStatusCode.ServiceUnavailable);
|
||||
rig.Handler.QueueStatus(HttpStatusCode.OK);
|
||||
|
||||
var before = DateTime.UtcNow;
|
||||
await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
var after = DateTime.UtcNow;
|
||||
|
||||
Assert.Equal(2, rig.Handler.Captured.Count);
|
||||
foreach (var req in rig.Handler.Captured)
|
||||
{
|
||||
Assert.True(req.TryGetHeader("X-Send-Time-Utc", out var raw));
|
||||
var parsed = DateTime.Parse(raw, null, System.Globalization.DateTimeStyles.RoundtripKind);
|
||||
Assert.Equal(DateTimeKind.Utc, parsed.Kind);
|
||||
Assert.InRange(parsed, before.AddSeconds(-1), after.AddSeconds(1));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================ Compression (bandwidth)
|
||||
|
||||
[Fact]
|
||||
|
||||
+71
@@ -144,6 +144,77 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Equal(1 + 2 + 3, snap.TotalBytesUp);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Captures_bytes_down_via_buffering_when_content_length_is_missing()
|
||||
{
|
||||
// AutomaticDecompression strips Content-Length on the backend's gzipped JSON —
|
||||
// the pre-fix handler recorded 0 for every such (healthy) response. A non-seekable
|
||||
// stream body reproduces the headerless shape.
|
||||
var metrics = new NetworkMetrics();
|
||||
var inner = new RecordingHttpMessageHandler();
|
||||
var payload = Encoding.UTF8.GetBytes("{\"ok\":true}"); // 11 bytes
|
||||
inner.QueueResponse(() => new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StreamContent(new NonSeekableReadStream(payload)),
|
||||
});
|
||||
using var client = BuildClient(metrics, inner);
|
||||
|
||||
using var resp = await client.SendAsync(NewPost(""));
|
||||
|
||||
var snap = metrics.GetSnapshot();
|
||||
Assert.Equal(payload.Length, snap.TotalBytesDown);
|
||||
// Buffering must not consume the body away from the caller.
|
||||
Assert.Equal("{\"ok\":true}", await resp.Content.ReadAsStringAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Bytes_down_fails_soft_to_zero_when_buffering_throws()
|
||||
{
|
||||
var metrics = new NetworkMetrics();
|
||||
var inner = new RecordingHttpMessageHandler();
|
||||
inner.QueueResponse(() => new HttpResponseMessage(HttpStatusCode.OK)
|
||||
{
|
||||
Content = new ThrowingContent(),
|
||||
});
|
||||
using var client = BuildClient(metrics, inner);
|
||||
|
||||
// ResponseHeadersRead keeps HttpClient's own post-pipeline buffering out of the
|
||||
// picture — the test isolates the handler's fallback catch (which must swallow
|
||||
// the serialization failure and record 0, never throw).
|
||||
using var resp = await client.SendAsync(NewPost(""), HttpCompletionOption.ResponseHeadersRead);
|
||||
|
||||
var snap = metrics.GetSnapshot();
|
||||
Assert.Equal(1, snap.RequestCount);
|
||||
Assert.Equal(0, snap.TotalBytesDown);
|
||||
}
|
||||
|
||||
/// <summary>Read-only, non-seekable stream — CanSeek=false keeps StreamContent from
|
||||
/// computing a Content-Length, mirroring a decompression-wrapped response body.</summary>
|
||||
private sealed class NonSeekableReadStream : System.IO.Stream
|
||||
{
|
||||
private readonly System.IO.MemoryStream _inner;
|
||||
public NonSeekableReadStream(byte[] data) { _inner = new System.IO.MemoryStream(data); }
|
||||
public override bool CanRead => true;
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => throw new NotSupportedException();
|
||||
public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
|
||||
public override void Flush() { }
|
||||
public override int Read(byte[] buffer, int offset, int count) => _inner.Read(buffer, offset, count);
|
||||
public override long Seek(long offset, System.IO.SeekOrigin origin) => throw new NotSupportedException();
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
protected override void Dispose(bool disposing) { if (disposing) _inner.Dispose(); base.Dispose(disposing); }
|
||||
}
|
||||
|
||||
/// <summary>Content whose serialization always fails — drives the fallback's catch path.</summary>
|
||||
private sealed class ThrowingContent : HttpContent
|
||||
{
|
||||
protected override Task SerializeToStreamAsync(System.IO.Stream stream, System.Net.TransportContext? context)
|
||||
=> throw new System.IO.IOException("connection dropped mid-body");
|
||||
protected override bool TryComputeLength(out long length) { length = 0; return false; }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Constructor_rejects_null_metrics()
|
||||
{
|
||||
|
||||
+11
@@ -40,6 +40,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues an arbitrary response — for shapes <see cref="QueueStatus"/> can't build,
|
||||
/// e.g. content without a Content-Length header (what AutomaticDecompression leaves
|
||||
/// behind on real compressed responses).
|
||||
/// </summary>
|
||||
public RecordingHttpMessageHandler QueueResponse(Func<HttpResponseMessage> factory)
|
||||
{
|
||||
_script.Enqueue(_ => factory());
|
||||
return this;
|
||||
}
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request,
|
||||
CancellationToken cancellationToken)
|
||||
|
||||
@@ -221,6 +221,81 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Equal(1, System.Threading.Volatile.Read(ref signalled));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PendingBytes_tracks_enqueue_and_shrinks_to_zero_when_all_uploaded()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
Assert.Equal(0, spool.PendingBytes);
|
||||
|
||||
spool.Enqueue(EventDraft("a"));
|
||||
spool.Enqueue(EventDraft("b"));
|
||||
spool.Enqueue(EventDraft("c"));
|
||||
|
||||
// PendingBytes must equal the full file size while nothing is uploaded yet
|
||||
// (every line written by Enqueue is still pending).
|
||||
Assert.Equal(spool.SpoolFileSizeBytes, spool.PendingBytes);
|
||||
|
||||
spool.MarkUploaded(2);
|
||||
Assert.Equal(0, spool.PendingBytes);
|
||||
// File keeps its append-only size — the two measures MUST diverge here,
|
||||
// otherwise the pressure trigger regresses to the file-size false positive.
|
||||
Assert.True(spool.SpoolFileSizeBytes > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PendingBytes_after_partial_upload_equals_remaining_tail_line_bytes()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(EventDraft("a"));
|
||||
spool.Enqueue(EventDraft("b"));
|
||||
spool.Enqueue(EventDraft("c-longer-rowkey-for-distinct-length"));
|
||||
|
||||
spool.MarkUploaded(1);
|
||||
|
||||
// Expected = on-disk byte length of the last line incl. its trailing \n.
|
||||
var lines = File.ReadAllLines(Path.Combine(tmp.Path, "spool.jsonl"), Encoding.UTF8);
|
||||
var expected = Encoding.UTF8.GetByteCount(lines[2]) + 1;
|
||||
Assert.Equal(expected, spool.PendingBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PendingBytes_is_rehydrated_on_restart_from_pending_tail_only()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("a"));
|
||||
s1.Enqueue(EventDraft("b"));
|
||||
s1.Enqueue(EventDraft("c"));
|
||||
s1.MarkUploaded(1);
|
||||
var expectedPendingBytes = s1.PendingBytes;
|
||||
Assert.True(expectedPendingBytes > 0);
|
||||
|
||||
// Restart: rehydrated PendingBytes must match the pre-restart value exactly
|
||||
// (uploaded prefix excluded), not the full file size.
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock());
|
||||
Assert.Equal(expectedPendingBytes, s2.PendingBytes);
|
||||
Assert.True(s2.PendingBytes < s2.SpoolFileSizeBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PendingBytes_rehydrate_ignores_corrupt_tail_line()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("a"));
|
||||
var cleanPendingBytes = s1.PendingBytes;
|
||||
|
||||
File.AppendAllText(
|
||||
Path.Combine(tmp.Path, "spool.jsonl"),
|
||||
"{\"Kind\":\"Event\",\"PartitionKey\":\"corrupt-no-",
|
||||
Encoding.UTF8);
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock());
|
||||
Assert.Equal(cleanPendingBytes, s2.PendingBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Enqueue_without_RequiresImmediateFlush_does_not_raise_ImmediateFlushRequested()
|
||||
{
|
||||
|
||||
@@ -109,6 +109,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// </summary>
|
||||
public bool EnableTimezoneAutoSet { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to set the Delivery Optimization group ID (DOGroupId policy value) from a
|
||||
/// deterministic network fingerprint (default gateway IP + MAC, RealmJoin-compatible).
|
||||
/// Existing DOGroupId/DOGroupIdSource policies are never overwritten.
|
||||
/// Default: false
|
||||
/// </summary>
|
||||
public bool EnableDoGroupIdAutoSet { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Continue-Anyway observation mode (operator-set tenant opt-in, delivered via
|
||||
/// remote config): stamped into the esp_config_detected decision-signal payload so
|
||||
@@ -254,6 +262,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// </summary>
|
||||
public bool UnrestrictedMode { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Mirror of <c>AnalyzerConfiguration.EnableRealmJoinWatcher</c> (remote, per tenant).
|
||||
/// Gates the RealmJoin built-in sections of the diagnostics package — the package
|
||||
/// service only sees this configuration, not the remote response. The RealmJoin host
|
||||
/// itself keeps reading the remote Analyzers block. Default: false.
|
||||
/// </summary>
|
||||
public bool EnableRealmJoinWatcher { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Sanitized command-line arguments for the agent_started event.
|
||||
/// Secrets (e.g. bootstrap token) are redacted before assignment.
|
||||
|
||||
@@ -94,12 +94,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
// a tenant admin can clear a previously-applied custom NTP server.
|
||||
agentConfig.NtpServer = remoteConfig.NtpServer;
|
||||
agentConfig.EnableTimezoneAutoSet = remoteConfig.EnableTimezoneAutoSet;
|
||||
agentConfig.EnableDoGroupIdAutoSet = remoteConfig.EnableDoGroupIdAutoSet;
|
||||
|
||||
// ---------------- Diagnostics — remote wins (including DiagnosticsUploadMode="Off"
|
||||
// which must clear any CLI-initial default).
|
||||
agentConfig.DiagnosticsUploadEnabled = remoteConfig.DiagnosticsUploadEnabled;
|
||||
agentConfig.DiagnosticsUploadMode = remoteConfig.DiagnosticsUploadMode;
|
||||
agentConfig.DiagnosticsLogPaths = remoteConfig.DiagnosticsLogPaths ?? new List<DiagnosticsLogPath>();
|
||||
// The diagnostics package gates its RealmJoin sections on the tenant's watcher
|
||||
// toggle but never sees the remote response — mirror the one Analyzers knob here.
|
||||
// The remaining Analyzers flags flow untouched to DefaultComponentFactory.
|
||||
agentConfig.EnableRealmJoinWatcher = remoteConfig.Analyzers?.EnableRealmJoinWatcher ?? false;
|
||||
|
||||
agentConfig.SendTraceEvents = remoteConfig.SendTraceEvents;
|
||||
|
||||
|
||||
+7
-4
@@ -10,10 +10,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// the <c>app_tracking_summary</c> terminal event.
|
||||
/// <para>
|
||||
/// <b>StartedAtUtc</b> is recorded on the first lifecycle transition into Downloading /
|
||||
/// Installing / InProgress and never overwritten. <b>CompletedAtUtc</b> is recorded on a
|
||||
/// transition to a terminal state (Installed / Skipped / Postponed / Error) and re-armed
|
||||
/// to <c>null</c> when the app goes terminal → active again (IME retry / re-evaluation,
|
||||
/// audit Q1) — so the final terminal event carries first-start → last-terminal.
|
||||
/// Installing / InProgress. <b>CompletedAtUtc</b> is recorded on a transition to a
|
||||
/// terminal state (Installed / Skipped / Postponed / Error). When the app goes
|
||||
/// terminal → active again (IME retry / re-evaluation pass, audit Q1) BOTH endpoints
|
||||
/// re-arm — CompletedAtUtc to <c>null</c>, StartedAtUtc to the re-entry time — so the
|
||||
/// final terminal event carries the status-defining ATTEMPT's window (2026-08
|
||||
/// attempt-duration change, matching the backend's AppInstallSummaries semantics),
|
||||
/// never the first-start → last-terminal span across install passes.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class AppInstallTiming
|
||||
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
{
|
||||
/// <summary>How the UTC value of a CMTrace line was arrived at. Recorded as evidence.</summary>
|
||||
public enum CmTraceOffsetOrigin
|
||||
{
|
||||
/// <summary>No offset known — the caller fell back to its own clock.</summary>
|
||||
None = 0,
|
||||
|
||||
/// <summary>The writer declared its offset in the line ("+480"). Authoritative.</summary>
|
||||
Bias = 1,
|
||||
|
||||
/// <summary>
|
||||
/// A per-FILE measured offset was applied. Retired by the 2026-08-20 revert (04b1a7c6):
|
||||
/// one offset per file is wrong when a file holds lines from two writer eras. Kept so
|
||||
/// stored events from agent 2.0.1410 remain interpretable.
|
||||
/// </summary>
|
||||
Calibrated = 2,
|
||||
|
||||
/// <summary>
|
||||
/// The line anchored ITSELF: it was read in a pass whose lines are provably fresh
|
||||
/// (written since the previous poll), so its own distance to the agent clock, rounded
|
||||
/// to the 15-minute offset grid, IS the writer's offset — no cross-line state involved.
|
||||
/// Immune to interleaved writer eras by construction. See
|
||||
/// <see cref="CmTraceOffsetCalibrator.TryMeasureOffset"/>.
|
||||
/// </summary>
|
||||
LineAnchored = 3,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures the UTC offset that the process WRITING a CMTrace log believes it is in.
|
||||
///
|
||||
/// <para>
|
||||
/// A CMTrace line carries local time and no offset — IME 1.104 writes
|
||||
/// <c>DateTime.Now.TimeOfDay</c> in both of its trace listeners. Converting such a line with
|
||||
/// the agent's own <c>TimeZoneInfo.Local</c> assumes the two processes agree, and they do not:
|
||||
/// both cache their zone at process start and neither follows a later <c>tzutil</c> or a
|
||||
/// Windows auto-timezone change. The resulting error is
|
||||
/// <c>Offset_writer - Offset_agent</c> and is zero only by coincidence. Field measurement over
|
||||
/// 11,068 sessions found 26 sessions carrying a real zone-offset error, from +1 h
|
||||
/// (GMT to W. Europe) to -17 h (a writer still on the Pacific OOBE default while the agent had
|
||||
/// already moved to E. Australia).
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// So the offset is MEASURED, not assumed. The tracker tails the log on a 100 ms poll, so a
|
||||
/// line that appeared since the previous pass was written essentially "now":
|
||||
/// <code>
|
||||
/// candidate = lineLocalTime - agentUtcNow
|
||||
/// offset = round(candidate / 15 min) * 15 min
|
||||
/// lineUtc = lineLocalTime - offset
|
||||
/// </code>
|
||||
/// Every real UTC offset is a whole multiple of 15 minutes, so the rounding absorbs poll and
|
||||
/// flush latency without ambiguity.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// THE FRESHNESS REQUIREMENT IS WHAT MAKES THIS CORRECT. An anchor must be the newest line of
|
||||
/// a pass in which the file actually GREW. Without that rule a backlog line that happens to be
|
||||
/// an exact multiple of 15 minutes old (30 min, 45 min, ...) rounds to a clean value with zero
|
||||
/// residual and calibrates the offset to a wrong figure — the residual guard alone cannot
|
||||
/// catch it. Callers must only offer anchors they know to be fresh.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// Calibration is kept per source file: IntuneManagementExtension.log and AppWorkload.log are
|
||||
/// written by different processes and can hold different beliefs. Re-calibrating on every
|
||||
/// fresh anchor makes this self-healing — if the writing process restarts and picks up the
|
||||
/// current zone, the measurement follows within one poll.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class CmTraceOffsetCalibrator
|
||||
{
|
||||
/// <summary>Grid that every real UTC offset sits on.</summary>
|
||||
internal static readonly TimeSpan OffsetGrid = TimeSpan.FromMinutes(15);
|
||||
|
||||
/// <summary>
|
||||
/// How far a candidate may sit off the grid before it is rejected. Poll latency (100 ms)
|
||||
/// plus the writer's flush latency are far below this; anything larger means the anchor
|
||||
/// was not actually fresh.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan MaxGridResidual = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>Real UTC offsets span UTC-12 (Dateline) to UTC+14 (Line Islands).</summary>
|
||||
internal static readonly TimeSpan MinOffset = TimeSpan.FromHours(-12);
|
||||
|
||||
/// <summary>Upper bound of a real UTC offset. See <see cref="MinOffset"/>.</summary>
|
||||
internal static readonly TimeSpan MaxOffset = TimeSpan.FromHours(14);
|
||||
|
||||
private sealed class Anchor
|
||||
{
|
||||
public TimeSpan Offset;
|
||||
public DateTime LocalTimestamp;
|
||||
}
|
||||
|
||||
private readonly object _lock = new object();
|
||||
|
||||
private readonly Dictionary<string, Anchor> _bySource =
|
||||
new Dictionary<string, Anchor>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Offer a freshly appended line as a calibration anchor.
|
||||
/// <para>
|
||||
/// The caller MUST only pass a line it knows to be fresh — the newest line of a pass in
|
||||
/// which the file grew. See the freshness note on this class.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <param name="sourceKey">Log file identity; the file name is enough.</param>
|
||||
/// <param name="localTimestamp">The line's local time exactly as written (Kind is ignored).</param>
|
||||
/// <param name="agentUtcNow">The agent's UTC clock at parse time.</param>
|
||||
/// <returns><c>true</c> when the anchor was accepted and the offset updated.</returns>
|
||||
/// <summary>
|
||||
/// The pure measurement: the distance between a FRESH line's local time and the agent's
|
||||
/// UTC clock, rounded to the 15-minute offset grid, is the writer's UTC offset.
|
||||
/// <para>
|
||||
/// Freshness is the caller's obligation and is what makes this correct — for a line
|
||||
/// written within the last poll interval, <c>local - now</c> differs from the writer's
|
||||
/// true offset only by seconds, which the grid rounding absorbs. Returns <c>false</c>
|
||||
/// when the residual exceeds <see cref="MaxGridResidual"/> (the line is not fresh, or
|
||||
/// the writer's clock is broken) or the rounded value is not a real timezone offset.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Known, accepted edge: a freshly WRITTEN line carrying an old REPLAYED timestamp whose
|
||||
/// age happens to sit within the residual of an exact grid multiple measures that age as
|
||||
/// an "offset" — the line then resolves to roughly the read time instead of its replayed
|
||||
/// past. The error is bounded by construction (an anchored line always resolves to
|
||||
/// now ± <see cref="MaxGridResidual"/>), the case needs an age of exactly N×15 min ± 2 min,
|
||||
/// and sub-24h replays are treated as current by the historic-replay guard anyway.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static bool TryMeasureOffset(DateTime localTimestamp, DateTime agentUtcNow, out TimeSpan offset)
|
||||
{
|
||||
var candidate = DateTime.SpecifyKind(localTimestamp, DateTimeKind.Unspecified)
|
||||
- DateTime.SpecifyKind(agentUtcNow, DateTimeKind.Unspecified);
|
||||
|
||||
var gridUnits = Math.Round(candidate.TotalMinutes / OffsetGrid.TotalMinutes,
|
||||
MidpointRounding.AwayFromZero);
|
||||
offset = TimeSpan.FromMinutes(gridUnits * OffsetGrid.TotalMinutes);
|
||||
|
||||
// Off-grid by more than the tolerance: the line was not fresh, or the writer's own
|
||||
// clock is broken. Either way it must not become an offset.
|
||||
if ((candidate - offset).Duration() > MaxGridResidual) return false;
|
||||
|
||||
// Not a value any real timezone can produce.
|
||||
if (offset < MinOffset || offset > MaxOffset) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public bool TryCalibrate(string sourceKey, DateTime localTimestamp, DateTime agentUtcNow)
|
||||
{
|
||||
if (string.IsNullOrEmpty(sourceKey)) return false;
|
||||
|
||||
TimeSpan offset;
|
||||
if (!TryMeasureOffset(localTimestamp, agentUtcNow, out offset)) return false;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_bySource.TryGetValue(sourceKey, out var existing))
|
||||
{
|
||||
// Only ever move the anchor forward. A line older than the current anchor
|
||||
// carries nothing newer and could re-introduce a stale belief.
|
||||
if (localTimestamp <= existing.LocalTimestamp) return false;
|
||||
existing.Offset = offset;
|
||||
existing.LocalTimestamp = localTimestamp;
|
||||
}
|
||||
else
|
||||
{
|
||||
_bySource[sourceKey] = new Anchor
|
||||
{
|
||||
Offset = offset,
|
||||
LocalTimestamp = localTimestamp,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>The offset measured for this source, when one has been established.</summary>
|
||||
public bool TryGetOffset(string sourceKey, out TimeSpan offset)
|
||||
{
|
||||
offset = TimeSpan.Zero;
|
||||
if (string.IsNullOrEmpty(sourceKey)) return false;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
Anchor anchor;
|
||||
if (!_bySource.TryGetValue(sourceKey, out anchor)) return false;
|
||||
offset = anchor.Offset;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Convert a line's local time to UTC using the measured offset. Returns <c>false</c> when
|
||||
/// this source has not been calibrated yet — the caller then picks its own fallback and
|
||||
/// should mark the result as not source-grounded.
|
||||
/// </summary>
|
||||
public bool TryResolveUtc(string sourceKey, DateTime localTimestamp, out DateTime utc)
|
||||
{
|
||||
utc = default(DateTime);
|
||||
|
||||
TimeSpan offset;
|
||||
if (!TryGetOffset(sourceKey, out offset)) return false;
|
||||
|
||||
utc = DateTime.SpecifyKind(
|
||||
DateTime.SpecifyKind(localTimestamp, DateTimeKind.Unspecified) - offset,
|
||||
DateTimeKind.Utc);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
+231
-6
@@ -42,7 +42,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task CheckLogFilesAsync(CancellationToken token)
|
||||
/// <remarks>internal (not private) as a test seam: the offset calibration depends on the
|
||||
/// pass structure — first observation versus a later pass over a grown file — which only
|
||||
/// a real read cycle exercises. Production still reaches it solely from the poll loop.</remarks>
|
||||
internal async Task CheckLogFilesAsync(CancellationToken token)
|
||||
{
|
||||
if (!Directory.Exists(_logFolder))
|
||||
return;
|
||||
@@ -74,7 +77,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
var fileInfo = new FileInfo(filePath);
|
||||
if (!fileInfo.Exists) continue;
|
||||
|
||||
// Captured BEFORE MarkChecked below stamps this pass. Growth measured
|
||||
// against a previously observed state is what makes this pass's lines valid
|
||||
// "written now" anchors — for the per-file measurement AND per-line anchoring.
|
||||
var hadPreviousObservation = _positionTracker.HasSeen(filePath);
|
||||
var lastCheckedUtc = _positionTracker.GetLastCheckedUtc(filePath);
|
||||
var passNowUtc = UtcNowProvider();
|
||||
|
||||
var startPos = _positionTracker.GetSafePosition(filePath, fileInfo.Length);
|
||||
|
||||
// Every look counts, including "no new data" and an empty first sight: the
|
||||
// NEXT pass's freshness window is measured from here. Restored bookmarks
|
||||
// deliberately carry no LastCheckedUtc — the first pass after a restart reads
|
||||
// downtime backlog and must never count as fresh.
|
||||
_positionTracker.MarkChecked(filePath, passNowUtc);
|
||||
|
||||
if (startPos >= fileInfo.Length)
|
||||
{
|
||||
// M2: guard the interpolated string so it isn't built every 100 ms tick
|
||||
@@ -86,6 +103,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
if (_logger.LogLevel >= AgentLogLevel.Trace)
|
||||
_logger.Trace($"ImeLogTracker: reading {Path.GetFileName(filePath)} from pos {startPos} (size={fileInfo.Length}, delta={fileInfo.Length - startPos})");
|
||||
|
||||
_currentSourceFileName = Path.GetFileName(filePath);
|
||||
_currentPassLinesAreFresh = hadPreviousObservation
|
||||
&& lastCheckedUtc > DateTime.MinValue
|
||||
&& (passNowUtc - lastCheckedUtc) <= FreshLineMaxAge;
|
||||
|
||||
// Newest bias-less line of this pass — the calibration anchor. Bias-carrying
|
||||
// lines are skipped: they already state the writer's offset, so they need no
|
||||
// measurement and must not overwrite one.
|
||||
CmTraceLogEntry calibrationAnchor = null;
|
||||
|
||||
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
|
||||
{
|
||||
stream.Seek(startPos, SeekOrigin.Begin);
|
||||
@@ -147,6 +174,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
if (CmTraceLogParser.TryParseLine(line, out entry))
|
||||
{
|
||||
messageToMatch = entry.Message;
|
||||
if (entry.HasTimestamp && !entry.BiasMinutes.HasValue)
|
||||
calibrationAnchor = entry;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -160,7 +189,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
// Simulation mode delay
|
||||
if (SimulationMode && entry != null)
|
||||
{
|
||||
await ApplySimulationDelay(entry.Timestamp, token);
|
||||
await ApplySimulationDelay(ResolveEntryUtc(entry), token);
|
||||
}
|
||||
|
||||
// Match against active patterns
|
||||
@@ -185,6 +214,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
|
||||
_positionTracker.SetPosition(filePath, stream.Position);
|
||||
_stateDirty = true;
|
||||
|
||||
// Calibrate AFTER the pass: this pass's lines were resolved with the
|
||||
// offset established previously, at most one poll (100 ms) old. Buffering
|
||||
// the pass to calibrate first is not an option — the first pass of
|
||||
// AppWorkload.log can be hundreds of MB. The cost is a warm-up of one
|
||||
// growing pass, during which lines fall back to the reader zone and are
|
||||
// flagged as such.
|
||||
if (hadPreviousObservation && calibrationAnchor != null)
|
||||
CalibrateFrom(_currentSourceFileName, Path.GetFileName(filePath), calibrationAnchor);
|
||||
}
|
||||
}
|
||||
catch (FileNotFoundException) { }
|
||||
@@ -222,7 +260,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
private void HandlePatternMatch(CompiledPattern pattern, Match match, string message, CmTraceLogEntry entry)
|
||||
{
|
||||
LastMatchedPatternId = pattern.PatternId;
|
||||
LastMatchedLogTimestamp = entry?.Timestamp;
|
||||
if (entry == null)
|
||||
{
|
||||
LastMatchedLogTimestamp = null;
|
||||
LastMatchedSourceLocalTimestamp = null;
|
||||
LastMatchedSourceOffsetMinutes = null;
|
||||
LastMatchedSourceOffsetOrigin = CmTraceOffsetOrigin.None;
|
||||
LastMatchedMeasuredWriterOffsetMinutes = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
CmTraceOffsetOrigin origin;
|
||||
int? offsetMinutes;
|
||||
LastMatchedLogTimestamp = ResolveEntryUtc(entry, out origin, out offsetMinutes);
|
||||
LastMatchedSourceLocalTimestamp = entry.HasTimestamp ? entry.LocalTimestamp : (DateTime?)null;
|
||||
LastMatchedSourceOffsetMinutes = offsetMinutes;
|
||||
LastMatchedSourceOffsetOrigin = origin;
|
||||
|
||||
// Observational: what the calibrator measured for this file, distinct from what
|
||||
// was applied above.
|
||||
TimeSpan measuredWriterOffset;
|
||||
LastMatchedMeasuredWriterOffsetMinutes =
|
||||
_currentSourceFileName != null
|
||||
&& OffsetCalibrator.TryGetOffset(_currentSourceFileName, out measuredWriterOffset)
|
||||
? (int)measuredWriterOffset.TotalMinutes
|
||||
: (int?)null;
|
||||
}
|
||||
|
||||
// Generic pattern-match hook (M4.4.4). Invoked before action-specific callbacks so
|
||||
// subscribers (e.g. ImeLogTrackerAdapter emitting WhiteGloveSealingPatternDetected)
|
||||
@@ -237,10 +300,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
// state nor the phase-change ignore list. SimulationMode (--replay-log-dir dev tool)
|
||||
// replays historic logs on purpose and bypasses the guard.
|
||||
var isStaleReplayLine = !SimulationMode && entry != null
|
||||
&& (UtcNowProvider() - NormalizeUtc(entry.Timestamp)) > HistoricReplayThreshold;
|
||||
&& (UtcNowProvider() - NormalizeUtc(ResolveEntryUtc(entry))) > HistoricReplayThreshold;
|
||||
if (isStaleReplayLine && AppMutatingActions.Contains(pattern.Action?.ToLower() ?? string.Empty))
|
||||
{
|
||||
_logger.Debug($"ImeLogTracker: skipped app action '{pattern.Action}' for historic line ({entry.Timestamp:o})");
|
||||
_logger.Debug($"ImeLogTracker: skipped app action '{pattern.Action}' for historic line ({ResolveEntryUtc(entry):o})");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -524,8 +587,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
internal void ProcessLogMessageForTest(string message, DateTime? sourceTimestampUtc = null)
|
||||
{
|
||||
if (string.IsNullOrEmpty(message)) return;
|
||||
// The test seam hands in an already-resolved UTC instant, so it takes the same route a
|
||||
// writer-declared bias does: TimestampUtc set, no zone left to guess.
|
||||
var entry = sourceTimestampUtc.HasValue
|
||||
? new CmTraceLogEntry { Timestamp = sourceTimestampUtc.Value, Message = message }
|
||||
? new CmTraceLogEntry
|
||||
{
|
||||
TimestampUtc = sourceTimestampUtc.Value,
|
||||
LocalTimestamp = DateTime.SpecifyKind(sourceTimestampUtc.Value, DateTimeKind.Unspecified),
|
||||
HasTimestamp = true,
|
||||
Message = message,
|
||||
}
|
||||
: null;
|
||||
foreach (var pattern in _activePatterns)
|
||||
{
|
||||
@@ -543,5 +614,159 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// Parses [DO TEL] JSON and links it to the correct app via FileId.
|
||||
/// The FileId contains the app GUID in the format: ...intunewin-bin_{appGuid}_{number}
|
||||
/// </summary>
|
||||
|
||||
/// <summary>
|
||||
/// Resolve a parsed CMTrace entry to UTC.
|
||||
///
|
||||
/// <para>
|
||||
/// Order of preference: a writer-declared bias (authoritative) > this process's own zone
|
||||
/// (a fallback carrying a known defect, see
|
||||
/// <see cref="CmTraceLogParser.ResolveUtcAssumingReaderZone"/>) > the agent clock for a
|
||||
/// line with no parseable timestamp.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// The reader-zone step is where the tracker is still wrong when IME's process holds a
|
||||
/// different zone belief than ours. It is preserved verbatim here so that splitting the
|
||||
/// parser changes no behaviour; the measured offset from <c>CmTraceOffsetCalibrator</c>
|
||||
/// slots in at exactly this point in the follow-up change.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private DateTime ResolveEntryUtc(CmTraceLogEntry entry)
|
||||
{
|
||||
CmTraceOffsetOrigin origin;
|
||||
int? offsetMinutes;
|
||||
return ResolveEntryUtc(entry, out origin, out offsetMinutes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolve a parsed entry to UTC and report HOW the offset was obtained, so the emitting
|
||||
/// side can attach the evidence (P8). See <see cref="CmTraceOffsetOrigin"/>.
|
||||
/// </summary>
|
||||
private DateTime ResolveEntryUtc(CmTraceLogEntry entry, out CmTraceOffsetOrigin origin, out int? offsetMinutes)
|
||||
{
|
||||
origin = CmTraceOffsetOrigin.None;
|
||||
offsetMinutes = null;
|
||||
|
||||
if (entry == null) return UtcNowProvider();
|
||||
|
||||
// Writer declared its own offset — nothing left to measure.
|
||||
if (entry.TimestampUtc.HasValue)
|
||||
{
|
||||
origin = CmTraceOffsetOrigin.Bias;
|
||||
// Bias uses the GetTimeZoneInformation convention (UTC = local + bias); report the
|
||||
// value in the same sense as a measured offset (local = UTC + offset).
|
||||
offsetMinutes = entry.BiasMinutes.HasValue ? -entry.BiasMinutes.Value : (int?)null;
|
||||
return entry.TimestampUtc.Value;
|
||||
}
|
||||
|
||||
if (!entry.HasTimestamp) return UtcNowProvider();
|
||||
|
||||
// ERA-AWARE RESOLUTION (2026-08-20, second attempt): a FRESH line anchors ITSELF.
|
||||
//
|
||||
// The first attempt (e9dba11b, reverted in 04b1a7c6) applied one measured offset per
|
||||
// FILE — wrong, because a single log file holds lines from multiple writer eras:
|
||||
// AgentExecutor.log is written by short-lived child processes whose zone belief
|
||||
// flips per process, interleaved in one file (fixture
|
||||
// tests/fixtures/cmtrace-logs/agentexecutor-two-writer-eras-v1.cmtrace). Any
|
||||
// cross-line anchor — per file, or nearest-in-write-order — inherits that trap.
|
||||
//
|
||||
// Per-line self-anchoring does not: at a 100 ms poll every line of a growing pass
|
||||
// was written essentially "now", so its own distance to the agent clock IS its
|
||||
// writer's offset, era by era. The line's timestamp still contributes the sub-poll
|
||||
// precision and ordering (two lines 6 ms apart stay 6 ms apart — a plain
|
||||
// occurredAt=now could not do that), while the offset grid absorbs poll latency.
|
||||
// Freshness is load-bearing (see FreshLineMaxAge) — backlog passes, restart
|
||||
// catch-up and replay logs must never anchor, because an old line's AGE can round
|
||||
// onto the grid.
|
||||
if (_currentPassLinesAreFresh)
|
||||
{
|
||||
TimeSpan lineOffset;
|
||||
if (CmTraceOffsetCalibrator.TryMeasureOffset(entry.LocalTimestamp, UtcNowProvider(), out lineOffset))
|
||||
{
|
||||
origin = CmTraceOffsetOrigin.LineAnchored;
|
||||
offsetMinutes = (int)lineOffset.TotalMinutes;
|
||||
return DateTime.SpecifyKind(entry.LocalTimestamp - lineOffset, DateTimeKind.Utc);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback for everything not provably fresh: assume the reader's zone, UNIFORMLY.
|
||||
// Wrong in absolute terms whenever the writer held a different belief, but
|
||||
// self-consistent — both ends of a duration are wrong by the same amount, so derived
|
||||
// durations stay right. That asymmetry is the lesson of the 04b1a7c6 revert:
|
||||
// partially corrected is strictly worse than uniformly wrong. Origin stays None so
|
||||
// the emitted event says so.
|
||||
var readerZoneOffset = TimeZoneInfo.Local.GetUtcOffset(entry.LocalTimestamp);
|
||||
offsetMinutes = (int)readerZoneOffset.TotalMinutes;
|
||||
return CmTraceLogParser.ResolveUtcAssumingReaderZone(entry.LocalTimestamp);
|
||||
}
|
||||
|
||||
// Field-forensics (tripwire, 2026-08-20): sessions e9753578 and 067797d8 both logged a
|
||||
// measurement whose LABEL and ANCHOR could not have come from the same file iteration —
|
||||
// 067797d8's anchor value (19:51:26.948) does not even exist in ANY log file on the
|
||||
// device, and IntuneManagementExtension.log never produced its expected first
|
||||
// measurement at all. Mechanism unexplained; resolution is immune (per-line anchoring),
|
||||
// but the observational layer must convict the culprit on the next occurrence. Hence:
|
||||
// the stream's file name travels alongside the label (a mismatch is THE smoking gun),
|
||||
// and the first rejected anchor per file becomes visible instead of silent.
|
||||
private readonly HashSet<string> _calibrationRejectLogged = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Feed the pass's newest bias-less line to the calibrator and report a measured offset
|
||||
/// that disagrees with this process's own zone — that disagreement is precisely the
|
||||
/// condition that used to corrupt every IME-derived timestamp silently.
|
||||
/// </summary>
|
||||
/// <param name="sourceFileName">The calibration key (<c>_currentSourceFileName</c>).</param>
|
||||
/// <param name="streamFileName">The file the enclosing iteration actually read. Must equal
|
||||
/// <paramref name="sourceFileName"/> — logged loudly when it does not (see tripwire note).</param>
|
||||
private void CalibrateFrom(string sourceFileName, string streamFileName, CmTraceLogEntry anchor)
|
||||
{
|
||||
if (!string.Equals(sourceFileName, streamFileName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger?.Warning(
|
||||
$"ImeLogTracker: CALIBRATION LABEL MISMATCH — key='{sourceFileName}' but the iteration read '{streamFileName}' " +
|
||||
$"(anchor local={anchor.LocalTimestamp:yyyy-MM-ddTHH:mm:ss.fffffff}). Skipping this anchor; see tripwire note.");
|
||||
return;
|
||||
}
|
||||
|
||||
TimeSpan previous;
|
||||
var hadOffset = OffsetCalibrator.TryGetOffset(sourceFileName, out previous);
|
||||
|
||||
if (!OffsetCalibrator.TryCalibrate(sourceFileName, anchor.LocalTimestamp, UtcNowProvider()))
|
||||
{
|
||||
// Once per file: why a file that visibly grows never produces a measurement.
|
||||
// 067797d8 ran 24 s of growing IME.log passes with no measurement at all — this
|
||||
// silence is exactly what made the phenomenon undiagnosable from the log.
|
||||
if (_calibrationRejectLogged.Add(sourceFileName))
|
||||
{
|
||||
_logger?.Debug(
|
||||
$"ImeLogTracker: {sourceFileName} anchor rejected by calibrator " +
|
||||
$"(anchor local={anchor.LocalTimestamp:yyyy-MM-ddTHH:mm:ss.fffffff}, now={UtcNowProvider():HH:mm:ss.fff}Z) — first rejection for this file, further ones stay silent.");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
TimeSpan measured;
|
||||
if (!OffsetCalibrator.TryGetOffset(sourceFileName, out measured)) return;
|
||||
if (hadOffset && measured == previous) return;
|
||||
|
||||
// The anchor's local timestamp is part of the record ON PURPOSE (tripwire, 2026-08-20):
|
||||
// session e9753578 logged "+02:00 measured for IntuneManagementExtension.log" although
|
||||
// that file provably held no +2-era line — an anchor/label pairing the committed code
|
||||
// could not be shown to produce. Should it ever happen again, the logged anchor value
|
||||
// itself will prove (or refute) the crossing without needing the device's log files.
|
||||
var readerZoneOffset = TimeZoneInfo.Local.GetUtcOffset(UtcNowProvider());
|
||||
if (measured == readerZoneOffset)
|
||||
{
|
||||
_logger?.Debug(
|
||||
$"ImeLogTracker: {sourceFileName} writer offset measured {measured} (matches this process, anchor local={anchor.LocalTimestamp:yyyy-MM-ddTHH:mm:ss.fff}).");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger?.Info(
|
||||
$"ImeLogTracker: {sourceFileName} writer offset measured {measured}, this process believes {readerZoneOffset} " +
|
||||
$"(anchor local={anchor.LocalTimestamp:yyyy-MM-ddTHH:mm:ss.fff}). Measurement is observational — resolution anchors per line.");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +85,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// Unspecified depending on the parser path. Normalize via SpecifyKind=Utc rather than
|
||||
/// ToUniversalTime() to avoid double-conversion when Kind is already UTC.
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Measures the UTC offset the IME process believes it is in, per log file.
|
||||
/// <para>
|
||||
/// Only this tracker may WRITE to it: calibration requires tailing with growth detection
|
||||
/// so the freshness rule can be honoured, and no other collector tails. Exposed so the
|
||||
/// snapshot-reading collectors can eventually READ from a shared instance
|
||||
/// (tasks/todo.md, P3b).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal CmTraceOffsetCalibrator OffsetCalibrator { get; } = new CmTraceOffsetCalibrator();
|
||||
|
||||
/// <summary>
|
||||
/// File whose lines are currently being processed. The polling loop walks files strictly
|
||||
/// sequentially on a single task, so a field is sufficient to give <c>ResolveEntryUtc</c>
|
||||
/// and the handlers the calibration key without threading it through every signature.
|
||||
/// </summary>
|
||||
private string _currentSourceFileName;
|
||||
|
||||
/// <summary>
|
||||
/// Whether every line of the file iteration currently in flight is provably FRESH —
|
||||
/// written after the previous look at this file, which happened at most
|
||||
/// <see cref="FreshLineMaxAge"/> ago in THIS process. Only then may a line anchor its
|
||||
/// own UTC offset (<see cref="CmTraceOffsetCalibrator.TryMeasureOffset"/>): the grid
|
||||
/// rounding is safe when the line's age is bounded by seconds, and becomes a trap on
|
||||
/// backlog whose age can hit an exact grid multiple. Same single-task justification as
|
||||
/// <see cref="_currentSourceFileName"/>.
|
||||
/// </summary>
|
||||
private bool _currentPassLinesAreFresh;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum gap between two looks at a file for its new bytes to count as fresh. Far
|
||||
/// above the 100 ms poll (rides out GC pauses and ESP CPU starvation), far below the
|
||||
/// 13-minute danger zone where a line's AGE could round onto the 15-minute offset grid.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan FreshLineMaxAge = TimeSpan.FromSeconds(30);
|
||||
|
||||
internal static DateTime NormalizeUtc(DateTime value) =>
|
||||
value.Kind == DateTimeKind.Utc ? value : DateTime.SpecifyKind(value, DateTimeKind.Utc);
|
||||
|
||||
@@ -191,6 +227,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
public string LastMatchedPatternId { get; internal set; }
|
||||
public DateTime? LastMatchedLogTimestamp { get; internal set; }
|
||||
|
||||
/// <summary>
|
||||
/// Raw local time of the last matched line, exactly as the writer wrote it — no zone
|
||||
/// attached, because the line carries none.
|
||||
/// <para>
|
||||
/// Recorded so an emitted event can carry the evidence for its own timestamp. Without it
|
||||
/// a conversion error is only provable from the diagnostics archive, and of the 5,386
|
||||
/// sessions that went through a timezone change essentially none had one — the field
|
||||
/// analysis had to fall back to median statistics over thousands of sessions to show
|
||||
/// what one stored value would have made obvious per session.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public DateTime? LastMatchedSourceLocalTimestamp { get; internal set; }
|
||||
|
||||
/// <summary>Offset applied to <see cref="LastMatchedSourceLocalTimestamp"/>, in minutes.</summary>
|
||||
public int? LastMatchedSourceOffsetMinutes { get; internal set; }
|
||||
|
||||
/// <summary>How that offset was arrived at — declared, measured, or neither.</summary>
|
||||
public CmTraceOffsetOrigin LastMatchedSourceOffsetOrigin { get; internal set; }
|
||||
|
||||
/// <summary>
|
||||
/// The offset the calibrator MEASURED for the source file, which is not necessarily the
|
||||
/// one applied.
|
||||
/// <para>
|
||||
/// Since the 2026-08-20 revert the measurement is observational only: it is recorded so the
|
||||
/// field tells us what an era-aware calibrator would have to work with, without acting on
|
||||
/// it. Reported separately from <see cref="LastMatchedSourceOffsetMinutes"/> precisely so
|
||||
/// the two can never be confused — that event said "calibrated" while carrying a wrong
|
||||
/// value, and an honest record has to distinguish measured from applied.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public int? LastMatchedMeasuredWriterOffsetMinutes { get; internal set; }
|
||||
|
||||
// Callbacks to EnrollmentTracker
|
||||
public Action<string> OnEspPhaseChanged { get; set; }
|
||||
public Action<string> OnImeAgentVersion { get; set; }
|
||||
|
||||
+62
-1
@@ -19,6 +19,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// has been rotated - resets to 0 to read from beginning.
|
||||
/// Returns 0 if no position has been recorded yet.
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Whether this file was already observed in an earlier pass.
|
||||
/// <para>
|
||||
/// The offset calibration depends on this: growth measured against a size we saw before
|
||||
/// means the new bytes were written within one poll interval, so the last line of that
|
||||
/// pass is a valid "written now" anchor. On the FIRST sight of a file we read it from
|
||||
/// position 0 and its last line may be arbitrarily old — anchoring on that would measure
|
||||
/// the line's age instead of the writer's timezone offset.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public bool HasSeen(string filePath) => _positions.ContainsKey(filePath);
|
||||
|
||||
public long GetSafePosition(string filePath, long currentFileSize)
|
||||
{
|
||||
FilePositionState state;
|
||||
@@ -53,6 +65,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
state.LastReadTime = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records that the file was looked at in a poll pass — whether or not it had new data.
|
||||
/// Creates the entry on first sight (position 0), so a file first seen EMPTY counts as
|
||||
/// observed and its first content is provably fresh.
|
||||
/// <para>
|
||||
/// This timestamp is the freshness reference for per-line offset anchoring: bytes found
|
||||
/// in the NEXT pass were written after this instant, so
|
||||
/// <c>now - LastCheckedUtc</c> bounds the age of every line in that pass's chunk.
|
||||
/// The caller supplies the clock (the tracker's <c>UtcNowProvider</c>) so replay/test
|
||||
/// clocks stay consistent with the freshness comparison.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public void MarkChecked(string filePath, DateTime nowUtc)
|
||||
{
|
||||
FilePositionState state;
|
||||
if (!_positions.TryGetValue(filePath, out state))
|
||||
{
|
||||
state = new FilePositionState();
|
||||
_positions[filePath] = state;
|
||||
}
|
||||
|
||||
state.LastCheckedUtc = nowUtc;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// When the file was last looked at, or <see cref="DateTime.MinValue"/> when it never
|
||||
/// was in THIS process — including entries restored from persisted state, whose
|
||||
/// bookmark survives an agent restart but whose freshness deliberately does not
|
||||
/// (the restart gap can hold arbitrarily old backlog).
|
||||
/// </summary>
|
||||
public DateTime GetLastCheckedUtc(string filePath)
|
||||
{
|
||||
FilePositionState state;
|
||||
return _positions.TryGetValue(filePath, out state) ? state.LastCheckedUtc : DateTime.MinValue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the stored position for a file, or 0 if not tracked.
|
||||
/// Does not perform rollover detection.
|
||||
@@ -77,6 +125,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// <summary>
|
||||
/// Restores a previously persisted position for a file.
|
||||
/// Used on agent restart to continue reading from the last known position.
|
||||
/// <para>
|
||||
/// <see cref="FilePositionState.LastCheckedUtc"/> is deliberately NOT restored: the
|
||||
/// bookmark survives the restart, the freshness guarantee does not — the first pass
|
||||
/// after a restart reads the whole downtime backlog and must never anchor per-line
|
||||
/// offsets on it.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public void RestorePosition(string filePath, long position, long lastKnownSize)
|
||||
{
|
||||
@@ -84,7 +138,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
{
|
||||
Position = position,
|
||||
LastKnownSize = lastKnownSize,
|
||||
LastReadTime = DateTime.UtcNow
|
||||
LastReadTime = DateTime.UtcNow,
|
||||
LastCheckedUtc = DateTime.MinValue
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -94,5 +149,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
public long Position { get; set; }
|
||||
public long LastKnownSize { get; set; }
|
||||
public DateTime LastReadTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When this file was last looked at by a poll pass IN THIS PROCESS. In-memory only —
|
||||
/// never persisted, so a restart resets it (see <see cref="LogFilePositionTracker.RestorePosition"/>).
|
||||
/// </summary>
|
||||
public DateTime LastCheckedUtc { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
+106
-9
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Shared;
|
||||
@@ -46,7 +47,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
|
||||
// Session caa6cf50 gate-starvation fix (2026-06-11) — fire-once guard for the
|
||||
// user-ESP-apps-settled AccountSetup synthesis (see MaybeSynthesizeAccountSetupComplete).
|
||||
private bool _userAppsSettledSynthesisFired;
|
||||
// Interlocked rather than a plain bool: since the sits-d Cloud-PC fix the synthesis is
|
||||
// ALSO re-evaluated from the IME app-state-change thread, so two threads can reach the
|
||||
// guard concurrently (Shell-Core watcher thread + IME log thread).
|
||||
private int _userAppsSettledSynthesisFired;
|
||||
|
||||
// sits-d Cloud-PC fix (2026-08-19) — set once a CONFIRMED post-AccountSetup Shell-Core
|
||||
// exit has been observed (live OR backfill). The synthesis needs two facts: the ESP exit
|
||||
// is an EDGE that happens once, the settled user-ESP apps are a LEVEL that may only be
|
||||
// reached minutes later. Recording the edge lets ReevaluateUserAppsSettledSynthesis
|
||||
// re-check the level afterwards instead of losing the completion for the rest of the
|
||||
// session.
|
||||
//
|
||||
// Codex review P1 (2026-08-19): "confirmed" is load-bearing. Shell-Core raises 62407 at
|
||||
// EVERY ESP phase transition, DeviceSetup→AccountSetup included. Remembering that
|
||||
// intermediate exit would let the deferred re-check open the strong AccountSetup gate the
|
||||
// moment the last user app settles — while the AccountSetup page is still up and its other
|
||||
// subcategories are still running. See IsConfirmedPostAccountSetupExit.
|
||||
private volatile bool _espExitObserved;
|
||||
|
||||
// Liveness plan PR3 — one-shot-per-appId dedupe for app_install_starved emissions.
|
||||
// Written from the Shell-Core watcher thread (esp_exited path), read at termination by
|
||||
@@ -314,11 +332,40 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// </summary>
|
||||
public void ResetForEspResumption() => _helloTracker?.ResetForEspResumption();
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Backfills recent ESP exit and failure events from Shell-Core log on startup.
|
||||
/// Secondary recovery mechanism when state persistence is unavailable.
|
||||
/// Replays the recent Shell-Core Hello-wizard start with an explicit lookback window.
|
||||
/// Called by <c>EspAndHelloHost</c> right after <see cref="Start"/> — see the host for why
|
||||
/// the window is sized to the previous run's downtime, and
|
||||
/// <see cref="ShellCoreTracker.ReplayBackfillRecords"/> for why the ESP exit is not part
|
||||
/// of the replay.
|
||||
/// </summary>
|
||||
public void BackfillRecentEspExitEvents() => _shellCoreTracker?.BackfillRecentEspExitEvents();
|
||||
public void BackfillRecentHelloWizardStart(int lookbackMinutes)
|
||||
=> _shellCoreTracker?.BackfillRecentHelloWizardStart(lookbackMinutes);
|
||||
|
||||
/// <summary>
|
||||
/// sits-d Cloud-PC fix (2026-08-19) — re-check the user-apps-settled AccountSetup
|
||||
/// synthesis after the ESP exit has already been observed. Wired to the IME tracker's
|
||||
/// app-state-change callback in <c>DefaultComponentFactory</c>.
|
||||
/// <para>
|
||||
/// Before this, the synthesis ran ONLY on the <c>EspExited</c> edge. When the ESP page
|
||||
/// exits while user-ESP apps are still in flight (the normal case on a tenant with a
|
||||
/// large required-app set), the single attempt misses and nothing re-checks — even once
|
||||
/// every tracked app has reached a terminal state minutes later. Sessions 8110e262 and
|
||||
/// a89aac2d reached 138/138 apps with 0 failed and still never completed.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Adds NO new completion path: the gate conditions inside the synthesis are unchanged
|
||||
/// (real ESP exit + every required user-ESP app terminal + zero failures). This only
|
||||
/// grants the existing, deliberately conservative check a second look.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public void ReevaluateUserAppsSettledSynthesis()
|
||||
{
|
||||
if (!_espExitObserved) return;
|
||||
if (Volatile.Read(ref _userAppsSettledSynthesisFired) != 0) return;
|
||||
MaybeSynthesizeAccountSetupCompleteFromSettledUserApps(emitStarvedOnMiss: false);
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
// Lifecycle
|
||||
@@ -536,6 +583,40 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
return skipUser == false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Codex review P1 (2026-08-19) — may THIS <c>esp_exiting</c> be remembered as the edge the
|
||||
/// deferred user-apps-settled re-check builds on?
|
||||
/// <para>
|
||||
/// Deliberately STRICTER than <c>!IsIntermediateDeviceEspExit()</c>: that helper forwards
|
||||
/// on an unknown SkipUser (forwarding a signal early is cheap), whereas remembering the
|
||||
/// wrong edge here can open a completion gate minutes later with no second chance to take
|
||||
/// it back. So this one demands positive evidence and treats "unknown" as "not confirmed":
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item>the provisioning tracker has seen AccountSetup activity — the page that just
|
||||
/// tore down is demonstrably the AccountSetup one; or</item>
|
||||
/// <item>SkipUser is explicitly <c>true</c> — the profile has no user ESP at all, so the
|
||||
/// Device-ESP exit IS the final one and no second exit is coming.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// Erring strict costs at worst today's behaviour (the session stalls as it did before the
|
||||
/// fix); erring loose costs a premature Succeeded, which is unrecoverable.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private bool IsConfirmedPostAccountSetupExit()
|
||||
{
|
||||
bool accountSetupSeen;
|
||||
try { accountSetupSeen = _accountSetupActivityProbe(); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"EspAndHelloTracker: account-setup-activity probe threw: {ex.Message}");
|
||||
accountSetupSeen = false;
|
||||
}
|
||||
if (accountSetupSeen) return true;
|
||||
|
||||
return GetSkipUserEspCached() == true;
|
||||
}
|
||||
|
||||
private bool? GetSkipUserEspCached()
|
||||
{
|
||||
lock (_skipConfigLock)
|
||||
@@ -675,6 +756,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
try { EspExited?.Invoke(this, args); }
|
||||
catch (Exception ex) { _logger.Error("Error forwarding EspExited", ex); }
|
||||
|
||||
// Every exit reaching this handler is one the agent observed LIVE: the Shell-Core
|
||||
// replay never re-raises 62407 (see ShellCoreTracker.ReplayBackfillRecords — a
|
||||
// replayed exit cannot be placed in time and is therefore kept out of the decision
|
||||
// stream entirely). That is what makes the AccountSetup read below a valid
|
||||
// ordering fact: the agent was continuously observing up to this instant.
|
||||
//
|
||||
// Record the edge BEFORE the synthesis attempt so a later app-state change can
|
||||
// re-run it even when the apps are not settled at this instant.
|
||||
if (IsConfirmedPostAccountSetupExit())
|
||||
_espExitObserved = true;
|
||||
|
||||
// Session caa6cf50 gate-starvation fix: a Shell-Core normal exit while IME's
|
||||
// user-ESP app tracking is fully settled is alternative evidence that
|
||||
// AccountSetup completed. Raised AFTER the EspExited forward so the reducer
|
||||
@@ -704,9 +796,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// conservative stall behaviour.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void MaybeSynthesizeAccountSetupCompleteFromSettledUserApps()
|
||||
private void MaybeSynthesizeAccountSetupCompleteFromSettledUserApps(bool emitStarvedOnMiss = true)
|
||||
{
|
||||
if (_userAppsSettledSynthesisFired) return;
|
||||
if (Volatile.Read(ref _userAppsSettledSynthesisFired) != 0) return;
|
||||
|
||||
bool settled;
|
||||
try { settled = _userEspAppsSettledProbe(); }
|
||||
@@ -722,11 +814,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// operator with an anonymous "session hangs in AccountSetup". One-shot per
|
||||
// appId; apps that are alive (Downloading/Installing) or failed are excluded
|
||||
// by the probe itself.
|
||||
EmitStarvedUserEspApps(trigger: "esp_exited_user_apps_not_settled");
|
||||
// Only on the ESP-exit edge: the re-evaluation path passes false so a repeated
|
||||
// miss cannot re-walk the probe on every single app-state change (the per-appId
|
||||
// dedupe would swallow the events anyway — this keeps the work off the hot path).
|
||||
if (emitStarvedOnMiss)
|
||||
EmitStarvedUserEspApps(trigger: "esp_exited_user_apps_not_settled");
|
||||
return;
|
||||
}
|
||||
|
||||
_userAppsSettledSynthesisFired = true;
|
||||
// Claim the single synthesis slot; a concurrent caller that loses the race returns.
|
||||
if (Interlocked.CompareExchange(ref _userAppsSettledSynthesisFired, 1, 0) != 0) return;
|
||||
_logger.Warning(
|
||||
"EspAndHelloTracker: ESP exited normally with all tracked user-ESP apps terminal (0 failed) " +
|
||||
"but AccountSetupCategory.Status never confirmed categorySucceeded — treating AccountSetup as complete " +
|
||||
@@ -820,6 +917,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
|
||||
// Test seam for the user-apps-settled synthesis — drives OnEspExited with the live
|
||||
// handler signature, mirroring TriggerEspExitedForTest's contract.
|
||||
internal bool UserAppsSettledSynthesisFiredForTest => _userAppsSettledSynthesisFired;
|
||||
internal bool UserAppsSettledSynthesisFiredForTest => Volatile.Read(ref _userAppsSettledSynthesisFired) != 0;
|
||||
}
|
||||
}
|
||||
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Runtime;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Pure state machine behind <c>PowerStateWatcherHost</c>: diffs successive
|
||||
/// <see cref="PowerStateResult"/> snapshots into <c>power_state_change</c> emissions.
|
||||
/// No WMI, no timers, no logging — fully deterministic and unit-testable.
|
||||
/// <para>
|
||||
/// Semantics:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>AC↔battery transitions</b> — one emission per accepted edge
|
||||
/// (<c>ac_to_battery</c> Warning/immediate, <c>battery_to_ac</c> Info).</item>
|
||||
/// <item><b>Threshold ladder</b> (default 50/30/15 %) — evaluated only while ON battery;
|
||||
/// each level latches once per tracker lifetime (no re-emit when the device charges
|
||||
/// back up and drains again — avoids plug-cycle alarm spam). A multi-level jump
|
||||
/// emits only the LOWEST newly-crossed level. Entering battery power already below
|
||||
/// a level counts as crossing it (covers "unplugged at 12 %" and, via
|
||||
/// <see cref="Baseline"/>, "enrollment started at 12 %").</item>
|
||||
/// <item><b>Storm cap</b> — a lifetime emission cap (default 20) as backstop against a
|
||||
/// flapping dock / dying battery controller; <see cref="EmissionCapReached"/> flips
|
||||
/// once so the host can log a single warning.</item>
|
||||
/// </list>
|
||||
/// A snapshot with <see cref="PowerStateResult.ProbeError"/> is ignored (previous state
|
||||
/// preserved); a null <see cref="PowerStateResult.BatteryPercent"/> skips threshold
|
||||
/// evaluation but AC diffing still works.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal sealed class PowerStateTransitionTracker
|
||||
{
|
||||
public const string TransitionAcToBattery = "ac_to_battery";
|
||||
public const string TransitionBatteryToAc = "battery_to_ac";
|
||||
public const string TransitionThresholdCrossed = "threshold_crossed";
|
||||
|
||||
internal static readonly int[] DefaultThresholdLadder = { 50, 30, 15 };
|
||||
internal const int DefaultMaxEmissions = 20;
|
||||
|
||||
private readonly int[] _ladder; // strictly descending
|
||||
private readonly int _maxEmissions;
|
||||
private readonly HashSet<int> _latchedLevels = new HashSet<int>();
|
||||
|
||||
private bool _hasState;
|
||||
private bool _lastOnAc;
|
||||
private int _emitted;
|
||||
|
||||
public PowerStateTransitionTracker(IReadOnlyList<int>? thresholdLadder = null, int maxEmissions = DefaultMaxEmissions)
|
||||
{
|
||||
_ladder = (thresholdLadder ?? DefaultThresholdLadder).OrderByDescending(l => l).ToArray();
|
||||
_maxEmissions = maxEmissions;
|
||||
}
|
||||
|
||||
/// <summary>True once the lifetime emission cap suppressed at least one emission.</summary>
|
||||
public bool EmissionCapReached { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Latches the arm-time state silently — except when the device is already on battery
|
||||
/// at/below a ladder level, which yields one threshold emission for the lowest crossed
|
||||
/// level (an enrollment STARTING at 12 % must still produce the event the analyze rule
|
||||
/// matches on). No AC-transition emission is ever produced from a baseline.
|
||||
/// </summary>
|
||||
public IReadOnlyList<PowerStateEmission> Baseline(PowerStateResult probe)
|
||||
{
|
||||
if (probe == null || probe.ProbeError != null || !probe.HasBattery)
|
||||
return Array.Empty<PowerStateEmission>();
|
||||
|
||||
_hasState = true;
|
||||
_lastOnAc = probe.OnAcPower;
|
||||
|
||||
var emissions = new List<PowerStateEmission>();
|
||||
EvaluateThresholds(probe, emissions);
|
||||
return emissions;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Diffs a fresh (host-debounced) snapshot against the last accepted state and returns
|
||||
/// 0..n emissions — an AC transition and a threshold crossing can co-occur (unplugged
|
||||
/// at 10 % ⇒ <c>ac_to_battery</c> + <c>threshold_crossed:15</c>).
|
||||
/// </summary>
|
||||
public IReadOnlyList<PowerStateEmission> Evaluate(PowerStateResult probe)
|
||||
{
|
||||
if (probe == null || probe.ProbeError != null)
|
||||
return Array.Empty<PowerStateEmission>();
|
||||
if (!_hasState)
|
||||
return Baseline(probe);
|
||||
|
||||
var emissions = new List<PowerStateEmission>();
|
||||
|
||||
if (_lastOnAc && !probe.OnAcPower)
|
||||
{
|
||||
Emit(emissions, new PowerStateEmission(
|
||||
transition: TransitionAcToBattery,
|
||||
thresholdPercent: null,
|
||||
probe: probe,
|
||||
severity: EventSeverity.Warning,
|
||||
immediateUpload: true,
|
||||
message: $"Power source switched from AC to battery ({FormatPercent(probe)} remaining) during enrollment"));
|
||||
}
|
||||
else if (!_lastOnAc && probe.OnAcPower)
|
||||
{
|
||||
Emit(emissions, new PowerStateEmission(
|
||||
transition: TransitionBatteryToAc,
|
||||
thresholdPercent: null,
|
||||
probe: probe,
|
||||
severity: EventSeverity.Info,
|
||||
immediateUpload: false,
|
||||
message: $"Power source switched back to AC ({FormatPercent(probe)}{(probe.IsCharging ? ", charging" : string.Empty)})"));
|
||||
}
|
||||
|
||||
_lastOnAc = probe.OnAcPower;
|
||||
EvaluateThresholds(probe, emissions);
|
||||
return emissions;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// While on battery with a known percentage: emit the LOWEST ladder level at/above the
|
||||
/// current charge that is not yet latched, then latch every level at/above the charge
|
||||
/// (a 60 %→10 % jump emits threshold 15 only, latching 50/30/15).
|
||||
/// </summary>
|
||||
private void EvaluateThresholds(PowerStateResult probe, List<PowerStateEmission> emissions)
|
||||
{
|
||||
if (probe.OnAcPower || !probe.BatteryPercent.HasValue) return;
|
||||
var percent = probe.BatteryPercent.Value;
|
||||
|
||||
int? lowestCrossed = null;
|
||||
foreach (var level in _ladder) // descending — the last match is the lowest crossed level
|
||||
{
|
||||
if (percent <= level && !_latchedLevels.Contains(level))
|
||||
lowestCrossed = level;
|
||||
}
|
||||
if (!lowestCrossed.HasValue) return;
|
||||
|
||||
foreach (var level in _ladder)
|
||||
{
|
||||
if (percent <= level) _latchedLevels.Add(level);
|
||||
}
|
||||
|
||||
var threshold = lowestCrossed.Value;
|
||||
var severity = threshold <= 15 ? EventSeverity.Error
|
||||
: threshold <= 30 ? EventSeverity.Warning
|
||||
: EventSeverity.Info;
|
||||
Emit(emissions, new PowerStateEmission(
|
||||
transition: TransitionThresholdCrossed,
|
||||
thresholdPercent: threshold,
|
||||
probe: probe,
|
||||
severity: severity,
|
||||
immediateUpload: severity >= EventSeverity.Warning,
|
||||
message: $"Battery dropped below {threshold}% ({FormatPercent(probe)} remaining) while enrolling on battery power"));
|
||||
}
|
||||
|
||||
private void Emit(List<PowerStateEmission> emissions, PowerStateEmission emission)
|
||||
{
|
||||
if (_emitted >= _maxEmissions)
|
||||
{
|
||||
EmissionCapReached = true;
|
||||
return;
|
||||
}
|
||||
_emitted++;
|
||||
emissions.Add(emission);
|
||||
}
|
||||
|
||||
private static string FormatPercent(PowerStateResult probe)
|
||||
=> probe.BatteryPercent.HasValue ? $"battery {probe.BatteryPercent.Value}%" : "battery level unknown";
|
||||
}
|
||||
|
||||
/// <summary>One decided <c>power_state_change</c> emission — the host maps this 1:1 onto an event.</summary>
|
||||
internal sealed class PowerStateEmission
|
||||
{
|
||||
public PowerStateEmission(string transition, int? thresholdPercent, PowerStateResult probe,
|
||||
EventSeverity severity, bool immediateUpload, string message)
|
||||
{
|
||||
Transition = transition;
|
||||
ThresholdPercent = thresholdPercent;
|
||||
OnAcPower = probe.OnAcPower;
|
||||
BatteryPercent = probe.BatteryPercent;
|
||||
IsCharging = probe.IsCharging;
|
||||
BatteryLifeMinutes = probe.BatteryLifeMinutes;
|
||||
Severity = severity;
|
||||
ImmediateUpload = immediateUpload;
|
||||
Message = message;
|
||||
}
|
||||
|
||||
public string Transition { get; }
|
||||
public int? ThresholdPercent { get; }
|
||||
public bool OnAcPower { get; }
|
||||
public int? BatteryPercent { get; }
|
||||
public bool IsCharging { get; }
|
||||
public int? BatteryLifeMinutes { get; }
|
||||
public EventSeverity Severity { get; }
|
||||
public bool ImmediateUpload { get; }
|
||||
public string Message { get; }
|
||||
}
|
||||
}
|
||||
+150
-46
@@ -34,6 +34,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
internal const int EventId_ShellCore_WebAppEvent = 62407;
|
||||
internal const int BackfillLookbackMinutes = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Upper bound for the caller-supplied backfill lookback. Matches the agent's max
|
||||
/// lifetime (360 min) — a window wider than the agent can ever live would only widen
|
||||
/// the blast radius of a stale Shell-Core record without recovering anything the
|
||||
/// current enrollment could still act on.
|
||||
/// </summary>
|
||||
internal const int BackfillLookbackMaxMinutes = 360;
|
||||
|
||||
private static readonly HashSet<int> TrackedShellCoreEventIds = new HashSet<int>
|
||||
{
|
||||
EventId_ShellCore_WebAppStarted,
|
||||
@@ -350,22 +358,39 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// Backfill (public — called by coordinator)
|
||||
// =====================================================================
|
||||
|
||||
/// <summary>Clamps a caller-supplied lookback into [1, <see cref="BackfillLookbackMaxMinutes"/>].</summary>
|
||||
internal static int ClampLookbackMinutes(int lookbackMinutes)
|
||||
{
|
||||
if (lookbackMinutes < 1) return 1;
|
||||
if (lookbackMinutes > BackfillLookbackMaxMinutes) return BackfillLookbackMaxMinutes;
|
||||
return lookbackMinutes;
|
||||
}
|
||||
|
||||
public void BackfillRecentHelloWizardStart() => BackfillRecentHelloWizardStart(BackfillLookbackMinutes);
|
||||
|
||||
/// <summary>
|
||||
/// Backfills recent ESP exit, ESP failure and Hello-wizard-start events from the
|
||||
/// Shell-Core log on startup. Secondary recovery mechanism when state persistence is
|
||||
/// unavailable — an agent restart while the user sits inside the Hello wizard would
|
||||
/// otherwise lose the wizard-start observation (session 772fe502).
|
||||
/// Same recovery with a caller-chosen lookback, so a restart can reach back over its whole
|
||||
/// downtime (after a mid-ESP reboot the agent is gone from the forced restart until the
|
||||
/// post-reboot logon relaunches it; after a crash, until the next boot — the scheduled task
|
||||
/// has a BootTrigger only). Clamped to [1, <see cref="BackfillLookbackMaxMinutes"/>].
|
||||
/// <para>
|
||||
/// The 62407 records in the window are read but deliberately NOT replayed — see
|
||||
/// <see cref="ReplayBackfillRecords"/>. They are counted and reported once as an
|
||||
/// <c>agent_trace</c> so the gap stays visible in the timeline without becoming
|
||||
/// decision-relevant.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public void BackfillRecentEspExitEvents()
|
||||
public void BackfillRecentHelloWizardStart(int lookbackMinutes)
|
||||
{
|
||||
try
|
||||
{
|
||||
var lookbackMs = BackfillLookbackMinutes * 60 * 1000;
|
||||
var lookbackMs = ClampLookbackMinutes(lookbackMinutes) * 60 * 1000;
|
||||
var query = new EventLogQuery(
|
||||
ShellCoreEventLogChannel,
|
||||
PathType.LogName,
|
||||
$"*[System[(EventID=62404 or EventID=62407) and TimeCreated[timediff(@SystemTime) <= {lookbackMs}]]]");
|
||||
|
||||
var records = new List<(int Id, string Description, DateTime OccurredAtUtc)>();
|
||||
using (var reader = new EventLogReader(query))
|
||||
{
|
||||
for (EventRecord record = reader.ReadEvent(); record != null; record = reader.ReadEvent())
|
||||
@@ -377,20 +402,135 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// (EspAndHelloTrackerAdapter) can stamp signals with the source time
|
||||
// rather than collapsing to wall-clock-now.
|
||||
var timestamp = (record.TimeCreated ?? DateTime.UtcNow).ToUniversalTime();
|
||||
HandleBackfillRecord(record.Id, description, timestamp);
|
||||
records.Add((record.Id, description, timestamp));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ReplayBackfillRecords(records);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"ESP exit/failure event backfill failed: {ex.Message}");
|
||||
_logger.Warning($"Shell-Core replay failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Internal backfill record handler — extracted for testability and to keep the
|
||||
/// backfill loop free of direct event-processing logic. The
|
||||
/// Replays a chronological batch of Shell-Core records. <b>Only the Hello-wizard start
|
||||
/// (62404) is replayed.</b> ESP exits and ESP failures (62407) are counted and reported,
|
||||
/// never re-raised.
|
||||
/// <para>
|
||||
/// Why the exit is excluded — the reasoning is the whole point of this method, so it lives
|
||||
/// here rather than in a commit message:
|
||||
/// </para>
|
||||
/// <list type="number">
|
||||
/// <item>Windows writes the IDENTICAL description
|
||||
/// (<c>CommercialOOBE_ESPProgress_Page_Exiting</c>) for the intermediate
|
||||
/// DeviceSetup→AccountSetup transition and for the final post-AccountSetup exit, so a
|
||||
/// replayed record carries no evidence of its own position.</item>
|
||||
/// <item>Everything that could order it after the fact — the AccountSetup registry
|
||||
/// probe, the settled-apps probe — reads state as it is NOW, not as it was at the
|
||||
/// event's time.</item>
|
||||
/// <item>The reducer orders exits by INGEST ORDINAL, not by timestamp
|
||||
/// (<c>IsPostAccountSetupFinalExit</c>). A replayed historic exit is assigned a fresher
|
||||
/// ordinal than reality, so it looks post-AccountSetup by construction.</item>
|
||||
/// <item><c>HandleEspExitingV1</c> passes <c>espFinalExitInFlight: true</c> for every
|
||||
/// arriving exit. With restored state (AccountSetupEntered + a genuine IME user
|
||||
/// session + desktop arrived) arm C of <c>ShouldTransitionToAwaitingHello</c> then
|
||||
/// opens on a historic intermediate exit — a completion built on a fact that never
|
||||
/// happened.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// There is therefore no honest way to classify a replayed exit, so it must not enter the
|
||||
/// decision stream at all. The same applies to a replayed ESP FAILURE, for the opposite
|
||||
/// reason: re-injecting a historic failure as fresh can fail a session that recovered
|
||||
/// (see ANALYZE-ESP-006, "ESP Failure Recovered After User Retry").
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The Hello-wizard start is different in kind and is the observation this replay was
|
||||
/// written for (session 772fe502): it is a CONSERVATIVE fact. It vetoes a premature
|
||||
/// "Hello is disabled" skip and can never by itself complete a session, so replaying it
|
||||
/// can only make the agent wait longer, never finish early.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal void ReplayBackfillRecords(
|
||||
IReadOnlyList<(int Id, string Description, DateTime OccurredAtUtc)> records)
|
||||
{
|
||||
if (records == null || records.Count == 0) return;
|
||||
|
||||
var skippedExits = 0;
|
||||
var skippedFailures = 0;
|
||||
DateTime? oldestSkipped = null;
|
||||
DateTime? newestSkipped = null;
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
var description = record.Description ?? string.Empty;
|
||||
|
||||
if (record.Id == EventId_ShellCore_WebAppStarted)
|
||||
{
|
||||
HandleBackfillRecord(record.Id, description, record.OccurredAtUtc);
|
||||
continue;
|
||||
}
|
||||
|
||||
var isFailure = HasEspFailurePattern(description);
|
||||
var isExit = !isFailure && EspExitingPattern.IsMatch(description);
|
||||
if (!isFailure && !isExit) continue;
|
||||
|
||||
if (isFailure) skippedFailures++; else skippedExits++;
|
||||
if (oldestSkipped == null || record.OccurredAtUtc < oldestSkipped.Value)
|
||||
oldestSkipped = record.OccurredAtUtc;
|
||||
if (newestSkipped == null || record.OccurredAtUtc > newestSkipped.Value)
|
||||
newestSkipped = record.OccurredAtUtc;
|
||||
}
|
||||
|
||||
if (skippedExits > 0 || skippedFailures > 0)
|
||||
EmitSkippedShellCoreRecords(skippedExits, skippedFailures, oldestSkipped, newestSkipped);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One informational <c>agent_trace</c> naming the 62407 records the replay deliberately
|
||||
/// did not re-raise. Decision-neutral by construction (informational events are exempt
|
||||
/// from the dispatch guard) — its only job is to keep the blind window visible to whoever
|
||||
/// debugs a session later, instead of the replay silently dropping evidence.
|
||||
/// </summary>
|
||||
private void EmitSkippedShellCoreRecords(
|
||||
int skippedExits, int skippedFailures, DateTime? oldestUtc, DateTime? newestUtc)
|
||||
{
|
||||
try
|
||||
{
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.AgentTrace,
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "ShellCoreTracker",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message =
|
||||
$"Shell-Core replay skipped {skippedExits} ESP exit(s) and {skippedFailures} ESP failure(s) " +
|
||||
"from the window in which no agent process was running — a replayed 62407 cannot be placed " +
|
||||
"in time and must not reach the decision engine.",
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
{ "skippedEspExits", skippedExits },
|
||||
{ "skippedEspFailures", skippedFailures },
|
||||
{ "oldestSkippedUtc", oldestUtc?.ToString("o") ?? string.Empty },
|
||||
{ "newestSkippedUtc", newestUtc?.ToString("o") ?? string.Empty },
|
||||
{ "reason", "replayed_62407_not_orderable" },
|
||||
},
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"ShellCoreTracker: skipped-records trace emit failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Internal backfill record handler — extracted for testability and to keep the replay
|
||||
/// loop free of direct event-processing logic. Handles the Hello-wizard start ONLY; see
|
||||
/// <see cref="ReplayBackfillRecords"/> for why 62407 is never replayed. The
|
||||
/// <paramref name="occurredAtUtc"/> is the original Shell-Core event time
|
||||
/// (<c>record.TimeCreated</c>); subscribers read it via
|
||||
/// <see cref="LastEventOccurredAtUtc"/> during their synchronous event handler.
|
||||
@@ -427,42 +567,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
return;
|
||||
}
|
||||
|
||||
if (EspExitingPattern.IsMatch(description))
|
||||
{
|
||||
bool shouldNotify = false;
|
||||
lock (_stateLock)
|
||||
{
|
||||
if (!_espExitDetected)
|
||||
{
|
||||
_espExitDetected = true;
|
||||
shouldNotify = true;
|
||||
}
|
||||
}
|
||||
if (shouldNotify)
|
||||
{
|
||||
_helloTracker?.NotifyEspExited();
|
||||
_logger.Info($"Backfill: ESP exit event found in recent Shell-Core logs (originalAt={occurredAtUtc:o})");
|
||||
LastEventOccurredAtUtc = occurredAtUtc;
|
||||
try
|
||||
{
|
||||
try { FinalizingSetupPhaseTriggered?.Invoke(this, "esp_exiting"); }
|
||||
catch (Exception ex) { _logger.Error("Backfill: FinalizingSetupPhaseTriggered handler failed", ex); }
|
||||
try { EspExited?.Invoke(this, new EspExitedEventArgs(occurredAtUtc)); }
|
||||
catch (Exception ex) { _logger.Error("Backfill: EspExited handler failed", ex); }
|
||||
}
|
||||
finally { LastEventOccurredAtUtc = null; }
|
||||
}
|
||||
}
|
||||
|
||||
if (HasEspFailurePattern(description))
|
||||
{
|
||||
var failureType = ExtractEspFailureType(description);
|
||||
_logger.Info($"Backfill: ESP failure event found in recent Shell-Core logs: {failureType} (originalAt={occurredAtUtc:o})");
|
||||
LastEventOccurredAtUtc = occurredAtUtc;
|
||||
try { EspFailureDetected?.Invoke(this, failureType); }
|
||||
catch (Exception ex) { _logger.Error($"Backfill: EspFailureDetected handler failed for '{failureType}'", ex); }
|
||||
finally { LastEventOccurredAtUtc = null; }
|
||||
}
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
|
||||
+16
-4
@@ -580,10 +580,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
var scanText = rawLine;
|
||||
if (CmTraceLogParser.TryParseLine(rawLine, out var entry))
|
||||
{
|
||||
// Provably stale → not progress. (TryParseLine falls back to now on an
|
||||
// unparseable timestamp, which keeps the line fresh — the safe default.)
|
||||
if (entry.Timestamp < cutoffUtc)
|
||||
continue;
|
||||
// Provably stale → not progress. A line without a parseable timestamp counts
|
||||
// as fresh — the safe default, since treating it as stale would hide progress.
|
||||
//
|
||||
// This collector reads a file snapshot rather than tailing it, so it cannot
|
||||
// measure the writer's UTC offset itself and falls back to this process's zone
|
||||
// (see ResolveUtcAssumingReaderZone). That fallback is wrong by exactly the
|
||||
// gap between the two processes' zone beliefs. It is the pre-existing
|
||||
// behaviour, kept deliberately unchanged here; the tracker-owned calibrator is
|
||||
// wired in separately so this commit shifts no behaviour.
|
||||
if (entry.HasTimestamp)
|
||||
{
|
||||
var entryUtc = entry.TimestampUtc
|
||||
?? CmTraceLogParser.ResolveUtcAssumingReaderZone(entry.LocalTimestamp);
|
||||
if (entryUtc < cutoffUtc)
|
||||
continue;
|
||||
}
|
||||
scanText = entry.Message;
|
||||
}
|
||||
|
||||
|
||||
+681
@@ -0,0 +1,681 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Eventing.Reader;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Xml.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
{
|
||||
/// <summary>
|
||||
/// Watches the <c>System</c> event log for two enrollment blind spots and forwards them as
|
||||
/// timeline ground truth:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>System clock steps</b> (Kernel-General EventID 1) → <c>system_clock_changed</c>.
|
||||
/// The payload carries <c>oldTime</c>/<c>newTime</c>/<c>timeDeltaMs</c> plus the
|
||||
/// setting process — the "clock set from X to Y" fact that today has to be inferred
|
||||
/// from timestamp discontinuities. w32time also logs frequent 1 ms micro-slews on the
|
||||
/// same EventID, so emissions are gated on <see cref="MinClockDeltaMs"/>.</item>
|
||||
/// <item><b>Completed sleep episodes</b> → <c>system_sleep_episode</c>. Classic S3/S4 comes
|
||||
/// from Power-Troubleshooter EventID 1 (one event per episode at wake, carrying
|
||||
/// SleepTime + WakeTime). Modern Standby comes from Kernel-Power EventID 507 (exit),
|
||||
/// whose payload carries the scenario duration and the actual sleep residency; the
|
||||
/// enter time is derived as exit − duration. Kernel-Power 506 (enter) is deliberately
|
||||
/// NOT consumed: 507 already carries everything, and pairing 506/507 would need an
|
||||
/// orphan-tolerant state machine across agent restarts for zero information gain —
|
||||
/// an episode without a 507 never completed and would not be emitted under any design.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// <b>Payload times are authoritative.</b> The decisive clock correction (w32time fixing a
|
||||
/// wrong BIOS clock) happens exactly when the device clock is untrustworthy, and the backend
|
||||
/// clamps implausible event timestamps to server time. All semantic instants therefore travel
|
||||
/// as explicit ISO-8601 payload fields, never only as the event's own <c>Timestamp</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Backfill + cross-restart dedup</b> mirror <see cref="WindowsUpdateTracker"/>: the System
|
||||
/// .evtx persists across the OOBE reboots, so a startup scan recovers pre-agent clock steps
|
||||
/// and standby episodes; a persisted RecordId watermark plus an intra-run seen-set (a HashSet,
|
||||
/// NOT a high-water mark — see the field comment) keep restarts from re-emitting.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal sealed class SystemTimelineTracker : IDisposable
|
||||
{
|
||||
internal const string Channel = "System";
|
||||
|
||||
internal const string ProviderKernelGeneral = "Microsoft-Windows-Kernel-General"; // EventID 1: system time changed
|
||||
internal const string ProviderPowerTroubleshooter = "Microsoft-Windows-Power-Troubleshooter"; // EventID 1: resumed from classic sleep/hibernate
|
||||
internal const string ProviderKernelPower = "Microsoft-Windows-Kernel-Power"; // EventID 507: Modern Standby exit
|
||||
|
||||
internal const int EventId_ClockChange = 1;
|
||||
internal const int EventId_ClassicResume = 1;
|
||||
internal const int EventId_ModernStandbyExit = 507;
|
||||
|
||||
internal const string WatermarkStateFileName = "system-timeline-watermark.json";
|
||||
|
||||
// w32time logs Kernel-General 1 for routine 1 ms slew corrections (verified live: several
|
||||
// per hour). Only genuine steps are timeline-relevant; anything below this is claimed in
|
||||
// the dedup set but never emitted.
|
||||
internal const long MinClockDeltaMs = 2000;
|
||||
|
||||
// A step of at least 5 minutes can invalidate token/cert validation windows and visibly
|
||||
// reorders the timeline (matches the web's TimeJumpBadge threshold) — surface it fast.
|
||||
internal const long ClockDeltaWarningMs = 5 * 60 * 1000;
|
||||
|
||||
// Modern Standby logs a 506/507 pair for every screen-off scenario, including seconds-long
|
||||
// ones where the device never actually slept (SleepDurationInUs 0). Only episodes with
|
||||
// real sleep residency are emitted. Classic Power-Troubleshooter episodes have no floor:
|
||||
// they only exist for genuinely completed S3/S4 transitions.
|
||||
internal const long MinSleepDurationSeconds = 60;
|
||||
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly string _sessionId;
|
||||
private readonly string _tenantId;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly bool _backfillEnabled;
|
||||
private readonly int _backfillLookbackMinutes;
|
||||
private readonly string _stateDirectory;
|
||||
|
||||
private EventLogWatcher _watcher;
|
||||
|
||||
// Dedup. Guarded because EventRecordWritten callbacks run concurrently with the backfill scan.
|
||||
//
|
||||
// Cross-restart boundary loaded from disk: the highest RecordId emitted in a PRIOR run.
|
||||
// Immutable during this run — records at/below it were already emitted before this process
|
||||
// started and must not be re-emitted by the backfill scan.
|
||||
private long _restartWatermark = -1;
|
||||
// RecordIds claimed in THIS run — the intra-run dedup set. Deliberately a HashSet, NOT a
|
||||
// high-water mark: the live watcher is armed before the backfill runs, so a live record with
|
||||
// a HIGHER RecordId can be processed before/during backfill. A high-water mark would then
|
||||
// make the dedup skip every older, never-emitted backfill record — silently dropping exactly
|
||||
// the pre-agent clock corrections and standby episodes this tracker exists for. The targeted
|
||||
// records are low-volume, so the set stays tiny.
|
||||
private readonly HashSet<long> _seenThisRun = new HashSet<long>();
|
||||
// Highest RecordId claimed so far (max of restart watermark + this run). Persisted so the
|
||||
// next run's restart watermark is correct.
|
||||
private long _maxEmittedRecordId = -1;
|
||||
private readonly object _watermarkLock = new object();
|
||||
|
||||
public SystemTimelineTracker(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
InformationalEventPost post,
|
||||
AgentLogger logger,
|
||||
bool backfillEnabled = true,
|
||||
int backfillLookbackMinutes = 1440,
|
||||
string stateDirectory = null)
|
||||
{
|
||||
_sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
_post = post ?? throw new ArgumentNullException(nameof(post));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_backfillEnabled = backfillEnabled;
|
||||
_backfillLookbackMinutes = backfillLookbackMinutes;
|
||||
_stateDirectory = stateDirectory != null ? Environment.ExpandEnvironmentVariables(stateDirectory) : null;
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
LoadWatermark();
|
||||
StartWatcher();
|
||||
|
||||
if (_backfillEnabled && _backfillLookbackMinutes > 0)
|
||||
BackfillRecentEvents();
|
||||
else
|
||||
_logger.Info("SystemTimeline backfill disabled by config");
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
if (_watcher == null) return;
|
||||
try
|
||||
{
|
||||
_watcher.Enabled = false;
|
||||
_watcher.Dispose();
|
||||
_logger.Info("SystemTimeline watcher stopped");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error("Error stopping SystemTimeline watcher", ex);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_watcher = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() => Stop();
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Watcher lifecycle
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void StartWatcher()
|
||||
{
|
||||
try
|
||||
{
|
||||
var query = new EventLogQuery(Channel, PathType.LogName, BuildXPath());
|
||||
_watcher = new EventLogWatcher(query);
|
||||
_watcher.EventRecordWritten += OnEventRecordWritten;
|
||||
_watcher.Enabled = true;
|
||||
_logger.Info($"SystemTimeline watcher started: {Channel} (clock changes + sleep episodes)");
|
||||
}
|
||||
catch (EventLogNotFoundException)
|
||||
{
|
||||
_logger.Warning($"SystemTimeline event log not found: {Channel} (normal on non-Windows test environments)");
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
_logger.Warning($"SystemTimeline watcher access denied for {Channel}: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error($"Failed to start SystemTimeline watcher for {Channel}", ex);
|
||||
CollectorDegradationReporter.Report(_post, _sessionId, _tenantId,
|
||||
collectorName: "SystemTimelineTracker", reason: $"watcher_arm_failed:{Channel}", ex: ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provider-qualified XPath for the three targeted signals. Provider filtering is mandatory
|
||||
/// on the System channel — EventID 1 alone matches many unrelated providers.
|
||||
/// Exposed as internal for tests.
|
||||
/// </summary>
|
||||
internal static string BuildXPath() =>
|
||||
"*[System[" + ProviderIdClauses() + "]]";
|
||||
|
||||
/// <summary>Backfill variant of <see cref="BuildXPath"/> with the lookback window applied.</summary>
|
||||
internal static string BuildBackfillXPath(long lookbackMs) =>
|
||||
"*[System[(" + ProviderIdClauses() + $") and TimeCreated[timediff(@SystemTime) <= {lookbackMs}]]]";
|
||||
|
||||
private static string ProviderIdClauses() =>
|
||||
$"(Provider[@Name='{ProviderKernelGeneral}'] and EventID={EventId_ClockChange})" +
|
||||
$" or (Provider[@Name='{ProviderPowerTroubleshooter}'] and EventID={EventId_ClassicResume})" +
|
||||
$" or (Provider[@Name='{ProviderKernelPower}'] and EventID={EventId_ModernStandbyExit})";
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Event processing
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void OnEventRecordWritten(object sender, EventRecordWrittenEventArgs e)
|
||||
{
|
||||
if (e.EventRecord == null) return;
|
||||
try
|
||||
{
|
||||
ProcessRecord(e.EventRecord, isBackfill: false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error("Error processing SystemTimeline event", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void ProcessRecord(EventRecord record, bool isBackfill)
|
||||
{
|
||||
var recordId = record.RecordId ?? -1;
|
||||
|
||||
// Cheap dedup gate BEFORE the XML rendering work.
|
||||
if (IsAlreadyProcessed(recordId))
|
||||
return;
|
||||
|
||||
string xml = null;
|
||||
try { xml = record.ToXml(); }
|
||||
catch { /* fall through with empty EventData */ }
|
||||
|
||||
ProcessEvent(
|
||||
providerName: record.ProviderName,
|
||||
eventId: record.Id,
|
||||
recordId: recordId,
|
||||
timeCreatedUtc: record.TimeCreated?.ToUniversalTime(),
|
||||
eventData: ParseEventData(xml),
|
||||
isBackfill: isBackfill);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Core processing extracted to primitive inputs so tests can drive it without synthesizing
|
||||
/// an abstract, Windows-only <see cref="EventRecord"/> (WindowsUpdateTracker test-seam
|
||||
/// pattern; a plain dictionary instead of named args because three providers with ~10
|
||||
/// payload fields each make a fixed signature impractical).
|
||||
/// </summary>
|
||||
internal void ProcessEvent(
|
||||
string providerName,
|
||||
int eventId,
|
||||
long recordId,
|
||||
DateTime? timeCreatedUtc,
|
||||
Dictionary<string, string> eventData,
|
||||
bool isBackfill)
|
||||
{
|
||||
if (!MarkProcessed(recordId))
|
||||
return; // already emitted (cross-restart or duplicate delivery)
|
||||
|
||||
eventData = eventData ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (string.Equals(providerName, ProviderKernelGeneral, StringComparison.OrdinalIgnoreCase) && eventId == EventId_ClockChange)
|
||||
HandleClockChange(recordId, timeCreatedUtc, eventData, isBackfill);
|
||||
else if (string.Equals(providerName, ProviderPowerTroubleshooter, StringComparison.OrdinalIgnoreCase) && eventId == EventId_ClassicResume)
|
||||
HandleClassicResume(recordId, timeCreatedUtc, eventData, isBackfill);
|
||||
else if (string.Equals(providerName, ProviderKernelPower, StringComparison.OrdinalIgnoreCase) && eventId == EventId_ModernStandbyExit)
|
||||
HandleModernStandbyExit(recordId, timeCreatedUtc, eventData, isBackfill);
|
||||
// Anything else slipped past the XPath filter — ignore.
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// system_clock_changed (Kernel-General EventID 1)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void HandleClockChange(long recordId, DateTime? timeCreatedUtc, Dictionary<string, string> eventData, bool isBackfill)
|
||||
{
|
||||
eventData.TryGetValue("NewTime", out var newTimeRaw);
|
||||
eventData.TryGetValue("OldTime", out var oldTimeRaw);
|
||||
var newTime = TryParseUtc(newTimeRaw);
|
||||
var oldTime = TryParseUtc(oldTimeRaw);
|
||||
|
||||
// Signed delta preferably from the authoritative instants (NewTime − OldTime carries
|
||||
// the direction unambiguously); TimeDeltaInMs is the fallback when either fails to
|
||||
// parse. Records with neither are unusable and skipped (already claimed by dedup).
|
||||
long deltaMs;
|
||||
if (newTime.HasValue && oldTime.HasValue)
|
||||
{
|
||||
deltaMs = (long)Math.Round((newTime.Value - oldTime.Value).TotalMilliseconds);
|
||||
}
|
||||
else if (eventData.TryGetValue("TimeDeltaInMs", out var deltaRaw) &&
|
||||
long.TryParse(deltaRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedDelta))
|
||||
{
|
||||
deltaMs = parsedDelta;
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.Debug($"SystemTimeline: clock-change record {recordId} has no usable times/delta — skipped");
|
||||
return;
|
||||
}
|
||||
|
||||
if (Math.Abs(deltaMs) < MinClockDeltaMs)
|
||||
return; // w32time micro-slew — claimed (MarkProcessed) but never emitted
|
||||
|
||||
eventData.TryGetValue("Reason", out var reasonRaw);
|
||||
int.TryParse(reasonRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var reason);
|
||||
var reasonText = reason == 1 ? "application_set"
|
||||
: reason == 2 ? "hardware_clock_sync"
|
||||
: "unknown";
|
||||
|
||||
eventData.TryGetValue("ProcessName", out var processName);
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "timeDeltaMs", deltaMs },
|
||||
{ "reason", reason },
|
||||
{ "reasonText", reasonText },
|
||||
{ "backfilled", isBackfill },
|
||||
};
|
||||
// Authoritative instants: re-normalized ISO-8601 when parseable, raw pass-through
|
||||
// otherwise — the payload must survive even a malformed provider rendering.
|
||||
data["newTime"] = newTime.HasValue ? newTime.Value.ToString("o") : (object)(newTimeRaw ?? string.Empty);
|
||||
data["oldTime"] = oldTime.HasValue ? oldTime.Value.ToString("o") : (object)(oldTimeRaw ?? string.Empty);
|
||||
if (recordId >= 0) data["recordId"] = recordId;
|
||||
if (timeCreatedUtc.HasValue) data["timeCreated"] = timeCreatedUtc.Value.ToString("o");
|
||||
if (!string.IsNullOrEmpty(processName)) data["processName"] = processName;
|
||||
if (eventData.TryGetValue("ProcessID", out var pidRaw) &&
|
||||
int.TryParse(pidRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var pid))
|
||||
{
|
||||
data["processId"] = pid;
|
||||
}
|
||||
|
||||
var direction = deltaMs >= 0 ? "forward" : "backward";
|
||||
var processLeaf = ProcessLeafName(processName);
|
||||
var message = $"System clock stepped {direction} {FormatDuration(TimeSpan.FromMilliseconds(Math.Abs(deltaMs)))}" +
|
||||
$" (reason: {reasonText}" + (processLeaf != null ? $", process: {processLeaf})" : ")");
|
||||
|
||||
var isLargeStep = Math.Abs(deltaMs) >= ClockDeltaWarningMs;
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
// The record's own creation time, not UtcNow: backfilled pre-agent clock steps must
|
||||
// land on the timeline where they happened (WindowsUpdateTracker convention).
|
||||
Timestamp = timeCreatedUtc ?? DateTime.UtcNow,
|
||||
EventType = Constants.EventTypes.SystemClockChanged,
|
||||
Severity = isLargeStep ? EventSeverity.Warning : EventSeverity.Info,
|
||||
Source = "SystemTimelineWatcher",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = message,
|
||||
Data = data,
|
||||
ImmediateUpload = isLargeStep,
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// system_sleep_episode — classic sleep/hibernate (Power-Troubleshooter EventID 1)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void HandleClassicResume(long recordId, DateTime? timeCreatedUtc, Dictionary<string, string> eventData, bool isBackfill)
|
||||
{
|
||||
eventData.TryGetValue("SleepTime", out var sleepRaw);
|
||||
eventData.TryGetValue("WakeTime", out var wakeRaw);
|
||||
var enteredAt = TryParseUtc(sleepRaw);
|
||||
var exitedAt = TryParseUtc(wakeRaw);
|
||||
|
||||
// EffectiveState 5 = hibernate (S4); TargetState is the fallback when the effective
|
||||
// state is missing (older event versions).
|
||||
eventData.TryGetValue("EffectiveState", out var effectiveState);
|
||||
eventData.TryGetValue("TargetState", out var targetState);
|
||||
var stateValue = !string.IsNullOrEmpty(effectiveState) ? effectiveState : targetState;
|
||||
var kind = stateValue == "5" ? "hibernate" : "sleep";
|
||||
|
||||
long? durationSeconds = null;
|
||||
if (enteredAt.HasValue && exitedAt.HasValue && exitedAt.Value > enteredAt.Value)
|
||||
durationSeconds = (long)Math.Round((exitedAt.Value - enteredAt.Value).TotalSeconds);
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "kind", kind },
|
||||
{ "backfilled", isBackfill },
|
||||
};
|
||||
data["enteredAt"] = enteredAt.HasValue ? enteredAt.Value.ToString("o") : (object)(sleepRaw ?? string.Empty);
|
||||
data["exitedAt"] = exitedAt.HasValue ? exitedAt.Value.ToString("o") : (object)(wakeRaw ?? string.Empty);
|
||||
if (durationSeconds.HasValue) data["durationSeconds"] = durationSeconds.Value;
|
||||
if (recordId >= 0) data["recordId"] = recordId;
|
||||
if (timeCreatedUtc.HasValue) data["timeCreated"] = timeCreatedUtc.Value.ToString("o");
|
||||
if (eventData.TryGetValue("WakeSourceType", out var wakeTypeRaw) &&
|
||||
int.TryParse(wakeTypeRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var wakeType))
|
||||
{
|
||||
data["wakeSourceType"] = wakeType;
|
||||
}
|
||||
eventData.TryGetValue("WakeSourceText", out var wakeText);
|
||||
if (!string.IsNullOrEmpty(wakeText)) data["wakeSourceText"] = wakeText;
|
||||
|
||||
var durationLabel = durationSeconds.HasValue
|
||||
? " after " + FormatDuration(TimeSpan.FromSeconds(durationSeconds.Value))
|
||||
: string.Empty;
|
||||
var wakeLabel = !string.IsNullOrEmpty(wakeText) ? $" (wake source: {wakeText})" : string.Empty;
|
||||
|
||||
EmitSleepEpisode(timeCreatedUtc, data,
|
||||
$"Device resumed from {kind}{durationLabel}{wakeLabel}");
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// system_sleep_episode — Modern Standby (Kernel-Power EventID 507)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void HandleModernStandbyExit(long recordId, DateTime? timeCreatedUtc, Dictionary<string, string> eventData, bool isBackfill)
|
||||
{
|
||||
eventData.TryGetValue("SleepDurationInUs", out var sleepUsRaw);
|
||||
long.TryParse(sleepUsRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var sleepUs);
|
||||
var sleepDurationSeconds = sleepUs / 1_000_000;
|
||||
|
||||
// Every screen-off scenario logs a 507, including seconds-long ones where the device
|
||||
// never slept (SleepDurationInUs 0, incl. SleepEntered=false) — one gate covers both.
|
||||
if (sleepDurationSeconds < MinSleepDurationSeconds)
|
||||
return; // claimed but not an episode worth the timeline
|
||||
|
||||
eventData.TryGetValue("DurationInUs", out var durationUsRaw);
|
||||
long.TryParse(durationUsRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var durationUs);
|
||||
if (durationUs < sleepUs) durationUs = sleepUs; // scenario duration can never undercut its sleep residency
|
||||
|
||||
var durationSeconds = durationUs / 1_000_000;
|
||||
var exitedAt = timeCreatedUtc; // 507 is written at scenario exit
|
||||
DateTime? enteredAt = exitedAt.HasValue
|
||||
? exitedAt.Value.AddMilliseconds(-(durationUs / 1000.0))
|
||||
: (DateTime?)null;
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "kind", "modern_standby" },
|
||||
{ "durationSeconds", durationSeconds },
|
||||
{ "sleepDurationSeconds", sleepDurationSeconds },
|
||||
{ "backfilled", isBackfill },
|
||||
};
|
||||
if (enteredAt.HasValue) data["enteredAt"] = enteredAt.Value.ToString("o");
|
||||
if (exitedAt.HasValue) data["exitedAt"] = exitedAt.Value.ToString("o");
|
||||
if (recordId >= 0) data["recordId"] = recordId;
|
||||
if (timeCreatedUtc.HasValue) data["timeCreated"] = timeCreatedUtc.Value.ToString("o");
|
||||
if (eventData.TryGetValue("Reason", out var reasonRaw) &&
|
||||
int.TryParse(reasonRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var reason))
|
||||
{
|
||||
data["reason"] = reason;
|
||||
}
|
||||
bool? onAcPower = null;
|
||||
if (eventData.TryGetValue("PowerStateAc", out var acRaw) && bool.TryParse(acRaw, out var ac))
|
||||
{
|
||||
onAcPower = ac;
|
||||
data["onAcPower"] = ac;
|
||||
}
|
||||
if (eventData.TryGetValue("BatteryRemainingCapacityOnExit", out var battRaw) &&
|
||||
long.TryParse(battRaw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var batt))
|
||||
{
|
||||
data["batteryRemainingCapacityOnExit"] = batt;
|
||||
}
|
||||
|
||||
var powerLabel = onAcPower.HasValue ? (onAcPower.Value ? ", on AC" : ", on battery") : string.Empty;
|
||||
EmitSleepEpisode(timeCreatedUtc, data,
|
||||
$"Device exited Modern Standby after {FormatDuration(TimeSpan.FromSeconds(durationSeconds))}" +
|
||||
$" (slept {FormatDuration(TimeSpan.FromSeconds(sleepDurationSeconds))}{powerLabel})");
|
||||
}
|
||||
|
||||
private void EmitSleepEpisode(DateTime? timeCreatedUtc, Dictionary<string, object> data, string message)
|
||||
{
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
// Wake/exit moment (the record's creation time) — backfilled pre-agent episodes
|
||||
// must land on the timeline where the device actually woke.
|
||||
Timestamp = timeCreatedUtc ?? DateTime.UtcNow,
|
||||
EventType = Constants.EventTypes.SystemSleepEpisode,
|
||||
Severity = EventSeverity.Info,
|
||||
Source = "SystemTimelineWatcher",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = message,
|
||||
Data = data,
|
||||
ImmediateUpload = false,
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Parsing helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Extracts <c><Data Name="...">value</Data></c> pairs from an event's rendered
|
||||
/// XML, keyed case-insensitively. Namespace-agnostic (matches on local name). Returns an
|
||||
/// empty map on null/malformed XML — never throws.
|
||||
/// </summary>
|
||||
internal static Dictionary<string, string> ParseEventData(string xml)
|
||||
{
|
||||
var result = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (string.IsNullOrEmpty(xml)) return result;
|
||||
|
||||
try
|
||||
{
|
||||
var doc = XDocument.Parse(xml);
|
||||
foreach (var el in doc.Descendants().Where(e => e.Name.LocalName == "Data"))
|
||||
{
|
||||
var nameAttr = el.Attribute("Name");
|
||||
if (nameAttr == null || string.IsNullOrEmpty(nameAttr.Value)) continue;
|
||||
if (!result.ContainsKey(nameAttr.Value))
|
||||
result[nameAttr.Value] = el.Value;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Malformed / unexpected XML — best effort, return what we have.
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
internal static DateTime? TryParseUtc(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value)) return null;
|
||||
return DateTime.TryParse(value, CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
|
||||
? parsed
|
||||
: (DateTime?)null;
|
||||
}
|
||||
|
||||
/// <summary>Compact human duration for messages: "2h 05m", "56m 12s", "45s".</summary>
|
||||
internal static string FormatDuration(TimeSpan span)
|
||||
{
|
||||
if (span.TotalHours >= 1)
|
||||
return $"{(long)span.TotalHours}h {span.Minutes:D2}m";
|
||||
if (span.TotalMinutes >= 1)
|
||||
return $"{span.Minutes}m {span.Seconds:D2}s";
|
||||
return $"{(long)Math.Ceiling(span.TotalSeconds)}s";
|
||||
}
|
||||
|
||||
/// <summary>Leaf file name of a kernel device path like <c>\Device\HarddiskVolume3\Windows\System32\svchost.exe</c>.</summary>
|
||||
internal static string ProcessLeafName(string devicePath)
|
||||
{
|
||||
if (string.IsNullOrEmpty(devicePath)) return null;
|
||||
var idx = devicePath.LastIndexOf('\\');
|
||||
var leaf = idx >= 0 ? devicePath.Substring(idx + 1) : devicePath;
|
||||
return string.IsNullOrEmpty(leaf) ? null : leaf;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Backfill
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void BackfillRecentEvents()
|
||||
{
|
||||
try
|
||||
{
|
||||
var lookbackMs = (long)_backfillLookbackMinutes * 60 * 1000;
|
||||
_logger.Info($"SystemTimeline backfill: scanning {Channel} " +
|
||||
$"(lookback={_backfillLookbackMinutes}min, restartWatermark={_restartWatermark})");
|
||||
|
||||
var query = new EventLogQuery(Channel, PathType.LogName, BuildBackfillXPath(lookbackMs));
|
||||
|
||||
int processed = 0;
|
||||
using (var reader = new EventLogReader(query))
|
||||
{
|
||||
EventRecord record;
|
||||
while ((record = reader.ReadEvent()) != null)
|
||||
{
|
||||
using (record)
|
||||
{
|
||||
ProcessRecord(record, isBackfill: true);
|
||||
processed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (processed == 0)
|
||||
_logger.Debug($"SystemTimeline backfill: no targeted events in last {_backfillLookbackMinutes} minutes");
|
||||
else
|
||||
_logger.Info($"SystemTimeline backfill: scanned {processed} record(s)");
|
||||
}
|
||||
catch (EventLogNotFoundException)
|
||||
{
|
||||
_logger.Warning($"SystemTimeline event log not found during backfill: {Channel} (normal on non-Windows test environments)");
|
||||
}
|
||||
catch (UnauthorizedAccessException ex)
|
||||
{
|
||||
_logger.Warning($"SystemTimeline backfill access denied: {ex.Message}");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"SystemTimeline backfill failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Watermark dedup (cross-restart)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private bool IsAlreadyProcessed(long recordId)
|
||||
{
|
||||
if (recordId < 0) return false; // no RecordId → cannot dedup, let it through
|
||||
lock (_watermarkLock)
|
||||
{
|
||||
return recordId <= _restartWatermark || _seenThisRun.Contains(recordId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Atomically claims <paramref name="recordId"/>. Returns true if this is the first time we
|
||||
/// see it (caller should evaluate/emit), false if it was already processed — either in a
|
||||
/// PRIOR run (at/below the restart watermark) or already this run (in the seen-set). Only
|
||||
/// advances + persists the max on a genuinely-new, higher RecordId. Records without a
|
||||
/// RecordId (-1) are always processed, never tracked or persisted. Note: threshold-suppressed
|
||||
/// records are claimed too, so a restart's backfill does not re-evaluate them.
|
||||
/// </summary>
|
||||
private bool MarkProcessed(long recordId)
|
||||
{
|
||||
if (recordId < 0) return true;
|
||||
|
||||
long toPersist = -1;
|
||||
lock (_watermarkLock)
|
||||
{
|
||||
if (recordId <= _restartWatermark || !_seenThisRun.Add(recordId))
|
||||
return false;
|
||||
|
||||
if (recordId > _maxEmittedRecordId)
|
||||
{
|
||||
_maxEmittedRecordId = recordId;
|
||||
toPersist = _maxEmittedRecordId;
|
||||
}
|
||||
}
|
||||
if (toPersist >= 0) PersistWatermark(toPersist);
|
||||
return true;
|
||||
}
|
||||
|
||||
internal void LoadWatermark()
|
||||
{
|
||||
if (string.IsNullOrEmpty(_stateDirectory)) return;
|
||||
|
||||
var filePath = Path.Combine(_stateDirectory, WatermarkStateFileName);
|
||||
if (!File.Exists(filePath)) return;
|
||||
|
||||
try
|
||||
{
|
||||
var json = File.ReadAllText(filePath);
|
||||
var state = JsonConvert.DeserializeObject<WatermarkState>(json);
|
||||
if (state != null)
|
||||
{
|
||||
lock (_watermarkLock)
|
||||
{
|
||||
_restartWatermark = state.LastRecordId;
|
||||
_maxEmittedRecordId = state.LastRecordId;
|
||||
}
|
||||
_logger.Info($"SystemTimeline watermark loaded: lastRecordId={state.LastRecordId} (persisted {state.PersistedUtc:O})");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"Failed to load SystemTimeline watermark: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private void PersistWatermark(long recordId)
|
||||
{
|
||||
if (string.IsNullOrEmpty(_stateDirectory)) return;
|
||||
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(_stateDirectory);
|
||||
var filePath = Path.Combine(_stateDirectory, WatermarkStateFileName);
|
||||
var state = new WatermarkState { LastRecordId = recordId, PersistedUtc = DateTime.UtcNow };
|
||||
var json = JsonConvert.SerializeObject(state, Formatting.Indented);
|
||||
var tempPath = filePath + ".tmp";
|
||||
File.WriteAllText(tempPath, json);
|
||||
File.Copy(tempPath, filePath, overwrite: true);
|
||||
try { File.Delete(tempPath); } catch { }
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"Failed to persist SystemTimeline watermark: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class WatermarkState
|
||||
{
|
||||
public long LastRecordId { get; set; }
|
||||
public DateTime PersistedUtc { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
{
|
||||
/// <summary>
|
||||
/// ARP resolution via <c>iphlpapi!SendARP</c> — resolves the MAC address of an on-link IPv4
|
||||
/// host (in practice: the default gateway). IPv4 only by API design; the OS answers from its
|
||||
/// ARP cache or sends a real ARP request, so the target must be on the local subnet.
|
||||
/// </summary>
|
||||
internal static class ArpNativeMethods
|
||||
{
|
||||
[DllImport("iphlpapi.dll", ExactSpelling = true)]
|
||||
private static extern int SendARP(uint destIP, uint srcIP, byte[] macAddress, ref uint macAddressLength);
|
||||
|
||||
/// <summary>
|
||||
/// Fail-soft MAC resolution. Returns false (with a diagnostic in <paramref name="error"/>)
|
||||
/// for non-IPv4 addresses, SendARP failures, or unexpected reply lengths — never throws.
|
||||
/// </summary>
|
||||
internal static bool TryResolveMac(IPAddress? ipv4Address, out byte[] mac, out string? error)
|
||||
{
|
||||
mac = new byte[6];
|
||||
error = null;
|
||||
try
|
||||
{
|
||||
if (ipv4Address == null || ipv4Address.AddressFamily != AddressFamily.InterNetwork)
|
||||
{
|
||||
error = "not an IPv4 address";
|
||||
return false;
|
||||
}
|
||||
|
||||
// SendARP takes the IPv4 address as a uint in network byte order (first octet in
|
||||
// the lowest byte).
|
||||
var addressBytes = ipv4Address.GetAddressBytes();
|
||||
uint dest = ((uint)addressBytes[3] << 24)
|
||||
| ((uint)addressBytes[2] << 16)
|
||||
| ((uint)addressBytes[1] << 8)
|
||||
| addressBytes[0];
|
||||
|
||||
uint len = (uint)mac.Length;
|
||||
var err = SendARP(dest, 0, mac, ref len);
|
||||
if (err != 0)
|
||||
{
|
||||
error = $"SendARP returned {err}";
|
||||
return false;
|
||||
}
|
||||
if (len != 6)
|
||||
{
|
||||
error = $"SendARP returned unexpected MAC length {len}";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
error = ex.Message;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#nullable enable
|
||||
using System.Net;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Shared "which NIC is the device actually using" heuristic: Up, not Loopback/Tunnel,
|
||||
/// carries a non-0.0.0.0 default gateway. Fail-soft — any enumeration error yields null.
|
||||
/// </summary>
|
||||
internal static class NetworkInterfaceLocator
|
||||
{
|
||||
/// <summary>
|
||||
/// Finds the active network interface: Up, not Loopback/Tunnel, has a non-0.0.0.0 gateway.
|
||||
/// </summary>
|
||||
internal static NetworkInterface? FindActiveNetworkInterface()
|
||||
{
|
||||
try
|
||||
{
|
||||
var interfaces = NetworkInterface.GetAllNetworkInterfaces();
|
||||
foreach (var nic in interfaces)
|
||||
{
|
||||
if (nic.OperationalStatus != OperationalStatus.Up)
|
||||
continue;
|
||||
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback ||
|
||||
nic.NetworkInterfaceType == NetworkInterfaceType.Tunnel)
|
||||
continue;
|
||||
|
||||
var ipProps = nic.GetIPProperties();
|
||||
foreach (var gw in ipProps.GatewayAddresses)
|
||||
{
|
||||
if (gw.Address.ToString() != "0.0.0.0")
|
||||
return nic;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Caller handles null
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>First IPv4 default gateway of the interface (0.0.0.0 excluded), or null.</summary>
|
||||
internal static IPAddress? GetIpv4Gateway(NetworkInterface nic)
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var gw in nic.GetIPProperties().GatewayAddresses)
|
||||
{
|
||||
var address = gw?.Address;
|
||||
if (address != null
|
||||
&& address.AddressFamily == AddressFamily.InterNetwork
|
||||
&& address.ToString() != "0.0.0.0")
|
||||
{
|
||||
return address;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Caller handles null
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+125
-88
@@ -10,7 +10,9 @@ using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
@@ -124,62 +126,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
private readonly string _imeLogFolder;
|
||||
private readonly string _agentLogFolder;
|
||||
private readonly string _agentStateFolder;
|
||||
private readonly string _agentSpoolFolder;
|
||||
private readonly string _agentDataFolder;
|
||||
// Built-in sections come from the Shared catalog (DiagnosticsBuiltInSections.All) — the
|
||||
// same list the backend serves to the portal, so what administrators see is exactly
|
||||
// what this service collects. Test seam: section id → folder; production leaves the
|
||||
// map empty and every section resolves from the catalog's (unexpanded) source folder.
|
||||
private readonly IReadOnlyDictionary<string, string> _sectionFolderOverrides;
|
||||
|
||||
private static readonly string DefaultImeLogFolder =
|
||||
Environment.ExpandEnvironmentVariables(@"%ProgramData%\Microsoft\IntuneManagementExtension\Logs");
|
||||
|
||||
private static readonly string DefaultAgentLogFolder =
|
||||
Environment.ExpandEnvironmentVariables(Constants.LogDirectory);
|
||||
|
||||
private static readonly string DefaultAgentStateFolder =
|
||||
Environment.ExpandEnvironmentVariables(Constants.StateDirectory);
|
||||
|
||||
private static readonly string DefaultAgentSpoolFolder =
|
||||
Environment.ExpandEnvironmentVariables(Constants.SpoolDirectory);
|
||||
|
||||
private static readonly string DefaultAgentDataFolder =
|
||||
Environment.ExpandEnvironmentVariables(@"%ProgramData%\AutopilotMonitor");
|
||||
|
||||
/// <summary>
|
||||
/// File extensions collected from built-in log folders (Agent + IME).
|
||||
/// Covers standard logs, structured data, traces, event logs, and diagnostics archives.
|
||||
/// </summary>
|
||||
private static readonly string[] LogFilePatterns = new[]
|
||||
{
|
||||
"*.log", "*.txt", "*.json", "*.jsonl",
|
||||
"*.etl", "*.evtx", "*.xml", "*.csv", "*.cab"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Patterns collected from the agent state folder. Adds completion markers on top of
|
||||
/// the standard log patterns so files like <c>enrollment-complete.marker</c> and
|
||||
/// <c>whiteglove-backfill-state.json</c> reach the diagnostics archive.
|
||||
/// </summary>
|
||||
private static readonly string[] StateFilePatterns = new[]
|
||||
{
|
||||
"*.log", "*.txt", "*.json", "*.jsonl",
|
||||
"*.etl", "*.evtx", "*.xml", "*.csv", "*.cab",
|
||||
"*.complete", "*.marker"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Marker patterns from the top-level data directory. Kept tight on purpose: only
|
||||
/// completion/exit markers, no session.id / bootstrap.json (those are config, not
|
||||
/// forensic state).
|
||||
/// </summary>
|
||||
private static readonly string[] RootMarkerPatterns = new[] { "*.complete", "*.marker" };
|
||||
|
||||
/// <summary>
|
||||
/// Telemetry spool: only the JSON-shaped files. The spool folder may also contain
|
||||
/// quarantine subfolders, so subfolders are excluded to keep the archive focused
|
||||
/// on what is actually pending upload.
|
||||
/// </summary>
|
||||
private static readonly string[] SpoolFilePatterns = new[] { "*.jsonl", "*.json" };
|
||||
// Scenario oracle for DiagnosticsSectionCondition.DevicePreparation. Production reads
|
||||
// the deterministic WDP registry marker (the same gate every other WDP switch uses);
|
||||
// tests inject a constant so they never touch the registry.
|
||||
private readonly Func<bool> _devicePreparationProbe;
|
||||
|
||||
public DiagnosticsPackageService(AgentConfiguration configuration, AgentLogger logger, BackendApiClient apiClient)
|
||||
: this(configuration, logger, apiClient, null, null, null, null, null)
|
||||
@@ -188,7 +144,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
|
||||
// Test seam: allows xUnit fixtures to redirect log/state/spool/data folders to a
|
||||
// temp dir without touching real %ProgramData% paths. Production callers always go
|
||||
// through the public ctor.
|
||||
// through the public ctor. The five named overrides map onto the catalog's section
|
||||
// ids; sectionFolderOverrides covers every other section (RealmJoin*, ImeBootstrapper…).
|
||||
internal DiagnosticsPackageService(
|
||||
AgentConfiguration configuration,
|
||||
AgentLogger logger,
|
||||
@@ -197,18 +154,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
string imeLogFolderOverride,
|
||||
string agentStateFolderOverride,
|
||||
string agentSpoolFolderOverride,
|
||||
string agentDataFolderOverride)
|
||||
string agentDataFolderOverride,
|
||||
IReadOnlyDictionary<string, string> sectionFolderOverrides = null,
|
||||
Func<bool> devicePreparationProbe = null)
|
||||
{
|
||||
_configuration = configuration;
|
||||
_logger = logger;
|
||||
_apiClient = apiClient;
|
||||
_httpClient = new HttpClient { Timeout = TimeSpan.FromMinutes(5) };
|
||||
|
||||
_agentLogFolder = agentLogFolderOverride ?? DefaultAgentLogFolder;
|
||||
_imeLogFolder = imeLogFolderOverride ?? DefaultImeLogFolder;
|
||||
_agentStateFolder = agentStateFolderOverride ?? DefaultAgentStateFolder;
|
||||
_agentSpoolFolder = agentSpoolFolderOverride ?? DefaultAgentSpoolFolder;
|
||||
_agentDataFolder = agentDataFolderOverride ?? DefaultAgentDataFolder;
|
||||
var overrides = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
void Map(string sectionId, string folder)
|
||||
{
|
||||
if (folder != null) overrides[sectionId] = folder;
|
||||
}
|
||||
Map("AgentLogs", agentLogFolderOverride);
|
||||
Map("ImeLogs", imeLogFolderOverride);
|
||||
Map("AgentState", agentStateFolderOverride);
|
||||
Map("AgentSpool", agentSpoolFolderOverride);
|
||||
Map("AgentMarkers", agentDataFolderOverride);
|
||||
if (sectionFolderOverrides != null)
|
||||
{
|
||||
foreach (var kv in sectionFolderOverrides)
|
||||
overrides[kv.Key] = kv.Value;
|
||||
}
|
||||
_sectionFolderOverrides = overrides;
|
||||
|
||||
// Tests that pass folder overrides but no probe must stay registry-free: a seam
|
||||
// caller gets "not WDP" by default, the production ctor gets the real marker.
|
||||
_devicePreparationProbe = devicePreparationProbe
|
||||
?? (overrides.Count == 0
|
||||
? (Func<bool>)EnrollmentRegistryDetector.IsDeterministicDevicePreparation
|
||||
: () => false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -370,33 +347,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
// 1. sessioninfo.txt — informational metadata, not subject to caps.
|
||||
AddSessionInfo(archive, enrollmentSucceeded);
|
||||
|
||||
// 2. Agent logs
|
||||
foreach (var pattern in LogFilePatterns)
|
||||
AddLogFiles(archive, _agentLogFolder, "AgentLogs", pattern, tracker);
|
||||
// 2. Built-in sections — ONE catalog shared with the backend (the portal
|
||||
// shows exactly this list): agent logs, IME logs (+ the Device Preparation
|
||||
// bootstrapper event log on WDP), agent state (recursive: `.quarantine`
|
||||
// and `.part1-<utc>/` buckets ride along), telemetry spool, top-level
|
||||
// markers, and the RealmJoin logs behind the tenant's watcher toggle.
|
||||
// The scenario probe is a registry read — evaluate it once per build.
|
||||
var isDevicePreparation = ProbeDevicePreparation();
|
||||
ManifestLine($"SCENARIO: devicePreparation={isDevicePreparation} realmJoinWatcher={_configuration.EnableRealmJoinWatcher}");
|
||||
foreach (var section in DiagnosticsBuiltInSections.All)
|
||||
AddBuiltInSection(archive, section, tracker, isDevicePreparation);
|
||||
|
||||
// 3. IME logs (all relevant files in the IME log folder)
|
||||
foreach (var pattern in LogFilePatterns)
|
||||
AddLogFiles(archive, _imeLogFolder, "ImeLogs", pattern, tracker);
|
||||
|
||||
// 4. Agent state: snapshot, journal, signal-log, ime-tracker-state,
|
||||
// enrollment-complete.marker, final-status.json, whiteglove-backfill-state.json.
|
||||
// Recursive to include `.quarantine` subfolders AND `.part1-<utc>/` buckets
|
||||
// that StateArchiver creates on WG Part-2 boot — both ride along under
|
||||
// AgentState/ via the same recursive walk.
|
||||
foreach (var pattern in StateFilePatterns)
|
||||
AddLogFiles(archive, _agentStateFolder, "AgentState", pattern, tracker, includeSubfolders: true);
|
||||
|
||||
// 5. Telemetry spool — pending uploads + upload cursor (forensic: what didn't ship?)
|
||||
foreach (var pattern in SpoolFilePatterns)
|
||||
AddLogFiles(archive, _agentSpoolFolder, "AgentSpool", pattern, tracker, includeSubfolders: false);
|
||||
|
||||
// 6. Top-level markers (whiteglove.complete, clean-exit marker) — Part-1/Part-2
|
||||
// bridge + ghost-restart guard. Top-directory only to avoid re-zipping
|
||||
// State/Spool/Logs/Agent which already have their own sections.
|
||||
foreach (var pattern in RootMarkerPatterns)
|
||||
AddLogFiles(archive, _agentDataFolder, "AgentMarkers", pattern, tracker, includeSubfolders: false);
|
||||
|
||||
// 7. Configured additional log paths (global + tenant, validated by guards)
|
||||
// 3. Configured additional log paths (global + tenant, validated by guards)
|
||||
foreach (var entry in _configuration.DiagnosticsLogPaths ?? new System.Collections.Generic.List<Shared.Models.DiagnosticsLogPath>())
|
||||
{
|
||||
// Resolve %LOGGED_ON_USER_PROFILE% token and get profile path for guard exception
|
||||
@@ -447,6 +409,81 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
private bool ProbeDevicePreparation()
|
||||
{
|
||||
try
|
||||
{
|
||||
return _devicePreparationProbe();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Fail toward Classic like every other WDP gate: the evtx export is skipped,
|
||||
// never the whole package.
|
||||
_logger.Warning($"Device Preparation probe failed — treating as Classic: {ex.Message}");
|
||||
ManifestLine($"SCENARIO PROBE FAILED (treated as Classic): {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure gate: is this built-in section collected under the given configuration and
|
||||
/// enrollment scenario? Unknown conditions (a newer Shared catalog than this agent)
|
||||
/// fail closed.
|
||||
/// </summary>
|
||||
internal static bool IsSectionActive(DiagnosticsBuiltInSection section, AgentConfiguration configuration, bool isDevicePreparation)
|
||||
{
|
||||
switch (section.Condition)
|
||||
{
|
||||
case DiagnosticsSectionCondition.Always:
|
||||
return true;
|
||||
case DiagnosticsSectionCondition.RealmJoinWatcher:
|
||||
return configuration.EnableRealmJoinWatcher;
|
||||
case DiagnosticsSectionCondition.DevicePreparation:
|
||||
return isDevicePreparation;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string DescribeInactiveCondition(DiagnosticsSectionCondition condition)
|
||||
{
|
||||
switch (condition)
|
||||
{
|
||||
case DiagnosticsSectionCondition.RealmJoinWatcher:
|
||||
return "RealmJoin Watcher disabled";
|
||||
case DiagnosticsSectionCondition.DevicePreparation:
|
||||
return "not a Device Preparation enrollment";
|
||||
default:
|
||||
return $"unknown condition {condition}";
|
||||
}
|
||||
}
|
||||
|
||||
private void AddBuiltInSection(ZipArchive archive, DiagnosticsBuiltInSection section, BudgetTracker tracker, bool isDevicePreparation)
|
||||
{
|
||||
if (!IsSectionActive(section, _configuration, isDevicePreparation))
|
||||
{
|
||||
ManifestLine($"BUILT-IN SKIPPED ({DescribeInactiveCondition(section.Condition)}): {section.Id}");
|
||||
return;
|
||||
}
|
||||
|
||||
// Override (test seam) or catalog source folder with %ProgramData% /
|
||||
// %LOGGED_ON_USER_PROFILE% expanded; null means the token is present but no
|
||||
// interactive user has been detected yet — same skip as configured paths.
|
||||
var folder = _sectionFolderOverrides.TryGetValue(section.Id, out var overrideFolder)
|
||||
? overrideFolder
|
||||
: UserProfileResolver.ExpandCustomTokens(section.SourceFolder);
|
||||
if (folder == null)
|
||||
{
|
||||
_logger.Warning($"Built-in diagnostics section skipped (no user session for token): {section.Id}");
|
||||
ManifestLine($"BUILT-IN SKIPPED (no user session for token): {section.Id} '{section.SourceFolder}'");
|
||||
return;
|
||||
}
|
||||
|
||||
ManifestLine($"BUILT-IN: {section.Id} -> folder='{folder}' zip='{section.ZipFolder}' recursive={section.IncludeSubfolders}");
|
||||
foreach (var pattern in section.Patterns)
|
||||
AddLogFiles(archive, folder, section.ZipFolder, pattern, tracker, section.IncludeSubfolders);
|
||||
}
|
||||
|
||||
private void AddSessionInfo(ZipArchive archive, bool? enrollmentSucceeded)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
|
||||
@@ -20,7 +20,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
/// </summary>
|
||||
public static class UserProfileResolver
|
||||
{
|
||||
public const string Token = "%LOGGED_ON_USER_PROFILE%";
|
||||
// Spelled once, in Shared: the built-in diagnostics catalog uses the same token.
|
||||
public const string Token = Shared.Models.DiagnosticsBuiltInSections.UserProfileToken;
|
||||
|
||||
private static readonly object Lock = new object();
|
||||
private static bool _resolved;
|
||||
|
||||
+2
-30
@@ -922,36 +922,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the active network interface: Up, not Loopback/Tunnel, has a non-0.0.0.0 gateway.
|
||||
/// </summary>
|
||||
private static NetworkInterface FindActiveNetworkInterface()
|
||||
{
|
||||
try
|
||||
{
|
||||
var interfaces = NetworkInterface.GetAllNetworkInterfaces();
|
||||
foreach (var nic in interfaces)
|
||||
{
|
||||
if (nic.OperationalStatus != OperationalStatus.Up)
|
||||
continue;
|
||||
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback ||
|
||||
nic.NetworkInterfaceType == NetworkInterfaceType.Tunnel)
|
||||
continue;
|
||||
|
||||
var ipProps = nic.GetIPProperties();
|
||||
foreach (var gw in ipProps.GatewayAddresses)
|
||||
{
|
||||
if (gw.Address.ToString() != "0.0.0.0")
|
||||
return nic;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Caller handles null
|
||||
}
|
||||
return null;
|
||||
}
|
||||
private static NetworkInterface FindActiveNetworkInterface() =>
|
||||
Monitoring.NetworkInterfaceLocator.FindActiveNetworkInterface();
|
||||
|
||||
/// <summary>
|
||||
/// Collects WiFi signal info via the native WLAN API (<see cref="WifiInfoReader"/>,
|
||||
|
||||
+11
-2
@@ -331,7 +331,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather.Collectors
|
||||
data[groupName] = group.Value;
|
||||
}
|
||||
|
||||
data["logTimestamp"] = entry.Timestamp.ToString("o");
|
||||
// Gather-rule logs are arbitrary files this collector does not
|
||||
// tail, so no writer offset can be measured for them; the
|
||||
// reader-zone fallback carries its known defect (see
|
||||
// ResolveUtcAssumingReaderZone). Behaviour unchanged from before
|
||||
// the parser split.
|
||||
var entryUtc = entry.TimestampUtc
|
||||
?? (entry.HasTimestamp
|
||||
? CmTraceLogParser.ResolveUtcAssumingReaderZone(entry.LocalTimestamp)
|
||||
: DateTime.UtcNow);
|
||||
data["logTimestamp"] = entryUtc.ToString("o");
|
||||
data["logComponent"] = entry.Component;
|
||||
data["logType"] = entry.Type;
|
||||
data["logMessage"] = TruncateMessage(entry.Message, 500);
|
||||
@@ -348,7 +357,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather.Collectors
|
||||
{
|
||||
SessionId = context.SessionId,
|
||||
TenantId = context.TenantId,
|
||||
Timestamp = entry.Timestamp,
|
||||
Timestamp = entryUtc,
|
||||
EventType = eventType,
|
||||
Severity = severity,
|
||||
Source = "GatherRuleExecutor",
|
||||
|
||||
+25
-19
@@ -22,11 +22,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
// Pressure thresholds — once either is crossed within a session, a one-shot
|
||||
// `spool_pressure_detected` event is emitted with RequiresImmediateFlush=true
|
||||
// so the backend can surface the condition without waiting for the next snapshot
|
||||
// to be queried out of Storage. Sized so a healthy 6h provisioning session never
|
||||
// trips them; tripping is a strong "upload backlog is growing or downstream stalled"
|
||||
// signal worth investigating in the field.
|
||||
// to be queried out of Storage. Both measure the PENDING backlog (items not yet
|
||||
// marked uploaded), never the append-only spool file size — a healthy long session
|
||||
// grows the file past any fixed size while its backlog stays near zero (field data
|
||||
// 2026-08: 99/100 pressure events were file-size-only false positives). Tripping is
|
||||
// therefore a genuine "upload is stalled or falling behind" signal.
|
||||
internal const int PressurePendingItemThreshold = 2000;
|
||||
internal const long PressureFileSizeBytesThreshold = 5L * 1024 * 1024; // 5 MB
|
||||
internal const long PressurePendingBytesThreshold = 5L * 1024 * 1024; // 5 MB
|
||||
|
||||
private readonly string _agentVersion;
|
||||
private readonly NetworkMetrics _networkMetrics;
|
||||
@@ -77,11 +79,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
|
||||
protected override void Collect()
|
||||
{
|
||||
// Full path writes 19 keys: agent_version + process metrics (5: cpu, ws, private,
|
||||
// threads, handles) + spool stats (4) + network delta (9: requests, failures,
|
||||
// bytes_up/down, avg_latency, total_up/down/requests/latency). cap=19 →
|
||||
// HashHelpers.GetPrime(19)=23 buckets → no resize on the 19th key.
|
||||
var data = new Dictionary<string, object>(capacity: 19, StringComparer.Ordinal)
|
||||
// Full path writes 20 keys: agent_version + process metrics (5: cpu, ws, private,
|
||||
// threads, handles) + spool stats (5) + network delta (9: requests, failures,
|
||||
// bytes_up/down, avg_latency, total_up/down/requests/latency). cap=20 →
|
||||
// HashHelpers.GetPrime(20)=23 buckets → no resize on the 20th key.
|
||||
var data = new Dictionary<string, object>(capacity: 20, StringComparer.Ordinal)
|
||||
{
|
||||
{ "agent_version", _agentVersion }
|
||||
};
|
||||
@@ -126,17 +128,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
// EnrollmentOrchestrator.Start into IComponentFactory.CreateCollectorHosts
|
||||
// (ARCH-F4). Null only on test fakes that don't construct a real spool.
|
||||
int pendingItemCount = 0;
|
||||
long pendingBytes = 0L;
|
||||
long spoolFileSizeBytes = 0L;
|
||||
if (_telemetrySpool != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
pendingItemCount = _telemetrySpool.PendingItemCount;
|
||||
pendingBytes = _telemetrySpool.PendingBytes;
|
||||
spoolFileSizeBytes = _telemetrySpool.SpoolFileSizeBytes;
|
||||
var peakPending = _telemetrySpool.PeakPendingItemCount;
|
||||
var totalEnqueued = _telemetrySpool.LastAssignedItemId + 1; // -1 sentinel → 0
|
||||
|
||||
data["spool_pending_item_count"] = pendingItemCount;
|
||||
data["spool_pending_bytes"] = pendingBytes;
|
||||
data["spool_peak_pending_item_count"] = peakPending;
|
||||
data["spool_file_size_bytes"] = spoolFileSizeBytes;
|
||||
data["spool_total_enqueued_count"] = totalEnqueued;
|
||||
@@ -194,20 +199,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
});
|
||||
}
|
||||
|
||||
// One-shot pressure event \u2014 fires once per session when the spool grows past
|
||||
// either threshold. ImmediateUpload=true so it shows up promptly on the
|
||||
// backend without waiting for the next batch drain.
|
||||
// One-shot pressure event \u2014 fires once per session when the pending backlog
|
||||
// grows past either threshold. ImmediateUpload=true so it shows up promptly on
|
||||
// the backend without waiting for the next batch drain.
|
||||
if (_telemetrySpool != null
|
||||
&& (pendingItemCount > PressurePendingItemThreshold
|
||||
|| spoolFileSizeBytes > PressureFileSizeBytesThreshold)
|
||||
|| pendingBytes > PressurePendingBytesThreshold)
|
||||
&& Interlocked.CompareExchange(ref _pressureEmitted, 1, 0) == 0)
|
||||
{
|
||||
var pressureData = new Dictionary<string, object>(capacity: 8, StringComparer.Ordinal)
|
||||
{
|
||||
{ "pendingItemCount", pendingItemCount },
|
||||
{ "fileSizeBytes", spoolFileSizeBytes },
|
||||
{ "pendingBytes", pendingBytes },
|
||||
{ "fileSizeBytes", spoolFileSizeBytes }, // informational only, not a trigger
|
||||
{ "pendingThreshold", PressurePendingItemThreshold },
|
||||
{ "fileSizeThresholdBytes", PressureFileSizeBytesThreshold },
|
||||
{ "pendingBytesThreshold", PressurePendingBytesThreshold },
|
||||
{ "totalEnqueuedCount", _telemetrySpool.LastAssignedItemId + 1 },
|
||||
{ "lastUploadedItemId", _telemetrySpool.LastUploadedItemId },
|
||||
{ "ImmediateUpload", true }
|
||||
@@ -222,15 +228,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
Severity = EventSeverity.Warning,
|
||||
Source = "AgentSelfMetricsCollector",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"Telemetry spool pressure detected: pending={pendingItemCount}, " +
|
||||
$"fileBytes={spoolFileSizeBytes} \u2014 upload may be stalled or session " +
|
||||
$"is unusually long.",
|
||||
Message = $"Telemetry spool pressure detected: pending={pendingItemCount} items / " +
|
||||
$"{pendingBytes} bytes not yet uploaded \u2014 upload is stalled or " +
|
||||
$"falling behind.",
|
||||
Data = pressureData
|
||||
});
|
||||
|
||||
Logger.Warning(
|
||||
$"AgentSelfMetricsCollector: spool pressure detected " +
|
||||
$"(pending={pendingItemCount}, fileBytes={spoolFileSizeBytes}). " +
|
||||
$"(pending={pendingItemCount}, pendingBytes={pendingBytes}). " +
|
||||
$"Emitted spool_pressure_detected (one-shot per session).");
|
||||
}
|
||||
}
|
||||
|
||||
+2
-30
@@ -420,36 +420,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
// Helpers
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Finds the active network interface: Up, not Loopback/Tunnel, has a non-0.0.0.0 gateway.
|
||||
/// </summary>
|
||||
private static NetworkInterface FindActiveNetworkInterface()
|
||||
{
|
||||
try
|
||||
{
|
||||
var interfaces = NetworkInterface.GetAllNetworkInterfaces();
|
||||
foreach (var nic in interfaces)
|
||||
{
|
||||
if (nic.OperationalStatus != OperationalStatus.Up)
|
||||
continue;
|
||||
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback ||
|
||||
nic.NetworkInterfaceType == NetworkInterfaceType.Tunnel)
|
||||
continue;
|
||||
|
||||
var ipProps = nic.GetIPProperties();
|
||||
foreach (var gw in ipProps.GatewayAddresses)
|
||||
{
|
||||
if (gw.Address.ToString() != "0.0.0.0")
|
||||
return nic;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Caller handles null
|
||||
}
|
||||
return null;
|
||||
}
|
||||
private static NetworkInterface FindActiveNetworkInterface() =>
|
||||
NetworkInterfaceLocator.FindActiveNetworkInterface();
|
||||
|
||||
private static string BuildChangeMessage(NetworkStateSnapshot before, NetworkStateSnapshot after)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic;
|
||||
using Microsoft.Win32;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
{
|
||||
public class DoGroupIdSetResult
|
||||
{
|
||||
/// <summary>The DOGroupId value is the computed one after this run (written or already there).</summary>
|
||||
public bool Success { get; set; }
|
||||
|
||||
/// <summary>True when this run actually wrote the registry value (false for the idempotent no-op).</summary>
|
||||
public bool Written { get; set; }
|
||||
|
||||
/// <summary>Deliberate non-action (existing foreign policy/value, no usable network) — not an error.</summary>
|
||||
public bool Skipped { get; set; }
|
||||
public string? SkipReason { get; set; }
|
||||
|
||||
public string? GroupId { get; set; }
|
||||
public string? PreviousValue { get; set; }
|
||||
public string? GatewayIp { get; set; }
|
||||
public string? GatewayMac { get; set; }
|
||||
public string? Error { get; set; }
|
||||
}
|
||||
|
||||
internal enum DoGroupIdAction
|
||||
{
|
||||
Write,
|
||||
NoOp,
|
||||
SkipForeign,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the Delivery Optimization group ID policy value (<c>DOGroupId</c> under the GPO
|
||||
/// DeliveryOptimization key) from a network fingerprint: a deterministic GUID derived from the
|
||||
/// default gateway's IPv4 address and MAC address, so all devices behind the same gateway land
|
||||
/// in the same DO peering group. The byte layout is RealmJoin-compatible — devices managed by
|
||||
/// RealmJoin on the same network compute the identical GUID and peer with ours.
|
||||
/// <para>
|
||||
/// Deliberately conservative where RealmJoin is not: no random-GUID fallback (a per-device
|
||||
/// random ID would isolate the device in a one-member group — worse than the OS default
|
||||
/// grouping), and existing foreign configuration always wins: a DOGroupIdSource policy, an
|
||||
/// MDM-managed DOGroupId, or a GPO value we did not write ourselves all result in a skip.
|
||||
/// Never throws.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class DoGroupIdService
|
||||
{
|
||||
internal const string GroupIdValueName = "DOGroupId";
|
||||
internal const string GroupIdSourceValueName = "DOGroupIdSource";
|
||||
|
||||
// RealmJoin's fingerprint-GUID prefix — part of the compatible byte layout, do not change.
|
||||
private static readonly byte[] GuidPrefix = { 101, 48, 6 };
|
||||
|
||||
/// <summary>
|
||||
/// Computes and applies the fingerprint GroupId. <paramref name="lastWrittenGroupId"/> is
|
||||
/// the GUID a previous agent run wrote (null when none is recorded) — it lets us update our
|
||||
/// own value after a gateway change while never touching a foreign one.
|
||||
/// </summary>
|
||||
public static DoGroupIdSetResult TrySetGroupId(string? lastWrittenGroupId, AgentLogger logger)
|
||||
{
|
||||
var result = new DoGroupIdSetResult();
|
||||
|
||||
try
|
||||
{
|
||||
// ---- Network fingerprint -------------------------------------------------
|
||||
var nic = NetworkInterfaceLocator.FindActiveNetworkInterface();
|
||||
var gateway = nic == null ? null : NetworkInterfaceLocator.GetIpv4Gateway(nic);
|
||||
if (gateway == null)
|
||||
{
|
||||
result.Skipped = true;
|
||||
result.SkipReason = "no_ipv4_gateway";
|
||||
logger.Info("DO GroupId auto-set: no active interface with an IPv4 default gateway — skipping.");
|
||||
return result;
|
||||
}
|
||||
result.GatewayIp = gateway.ToString();
|
||||
|
||||
if (!ArpNativeMethods.TryResolveMac(gateway, out var mac, out var arpError))
|
||||
{
|
||||
result.Skipped = true;
|
||||
result.SkipReason = "arp_failed";
|
||||
result.Error = arpError;
|
||||
logger.Warning($"DO GroupId auto-set: could not resolve gateway MAC ({arpError}) — skipping.");
|
||||
return result;
|
||||
}
|
||||
result.GatewayMac = BitConverter.ToString(mac);
|
||||
|
||||
var computed = ComputeGroupId(gateway.GetAddressBytes(), mac).ToString();
|
||||
result.GroupId = computed;
|
||||
|
||||
// ---- Foreign-configuration checks ----------------------------------------
|
||||
// DOGroupIdSource (either store) selects a dynamic group source — a fingerprint
|
||||
// GroupId would fight it. An MDM-managed DOGroupId means Intune owns the setting.
|
||||
if (ReadPolicyValue(DoThrottlePolicyReader.GpoKeyPath, GroupIdSourceValueName) != null
|
||||
|| ReadPolicyValue(DoThrottlePolicyReader.MdmKeyPath, GroupIdSourceValueName) != null)
|
||||
{
|
||||
result.Skipped = true;
|
||||
result.SkipReason = "group_id_source_policy";
|
||||
logger.Info("DO GroupId auto-set: DOGroupIdSource policy is configured — skipping.");
|
||||
return result;
|
||||
}
|
||||
if (ReadPolicyValue(DoThrottlePolicyReader.MdmKeyPath, GroupIdValueName) != null)
|
||||
{
|
||||
result.Skipped = true;
|
||||
result.SkipReason = "mdm_policy_present";
|
||||
logger.Info("DO GroupId auto-set: DOGroupId is managed via MDM policy — skipping.");
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- Ownership decision ---------------------------------------------------
|
||||
var existing = ReadPolicyValue(DoThrottlePolicyReader.GpoKeyPath, GroupIdValueName) as string;
|
||||
result.PreviousValue = existing;
|
||||
|
||||
switch (DecideAction(existing, computed, lastWrittenGroupId))
|
||||
{
|
||||
case DoGroupIdAction.NoOp:
|
||||
result.Success = true;
|
||||
logger.Info($"DO GroupId auto-set: already set to {computed} — nothing to do.");
|
||||
return result;
|
||||
|
||||
case DoGroupIdAction.SkipForeign:
|
||||
result.Skipped = true;
|
||||
result.SkipReason = "foreign_value_present";
|
||||
logger.Info($"DO GroupId auto-set: existing DOGroupId '{existing}' was not written by us — skipping.");
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- Write + readback -----------------------------------------------------
|
||||
using (var hklm = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry64))
|
||||
using (var key = hklm.CreateSubKey(DoThrottlePolicyReader.GpoKeyPath))
|
||||
{
|
||||
key.SetValue(GroupIdValueName, computed, RegistryValueKind.String);
|
||||
}
|
||||
|
||||
var readback = ReadPolicyValue(DoThrottlePolicyReader.GpoKeyPath, GroupIdValueName) as string;
|
||||
if (!string.Equals(readback, computed, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
result.Error = $"readback mismatch: wrote '{computed}', read '{readback ?? "<null>"}'";
|
||||
logger.Warning($"DO GroupId auto-set failed: {result.Error}");
|
||||
return result;
|
||||
}
|
||||
|
||||
result.Success = true;
|
||||
result.Written = true;
|
||||
logger.Info($"DO GroupId auto-set: DOGroupId set to {computed} (gateway {result.GatewayIp}, previous: {existing ?? "<none>"})");
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Error = ex.Message;
|
||||
logger.Warning($"DO GroupId auto-set failed: {ex.Message}");
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// RealmJoin-compatible fingerprint GUID: 16 bytes laid out as
|
||||
/// prefix {101,48,6} + {0,0,0} + {ip[1],0,ip[2],ip[3]} + mac[6], passed to
|
||||
/// <c>new Guid(byte[])</c>. The IP byte shuffle mirrors RealmJoin's "swap for
|
||||
/// new Guid()" step — the layout must stay bit-identical for cross-product peering.
|
||||
/// </summary>
|
||||
internal static Guid ComputeGroupId(byte[] ipv4Bytes, byte[] macBytes)
|
||||
{
|
||||
if (ipv4Bytes == null || ipv4Bytes.Length != 4)
|
||||
throw new ArgumentException("IPv4 address bytes required", nameof(ipv4Bytes));
|
||||
if (macBytes == null || macBytes.Length != 6)
|
||||
throw new ArgumentException("6-byte MAC required", nameof(macBytes));
|
||||
|
||||
var bytes = new byte[16];
|
||||
Array.Copy(GuidPrefix, 0, bytes, 0, GuidPrefix.Length);
|
||||
// bytes[3..5] stay zero (RealmJoin's zero padding)
|
||||
bytes[6] = ipv4Bytes[1];
|
||||
// bytes[7] stays zero
|
||||
bytes[8] = ipv4Bytes[2];
|
||||
bytes[9] = ipv4Bytes[3];
|
||||
Array.Copy(macBytes, 0, bytes, 10, macBytes.Length);
|
||||
return new Guid(bytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure ownership decision. GUID comparison tolerates case and braces so a value written
|
||||
/// by another tool in "{...}" form still matches semantically.
|
||||
/// </summary>
|
||||
internal static DoGroupIdAction DecideAction(string? existing, string computed, string? lastWrittenByUs)
|
||||
{
|
||||
if (string.IsNullOrEmpty(existing))
|
||||
return DoGroupIdAction.Write;
|
||||
if (GuidEquals(existing, computed))
|
||||
return DoGroupIdAction.NoOp;
|
||||
if (GuidEquals(existing, lastWrittenByUs))
|
||||
return DoGroupIdAction.Write; // our own stale value — gateway changed
|
||||
return DoGroupIdAction.SkipForeign;
|
||||
}
|
||||
|
||||
private static bool GuidEquals(string? a, string? b)
|
||||
{
|
||||
if (string.IsNullOrEmpty(a) || string.IsNullOrEmpty(b)) return false;
|
||||
return Guid.TryParse(a, out var ga) && Guid.TryParse(b, out var gb) && ga == gb;
|
||||
}
|
||||
|
||||
private static object? ReadPolicyValue(string keyPath, string valueName)
|
||||
{
|
||||
// Registry64 explicitly — SOFTWARE\Policies is WOW64-redirected, and the policy
|
||||
// stores live in the 64-bit view regardless of the agent's bitness.
|
||||
using (var hklm = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry64))
|
||||
using (var key = hklm.OpenSubKey(keyPath))
|
||||
{
|
||||
return key?.GetValue(valueName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+28
-3
@@ -21,12 +21,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// <remarks>
|
||||
/// <b>bytes-up</b> is read from <see cref="System.Net.Http.Headers.HttpContentHeaders.ContentLength"/>
|
||||
/// on the request content (set eagerly by <c>ByteArrayContent</c>/<c>StringContent</c>).
|
||||
/// <b>bytes-down</b> is read from the response Content-Length header — when
|
||||
/// <c>AutomaticDecompression</c> is enabled the framework may strip this, so the value
|
||||
/// is best-effort (degrades to 0). The request count and failure count are always exact.
|
||||
/// <b>bytes-down</b> prefers the response Content-Length header; the pipeline's
|
||||
/// <c>AutomaticDecompression</c> strips that header on compressed responses (i.e. on
|
||||
/// virtually every backend JSON response), so when it is absent the content is buffered
|
||||
/// and the DECOMPRESSED payload size is recorded instead — before the fallback the
|
||||
/// counter read 0 on healthy sessions. <c>HttpClient</c> buffers responses anyway
|
||||
/// (default <c>HttpCompletionOption.ResponseContentRead</c>), so this adds no real
|
||||
/// memory cost. The request count and failure count are always exact.
|
||||
/// </remarks>
|
||||
internal sealed class NetworkMetricsRecordingHandler : DelegatingHandler
|
||||
{
|
||||
// Buffering cap for the bytes-down fallback. Backend responses are small JSON
|
||||
// (config, ingest acks); anything larger is counted as 0 rather than buffered.
|
||||
private const long MaxCountedResponseBytes = 16 * 1024 * 1024;
|
||||
|
||||
private readonly NetworkMetrics _metrics;
|
||||
|
||||
public NetworkMetricsRecordingHandler(NetworkMetrics metrics)
|
||||
@@ -54,6 +62,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
{
|
||||
response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
bytesDown = response?.Content?.Headers?.ContentLength ?? 0L;
|
||||
if (bytesDown == 0 && response?.Content != null)
|
||||
{
|
||||
// AutomaticDecompression stripped Content-Length (the norm for the
|
||||
// backend's gzipped JSON). Buffer the content — after buffering the
|
||||
// header reflects the decompressed length. Fail-soft: a buffering
|
||||
// error (too large, connection dropped mid-body) keeps 0 and leaves
|
||||
// the response for the caller to observe the failure itself.
|
||||
try
|
||||
{
|
||||
await response.Content.LoadIntoBufferAsync(MaxCountedResponseBytes).ConfigureAwait(false);
|
||||
bytesDown = response.Content.Headers.ContentLength ?? 0L;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// keep bytesDown = 0
|
||||
}
|
||||
}
|
||||
if (response != null && !response.IsSuccessStatusCode)
|
||||
{
|
||||
failed = true;
|
||||
|
||||
@@ -74,13 +74,37 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
private readonly string _stateDirectory;
|
||||
private readonly Persistence.StartupEventGate? _startupEventGate;
|
||||
|
||||
/// <summary>
|
||||
/// How the previous agent run in this enrollment ended (<c>Program.PreviousExitSummary.ExitType</c>:
|
||||
/// <c>first_run</c> | <c>clean</c> | <c>exception_crash</c> | <c>hard_kill</c> |
|
||||
/// <c>reboot_kill</c>). Decides whether the Shell-Core ESP-exit replay runs at all — see
|
||||
/// <see cref="ResolveEspExitBackfillLookbackMinutes"/>.
|
||||
/// </summary>
|
||||
private readonly string? _previousExitType;
|
||||
|
||||
/// <summary>
|
||||
/// OS boot time when the previous agent run was killed by a reboot (<c>reboot_kill</c>),
|
||||
/// otherwise null. One of the two inputs that size the Shell-Core ESP-exit backfill.
|
||||
/// </summary>
|
||||
private readonly DateTime? _previousBootUtc;
|
||||
|
||||
/// <summary>
|
||||
/// Last moment the previous agent run is known to have been alive — the mtime of its
|
||||
/// reducer snapshot, which is rewritten on every decision step. Independent of HOW that
|
||||
/// run ended, which makes it the more broadly usable of the two window inputs.
|
||||
/// </summary>
|
||||
private readonly DateTime? _previousRunLastAliveUtc;
|
||||
|
||||
public DefaultComponentFactory(
|
||||
AgentConfiguration agentConfig,
|
||||
AgentConfigResponse remoteConfig,
|
||||
NetworkMetrics? networkMetrics,
|
||||
string agentVersion,
|
||||
string stateDirectory,
|
||||
Persistence.StartupEventGate? startupEventGate = null)
|
||||
Persistence.StartupEventGate? startupEventGate = null,
|
||||
DateTime? previousBootUtc = null,
|
||||
string? previousExitType = null,
|
||||
DateTime? previousRunLastAliveUtc = null)
|
||||
{
|
||||
_agentConfig = agentConfig ?? throw new ArgumentNullException(nameof(agentConfig));
|
||||
_remoteConfig = remoteConfig ?? throw new ArgumentNullException(nameof(remoteConfig));
|
||||
@@ -88,6 +112,88 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
_agentVersion = string.IsNullOrEmpty(agentVersion) ? "unknown" : agentVersion;
|
||||
_stateDirectory = stateDirectory ?? throw new ArgumentNullException(nameof(stateDirectory));
|
||||
_startupEventGate = startupEventGate;
|
||||
_previousBootUtc = previousBootUtc;
|
||||
_previousExitType = previousExitType;
|
||||
_previousRunLastAliveUtc = previousRunLastAliveUtc;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lookback in minutes for the Shell-Core ESP-exit replay; <c>0</c> means "do not replay".
|
||||
/// <para>
|
||||
/// The replay is a RESTART-RECOVERY mechanism: it re-reads observations the agent could not
|
||||
/// make because no agent process was running. That only ever applies when a previous run
|
||||
/// existed. On <c>first_run</c> — the happy path — it stays off, so the agent behaves
|
||||
/// exactly as it did before the recovery was added.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Otherwise the window is measured rather than guessed: it reaches back to the last moment
|
||||
/// the previous run is known to have been alive. Two independent inputs feed it and the
|
||||
/// wider one wins, because either can be missing:
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item><b>Snapshot mtime</b> — the reducer snapshot is rewritten on every decision
|
||||
/// step, so its last-write time is "when we last knew what was happening", regardless
|
||||
/// of how that run ended. This is what makes <c>exception_crash</c> safe: the scheduled
|
||||
/// task carries a BootTrigger only, with no restart-on-failure, so a crashed agent does
|
||||
/// not come back until the next boot — possibly hours later, long after Windows wrote
|
||||
/// the final ESP exit. A fixed five minutes would lose it every time.</item>
|
||||
/// <item><b>Last boot</b> — populated for <c>hard_kill</c> / <c>reboot_kill</c>; covers
|
||||
/// the case where the snapshot is missing or was never written.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// The default stays the floor (a restart that returns in seconds still gets its five
|
||||
/// minutes) and <c>ShellCoreTracker.ClampLookbackMinutes</c> is the ceiling, so policy and
|
||||
/// backstop together can never build an unbounded event-log query.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static int ResolveEspExitBackfillLookbackMinutes(
|
||||
string? previousExitType,
|
||||
DateTime? previousBootUtc,
|
||||
DateTime? previousRunLastAliveUtc,
|
||||
DateTime utcNow)
|
||||
{
|
||||
if (string.IsNullOrEmpty(previousExitType)
|
||||
|| string.Equals(previousExitType, "first_run", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var lookback = ShellCoreTracker.BackfillLookbackMinutes;
|
||||
|
||||
foreach (var since in new[] { previousRunLastAliveUtc, previousBootUtc })
|
||||
{
|
||||
if (!since.HasValue) continue;
|
||||
|
||||
var elapsed = utcNow - since.Value;
|
||||
if (elapsed <= TimeSpan.Zero) continue; // clock skew — never build a negative window
|
||||
|
||||
var minutes = (int)Math.Ceiling(elapsed.TotalMinutes) + 1;
|
||||
if (minutes > lookback) lookback = minutes;
|
||||
}
|
||||
|
||||
return lookback;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Last-write time of the previous run's reducer snapshot, or null when it does not exist
|
||||
/// (first run) or cannot be read. Best-effort by design — a missing timestamp only means
|
||||
/// the replay window falls back to the other inputs.
|
||||
/// </summary>
|
||||
internal static DateTime? ReadPreviousRunLastAliveUtc(string stateDirectory)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrEmpty(stateDirectory)) return null;
|
||||
var snapshotPath = Path.Combine(
|
||||
stateDirectory, Persistence.DeathRattlePrelude.SnapshotFileName);
|
||||
return File.Exists(snapshotPath)
|
||||
? (DateTime?)File.GetLastWriteTimeUtc(snapshotPath)
|
||||
: null;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public CollectorSurfaces CreateCollectorHosts(
|
||||
@@ -150,6 +256,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
// on esp_exiting events long after Start.
|
||||
ImeLogHost? imeLogHostRef = null;
|
||||
|
||||
// sits-d Cloud-PC fix (2026-08-19), narrowed after review: the Shell-Core ESP-exit
|
||||
// replay is restart recovery, so it stays OFF on a first run and the happy path keeps
|
||||
// its pre-fix behaviour exactly. See ResolveEspExitBackfillLookbackMinutes.
|
||||
var espExitBackfillLookbackMinutes = ResolveEspExitBackfillLookbackMinutes(
|
||||
_previousExitType, _previousBootUtc, _previousRunLastAliveUtc, DateTime.UtcNow);
|
||||
if (espExitBackfillLookbackMinutes > 0)
|
||||
{
|
||||
logger.Info(
|
||||
$"DefaultComponentFactory: previous run ended as '{_previousExitType}' — Shell-Core " +
|
||||
$"ESP-exit replay enabled with a {espExitBackfillLookbackMinutes} min lookback");
|
||||
}
|
||||
|
||||
var espAndHelloHost = new EspAndHelloHost(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
@@ -174,7 +292,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
// covers Store/WinGet apps invisible to the starved/correlation paths.
|
||||
packageStatesProbe: () =>
|
||||
imeLogHostRef?.AllKnownPackageStates
|
||||
?? Array.Empty<Monitoring.Enrollment.Ime.AppPackageState>());
|
||||
?? Array.Empty<Monitoring.Enrollment.Ime.AppPackageState>(),
|
||||
espExitBackfillLookbackMinutes: espExitBackfillLookbackMinutes);
|
||||
hosts.Add(espAndHelloHost);
|
||||
|
||||
// Hybrid User-Driven completion-gap fix (2026-05-01): the AadJoinHost notifies
|
||||
@@ -305,10 +424,41 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
stateDirectory: _stateDirectory,
|
||||
whiteGloveSealingPatternIds: whiteGloveSealingPatternIds,
|
||||
simulationMode: simulationMode,
|
||||
simulationSpeedFactor: _agentConfig.ReplaySpeedFactor);
|
||||
simulationSpeedFactor: _agentConfig.ReplaySpeedFactor,
|
||||
// Codex review P2 (2026-08-19): the IME tracker restores its persisted app states
|
||||
// inside Start(), silently (LoadState raises no OnAppStateChanged). This host starts
|
||||
// AFTER EspAndHelloHost, so the Shell-Core ESP-exit replay has already run against
|
||||
// an empty app list. Nudge the synthesis once the restored level is in place —
|
||||
// otherwise the exact reboot case (apps terminal BEFORE the reboot, only the final
|
||||
// exit lost to the downtime) still never completes.
|
||||
onStateRestored: () => espAndHelloHost.ReevaluateUserAppsSettledSynthesis());
|
||||
hosts.Add(imeLogHost);
|
||||
imeLogHostRef = imeLogHost;
|
||||
|
||||
// sits-d Cloud-PC fix (2026-08-19): the user-apps-settled AccountSetup synthesis used
|
||||
// to run ONLY on the EspExited edge. On a tenant with a large required-app set the ESP
|
||||
// page exits while apps are still in flight, the single attempt misses, and nothing
|
||||
// ever re-checks — sessions 8110e262 / a89aac2d reached 138/138 apps with 0 failed and
|
||||
// still never completed. Chain into the tracker's app-state callback (same
|
||||
// preserve-previous pattern DeliveryOptimizationHost uses) so every terminal app
|
||||
// transition gives the synthesis another look. Cheap: the re-check returns immediately
|
||||
// unless the ESP exit was already observed and the synthesis has not fired yet.
|
||||
var previousAppStateChanged = imeLogHost.Tracker.OnAppStateChanged;
|
||||
imeLogHost.Tracker.OnAppStateChanged = (pkg, oldState, newState) =>
|
||||
{
|
||||
try { previousAppStateChanged?.Invoke(pkg, oldState, newState); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"DefaultComponentFactory: previous OnAppStateChanged handler threw: {ex.Message}");
|
||||
}
|
||||
|
||||
try { espAndHelloHost.ReevaluateUserAppsSettledSynthesis(); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"DefaultComponentFactory: user-apps-settled re-check threw: {ex.Message}");
|
||||
}
|
||||
};
|
||||
|
||||
if (collectors.StallProbeEnabled)
|
||||
{
|
||||
hosts.Add(new StallProbeHost(
|
||||
@@ -407,6 +557,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
stateDirectory: _stateDirectory));
|
||||
}
|
||||
|
||||
// System timeline watcher — System channel: Kernel-General 1 (system clock stepped,
|
||||
// ground truth for time jumps incl. old/new time + setting process) and completed
|
||||
// sleep/standby episodes (Power-Troubleshooter 1, Kernel-Power 507). Backfill recovers
|
||||
// the pre-agent OOBE clock correction and standby episodes from the persistent .evtx.
|
||||
if (collectors.EnableSystemTimelineWatcher)
|
||||
{
|
||||
hosts.Add(new SystemTimelineWatcherHost(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
logger: logger,
|
||||
ingress: ingress,
|
||||
clock: clock,
|
||||
backfillLookbackMinutes: collectors.SystemEventBackfillLookbackMinutes,
|
||||
stateDirectory: _stateDirectory));
|
||||
}
|
||||
|
||||
// ----- Peripheral hosts (event-only; driven by remote-config toggles) --------------
|
||||
|
||||
// V1 parity (PeriodicCollectorManager) — combine Performance + AgentSelfMetrics under
|
||||
@@ -445,6 +611,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
logger: logger,
|
||||
apiBaseUrl: _agentConfig.ApiBaseUrl));
|
||||
|
||||
// Live AC/battery watcher — WMI Win32_PowerManagementEvent push (no polling), emits
|
||||
// power_state_change on AC↔battery transitions and 50/30/15 % downward threshold
|
||||
// crossings. Deliberately NOT under PeriodicCollectorLifecycleHost: it is push-based
|
||||
// (zero idle cost) and must stay alive precisely when the device sits idle on battery.
|
||||
// Never arms on devices without a battery. Kill-switchable via remote config.
|
||||
if (collectors.EnablePowerStateWatcher)
|
||||
{
|
||||
hosts.Add(new PowerStateWatcherHost(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
ingress: ingress,
|
||||
clock: clock,
|
||||
logger: logger));
|
||||
}
|
||||
|
||||
// Microsoft 365 Apps (Office C2R) install detector — event-driven (Rev 2): woken by a WMI
|
||||
// Win32_ProcessStartTrace push on OfficeC2RClient.exe (+ startup probe), progress via
|
||||
// RegNotifyChangeKeyValue, stop via Process.Exited. No idle polling. Constructed first so
|
||||
|
||||
@@ -14,6 +14,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
|
||||
private readonly EspAndHelloTracker _tracker;
|
||||
private readonly EspAndHelloTrackerAdapter _adapter;
|
||||
private readonly int _espExitBackfillLookbackMinutes;
|
||||
private int _disposed;
|
||||
|
||||
/// <summary>
|
||||
@@ -73,8 +74,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
bool isDevicePreparation = false,
|
||||
Func<bool>? userEspAppsSettledProbe = null,
|
||||
Func<System.Collections.Generic.IReadOnlyList<Monitoring.Enrollment.Ime.AppPackageState>>? starvedUserEspAppsProbe = null,
|
||||
Func<System.Collections.Generic.IReadOnlyList<Monitoring.Enrollment.Ime.AppPackageState>>? packageStatesProbe = null)
|
||||
Func<System.Collections.Generic.IReadOnlyList<Monitoring.Enrollment.Ime.AppPackageState>>? packageStatesProbe = null,
|
||||
int espExitBackfillLookbackMinutes = ShellCoreTracker.BackfillLookbackMinutes)
|
||||
{
|
||||
_espExitBackfillLookbackMinutes = espExitBackfillLookbackMinutes;
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
var post = new InformationalEventPost(ingress, clock);
|
||||
@@ -100,9 +103,56 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
_adapter = new EspAndHelloTrackerAdapter(_tracker, ingress, clock);
|
||||
}
|
||||
|
||||
public void Start() => _tracker.Start();
|
||||
/// <summary>
|
||||
/// Starts the tracker and replays the recent Shell-Core Hello-wizard start — restart
|
||||
/// recovery for an observation no agent process was around to make.
|
||||
/// <para>
|
||||
/// The backfill has existed since session 772fe502 but was never wired to a caller, so
|
||||
/// the Shell-Core watcher only ever saw events written AFTER <see cref="Start"/>. Any
|
||||
/// agent restart therefore lost an ESP exit (62407) or Hello-wizard start (62404) that
|
||||
/// happened while the agent was down — and after a mid-ESP reboot that is exactly the
|
||||
/// window the completion signal lands in (sits-d Cloud PCs, 2026-08-19: five sessions
|
||||
/// hung in AccountSetup until the server-side timeout while the devices were fine).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A lookback of zero disables the replay entirely. That is the FIRST run of an enrollment:
|
||||
/// there is no gap to recover because no earlier agent process existed, and any 62407
|
||||
/// already in the log belongs to a phase transition that happened before this agent was
|
||||
/// ever meant to observe anything. Replaying it would push an <c>EspExiting</c> into the
|
||||
/// reducer that the pre-fix agent never saw — a happy-path behaviour change for no gain.
|
||||
/// The caller (<c>DefaultComponentFactory</c>) owns that policy.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <para>
|
||||
/// Ordering is deliberate: Start first, then replay. A record written between the two is
|
||||
/// observed twice, which the wizard rail already tolerates (single-shot replay guard, plus
|
||||
/// the HelloTracker once-guard and the adapter's dedup flag downstream). The reverse order
|
||||
/// could drop a record entirely.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The replay covers the Hello-wizard start only — a conservative fact that can make the
|
||||
/// agent wait longer but never finish early. ESP exits are deliberately excluded because a
|
||||
/// replayed 62407 cannot be placed in time; see
|
||||
/// <c>ShellCoreTracker.ReplayBackfillRecords</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public void Start()
|
||||
{
|
||||
_tracker.Start();
|
||||
|
||||
if (_espExitBackfillLookbackMinutes > 0)
|
||||
_tracker.BackfillRecentHelloWizardStart(_espExitBackfillLookbackMinutes);
|
||||
}
|
||||
|
||||
public void Stop() => _tracker.Stop();
|
||||
|
||||
/// <summary>
|
||||
/// Re-checks the user-apps-settled AccountSetup synthesis — see
|
||||
/// <see cref="EspAndHelloTracker.ReevaluateUserAppsSettledSynthesis"/>. Driven by the IME
|
||||
/// tracker's app-state-change callback (wired in <c>DefaultComponentFactory</c>).
|
||||
/// </summary>
|
||||
public void ReevaluateUserAppsSettledSynthesis() => _tracker.ReevaluateUserAppsSettledSynthesis();
|
||||
|
||||
private void OnTrackerWhiteGloveCompleted(object sender, EventArgs e)
|
||||
=> WhiteGloveCompleted?.Invoke(this, e);
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
|
||||
private readonly ImeLogTracker _tracker;
|
||||
private readonly ImeProcessWatcher _processWatcher;
|
||||
private readonly Action? _onStateRestored;
|
||||
private readonly ImeLogTrackerAdapter _adapter;
|
||||
private readonly AgentLogger _logger;
|
||||
private int _disposed;
|
||||
@@ -96,9 +97,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
string stateDirectory,
|
||||
IReadOnlyCollection<string>? whiteGloveSealingPatternIds,
|
||||
bool simulationMode = false,
|
||||
double simulationSpeedFactor = 50)
|
||||
double simulationSpeedFactor = 50,
|
||||
Action? onStateRestored = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_onStateRestored = onStateRestored;
|
||||
var logFolder = string.IsNullOrEmpty(imeLogPathOverride) ? DefaultImeLogFolder : imeLogPathOverride!;
|
||||
var expandedMatchLogPath = string.IsNullOrEmpty(imeMatchLogPath)
|
||||
? null
|
||||
@@ -128,10 +131,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
_processWatcher = new ImeProcessWatcher(sessionId, tenantId, processWatcherPost, logger);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Codex review P2 (2026-08-19) — <see cref="ImeLogTracker.Start"/> restores the persisted
|
||||
/// package states (LoadState) and does NOT raise <c>OnAppStateChanged</c> for them. This
|
||||
/// host is started AFTER <c>EspAndHelloHost</c>, so a LIVE ESP exit observed before this
|
||||
/// restore completes would evaluate the user-apps-settled synthesis against an empty app
|
||||
/// list — and without a nudge here nothing would ever re-check the restored level. The
|
||||
/// <c>onStateRestored</c> callback fires once, after the restore, so the deferred
|
||||
/// synthesis gets its look. (It deliberately covers ONLY live exits: a 62407 that fired
|
||||
/// while no agent was running is never replayed — see
|
||||
/// <c>ShellCoreTracker.ReplayBackfillRecords</c> for why it cannot be placed in time.)
|
||||
/// </summary>
|
||||
public void Start()
|
||||
{
|
||||
_tracker.Start();
|
||||
_processWatcher.Start();
|
||||
|
||||
try { _onStateRestored?.Invoke(); }
|
||||
catch (Exception ex) { _logger.Warning($"ImeLogHost: onStateRestored callback threw: {ex.Message}"); }
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
|
||||
+332
@@ -0,0 +1,332 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Management;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Runtime;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// Live AC/battery watcher: subscribes to the WMI push event
|
||||
/// <c>Win32_PowerManagementEvent WHERE EventCode = 10</c> (PBT_APMPOWERSTATUSCHANGE — fires on
|
||||
/// AC↔DC switches and battery-percentage steps), falling back to the intrinsic
|
||||
/// <c>__InstanceModificationEvent WITHIN 30 … Win32_Battery</c> where the legacy push provider
|
||||
/// no longer loads (Windows 11 24H2/25H2, builds 26200+: activation fails WBEM_E_NOT_FOUND
|
||||
/// although the provider is still registered — session 161b838c). Either way the arriving
|
||||
/// event's payload is ignored; each arrival re-probes via <see cref="PowerStateProbe"/> and
|
||||
/// diffs the snapshots through the pure <see cref="PowerStateTransitionTracker"/> into
|
||||
/// <c>power_state_change</c> events. No polling in this process (the fallback's WITHIN
|
||||
/// sampling runs inside WinMgmt). Complements the one-shot startup
|
||||
/// <c>power_state_check</c> (<c>StartupEnvironmentProbes</c>), which stays the baseline event.
|
||||
/// <para>
|
||||
/// Devices without a battery (or with a failing probe) never arm the WMI subscription — zero
|
||||
/// overhead on desktops/VMs. WMI arming runs on a background thread (WinMgmt can hang in OOBE;
|
||||
/// same rationale as <c>ConsoleBypassWatcher</c>), and an arm failure is surfaced once as
|
||||
/// <c>collector_degraded</c>. A 5 s trailing-edge debounce collapses docking flaps to a
|
||||
/// no-diff; the tracker's lifetime emission cap is the storm backstop.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal sealed class PowerStateWatcherHost : ICollectorHost
|
||||
{
|
||||
public string Name => "PowerStateWatcher";
|
||||
|
||||
private static readonly TimeSpan DebounceDelay = TimeSpan.FromSeconds(5);
|
||||
|
||||
private readonly string _sessionId;
|
||||
private readonly string _tenantId;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly Func<PowerStateResult> _probe;
|
||||
private readonly PowerStateTransitionTracker _tracker = new PowerStateTransitionTracker();
|
||||
private readonly object _lock = new object();
|
||||
private readonly object _tickLock = new object();
|
||||
|
||||
private ManagementEventWatcher? _watcher;
|
||||
private Timer? _debounceTimer;
|
||||
private bool _started;
|
||||
private bool _disposed;
|
||||
private bool _capWarned;
|
||||
|
||||
/// <param name="probe">Power-state snapshot source; test seam, defaults to
|
||||
/// <see cref="PowerStateProbe.Probe"/>. The ctor never calls it (the factory constructs
|
||||
/// hosts eagerly; all probing happens in <see cref="Start"/>'s background thread).</param>
|
||||
public PowerStateWatcherHost(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
AgentLogger logger,
|
||||
Func<PowerStateResult>? probe = null)
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
_sessionId = sessionId;
|
||||
_tenantId = tenantId;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_post = new InformationalEventPost(ingress, clock, logger);
|
||||
_probe = probe ?? PowerStateProbe.Probe;
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (_started || _disposed) return;
|
||||
_started = true;
|
||||
}
|
||||
|
||||
// Baseline probe + WMI arming are synchronous WinMgmt-adjacent operations and WinMgmt
|
||||
// can hang during OOBE — never run them inline on the orchestrator's sequential
|
||||
// host-start loop (ConsoleBypassWatcher precedent). Disposal racing the background
|
||||
// start is handled by the _disposed re-checks inside StartCore.
|
||||
System.Threading.Tasks.Task.Run(() => StartCore(armWmi: true));
|
||||
}
|
||||
|
||||
/// <summary>Internal for tests: the synchronous start body; tests pass
|
||||
/// <paramref name="armWmi"/> = false so no real WMI subscription is created
|
||||
/// (precedent: ConsoleBypassWatcher leaves WMI arming untested).</summary>
|
||||
internal void StartCore(bool armWmi)
|
||||
{
|
||||
PowerStateResult baseline;
|
||||
try
|
||||
{
|
||||
baseline = _probe();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"[{Name}] baseline probe threw ({ex.Message}) — power-state watcher not armed");
|
||||
return;
|
||||
}
|
||||
|
||||
if (baseline.ProbeError != null)
|
||||
{
|
||||
_logger.Info($"[{Name}] power probe failed ({baseline.ProbeError}) — watcher not armed");
|
||||
return;
|
||||
}
|
||||
if (!baseline.HasBattery)
|
||||
{
|
||||
_logger.Info($"[{Name}] no battery present (desktop/VM) — watcher not armed");
|
||||
return;
|
||||
}
|
||||
|
||||
// Latch the baseline BEFORE arming so the first WMI push diffs against a real state.
|
||||
// A session already on battery below a ladder level emits its threshold event here.
|
||||
EmitAll(_tracker.Baseline(baseline));
|
||||
|
||||
if (!armWmi) return;
|
||||
|
||||
// Arm chain: the extrinsic push event first (zero WinMgmt cost, still alive on older
|
||||
// builds), then the intrinsic Win32_Battery modification event. On Windows 11 24H2/25H2
|
||||
// (observed on builds 26200/26220, session 161b838c) the MS_Power_Management_Event_Provider
|
||||
// is still REGISTERED but its activation fails with WBEM_E_NOT_FOUND — the legacy provider
|
||||
// no longer loads, so the push query throws on exactly the devices that enroll today.
|
||||
// The fallback's WITHIN polling runs inside WinMgmt (a Win32_Battery snapshot every 30 s),
|
||||
// NOT in this process; the handler path is identical (payload ignored, re-probe + diff),
|
||||
// worst-case detection latency ≈ 35 s incl. debounce.
|
||||
if (TryArmWatcher(
|
||||
"SELECT * FROM Win32_PowerManagementEvent WHERE EventCode = 10",
|
||||
"power_management_event (push)", baseline, out var pushError))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (TryArmWatcher(
|
||||
"SELECT * FROM __InstanceModificationEvent WITHIN 30 WHERE TargetInstance ISA 'Win32_Battery'",
|
||||
"battery_instance_poll (WinMgmt WITHIN 30s)", baseline, out var fallbackError))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.Warning($"[{Name}] could not arm any WMI power watcher — " +
|
||||
$"push: {Describe(pushError)}; fallback: {Describe(fallbackError)} — " +
|
||||
"live power-state transitions will not be observed this run");
|
||||
CollectorDegradationReporter.Report(_post, _sessionId, _tenantId, Name, "watcher_arm_failed",
|
||||
fallbackError ?? pushError);
|
||||
}
|
||||
|
||||
private static string Describe(Exception? ex)
|
||||
=> ex == null ? "n/a" : $"{ex.GetType().Name}: {ex.Message}";
|
||||
|
||||
/// <summary>
|
||||
/// One arm attempt for the given WQL event query, with the full dispose-race dance
|
||||
/// (ConsoleBypassWatcher pattern). Returns true when the chain should stop: the watcher
|
||||
/// armed, or disposal won the race (then arming anything would leak a subscription).
|
||||
/// Returns false with the captured exception so the caller can try the next query.
|
||||
/// </summary>
|
||||
private bool TryArmWatcher(string wql, string modeLabel, PowerStateResult baseline, out Exception? error)
|
||||
{
|
||||
error = null;
|
||||
try
|
||||
{
|
||||
var watcher = new ManagementEventWatcher(new WqlEventQuery(wql));
|
||||
watcher.EventArrived += OnPowerStatusChange;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
// Dispose won the race against the background start — don't arm.
|
||||
try { watcher.EventArrived -= OnPowerStatusChange; } catch { }
|
||||
try { watcher.Dispose(); } catch { }
|
||||
return true;
|
||||
}
|
||||
_watcher = watcher;
|
||||
}
|
||||
|
||||
watcher.Start();
|
||||
|
||||
// Re-check after arming: Dispose() may have run between the lock release above and
|
||||
// Start() — it then stopped a not-yet-armed watcher and this thread armed it right
|
||||
// back. If disposal won that race, tear the watcher down here.
|
||||
bool disposedDuringArm;
|
||||
lock (_lock)
|
||||
{
|
||||
disposedDuringArm = _disposed;
|
||||
if (disposedDuringArm) _watcher = null;
|
||||
}
|
||||
if (disposedDuringArm)
|
||||
{
|
||||
try { watcher.EventArrived -= OnPowerStatusChange; } catch { }
|
||||
try { watcher.Stop(); } catch { }
|
||||
try { watcher.Dispose(); } catch { }
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.Info($"[{Name}] armed via {modeLabel} — " +
|
||||
$"baseline: {(baseline.OnAcPower ? "AC" : "battery")}, {baseline.BatteryPercent?.ToString() ?? "?"}%");
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
error = ex;
|
||||
_logger.Info($"[{Name}] arm attempt '{modeLabel}' failed ({ex.GetType().Name}: {ex.Message})");
|
||||
lock (_lock) { _watcher = null; }
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnPowerStatusChange(object sender, EventArrivedEventArgs e)
|
||||
{
|
||||
// Push and fallback events are treated identically: the payload is ignored, the event
|
||||
// only tells us "power status may have changed". Debounce, then re-probe — never call
|
||||
// watcher.Stop() from inside this callback.
|
||||
try { ScheduleDebouncedTick(); }
|
||||
catch (Exception ex) { _logger.Debug($"[{Name}] power event handler error: {ex.Message}"); }
|
||||
}
|
||||
|
||||
private void ScheduleDebouncedTick()
|
||||
{
|
||||
lock (_lock) { if (_disposed) return; }
|
||||
// One trailing-edge tick per change burst (ImeRegistryAppStateHost pattern): a
|
||||
// dock-flap AC→battery→AC inside the window collapses to a no-diff in the tracker.
|
||||
var timer = _debounceTimer;
|
||||
if (timer == null)
|
||||
{
|
||||
var created = new Timer(_ => DebouncedTick(), null, DebounceDelay, Timeout.InfiniteTimeSpan);
|
||||
if (Interlocked.CompareExchange(ref _debounceTimer, created, null) != null)
|
||||
created.Dispose();
|
||||
else
|
||||
return;
|
||||
timer = _debounceTimer;
|
||||
}
|
||||
try { timer?.Change(DebounceDelay, Timeout.InfiniteTimeSpan); }
|
||||
catch (ObjectDisposedException) { /* stopping */ }
|
||||
}
|
||||
|
||||
/// <summary>Internal for tests: runs one debounced probe/diff/emit cycle synchronously,
|
||||
/// exactly what the trailing-edge timer callback executes after a WMI push.</summary>
|
||||
internal void TickForTest() => DebouncedTick();
|
||||
|
||||
private void DebouncedTick()
|
||||
{
|
||||
lock (_lock) { if (_disposed) return; }
|
||||
try
|
||||
{
|
||||
IReadOnlyList<PowerStateEmission> emissions;
|
||||
lock (_tickLock) // tracker is not thread-safe; ticks must not interleave
|
||||
{
|
||||
emissions = _tracker.Evaluate(_probe());
|
||||
}
|
||||
EmitAll(emissions);
|
||||
|
||||
if (_tracker.EmissionCapReached && !_capWarned)
|
||||
{
|
||||
_capWarned = true;
|
||||
_logger.Warning($"[{Name}] emission cap reached — further power_state_change events " +
|
||||
"suppressed this run (flapping power source or dying battery controller)");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"[{Name}] debounced power tick failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private void EmitAll(IReadOnlyList<PowerStateEmission> emissions)
|
||||
{
|
||||
foreach (var emission in emissions)
|
||||
{
|
||||
try
|
||||
{
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
{ "transition", emission.Transition },
|
||||
{ "onAcPower", emission.OnAcPower },
|
||||
{ "batteryPercent", emission.BatteryPercent.HasValue ? (object)emission.BatteryPercent.Value : "unknown" },
|
||||
{ "isCharging", emission.IsCharging },
|
||||
{ "batteryLifeMinutes", emission.BatteryLifeMinutes.HasValue ? (object)emission.BatteryLifeMinutes.Value : "unknown" },
|
||||
};
|
||||
if (emission.ThresholdPercent.HasValue)
|
||||
data["thresholdPercent"] = emission.ThresholdPercent.Value;
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.PowerStateChange,
|
||||
Severity = emission.Severity,
|
||||
Source = Name,
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = emission.Message,
|
||||
ImmediateUpload = emission.ImmediateUpload,
|
||||
Data = data,
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"[{Name}] failed to emit power_state_change: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
ManagementEventWatcher? watcher;
|
||||
lock (_lock)
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
watcher = _watcher;
|
||||
_watcher = null;
|
||||
}
|
||||
|
||||
if (watcher != null)
|
||||
{
|
||||
try { watcher.EventArrived -= OnPowerStatusChange; } catch { }
|
||||
try { watcher.Stop(); } catch { }
|
||||
try { watcher.Dispose(); } catch { }
|
||||
}
|
||||
|
||||
_debounceTimer?.Dispose();
|
||||
_debounceTimer = null;
|
||||
}
|
||||
|
||||
public void Dispose() => Stop();
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// Standalone host for the <see cref="SystemTimelineTracker"/> — watches the <c>System</c>
|
||||
/// event log for system clock steps (Kernel-General 1) and completed sleep/standby episodes
|
||||
/// (Power-Troubleshooter 1 / Kernel-Power 507) and forwards them via the shared single-rail
|
||||
/// ingress. Mirrors <see cref="WindowsUpdateWatcherHost"/>: builds its own
|
||||
/// <see cref="InformationalEventPost"/> and delegates Start/Stop to the tracker.
|
||||
/// </summary>
|
||||
internal sealed class SystemTimelineWatcherHost : ICollectorHost
|
||||
{
|
||||
public string Name => "SystemTimelineTracker";
|
||||
|
||||
private readonly SystemTimelineTracker _tracker;
|
||||
private readonly AgentLogger _logger;
|
||||
private int _disposed;
|
||||
|
||||
public SystemTimelineWatcherHost(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
AgentLogger logger,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
int backfillLookbackMinutes,
|
||||
string stateDirectory)
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
var post = new InformationalEventPost(ingress, clock, logger);
|
||||
_tracker = new SystemTimelineTracker(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
post: post,
|
||||
logger: logger,
|
||||
backfillEnabled: backfillLookbackMinutes > 0,
|
||||
backfillLookbackMinutes: backfillLookbackMinutes,
|
||||
stateDirectory: stateDirectory);
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
_tracker.Start();
|
||||
_logger.Info("SystemTimelineWatcherHost: started.");
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
try { _tracker.Stop(); }
|
||||
catch (Exception ex) { _logger.Warning($"SystemTimelineWatcherHost: stop failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) == 1) return;
|
||||
Stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
// real activity (rare, and worth keeping diagnostics running for).
|
||||
SharedConstants.EventTypes.ModernDeploymentLog,
|
||||
|
||||
// Environment observation, not device/enrollment progress: an idle device draining
|
||||
// its battery must not keep the periodic collectors alive or push out stall
|
||||
// detection — the exact scenario the power watcher exists to observe.
|
||||
SharedConstants.EventTypes.PowerStateChange,
|
||||
|
||||
// Same reasoning for the system timeline watcher: a clock step, a wake from standby,
|
||||
// or a backfilled episode at agent start is environment observation and must not
|
||||
// reset the idle/stall clocks.
|
||||
SharedConstants.EventTypes.SystemClockChanged,
|
||||
SharedConstants.EventTypes.SystemSleepEpisode,
|
||||
|
||||
// Agent health / control / transport — not device/enrollment progress.
|
||||
SharedConstants.EventTypes.CollectorDegraded,
|
||||
SharedConstants.EventTypes.TelemetryUploadPoisoned,
|
||||
|
||||
@@ -124,6 +124,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Persistence
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Read-only peek of the key's current fingerprint (possibly restored from a previous
|
||||
/// agent run), or null when none is recorded. Like <see cref="HasFingerprint"/> this makes
|
||||
/// NO claim — use it when the fingerprint carries state the caller needs to read back
|
||||
/// (e.g. the DO GroupId ownership marker).
|
||||
/// </summary>
|
||||
public string? GetFingerprint(string key)
|
||||
{
|
||||
if (string.IsNullOrEmpty(key)) return null;
|
||||
lock (_lock)
|
||||
{
|
||||
return _entries.TryGetValue(key, out var entry) ? entry.Fingerprint : null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Retry-until-success: true when <see cref="MarkSucceeded"/> latched this key in any run.</summary>
|
||||
public bool AlreadySucceeded(string key)
|
||||
{
|
||||
|
||||
@@ -25,6 +25,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Runtime
|
||||
/// Always emitted — offset >60s is Warning, smaller is Info, failure is Warning.</item>
|
||||
/// <item><b>power_state_check</b> — AC vs. battery snapshot via Win32 <c>GetSystemPowerStatus</c>.
|
||||
/// On battery with <80% charge is Warning, otherwise Info, probe failure is Warning.</item>
|
||||
/// <item><b>do_group_id_auto_set</b> — only when <c>EnableDoGroupIdAutoSet=true</c>. Sets the
|
||||
/// Delivery Optimization <c>DOGroupId</c> policy value from a network fingerprint via
|
||||
/// <see cref="DoGroupIdService"/>; skips (Info) rather than fight foreign policy values.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// Each probe runs best-effort: failures are logged + optionally emitted as a Warning event,
|
||||
@@ -78,6 +81,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Runtime
|
||||
await RunNtpCheck(configuration, logger, post, gate).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (configuration.EnableDoGroupIdAutoSet)
|
||||
RunDoGroupIdAutoSet(configuration, logger, post, gate);
|
||||
|
||||
RunPowerStateCheck(configuration, logger, post);
|
||||
}
|
||||
|
||||
@@ -331,6 +337,99 @@ namespace AutopilotMonitor.Agent.V2.Core.Runtime
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One-shot DO GroupId apply with change-based restart dedup: the gate fingerprint doubles
|
||||
/// as the ownership marker ("written:<guid>" records the GUID we last wrote, so a later
|
||||
/// run can distinguish our own stale value from a foreign one after a gateway change).
|
||||
/// Unchanged outcomes (same GUID, same skip reason) re-run silently without a duplicate
|
||||
/// event; errors never claim a fingerprint so each failing start keeps its retry trail.
|
||||
/// </summary>
|
||||
private static void RunDoGroupIdAutoSet(
|
||||
AgentConfiguration configuration,
|
||||
AgentLogger logger,
|
||||
Orchestration.InformationalEventPost post,
|
||||
Persistence.StartupEventGate? gate)
|
||||
{
|
||||
try
|
||||
{
|
||||
var lastWritten = ExtractLastWrittenGroupId(gate?.GetFingerprint(Constants.EventTypes.DoGroupIdAutoSet));
|
||||
var result = DoGroupIdService.TrySetGroupId(lastWritten, logger);
|
||||
|
||||
var fingerprint = ComputeDoGroupIdFingerprint(result);
|
||||
if (fingerprint != null && gate != null
|
||||
&& !gate.ShouldEmit(Constants.EventTypes.DoGroupIdAutoSet, fingerprint))
|
||||
{
|
||||
logger.Debug("StartupEnvironmentProbes: DO GroupId outcome unchanged — skipping duplicate event (restart dedup).");
|
||||
return;
|
||||
}
|
||||
|
||||
SafeEmit(post, logger, BuildDoGroupIdEvent(configuration, result));
|
||||
|
||||
if (fingerprint != null)
|
||||
gate?.MarkEmitted(Constants.EventTypes.DoGroupIdAutoSet);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"StartupEnvironmentProbes: DO GroupId auto-set threw: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
internal const string DoGroupIdWrittenFingerprintPrefix = "written:";
|
||||
|
||||
/// <summary>
|
||||
/// Success (written or already correct) fingerprints as "written:<guid>", a skip as
|
||||
/// "skipped:<reason>"; errors return null so they are never deduped.
|
||||
/// </summary>
|
||||
internal static string? ComputeDoGroupIdFingerprint(DoGroupIdSetResult result)
|
||||
{
|
||||
if (result.Success) return DoGroupIdWrittenFingerprintPrefix + result.GroupId;
|
||||
if (result.Skipped) return "skipped:" + result.SkipReason;
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>Ownership marker readback: the GUID from a "written:" fingerprint, else null.</summary>
|
||||
internal static string? ExtractLastWrittenGroupId(string? fingerprint) =>
|
||||
fingerprint != null && fingerprint.StartsWith(DoGroupIdWrittenFingerprintPrefix, StringComparison.Ordinal)
|
||||
? fingerprint.Substring(DoGroupIdWrittenFingerprintPrefix.Length)
|
||||
: null;
|
||||
|
||||
internal static EnrollmentEvent BuildDoGroupIdEvent(AgentConfiguration configuration, DoGroupIdSetResult result)
|
||||
{
|
||||
string message;
|
||||
if (result.Success && result.Written)
|
||||
message = $"Delivery Optimization GroupId set to {result.GroupId} (gateway {result.GatewayIp})";
|
||||
else if (result.Success)
|
||||
message = $"Delivery Optimization GroupId already set to {result.GroupId}";
|
||||
else if (result.Skipped)
|
||||
message = $"Delivery Optimization GroupId not set: {result.SkipReason}";
|
||||
else
|
||||
message = $"Delivery Optimization GroupId auto-set failed: {result.Error}";
|
||||
|
||||
return new EnrollmentEvent
|
||||
{
|
||||
SessionId = configuration.SessionId,
|
||||
TenantId = configuration.TenantId,
|
||||
EventType = Constants.EventTypes.DoGroupIdAutoSet,
|
||||
Severity = result.Success || result.Skipped ? EventSeverity.Info : EventSeverity.Warning,
|
||||
Source = "StartupEnvironmentProbes",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Timestamp = DateTime.UtcNow,
|
||||
Message = message,
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
{ "groupId", result.GroupId ?? "" },
|
||||
{ "previousValue", result.PreviousValue ?? "" },
|
||||
{ "gatewayIp", result.GatewayIp ?? "" },
|
||||
{ "gatewayMac", result.GatewayMac ?? "" },
|
||||
{ "written", result.Written },
|
||||
{ "skipped", result.Skipped },
|
||||
{ "skipReason", result.SkipReason ?? "" },
|
||||
{ "success", result.Success },
|
||||
{ "error", result.Error ?? "" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static void RunPowerStateCheck(
|
||||
AgentConfiguration configuration,
|
||||
AgentLogger logger,
|
||||
|
||||
@@ -124,6 +124,17 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
// in ImeLogTracker so the layers can never drift apart).
|
||||
private static readonly TimeSpan StaleSourceThreshold = ImeLogTracker.HistoricReplayThreshold;
|
||||
|
||||
/// <summary>
|
||||
/// How far ahead of the agent clock a source timestamp may sit before it is rejected.
|
||||
/// <para>
|
||||
/// Writer and reader share one system clock, so a correctly converted line is always at
|
||||
/// or behind "now"; this is pure jitter tolerance, not a skew allowance. Deliberately far
|
||||
/// below one hour — the value that a timezone mistake produces — so that such a mistake
|
||||
/// can actually trip it.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private static readonly TimeSpan MaxFutureSourceSkew = TimeSpan.FromMinutes(2);
|
||||
|
||||
// Matches the deterministic marker line the bootstrap script writes via Write-Log,
|
||||
// e.g. "Bootstrap script version: v2.0" — see scripts/Bootstrap/Install-AutopilotMonitor.ps1.
|
||||
// Same shape as the web-side regex in utils/bootstrapVersion.ts; kept in sync intentionally.
|
||||
@@ -329,12 +340,28 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
|
||||
var clockNow = _clock.UtcNow;
|
||||
var ageHours = (clockNow - asUtc).TotalHours;
|
||||
if (ageHours > StaleSourceThreshold.TotalHours || ageHours < -1.0)
|
||||
|
||||
// Future guard. A source timestamp can never legitimately be ahead of the agent
|
||||
// clock: the writer stamps it from the SAME system clock, and the tracker only
|
||||
// sees the line after it was written. Anything ahead means the local-to-UTC
|
||||
// conversion was wrong.
|
||||
//
|
||||
// The previous threshold was ageHours < -1.0, which could not fire for the single
|
||||
// most common timezone error: with a one-hour offset mistake the real value is
|
||||
// parseLag - 3600 s, i.e. just ABOVE -1.0. It sat exactly on the boundary of the
|
||||
// case it was meant to catch. It was also asymmetric — a past-shifted error had to
|
||||
// exceed 24 h before anything noticed, while the measured field errors reached
|
||||
// -17 h. The bound is now a small jitter tolerance in the direction that is
|
||||
// physically impossible.
|
||||
var isFutureSkewed = asUtc > clockNow + MaxFutureSourceSkew;
|
||||
|
||||
if (ageHours > StaleSourceThreshold.TotalHours || isFutureSkewed)
|
||||
{
|
||||
// Pathologically stale (>24h old) or in the future (>1h skew) — log,
|
||||
// fall back, and let the caller flag the event.
|
||||
var reason = isFutureSkewed
|
||||
? $"ahead of the agent clock by {(asUtc - clockNow).TotalMinutes:F1} min"
|
||||
: $"pathologically stale (ageHours={ageHours:F1})";
|
||||
_logger?.Warning(
|
||||
$"ImeAdapter: source timestamp {asUtc:o} rejected (ageHours={ageHours:F1}); falling back to clock");
|
||||
$"ImeAdapter: source timestamp {asUtc:o} rejected — {reason}; falling back to clock");
|
||||
derivedFromClock = true;
|
||||
return clockNow;
|
||||
}
|
||||
@@ -347,12 +374,69 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
return _clock.UtcNow;
|
||||
}
|
||||
|
||||
private static void TagDerivedTimestamp(IDictionary<string, string> data, bool derivedFromClock, DateTime? rawSourceTs)
|
||||
/// <summary>
|
||||
/// Attach the evidence for this event's timestamp: the raw local time as the writer wrote
|
||||
/// it, the offset applied, and how that offset was obtained.
|
||||
///
|
||||
/// <para>
|
||||
/// Without this the only record of a conversion is the converted value itself, so a wrong
|
||||
/// offset is indistinguishable from a correct one. Proving the field defect required the
|
||||
/// diagnostics archive — and of the 5,386 sessions that went through a timezone change,
|
||||
/// essentially none had one; the analysis had to fall back to median statistics across
|
||||
/// thousands of sessions. These three fields make each session answer the question on its
|
||||
/// own, and make a stored timestamp recomputable after the fact without an agent release.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// Only emitted for events whose time came from a CMTrace line. Agent-native events have
|
||||
/// no second clock frame, so there is nothing to record.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private void TagDerivedTimestamp(IDictionary<string, string> data, bool derivedFromClock, DateTime? rawSourceTs)
|
||||
{
|
||||
var culture = System.Globalization.CultureInfo.InvariantCulture;
|
||||
|
||||
var sourceLocal = _tracker.LastMatchedSourceLocalTimestamp;
|
||||
if (sourceLocal.HasValue)
|
||||
{
|
||||
// No zone suffix: the value genuinely carries none, and formatting one in would
|
||||
// re-introduce the very assumption this whole change removes.
|
||||
data["sourceLocalTs"] = sourceLocal.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", culture);
|
||||
data["sourceOffsetOrigin"] = OriginTag(_tracker.LastMatchedSourceOffsetOrigin);
|
||||
|
||||
var offsetMinutes = _tracker.LastMatchedSourceOffsetMinutes;
|
||||
if (offsetMinutes.HasValue)
|
||||
data["sourceOffsetMinutes"] = offsetMinutes.Value.ToString(culture);
|
||||
|
||||
// Observational only — what the calibrator measured, which since the 2026-08-20
|
||||
// revert is deliberately NOT what was applied. Kept separate so the two can never
|
||||
// be read as the same thing.
|
||||
var measured = _tracker.LastMatchedMeasuredWriterOffsetMinutes;
|
||||
if (measured.HasValue)
|
||||
data["measuredWriterOffsetMinutes"] = measured.Value.ToString(culture);
|
||||
}
|
||||
|
||||
if (!derivedFromClock) return;
|
||||
data["derivedTimestamp"] = "true";
|
||||
if (rawSourceTs.HasValue)
|
||||
data["rejectedSourceTimestamp"] = rawSourceTs.Value.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
|
||||
data["rejectedSourceTimestamp"] = rawSourceTs.Value.ToString("o", culture);
|
||||
}
|
||||
|
||||
private static string OriginTag(CmTraceOffsetOrigin origin)
|
||||
{
|
||||
switch (origin)
|
||||
{
|
||||
case CmTraceOffsetOrigin.Bias: return "bias";
|
||||
// Per-FILE offset application, retired by the 2026-08-20 revert (04b1a7c6).
|
||||
// Kept so stored 2.0.1410 events remain interpretable; no current path emits it.
|
||||
case CmTraceOffsetOrigin.Calibrated: return "calibrated";
|
||||
// The line anchored itself: read provably fresh, its own distance to the agent
|
||||
// clock (grid-rounded) was applied as the writer's offset. Era-safe per line.
|
||||
case CmTraceOffsetOrigin.LineAnchored: return "line-anchored";
|
||||
// The line was not provably fresh (backlog, restart catch-up, replay), so this
|
||||
// process's own zone was assumed — correct only if the writer happens to share it.
|
||||
default: return "reader-zone-fallback";
|
||||
}
|
||||
}
|
||||
|
||||
// CMTrace Kind normalization — shared implementation lives on the tracker (used by its
|
||||
@@ -720,12 +804,15 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
/// <see cref="AppInstallTiming"/> so the emitter can inline the values on the
|
||||
/// current event's payload without a second dictionary lookup.
|
||||
/// <para>
|
||||
/// Audit Q1: a terminal→active transition (the IME retry path — <see
|
||||
/// cref="AppPackageState.UpdateState"/> permits e.g. Error→Downloading and
|
||||
/// Skipped→Downloading) re-arms <c>Completed</c> to <c>null</c>, so the NEXT terminal
|
||||
/// transition stamps a fresh CompletedAt. StartedAt keeps its first value — the final
|
||||
/// terminal payload then covers first-start → last-terminal (total occupancy across
|
||||
/// attempts) instead of freezing at the first failure. Observed pre-fix: a user-scope
|
||||
/// Audit Q1 + 2026-08 attempt-duration change: a terminal→active transition (the IME
|
||||
/// retry / re-evaluation path — <see cref="AppPackageState.UpdateState"/> permits e.g.
|
||||
/// Error→Downloading and Skipped→Downloading) re-arms BOTH endpoints: <c>Completed</c>
|
||||
/// to <c>null</c> (the NEXT terminal transition stamps fresh) and <c>Started</c> to
|
||||
/// the re-entry time. The final terminal payload therefore describes the
|
||||
/// status-defining ATTEMPT — matching the backend's AppInstallSummaries semantics —
|
||||
/// instead of first-start → last-terminal across passes: the IME processes the app
|
||||
/// list repeatedly (device-ESP evaluation pass, user-context pass), and the idle gap
|
||||
/// between passes is not install time. Observed pre-fix (old bug shape): a user-scope
|
||||
/// app Skipped early in DeviceSetup, retried after sign-in, emitted its final
|
||||
/// Installed event with the stale early CompletedAt (durationSeconds=-2342).
|
||||
/// </para>
|
||||
@@ -736,11 +823,13 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
{
|
||||
_appTimings.TryGetValue(appId, out var current);
|
||||
|
||||
// Terminal→active (retry): the earlier terminal stamp no longer describes this
|
||||
// app's outcome — re-arm so the next terminal transition stamps fresh.
|
||||
// Terminal→active (retry / new pass): the earlier stamps no longer describe
|
||||
// this app's outcome — re-arm both endpoints so the next terminal transition
|
||||
// reports the fresh attempt's window.
|
||||
if (current.Completed != null && IsLifecycleActive(newState))
|
||||
{
|
||||
current.Completed = null;
|
||||
current.Started = now;
|
||||
}
|
||||
|
||||
// First lifecycle-active transition → record StartedAt.
|
||||
|
||||
+6
@@ -119,6 +119,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
AddSecurityHeaders(request);
|
||||
request.Headers.Add("X-Correlation-ID", correlationId);
|
||||
|
||||
// P14: device-clock send time, one value per request. Lets the backend separate
|
||||
// spool delay (SentAt − OccurredUtc, both device frame) from device-vs-server
|
||||
// clock offset (ReceivedAt − SentAt) — indistinguishable without this header.
|
||||
// Stamped per attempt on purpose: a retry after backoff is a NEW send moment.
|
||||
request.Headers.Add("X-Send-Time-Utc", DateTime.UtcNow.ToString("O"));
|
||||
|
||||
HttpResponseMessage response;
|
||||
try
|
||||
{
|
||||
|
||||
@@ -59,10 +59,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// </summary>
|
||||
int PeakPendingItemCount { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Serialized on-disk byte size of the pending (not-yet-uploaded) tail — the actual
|
||||
/// upload backlog. Unlike <see cref="SpoolFileSizeBytes"/>, which grows append-only
|
||||
/// over the whole session regardless of upload progress, this shrinks back to 0 as
|
||||
/// <see cref="MarkUploaded"/> advances the cursor, so it is the correct input for
|
||||
/// back-pressure thresholds.
|
||||
/// </summary>
|
||||
long PendingBytes { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Current size of the on-disk <c>spool.jsonl</c> file in bytes. Returns 0 when the
|
||||
/// file does not yet exist. Cheap (single <c>FileInfo.Length</c> call) and safe to
|
||||
/// poll from periodic collectors.
|
||||
/// poll from periodic collectors. Append-only cumulative — informational only, NOT a
|
||||
/// backlog measure (use <see cref="PendingBytes"/> for that).
|
||||
/// </summary>
|
||||
long SpoolFileSizeBytes { get; }
|
||||
|
||||
|
||||
@@ -56,9 +56,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// Populated from disk in the ctor and kept in sync by Enqueue / MarkUploaded. Peek
|
||||
// reads from here so the periodic drain loop is O(batchSize) instead of O(N) per
|
||||
// call. Disk JSONL stays the source of truth — the cache is rebuilt on every
|
||||
// process start.
|
||||
private readonly LinkedList<TelemetryItem> _pending = new LinkedList<TelemetryItem>();
|
||||
// process start. Each entry carries its on-disk line length (incl. trailing \n) so
|
||||
// MarkUploaded can decrement _pendingBytes exactly.
|
||||
private readonly LinkedList<PendingEntry> _pending = new LinkedList<PendingEntry>();
|
||||
private int _peakPendingCount;
|
||||
private long _pendingBytes;
|
||||
|
||||
private readonly struct PendingEntry
|
||||
{
|
||||
public PendingEntry(TelemetryItem item, int byteLength)
|
||||
{
|
||||
Item = item;
|
||||
ByteLength = byteLength;
|
||||
}
|
||||
|
||||
public TelemetryItem Item { get; }
|
||||
public int ByteLength { get; }
|
||||
}
|
||||
|
||||
public TelemetrySpool(string directoryPath, IClock clock, Logging.AgentLogger? logger = null)
|
||||
{
|
||||
@@ -105,6 +119,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
get { lock (_lock) return _peakPendingCount; }
|
||||
}
|
||||
|
||||
public long PendingBytes
|
||||
{
|
||||
get { lock (_lock) return _pendingBytes; }
|
||||
}
|
||||
|
||||
public long SpoolFileSizeBytes
|
||||
{
|
||||
get
|
||||
@@ -171,7 +190,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
}
|
||||
|
||||
_lastAssignedItemId = itemId;
|
||||
_pending.AddLast(item);
|
||||
_pending.AddLast(new PendingEntry(item, bytes.Length));
|
||||
_pendingBytes += bytes.Length;
|
||||
if (_pending.Count > _peakPendingCount)
|
||||
_peakPendingCount = _pending.Count;
|
||||
|
||||
@@ -209,7 +229,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
var node = _pending.First;
|
||||
while (node != null && batch.Count < max)
|
||||
{
|
||||
batch.Add(node.Value);
|
||||
batch.Add(node.Value.Item);
|
||||
node = node.Next;
|
||||
}
|
||||
return batch;
|
||||
@@ -233,25 +253,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// Trim the in-memory tail. The list is in TelemetryItemId-monotonic order
|
||||
// (Enqueue only appends), so we can drop from the front until the first
|
||||
// remaining item is past the cursor.
|
||||
while (_pending.First != null && _pending.First.Value.TelemetryItemId <= upToItemIdInclusive)
|
||||
while (_pending.First != null && _pending.First.Value.Item.TelemetryItemId <= upToItemIdInclusive)
|
||||
{
|
||||
_pendingBytes -= _pending.First.Value.ByteLength;
|
||||
_pending.RemoveFirst();
|
||||
}
|
||||
|
||||
// _pendingBytes drives the spool-pressure trigger: negative drift would
|
||||
// silently disarm it, a stale positive on an empty list would pin it on.
|
||||
// The list is authoritative — on drift, recompute from it and log loudly.
|
||||
if (_pendingBytes < 0 || (_pending.Count == 0 && _pendingBytes != 0))
|
||||
{
|
||||
long recomputed = 0;
|
||||
foreach (var entry in _pending) recomputed += entry.ByteLength;
|
||||
_logger?.Warning(
|
||||
$"TelemetrySpool: pending-bytes accounting drifted to {_pendingBytes} with {_pending.Count} pending items after MarkUploaded({upToItemIdInclusive}) — recomputed to {recomputed}.");
|
||||
_pendingBytes = recomputed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Single-pass scan of <c>spool.jsonl</c>: returns the highest <c>TelemetryItemId</c>
|
||||
/// found AND populates <see cref="_pending"/> with every item past
|
||||
/// <see cref="_lastUploadedItemId"/>. Called once from the ctor — replaces the
|
||||
/// previous per-Peek <c>File.ReadAllLines</c> hot path.
|
||||
/// Single-pass streaming scan of <c>spool.jsonl</c> (<c>File.ReadLines</c> — never
|
||||
/// materializes the whole file): returns the highest <c>TelemetryItemId</c> found AND
|
||||
/// populates <see cref="_pending"/> / <see cref="_pendingBytes"/> with every item past
|
||||
/// <see cref="_lastUploadedItemId"/>. Called once from the ctor.
|
||||
/// </summary>
|
||||
private long ScanSpoolAndRehydratePending()
|
||||
{
|
||||
if (!File.Exists(_spoolPath)) return -1;
|
||||
|
||||
long highest = -1;
|
||||
foreach (var line in File.ReadAllLines(_spoolPath, Encoding.UTF8))
|
||||
foreach (var line in File.ReadLines(_spoolPath, Encoding.UTF8))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line)) continue;
|
||||
TelemetryItem item;
|
||||
@@ -270,7 +303,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
if (item.TelemetryItemId > highest) highest = item.TelemetryItemId;
|
||||
if (item.TelemetryItemId > _lastUploadedItemId)
|
||||
{
|
||||
_pending.AddLast(item);
|
||||
// +1 = the trailing \n Enqueue writes after every line, so the rehydrated
|
||||
// byte count matches what Enqueue would have accumulated.
|
||||
var byteLength = Encoding.UTF8.GetByteCount(line) + 1;
|
||||
_pending.AddLast(new PendingEntry(item, byteLength));
|
||||
_pendingBytes += byteLength;
|
||||
}
|
||||
}
|
||||
return highest;
|
||||
|
||||
@@ -79,13 +79,27 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// write to the same state file; per-enrollment lifecycle like the other state files.
|
||||
var startupEventGate = new AutopilotMonitor.Agent.V2.Core.Persistence.StartupEventGate(stateSubdir, logger);
|
||||
|
||||
// sits-d Cloud-PC fix (2026-08-19): a reboot_kill means the agent was down across the
|
||||
// forced mid-ESP restart. Hand the boot timestamp to the factory so the Shell-Core
|
||||
// ESP-exit backfill can reach back over the whole downtime instead of 5 minutes.
|
||||
var previousBootUtc =
|
||||
string.Equals(previousExit?.ExitType, "reboot_kill", StringComparison.OrdinalIgnoreCase)
|
||||
? previousExit?.LastBootUtc
|
||||
: null;
|
||||
|
||||
var componentFactory = new DefaultComponentFactory(
|
||||
agentConfig: agentConfig,
|
||||
remoteConfig: remoteConfig,
|
||||
networkMetrics: auth.NetworkMetrics,
|
||||
agentVersion: agentVersion,
|
||||
stateDirectory: stateSubdir,
|
||||
startupEventGate: startupEventGate);
|
||||
startupEventGate: startupEventGate,
|
||||
previousBootUtc: previousBootUtc,
|
||||
previousExitType: previousExit?.ExitType,
|
||||
// Measured, not guessed: the reducer snapshot is rewritten on every decision step,
|
||||
// so its mtime is the last moment the previous run is known to have been alive —
|
||||
// and unlike LastBootUtc it exists for every exit type, exception_crash included.
|
||||
previousRunLastAliveUtc: DefaultComponentFactory.ReadPreviousRunLastAliveUtc(stateSubdir));
|
||||
|
||||
var whiteGloveSealingPatternIds = (System.Collections.Generic.IReadOnlyCollection<string>)remoteConfig.WhiteGloveSealingPatternIds
|
||||
?? Array.Empty<string>();
|
||||
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.DecisionCore.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// The reducer half of the "a replayed 62407 must never reach the decision engine" contract
|
||||
/// (2026-08-19). The agent half — ShellCoreTracker never re-raising a replayed exit — lives in
|
||||
/// <c>ShellCoreTrackerReplayScopeTests</c>; this file proves WHY that upstream restriction is
|
||||
/// the only place the problem can be solved.
|
||||
/// <para>
|
||||
/// The reducer is itself a completion gate, and by design it cannot tell a replayed exit from
|
||||
/// a live one:
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item><c>HandleEspExitingV1</c> passes <c>espFinalExitInFlight: true</c> for EVERY
|
||||
/// arriving exit — the signal carries no provenance.</item>
|
||||
/// <item><c>IsPostAccountSetupFinalExit</c> orders exits by INGEST ORDINAL rather than by
|
||||
/// timestamp (deliberately: replayed CMTrace lines carry backdated source times). A
|
||||
/// historic exit replayed today is assigned a fresher ordinal than reality, so it reads as
|
||||
/// post-AccountSetup by construction.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// After a restart the state is restored from the snapshot, so AccountSetupEntered, the IME
|
||||
/// user-session completion and the desktop arrival are all already present. Arm C of
|
||||
/// <c>ShouldTransitionToAwaitingHello</c> then needs exactly one more fact: an exit. These
|
||||
/// tests pin that it opens — which is correct behaviour for a live exit and would be a
|
||||
/// premature completion for a replayed one. Hence the upstream ban.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ClassicEspExitingOnRestoredStateTests
|
||||
{
|
||||
private static readonly DateTime Fixed = new DateTime(2026, 8, 19, 8, 43, 16, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public void Arm_C_opens_on_any_arriving_exit_once_restored_state_carries_the_other_three_facts()
|
||||
{
|
||||
// This is the danger, stated as a test: with restored facts in place, a single
|
||||
// EspExiting signal is all that separates EspAccountSetup from AwaitingHello. The
|
||||
// reducer has no way to ask "was that exit real, or did we dig it out of the event
|
||||
// log?" — so the answer has to be that a replayed exit is never turned into a signal.
|
||||
var engine = new DecisionEngine();
|
||||
var state = BuildRestoredState(
|
||||
accountSetupEntered: true,
|
||||
imeUserSessionCompleted: true,
|
||||
desktopArrived: true);
|
||||
|
||||
var step = engine.Reduce(state, MakeEspExitingSignal(ordinal: 10));
|
||||
|
||||
Assert.Equal(SessionStage.AwaitingHello, step.NewState.Stage);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// Each row drops exactly one of arm C's mandatory facts — the transition must not happen.
|
||||
[InlineData(false, true, true)]
|
||||
[InlineData(true, false, true)]
|
||||
[InlineData(true, true, false)]
|
||||
public void Arm_C_stays_shut_when_any_of_its_mandatory_facts_is_missing(
|
||||
bool accountSetupEntered, bool imeUserSessionCompleted, bool desktopArrived)
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = BuildRestoredState(accountSetupEntered, imeUserSessionCompleted, desktopArrived);
|
||||
|
||||
var step = engine.Reduce(state, MakeEspExitingSignal(ordinal: 10));
|
||||
|
||||
Assert.NotEqual(SessionStage.AwaitingHello, step.NewState.Stage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_historic_source_timestamp_does_not_hold_the_gate_shut()
|
||||
{
|
||||
// The exit's OccurredAtUtc is an hour older than the AccountSetup anchor — exactly the
|
||||
// shape a replayed record has. It makes no difference: the ordering that arm C uses is
|
||||
// the ingest ordinal, and the in-flight flag bypasses even that. Timestamps are not a
|
||||
// defence, which is why the fix cannot live in the reducer.
|
||||
var engine = new DecisionEngine();
|
||||
var state = BuildRestoredState(
|
||||
accountSetupEntered: true,
|
||||
imeUserSessionCompleted: true,
|
||||
desktopArrived: true);
|
||||
|
||||
var step = engine.Reduce(
|
||||
state,
|
||||
MakeEspExitingSignal(ordinal: 10, occurredAtUtc: Fixed.AddHours(-1)));
|
||||
|
||||
Assert.Equal(SessionStage.AwaitingHello, step.NewState.Stage);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
private static DecisionState BuildRestoredState(
|
||||
bool accountSetupEntered,
|
||||
bool imeUserSessionCompleted,
|
||||
bool desktopArrived)
|
||||
{
|
||||
var anchor = Fixed.AddMinutes(-90);
|
||||
|
||||
var builder = DecisionState.CreateInitial("s", "t", Fixed.AddDays(-1))
|
||||
.ToBuilder()
|
||||
.WithStage(SessionStage.EspAccountSetup)
|
||||
.WithStepIndex(452) // the restored StepIndex the sits-d sessions showed
|
||||
.WithLastAppliedSignalOrdinal(9);
|
||||
|
||||
if (accountSetupEntered)
|
||||
builder.AccountSetupEnteredUtc = new SignalFact<DateTime>(anchor, 1);
|
||||
|
||||
// IsImeUserSessionGenuine requires the completion to be at-or-after the AccountSetup
|
||||
// anchor (the defaultuser0-ghost guard).
|
||||
if (imeUserSessionCompleted)
|
||||
builder.ImeUserSessionCompletedUtc = new SignalFact<DateTime>(anchor.AddMinutes(5), 2);
|
||||
|
||||
if (desktopArrived)
|
||||
builder.DesktopArrivedUtc = new SignalFact<DateTime>(anchor.AddMinutes(6), 3);
|
||||
|
||||
return builder.Build();
|
||||
}
|
||||
|
||||
private static DecisionSignal MakeEspExitingSignal(long ordinal, DateTime? occurredAtUtc = null) =>
|
||||
new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: DecisionSignalKind.EspExiting,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: occurredAtUtc ?? Fixed,
|
||||
sourceOrigin: "EspAndHelloTracker",
|
||||
evidence: new Evidence(
|
||||
kind: EvidenceKind.Derived,
|
||||
identifier: "esp-hello-detector-v1",
|
||||
summary: "ESP exiting (coordinator-forwarded Shell-Core 62407)",
|
||||
derivationInputs: new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["subSource"] = "ShellCoreTracker",
|
||||
["eventSource"] = "Microsoft-Windows-Shell-Core",
|
||||
["eventId"] = "62407",
|
||||
}));
|
||||
}
|
||||
}
|
||||
+286
@@ -0,0 +1,286 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.DecisionCore.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// sits-d Cloud-PC fix (2026-08-20, sessions 8110e262 / a89aac2d / e7ba63c9 / cb4a485a /
|
||||
/// 3d6278fb). On a SkipUser=true flow Windows never renders the user ESP page: Shell-Core
|
||||
/// 62407 does not fire and the AccountSetup provisioning categories are never written —
|
||||
/// verified live (Shell-Core watcher armed across three agent runs, zero 62407 in six
|
||||
/// hours; SkipUserStatusPage read 10+ times across all runs, True every time). Arm B of
|
||||
/// <c>ShouldTransitionToAwaitingHello</c> was therefore open from the bootstrap
|
||||
/// EspConfigDetected onward, but every site that evaluates the gate hangs off a signal the
|
||||
/// flow structurally cannot produce, so five sessions sat at "waiting on: hello_resolution"
|
||||
/// until the max-lifetime watchdog.
|
||||
/// <para>
|
||||
/// The fix lets the observed skip stand in for the final-exit evidence inside
|
||||
/// <c>HandleImeUserSessionCompletedV1</c>'s completion attempt. These tests replay the real
|
||||
/// 8110e262 signal ordering end to end (including the Fix-10 AccountSetup bounce-back a
|
||||
/// restart re-emission triggers) and pin the negative space: no skip observed, no desktop,
|
||||
/// ghost IME completion, and Device Preparation flows are all unchanged.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ClassicSkipUserEspUserSessionCompletionTests
|
||||
{
|
||||
// 8110e262 run 1: agent_started 08:30:08Z.
|
||||
private static readonly DateTime T0 = new DateTime(2026, 8, 19, 8, 30, 8, DateTimeKind.Utc);
|
||||
|
||||
[Fact]
|
||||
public void Sits_d_signal_ordering_completes_via_hello_safety()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial(
|
||||
"8110e262-da57-4282-9290-bb0c96d05614", "5ca2b350", T0);
|
||||
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0, null)).NewState;
|
||||
|
||||
// Bootstrap EspConfigDetected — SkipUserStatusPage=True, read from FirstSync 2 s in.
|
||||
state = engine.Reduce(state, MakeEspConfigSignal(1, skipUser: "true")).NewState;
|
||||
Assert.True(state.ScenarioObservations.SkipUserEsp!.Value);
|
||||
|
||||
// Hello policy enabled (device-scoped CSP) — the reason the desktop fast-path
|
||||
// could never take this flow.
|
||||
state = engine.Reduce(state, MakeSignal(2, DecisionSignalKind.HelloPolicyDetected, T0.AddSeconds(1),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.HelloEnabled] = "true" })).NewState;
|
||||
|
||||
// Desktop 5 s after agent start (the user ESP page does not exist — the user is
|
||||
// simply logged on). Pins that desktop arrival alone does NOT promote: the knock
|
||||
// deliberately lives on the IME user-session edge, because a desktop-side promotion
|
||||
// would be undone by Fix 10's AccountSetup bounce-back two signals later (the IME
|
||||
// AccountSetup phase line lands AFTER the desktop on this flow).
|
||||
state = engine.Reduce(state, MakeSignal(3, DecisionSignalKind.DesktopArrived, T0.AddSeconds(5), null)).NewState;
|
||||
Assert.NotEqual(SessionStage.AwaitingHello, state.Stage);
|
||||
Assert.DoesNotContain(state.Deadlines, d => d.Name == DeadlineNames.HelloSafety);
|
||||
|
||||
// IME's AccountSetup phase line (source ts 08:29:56 — before agent start; the IME
|
||||
// logs the phase for its user-session app processing even though no page is shown).
|
||||
state = engine.Reduce(state, MakeSignal(4, DecisionSignalKind.EspPhaseChanged, T0.AddSeconds(-12),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
Assert.Equal(SessionStage.EspAccountSetup, state.Stage);
|
||||
|
||||
// 08:36:59 — IME user session completed (138/138 apps, 0 failed). THE knock:
|
||||
// skip-user stands in for the exit that cannot exist; Hello enabled + unresolved
|
||||
// promotes to AwaitingHello with HelloSafety armed.
|
||||
var promoted = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.ImeUserSessionCompleted,
|
||||
T0.AddMinutes(6).AddSeconds(51), null));
|
||||
Assert.Equal(SessionStage.AwaitingHello, promoted.NewState.Stage);
|
||||
var helloSafety = Assert.Single(promoted.NewState.Deadlines, d => d.Name == DeadlineNames.HelloSafety);
|
||||
Assert.Contains(promoted.Effects,
|
||||
e => e.Kind == DecisionEffectKind.ScheduleDeadline && e.Deadline?.Name == DeadlineNames.HelloSafety);
|
||||
|
||||
// The Hello wizard never appears over RDP — HelloSafety resolves synthetically.
|
||||
var afterSafety = engine.Reduce(promoted.NewState,
|
||||
MakeSignal(6, DecisionSignalKind.DeadlineFired, helloSafety.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.HelloSafety }));
|
||||
Assert.Equal(SessionStage.Finalizing, afterSafety.NewState.Stage);
|
||||
Assert.Equal("Timeout", afterSafety.NewState.HelloOutcome!.Value);
|
||||
|
||||
var completed = engine.Reduce(afterSafety.NewState,
|
||||
MakeSignal(7, DecisionSignalKind.DeadlineFired, helloSafety.DueAtUtc.AddSeconds(5),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.FinalizingGrace }));
|
||||
Assert.Equal(SessionStage.Completed, completed.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentComplete, completed.NewState.Outcome);
|
||||
Assert.Contains(completed.Effects,
|
||||
e => e.Kind == DecisionEffectKind.EmitEventTimelineEntry
|
||||
&& e.Parameters != null
|
||||
&& e.Parameters.TryGetValue("eventType", out var et) && et == "enrollment_complete");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Fix10_bounceback_is_healed_by_the_ime_reemission()
|
||||
{
|
||||
// Restart shape (8110e262 run 3): the promoted AwaitingHello gets bounced back by a
|
||||
// re-emitted AccountSetup phase line (Fix 10 cancels HelloSafety, deliberately — it
|
||||
// guards the premature-promotion case). The IME phase line always precedes the
|
||||
// user-session-complete line in log order, so the re-emitted completion re-knocks.
|
||||
var engine = new DecisionEngine();
|
||||
var state = ProgressToPromotedAwaitingHello(engine);
|
||||
|
||||
var bounced = engine.Reduce(state, MakeSignal(10, DecisionSignalKind.EspPhaseChanged,
|
||||
T0.AddMinutes(18).AddSeconds(10),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" }));
|
||||
Assert.Equal(SessionStage.EspAccountSetup, bounced.NewState.Stage);
|
||||
Assert.DoesNotContain(bounced.NewState.Deadlines, d => d.Name == DeadlineNames.HelloSafety);
|
||||
|
||||
var reknocked = engine.Reduce(bounced.NewState,
|
||||
MakeSignal(11, DecisionSignalKind.ImeUserSessionCompleted, T0.AddMinutes(6).AddSeconds(51), null));
|
||||
Assert.Equal(SessionStage.AwaitingHello, reknocked.NewState.Stage);
|
||||
Assert.Single(reknocked.NewState.Deadlines, d => d.Name == DeadlineNames.HelloSafety);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
// No EspConfigDetected at all — SkipUserEsp unknown.
|
||||
[InlineData(null)]
|
||||
// Explicit SkipUser=false (the two sits-d machines that succeeded had this shape and
|
||||
// a REAL page: exit + registry evidence. Without the exit the gate must stay shut.)
|
||||
[InlineData("false")]
|
||||
public void No_promotion_without_an_observed_skip(string? skipUser)
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("s", "t", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0, null)).NewState;
|
||||
if (skipUser != null)
|
||||
{
|
||||
state = engine.Reduce(state, MakeEspConfigSignal(1, skipUser)).NewState;
|
||||
}
|
||||
state = engine.Reduce(state, MakeSignal(2, DecisionSignalKind.HelloPolicyDetected, T0.AddSeconds(1),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.HelloEnabled] = "true" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(3, DecisionSignalKind.DesktopArrived, T0.AddSeconds(5), null)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(4, DecisionSignalKind.EspPhaseChanged, T0.AddSeconds(10),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state,
|
||||
MakeSignal(5, DecisionSignalKind.ImeUserSessionCompleted, T0.AddMinutes(7), null));
|
||||
|
||||
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
|
||||
Assert.DoesNotContain(step.NewState.Deadlines, d => d.Name == DeadlineNames.HelloSafety);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_promotion_without_the_desktop()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("s", "t", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0, null)).NewState;
|
||||
state = engine.Reduce(state, MakeEspConfigSignal(1, skipUser: "true")).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(2, DecisionSignalKind.EspPhaseChanged, T0.AddSeconds(10),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state,
|
||||
MakeSignal(3, DecisionSignalKind.ImeUserSessionCompleted, T0.AddMinutes(7), null));
|
||||
|
||||
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
|
||||
Assert.DoesNotContain(step.NewState.Deadlines, d => d.Name == DeadlineNames.HelloSafety);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_ghost_pre_anchor_ime_completion_does_not_knock()
|
||||
{
|
||||
// defaultuser0-ghost guard stays load-bearing: an IME completion whose source time
|
||||
// precedes the AccountSetup anchor is the OOBE/technician frame, not the user.
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("s", "t", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0, null)).NewState;
|
||||
state = engine.Reduce(state, MakeEspConfigSignal(1, skipUser: "true")).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(2, DecisionSignalKind.DesktopArrived, T0.AddSeconds(5), null)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(3, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
|
||||
var step = engine.Reduce(state,
|
||||
MakeSignal(4, DecisionSignalKind.ImeUserSessionCompleted, T0.AddMinutes(1), null));
|
||||
|
||||
Assert.Equal(SessionStage.EspAccountSetup, step.NewState.Stage);
|
||||
Assert.DoesNotContain(step.NewState.Deadlines, d => d.Name == DeadlineNames.HelloSafety);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Device_preparation_flows_keep_their_own_rails()
|
||||
{
|
||||
// WDP has no ESP at all — completion rides the Hello+Desktop conjunction and the
|
||||
// DevicePrepCompletion backstop (afee7ae0). Even a (spurious) SkipUser=true read
|
||||
// must not route WDP through the classic skip-user knock.
|
||||
var engine = new DecisionEngine();
|
||||
var seed = DecisionState.CreateInitial("s", "t", T0)
|
||||
.ToBuilder()
|
||||
.WithStage(SessionStage.SessionStarted)
|
||||
.WithStepIndex(5)
|
||||
.WithLastAppliedSignalOrdinal(4)
|
||||
.WithScenarioProfile(EnrollmentScenarioProfile.Empty.With(
|
||||
mode: EnrollmentMode.DevicePreparation,
|
||||
confidence: ProfileConfidence.High,
|
||||
evidenceOrdinal: 1,
|
||||
reason: "test"));
|
||||
seed.ScenarioObservations = EnrollmentScenarioObservations.Empty
|
||||
.WithSkipUserEsp(true, 1);
|
||||
seed.AccountSetupEnteredUtc = new SignalFact<DateTime>(T0.AddMinutes(1), 2);
|
||||
seed.DesktopArrivedUtc = new SignalFact<DateTime>(T0.AddMinutes(2), 3);
|
||||
var state = seed.Build();
|
||||
|
||||
var step = engine.Reduce(state,
|
||||
MakeSignal(5, DecisionSignalKind.ImeUserSessionCompleted, T0.AddMinutes(3), null));
|
||||
|
||||
Assert.DoesNotContain(step.NewState.Deadlines, d => d.Name == DeadlineNames.HelloSafety);
|
||||
Assert.NotEqual(SessionStage.AwaitingHello, step.NewState.Stage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Hello_disabled_skip_user_completes_directly_from_the_knock()
|
||||
{
|
||||
// HelloSatisfiedForCompletion holds (policy disabled, no wizard observed) — the
|
||||
// knock completes through Finalizing with the synthetic Skipped outcome instead of
|
||||
// parking 5 minutes in AwaitingHello.
|
||||
var engine = new DecisionEngine();
|
||||
var state = DecisionState.CreateInitial("s", "t", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0, null)).NewState;
|
||||
state = engine.Reduce(state, MakeEspConfigSignal(1, skipUser: "true")).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(2, DecisionSignalKind.HelloPolicyDetected, T0.AddSeconds(1),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.HelloEnabled] = "false" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(3, DecisionSignalKind.EspPhaseChanged, T0.AddSeconds(10),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
// Desktop arrives; the hello-disabled fast-path already promotes/completes via the
|
||||
// gate's arm B on this ordering, so the knock is not even needed here — assert the
|
||||
// flow terminates either way (belt for orderings where desktop precedes the config).
|
||||
state = engine.Reduce(state, MakeSignal(4, DecisionSignalKind.DesktopArrived, T0.AddMinutes(5), null)).NewState;
|
||||
|
||||
if (state.Stage != SessionStage.Finalizing && !state.Stage.IsTerminal())
|
||||
{
|
||||
var step = engine.Reduce(state,
|
||||
MakeSignal(5, DecisionSignalKind.ImeUserSessionCompleted, T0.AddMinutes(7), null));
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Equal("Skipped", step.NewState.HelloOutcome!.Value);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Equal(SessionStage.Finalizing, state.Stage);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
private static DecisionState ProgressToPromotedAwaitingHello(DecisionEngine engine)
|
||||
{
|
||||
var state = DecisionState.CreateInitial("s", "t", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0, null)).NewState;
|
||||
state = engine.Reduce(state, MakeEspConfigSignal(1, skipUser: "true")).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(2, DecisionSignalKind.HelloPolicyDetected, T0.AddSeconds(1),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.HelloEnabled] = "true" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(3, DecisionSignalKind.DesktopArrived, T0.AddSeconds(5), null)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(4, DecisionSignalKind.EspPhaseChanged, T0.AddSeconds(-12),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.ImeUserSessionCompleted,
|
||||
T0.AddMinutes(6).AddSeconds(51), null)).NewState;
|
||||
Assert.Equal(SessionStage.AwaitingHello, state.Stage);
|
||||
return state;
|
||||
}
|
||||
|
||||
private static DecisionSignal MakeEspConfigSignal(long ordinal, string skipUser) =>
|
||||
MakeSignal(ordinal, DecisionSignalKind.EspConfigDetected, T0.AddSeconds(ordinal),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[SignalPayloadKeys.SkipUserEsp] = skipUser,
|
||||
[SignalPayloadKeys.SkipDeviceEsp] = "false",
|
||||
});
|
||||
|
||||
private static DecisionSignal MakeSignal(
|
||||
long ordinal,
|
||||
DecisionSignalKind kind,
|
||||
DateTime occurredAtUtc,
|
||||
IReadOnlyDictionary<string, string>? payload)
|
||||
{
|
||||
return new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: kind,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: occurredAtUtc,
|
||||
sourceOrigin: "test",
|
||||
evidence: new Evidence(EvidenceKind.Synthetic, $"test-{kind}-{ordinal}", $"synthetic {kind}"),
|
||||
payload: payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -192,6 +192,33 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
Assert.Equal(SessionStage.Completed, completed.NewState.Stage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BackstopFired_WithoutDesktop_ParksInAwaitingDesktopUntilDesktopCompletes()
|
||||
{
|
||||
// The arming site (HandleDesktopArrivedV1) guarantees desktop-first, so this
|
||||
// fires only on a stale/replayed timer — Dispatch routes DeadlineFired purely on
|
||||
// the payload name, no armed-state check. The fallback must keep the handler
|
||||
// total: synthetic Hello timeout recorded, session parks in AwaitingDesktop, and
|
||||
// a later real desktop arrival still completes it.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SeedWdpSession(engine);
|
||||
|
||||
var fired = engine.Reduce(state, DeadlineFired(20, T0.AddMinutes(39), DeadlineNames.DevicePrepCompletion));
|
||||
|
||||
Assert.Equal(SessionStage.AwaitingDesktop, fired.NewState.Stage);
|
||||
Assert.NotNull(fired.NewState.HelloResolvedUtc);
|
||||
Assert.Equal("Timeout", fired.NewState.HelloOutcome!.Value);
|
||||
Assert.DoesNotContain(fired.NewState.Deadlines, d => d.Name == DeadlineNames.DevicePrepCompletion);
|
||||
|
||||
// Hello is satisfied (synthetically) — only the desktop is still missing.
|
||||
var waiting = Assert.Single(CompletionWaitingEffects(fired));
|
||||
Assert.Equal("desktop_arrival", waiting.Parameters!["missingPrerequisites"]);
|
||||
|
||||
var completed = engine.Reduce(fired.NewState,
|
||||
MakeSignal(21, DecisionSignalKind.DesktopArrived, T0.AddMinutes(45)));
|
||||
Assert.Equal(SessionStage.Finalizing, completed.NewState.Stage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HelloResolved_BeforeBackstopFires_CompletesNormally()
|
||||
{
|
||||
|
||||
@@ -372,6 +372,9 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
// Before RealmJoinPhaseChanged became a typed signal, LastDeploymentPhase was only
|
||||
// written at detection time — every realmjoin_timeout claimed "last phase: 0" no
|
||||
// matter how far RJ actually got, hiding this whole failure class from ops.
|
||||
// Since the activity-based extension, the PhaseChanged at T0+6 counts as activity,
|
||||
// so the first fire (59 min later) extends once; the timeout lands on the re-armed
|
||||
// fire a full inactivity window after the last activity.
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
@@ -384,7 +387,13 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
})).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.DesktopArrived, T0.AddMinutes(7))).NewState;
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.DeadlineFired, T0.AddMinutes(65),
|
||||
// First fire at the original due (T0+65): activity 59 min ago → extends to T0+66.
|
||||
state = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.DeadlineFired, T0.AddMinutes(65),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout })).NewState;
|
||||
var rearmed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
Assert.Equal(T0.AddMinutes(66), rearmed.DueAtUtc);
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(9, DecisionSignalKind.DeadlineFired, rearmed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout }));
|
||||
|
||||
var timeout = Assert.Single(step.Effects, e =>
|
||||
@@ -395,6 +404,248 @@ namespace AutopilotMonitor.DecisionCore.Tests
|
||||
Assert.Equal("101", timeout.Parameters["lastSeenPhase"]);
|
||||
}
|
||||
|
||||
// ============================================================== Activity-based extension (report 55e6afd61c9d)
|
||||
|
||||
[Fact]
|
||||
public void Timeout_fire_with_recent_first_deployment_activity_extends_instead_of_timing_out()
|
||||
{
|
||||
// Report 55e6afd61c9d (Douglas): timer armed at detection (phase 0, RJ agent MSI
|
||||
// install during DeviceSetup), first deployment only started 16 min later, Office
|
||||
// completed 3 s before the deadline — the hard cut truncated an actively working
|
||||
// deployment. The fire must now re-arm to lastActivity + 60 min instead.
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "0" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.DesktopArrived, T0.AddMinutes(6))).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(21),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "0",
|
||||
})).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.RealmJoinPackageCompleted, T0.AddMinutes(62),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PackageId] = "generic-microsoft-office-2016-proplus",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.Scope] = RealmJoinPackageFact.ScopeMachine,
|
||||
[DecisionEngine.RealmJoinPayloadKeys.Success] = "true",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.LastExitCode] = "0",
|
||||
})).NewState;
|
||||
|
||||
var armed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
var step = engine.Reduce(state, MakeSignal(9, DecisionSignalKind.DeadlineFired, armed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout }));
|
||||
|
||||
// No completion, no outcome — the gate stays closed and the window is re-armed.
|
||||
Assert.NotEqual(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Null(step.NewState.RealmJoinFacts.Outcome);
|
||||
Assert.Equal($"DeadlineFired:{DeadlineNames.RealmJoinTimeout}:Extended", step.Transition.Trigger);
|
||||
|
||||
var rearmed = Assert.Single(step.NewState.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
Assert.Equal(T0.AddMinutes(62 + 60), rearmed.DueAtUtc); // lastActivity + inactivity window
|
||||
|
||||
Assert.Single(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.ScheduleDeadline &&
|
||||
e.Deadline != null &&
|
||||
e.Deadline.Name == DeadlineNames.RealmJoinTimeout &&
|
||||
e.Deadline.DueAtUtc == rearmed.DueAtUtc);
|
||||
|
||||
var extended = Assert.Single(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_timeout_extended");
|
||||
Assert.Equal("Info", extended.Parameters!["severity"]);
|
||||
Assert.Equal("101", extended.Parameters["deploymentPhase"]);
|
||||
Assert.DoesNotContain(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_timeout");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resolved_after_extension_completes_via_classic_path()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.DesktopArrived, T0.AddMinutes(6))).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(40),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "100",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "101",
|
||||
})).NewState;
|
||||
var armed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
state = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.DeadlineFired, armed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout })).NewState;
|
||||
Assert.Null(state.RealmJoinFacts.Outcome); // extended, not timed out
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(9, DecisionSignalKind.RealmJoinResolved, T0.AddMinutes(80),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "110" }));
|
||||
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Equal("Resolved", step.NewState.RealmJoinFacts.Outcome!.Value);
|
||||
Assert.DoesNotContain(step.NewState.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Rearmed_fire_after_quiet_window_times_out_with_inactivity_reason()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "0" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.DesktopArrived, T0.AddMinutes(6))).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(30),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "0",
|
||||
})).NewState;
|
||||
var armed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
state = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.DeadlineFired, armed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout })).NewState;
|
||||
var rearmed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
Assert.Equal(T0.AddMinutes(90), rearmed.DueAtUtc); // lastActivity (T0+30) + 60 min
|
||||
|
||||
// No further activity — the re-armed fire must now time out for real.
|
||||
var step = engine.Reduce(state, MakeSignal(9, DecisionSignalKind.DeadlineFired, rearmed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout }));
|
||||
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Equal("Timeout", step.NewState.RealmJoinFacts.Outcome!.Value);
|
||||
var timeout = Assert.Single(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_timeout");
|
||||
Assert.Equal("inactivity", timeout.Parameters!["reason"]);
|
||||
Assert.Contains("no deployment activity", timeout.Parameters["message"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Extension_is_capped_at_absolute_ceiling_and_cap_fire_times_out_despite_activity()
|
||||
{
|
||||
// Late activity pushes the sliding window beyond detection + 4 h — the re-arm must
|
||||
// clamp to the cap, and the cap fire must time out even though activity is recent.
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.DesktopArrived, T0.AddMinutes(6))).NewState;
|
||||
// Package completion whose sliding window (T0+230+60) would exceed the cap (T0+245).
|
||||
state = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.RealmJoinPackageCompleted, T0.AddMinutes(230),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PackageId] = "generic-big-suite",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.Scope] = RealmJoinPackageFact.ScopeMachine,
|
||||
[DecisionEngine.RealmJoinPayloadKeys.Success] = "true",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.LastExitCode] = "0",
|
||||
})).NewState;
|
||||
|
||||
var armed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
state = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.DeadlineFired, armed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout })).NewState;
|
||||
|
||||
var rearmed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
Assert.Equal(T0.AddMinutes(5).AddHours(4), rearmed.DueAtUtc); // clamped to detection + cap
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(9, DecisionSignalKind.DeadlineFired, rearmed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout }));
|
||||
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Equal("Timeout", step.NewState.RealmJoinFacts.Outcome!.Value);
|
||||
var timeout = Assert.Single(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_timeout");
|
||||
Assert.Equal("absolute_cap", timeout.Parameters!["reason"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stale_fire_of_old_incarnation_after_rearm_is_dead_end()
|
||||
{
|
||||
// Race: the OLD deadline incarnation's fire was already queued when the extension
|
||||
// re-armed. The armed deadline is due LATER than the stale fire — dead-end, no
|
||||
// second extension evaluation, no timeout.
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "0" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(30),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "0",
|
||||
})).NewState;
|
||||
var armed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
state = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.DeadlineFired, armed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout })).NewState;
|
||||
Assert.Null(state.RealmJoinFacts.Outcome); // extended
|
||||
|
||||
var step = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.DeadlineFired, armed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout }));
|
||||
|
||||
Assert.False(step.Transition.Taken);
|
||||
Assert.Null(step.NewState.RealmJoinFacts.Outcome);
|
||||
Assert.Empty(step.Effects);
|
||||
var stillArmed = Assert.Single(step.NewState.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
Assert.Equal(T0.AddMinutes(90), stillArmed.DueAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Detection_phase_alone_is_not_activity_idle_case_times_out_at_60_min_unchanged()
|
||||
{
|
||||
// The phase captured AT detection must not count as activity — RJ detected in a
|
||||
// first-deployment phase that then never moves still times out after the original
|
||||
// 60 min (the extension exists for demonstrable progress, not for standing still).
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "101" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.DesktopArrived, T0.AddMinutes(6))).NewState;
|
||||
Assert.Null(state.RealmJoinFacts.LastActivityUtc);
|
||||
|
||||
var armed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
var step = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.DeadlineFired, armed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout }));
|
||||
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Equal("Timeout", step.NewState.RealmJoinFacts.Outcome!.Value);
|
||||
var timeout = Assert.Single(step.Effects, e =>
|
||||
e.Kind == DecisionEffectKind.EmitEventTimelineEntry &&
|
||||
e.Parameters != null &&
|
||||
e.Parameters.TryGetValue("eventType", out var et) && et == "realmjoin_timeout");
|
||||
Assert.Equal("hard_timeout", timeout.Parameters!["reason"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Regular_deployment_phase_with_activity_does_not_extend()
|
||||
{
|
||||
// Session 6f1959c0 shape: RJ stood at 210 when the agent booted and deploys
|
||||
// regular (non-first) packages afterwards. Phase 200/210 is outside the
|
||||
// first-deployment window — activity there must NOT hold the session hostage.
|
||||
var engine = new DecisionEngine();
|
||||
var state = PrimeClassicAwaitingDesktop(engine);
|
||||
state = engine.Reduce(state, MakeSignal(5, DecisionSignalKind.RealmJoinDetected, T0.AddMinutes(5),
|
||||
new Dictionary<string, string> { [DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "210" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(6, DecisionSignalKind.DesktopArrived, T0.AddMinutes(6))).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(7, DecisionSignalKind.RealmJoinPhaseChanged, T0.AddMinutes(40),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[DecisionEngine.RealmJoinPayloadKeys.DeploymentPhase] = "200",
|
||||
[DecisionEngine.RealmJoinPayloadKeys.PreviousPhase] = "210",
|
||||
})).NewState;
|
||||
|
||||
var armed = Assert.Single(state.Deadlines, d => d.Name == DeadlineNames.RealmJoinTimeout);
|
||||
var step = engine.Reduce(state, MakeSignal(8, DecisionSignalKind.DeadlineFired, armed.DueAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = DeadlineNames.RealmJoinTimeout }));
|
||||
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.Equal("Timeout", step.NewState.RealmJoinFacts.Outcome!.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SelfDeploying_deferred_terminal_released_by_first_deployment_incomplete()
|
||||
{
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user