mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 13:10:13 +00:00
Compare commits
50 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e8e350bf14 | |||
| 4e519fa4b7 | |||
| 52f6a177f9 | |||
| caafaeafd4 | |||
| bfb6c12384 | |||
| cd4199b1d1 | |||
| d203ceee32 | |||
| 59c012b42e | |||
| 86a122f31f | |||
| 14af0a05cd | |||
| ee41ea80cf | |||
| 20ae3d8c09 | |||
| d0f62a6aaf | |||
| 683142dd27 | |||
| e3f9ad5704 | |||
| 0f98d9ca17 | |||
| bfe345ca93 | |||
| 587ed74d96 | |||
| d5222fcb93 | |||
| 8d24c5680b | |||
| 818f5f1091 | |||
| 485bda3658 | |||
| 0b82faa7bd | |||
| c26542776e | |||
| 80cd6c52f4 | |||
| e40471e61a | |||
| 7908d54999 | |||
| 51aaa2e80a | |||
| 0a6779cd26 | |||
| 1a1c2b413e | |||
| 5e763375b4 | |||
| 9f4f456dc5 | |||
| af2f2c7abe | |||
| 98d87a8579 | |||
| 0709ce2f76 | |||
| 69d876b0a6 | |||
| 224eb668a1 | |||
| a6980a93f6 | |||
| eb57980b53 | |||
| c4c2415de8 | |||
| 9331366025 | |||
| 85a0cb3c31 | |||
| 5efa7d078c | |||
| 35d1445ebb | |||
| 23cd9050ea | |||
| 8615fb458f | |||
| 645f747191 | |||
| 5934ae9fc4 | |||
| 3d10011460 | |||
| edad842a0b |
@@ -38,6 +38,8 @@
|
||||
- Simplicity first: minimal, targeted changes. No temporary fixes — find the root cause.
|
||||
- Where architecture is flawed, state is duplicated, or patterns are inconsistent, propose and implement the structural fix. Ask what a perfectionist reviewer would reject, and fix that too.
|
||||
- On non-trivial changes, ask whether there is a more elegant way before presenting. Skip it for simple fixes — don't over-engineer.
|
||||
- Reuse before you write: search for an existing helper by what it does (its key call, literal or regex), not by a guessed name. Web helpers live in `lib/` (route-only ones in a `lib/` beside the route), backend helpers in `Functions/Helpers/`.
|
||||
- Dead and copied code is gated: the .NET build fails on unused private members and parameters (`.editorconfig`), `npm run knip` covers web and MCP, and `npm run dup` in `tools/quality` fails on new clones. CI only reports, so run them before calling a change done.
|
||||
- English for code comments and everything that carries knowledge.
|
||||
|
||||
Private half of these instructions (process, working files, technical docs, operations): @../internal/CLAUDE.md
|
||||
|
||||
+1
-1
Submodule .claude/commands updated: 1be456ea91...d7b195b1ae
@@ -0,0 +1,11 @@
|
||||
# Analyzer severities for the .NET build. Directory.Build.props runs the IDE rules in
|
||||
# `dotnet build` (EnforceCodeStyleInBuild), and the warning freeze makes every warning an error.
|
||||
root = true
|
||||
|
||||
[*.cs]
|
||||
# Unused private members, private members that are written but never read, and unused parameters
|
||||
# of non-public methods. An injected dependency nobody reads is a feature that silently never runs.
|
||||
dotnet_diagnostic.IDE0051.severity = warning
|
||||
dotnet_diagnostic.IDE0052.severity = warning
|
||||
dotnet_diagnostic.IDE0060.severity = warning
|
||||
dotnet_code_quality_unused_parameters = non_public
|
||||
@@ -81,6 +81,23 @@ updates:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
|
||||
# Duplicate-check tooling (jscpd for tools/quality/check-duplicates.mjs), installed by the
|
||||
# CI duplicates job from this lockfile. A jscpd update can change the clone fingerprints:
|
||||
# if its PR fails the job, regenerate the baseline (npm run dup:baseline) in that PR.
|
||||
- package-ecosystem: npm
|
||||
directory: /tools/quality
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
groups:
|
||||
quality-tooling:
|
||||
patterns: ['*']
|
||||
quality-tooling-security:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
|
||||
# Every action is referenced by commit SHA with the version in a trailing comment;
|
||||
# action-pins.yml fails any workflow change that reintroduces a mutable tag. Dependabot
|
||||
# is what keeps those pins current: it rewrites the SHA and the comment together, so the
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Fails when a workflow or a CI helper spells out a host instead of reading Get-Endpoints.ps1.
|
||||
|
||||
.DESCRIPTION
|
||||
The CI counterpart of the URL-registry guard tests (HardcodedUrlGuardTests,
|
||||
hardcodedUrls.guard.test.ts, hardcoded-urls.guard.test.ts). Enforced: every host the registry
|
||||
defines -- derived from it, so the list cannot fall behind -- plus any Azure storage, App Service
|
||||
or autopilotmonitor.com host, so a new or misspelled account is caught as well.
|
||||
|
||||
Scans .github/workflows/*.yml and .github/scripts/*.ps1 except the registry itself. Comment lines
|
||||
and PowerShell block comments do not count: comments may cite URLs.
|
||||
|
||||
.PARAMETER Root
|
||||
Repository root to scan. Defaults to the parent of .github/scripts.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Root
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not $Root) {
|
||||
$Root = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
|
||||
}
|
||||
|
||||
$endpoints = & (Join-Path $PSScriptRoot 'Get-Endpoints.ps1')
|
||||
$registryHosts = $endpoints.Values | ForEach-Object { ([uri]$_).Host } | Sort-Object -Unique
|
||||
$patterns = @($registryHosts | ForEach-Object { [regex]::Escape($_) }) + @(
|
||||
'[a-z0-9-]+\.(blob|table|queue|file)\.core\.windows\.net'
|
||||
'[a-z0-9-]+\.azurewebsites\.net'
|
||||
'([a-z0-9-]+\.)*autopilotmonitor\.com'
|
||||
)
|
||||
$hostPattern = [regex]::new('(' + ($patterns -join '|') + ')', 'IgnoreCase')
|
||||
|
||||
$files = @(
|
||||
Get-ChildItem -Path (Join-Path $Root '.github/workflows') -File -Filter '*.yml'
|
||||
Get-ChildItem -Path (Join-Path $Root '.github/workflows') -File -Filter '*.yaml'
|
||||
Get-ChildItem -Path (Join-Path $Root '.github/scripts') -File -Filter '*.ps1' |
|
||||
Where-Object Name -ne 'Get-Endpoints.ps1'
|
||||
)
|
||||
|
||||
$findings = 0
|
||||
foreach ($file in $files) {
|
||||
$relative = [System.IO.Path]::GetRelativePath($Root, $file.FullName) -replace '\\', '/'
|
||||
$isPowerShell = $file.Extension -eq '.ps1'
|
||||
$inBlock = $false
|
||||
$lineNumber = 0
|
||||
foreach ($line in [System.IO.File]::ReadAllLines($file.FullName)) {
|
||||
$lineNumber++
|
||||
$code = $line
|
||||
if ($isPowerShell) {
|
||||
if ($inBlock) {
|
||||
$end = $code.IndexOf('#>')
|
||||
if ($end -lt 0) { continue }
|
||||
$code = $code.Substring($end + 2)
|
||||
$inBlock = $false
|
||||
}
|
||||
$start = $code.IndexOf('<#')
|
||||
if ($start -ge 0) {
|
||||
$end = $code.IndexOf('#>', $start + 2)
|
||||
if ($end -lt 0) {
|
||||
$code = $code.Substring(0, $start)
|
||||
$inBlock = $true
|
||||
} else {
|
||||
$code = $code.Remove($start, $end + 2 - $start)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($code.TrimStart().StartsWith('#')) { continue }
|
||||
|
||||
$match = $hostPattern.Match($code)
|
||||
if ($match.Success) {
|
||||
Write-Host "::error file=$relative,line=$lineNumber::$($match.Value) is spelled out here -- take it from .github/scripts/Get-Endpoints.ps1."
|
||||
$findings++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($findings -gt 0) {
|
||||
throw "$findings host literal(s) outside the endpoint registry."
|
||||
}
|
||||
Write-Host "No host literal outside .github/scripts/Get-Endpoints.ps1 ($($files.Count) files scanned)."
|
||||
@@ -0,0 +1,71 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
The one place the workflows and the release scripts take their hosts from.
|
||||
|
||||
.DESCRIPTION
|
||||
CI's member of the URL registry (Shared Constants.cs, web lib/config.ts, MCP src/config.ts):
|
||||
every own host the workflows, the .github/scripts helpers and the local release build talk to,
|
||||
plus the Authenticode timestamp server, is defined here once. Assert-HostsFromRegistry.ps1
|
||||
(Workflow gates) fails on any of these hosts spelled out elsewhere under .github/.
|
||||
|
||||
The storage URLs derive from the account name, so blob and table can never point at two
|
||||
different accounts.
|
||||
|
||||
A file in the repository rather than a GitHub variable: the local release build reads it too,
|
||||
and bearer tokens are sent to these URLs -- where a token goes stays a reviewed line in git.
|
||||
|
||||
.PARAMETER ExportToGitHubEnv
|
||||
Write every entry to $GITHUB_ENV instead of returning it, so the following steps of the job
|
||||
read them as environment variables ($env:AGENT_CONTAINER_URL, ${{ env.AGENT_CONTAINER_URL }}).
|
||||
|
||||
.OUTPUTS
|
||||
[ordered] hashtable, keyed by the names the environment variables carry.
|
||||
|
||||
.EXAMPLE
|
||||
$endpoints = & "$PSScriptRoot/Get-Endpoints.ps1"
|
||||
Invoke-WebRequest -Uri "$($endpoints.AGENT_CONTAINER_URL)/version.json"
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[switch]$ExportToGitHubEnv
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$storageAccount = 'autopilotmonitoreu'
|
||||
# Pre-migration account in another tenant; only the fail-soft legacy mirror still writes there.
|
||||
$legacyStorageAccount = 'autopilotmonitor'
|
||||
|
||||
$blob = "https://$storageAccount.blob.core.windows.net"
|
||||
$table = "https://$storageAccount.table.core.windows.net"
|
||||
|
||||
$endpoints = [ordered]@{
|
||||
STORAGE_BLOB_URL = $blob
|
||||
STORAGE_TABLE_URL = $table
|
||||
AGENT_CONTAINER_URL = "$blob/agent"
|
||||
VERSIONS_CONTAINER_URL = "$blob/versions"
|
||||
STATS_CONTAINER_URL = "$blob/stats"
|
||||
# Platform settings row; CI writes only its agent and bootstrap version columns.
|
||||
ADMIN_CONFIG_URL = "$table/AdminConfiguration(PartitionKey='GlobalConfig',RowKey='config')"
|
||||
LEGACY_AGENT_CONTAINER_URL = "https://$legacyStorageAccount.blob.core.windows.net/agent"
|
||||
# Front Door in front of the agent container: what customer scripts download from.
|
||||
DOWNLOAD_ALIAS_URL = 'https://download.autopilotmonitor.com/agent'
|
||||
API_BASE_URL = 'https://autopilotmonitor-api-eu.azurewebsites.net'
|
||||
MCP_BASE_URL = 'https://mcp.autopilotmonitor.com'
|
||||
PORTAL_BASE_URL = 'https://portal.autopilotmonitor.com'
|
||||
WWW_BASE_URL = 'https://www.autopilotmonitor.com'
|
||||
DOCS_BASE_URL = 'https://docs.autopilotmonitor.com'
|
||||
CODE_SIGN_TIMESTAMP_URL = 'http://timestamp.acs.microsoft.com'
|
||||
}
|
||||
|
||||
if ($ExportToGitHubEnv) {
|
||||
if ([string]::IsNullOrWhiteSpace($env:GITHUB_ENV)) {
|
||||
throw '-ExportToGitHubEnv only works inside a GitHub Actions step.'
|
||||
}
|
||||
$endpoints.GetEnumerator() | ForEach-Object { "$($_.Key)=$($_.Value)" } |
|
||||
Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
$endpoints.GetEnumerator() | ForEach-Object { Write-Host "$($_.Key) = $($_.Value)" }
|
||||
return
|
||||
}
|
||||
|
||||
return $endpoints
|
||||
@@ -81,7 +81,6 @@ param(
|
||||
[string]$SigningToken,
|
||||
[string]$KeyVaultUrl,
|
||||
[string]$CertificateName,
|
||||
[string]$TimestampUrl = 'http://timestamp.acs.microsoft.com',
|
||||
[string]$ExpectedPublisher,
|
||||
[string]$SignedOutputDir,
|
||||
[switch]$DryRun
|
||||
@@ -93,10 +92,13 @@ if (-not $RepoRoot) {
|
||||
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path
|
||||
}
|
||||
|
||||
$ContainerUrl = 'https://autopilotmonitoreu.blob.core.windows.net/agent'
|
||||
$LegacyContainerUrl = 'https://autopilotmonitor.blob.core.windows.net/agent'
|
||||
$AliasUrl = 'https://download.autopilotmonitor.com/agent'
|
||||
$TableUrl = "https://autopilotmonitoreu.table.core.windows.net/AdminConfiguration(PartitionKey='GlobalConfig',RowKey='config')"
|
||||
# Every host, the timestamp server included, from the one endpoint registry.
|
||||
$Endpoints = & (Join-Path $PSScriptRoot 'Get-Endpoints.ps1')
|
||||
$ContainerUrl = $Endpoints.AGENT_CONTAINER_URL
|
||||
$LegacyContainerUrl = $Endpoints.LEGACY_AGENT_CONTAINER_URL
|
||||
$AliasUrl = $Endpoints.DOWNLOAD_ALIAS_URL
|
||||
$TableUrl = $Endpoints.ADMIN_CONFIG_URL
|
||||
$TimestampUrl = $Endpoints.CODE_SIGN_TIMESTAMP_URL
|
||||
|
||||
$BootstrapSource = Join-Path $RepoRoot 'scripts/Bootstrap/Install-AutopilotMonitor.ps1'
|
||||
$BootstrapBlob = 'Install-AutopilotMonitor.ps1'
|
||||
@@ -210,7 +212,7 @@ function Set-PublishSetSignature {
|
||||
--timestamp-rfc3161 $TimestampUrl `
|
||||
--file-digest sha256 `
|
||||
--description 'Autopilot Monitor bootstrap' `
|
||||
--description-url 'https://www.autopilotmonitor.com' `
|
||||
--description-url $Endpoints.WWW_BASE_URL `
|
||||
$paths
|
||||
|
||||
# Native tool: PowerShell does not throw on a non-zero exit, and an unnoticed signing
|
||||
@@ -327,8 +329,10 @@ $loaderVersion = Get-BootstrapScriptVersion -Content $loaderContent -Origin $Loa
|
||||
Write-Host "Loader script version: $loaderVersion"
|
||||
|
||||
# ------------------------------------------------------------------ 2. dev render
|
||||
$stableUrlLiteral = '$AgentDownloadUrl = "https://download.autopilotmonitor.com/agent/AutopilotMonitor-Agent.zip"'
|
||||
$devUrlLiteral = '$AgentDownloadUrl = "https://download.autopilotmonitor.com/agent/AutopilotMonitor-Agent-dev.zip"'
|
||||
# The anchors are the customer scripts' own text, built from the alias in the registry. Should
|
||||
# the alias ever move, the scripts have to move with it -- until they do, the render stops here.
|
||||
$stableUrlLiteral = '$AgentDownloadUrl = "' + $AliasUrl + '/AutopilotMonitor-Agent.zip"'
|
||||
$devUrlLiteral = '$AgentDownloadUrl = "' + $AliasUrl + '/AutopilotMonitor-Agent-dev.zip"'
|
||||
$stableManLiteral = '$VersionJsonName = "version.json"'
|
||||
$devManLiteral = '$VersionJsonName = "version-dev.json"'
|
||||
|
||||
@@ -339,8 +343,8 @@ $devContent = $sourceContent.Replace($stableUrlLiteral, $devUrlLiteral).Replace(
|
||||
Write-Host "Rendered $DevBlob (dev agent URL + version-dev.json)"
|
||||
|
||||
# The dev loader must fetch the dev bootstrap, or a dev device would run the stable chain.
|
||||
$stableLoaderLiteral = '$BootstrapUrl = "https://download.autopilotmonitor.com/agent/Install-AutopilotMonitor.ps1"'
|
||||
$devLoaderLiteral = '$BootstrapUrl = "https://download.autopilotmonitor.com/agent/Install-AutopilotMonitor-Dev.ps1"'
|
||||
$stableLoaderLiteral = '$BootstrapUrl = "' + $AliasUrl + '/Install-AutopilotMonitor.ps1"'
|
||||
$devLoaderLiteral = '$BootstrapUrl = "' + $AliasUrl + '/Install-AutopilotMonitor-Dev.ps1"'
|
||||
|
||||
if ($loaderContent.IndexOf($stableLoaderLiteral) -lt 0) {
|
||||
throw "Loader dev-render: anchor literal missing in $LoaderSource"
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
name: Workflow gates
|
||||
|
||||
# Two checks over the workflow files themselves:
|
||||
# Three checks over the workflow files and their helper scripts:
|
||||
# * verify-pins -- every third-party action is referenced by a full commit SHA plus a
|
||||
# version comment (stricter than zizmor's unpinned-uses audit).
|
||||
# * zizmor -- static analysis of every workflow (permissions, credential
|
||||
# persistence, template injection, ...). Configuration and the
|
||||
# reasoned ignores live in zizmor.yml at the repository root; the
|
||||
# same run is expected locally before a workflow change is committed.
|
||||
# * hosts -- no workflow or .github/scripts helper spells out a host; every one
|
||||
# comes from .github/scripts/Get-Endpoints.ps1, CI's member of the URL
|
||||
# registry (the C#, web and MCP registries have guard tests of their own).
|
||||
#
|
||||
# Every third-party action must be referenced by a full commit SHA, never by a tag.
|
||||
# Tags are mutable: whoever controls an action repository can repoint v4 at new code,
|
||||
@@ -34,10 +37,10 @@ name: Workflow gates
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['.github/workflows/**', '.github/dependabot.yml', 'zizmor.yml']
|
||||
paths: ['.github/workflows/**', '.github/scripts/**', '.github/dependabot.yml', 'zizmor.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['.github/workflows/**', '.github/dependabot.yml', 'zizmor.yml']
|
||||
paths: ['.github/workflows/**', '.github/scripts/**', '.github/dependabot.yml', 'zizmor.yml']
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -106,3 +109,17 @@ jobs:
|
||||
with:
|
||||
version: 1.30.1
|
||||
min-severity: low
|
||||
|
||||
hosts:
|
||||
name: Hosts only from the endpoint registry
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# The enforced host list is derived from the registry itself, plus every Azure storage,
|
||||
# App Service and autopilotmonitor.com host, so it cannot fall behind a new entry.
|
||||
- name: No host literal outside .github/scripts/Get-Endpoints.ps1
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Assert-HostsFromRegistry.ps1
|
||||
|
||||
@@ -32,10 +32,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
|
||||
# Every job states its own permissions. This default is what the build job runs with: it
|
||||
# executes third-party code (MSBuild targets, analyzers, test frameworks) and must never hold
|
||||
# more than a read-only token.
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write # keyless Sigstore signing (artifact attestation)
|
||||
attestations: write # store the provenance attestation
|
||||
contents: read
|
||||
|
||||
# Shared with publish-scripts.yml: both write version.json (this workflow the agent fields,
|
||||
# the script publisher bootstrapVersion under If-Match). Serialising them keeps a script push
|
||||
@@ -48,7 +49,9 @@ env:
|
||||
LINE_MAJOR: 2
|
||||
AGENT_PROJECT: src/Agent/AutopilotMonitor.Agent.V2/AutopilotMonitor.Agent.V2.csproj
|
||||
DIALOG_PROJECT: src/Agent/AutopilotMonitor.SummaryDialog/AutopilotMonitor.SummaryDialog.csproj
|
||||
TEST_PROJECT: src/Agent/AutopilotMonitor.DecisionCore.Tests/AutopilotMonitor.DecisionCore.Tests.csproj
|
||||
DECISIONCORE_TEST_PROJECT: src/Agent/AutopilotMonitor.DecisionCore.Tests/AutopilotMonitor.DecisionCore.Tests.csproj
|
||||
CORE_TEST_PROJECT: src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/AutopilotMonitor.Agent.V2.Core.Tests.csproj
|
||||
DIALOG_TEST_PROJECT: src/Agent/AutopilotMonitor.SummaryDialog.Tests/AutopilotMonitor.SummaryDialog.Tests.csproj
|
||||
CSPROJ_PATH: src/Agent/AutopilotMonitor.Agent.V2/AutopilotMonitor.Agent.V2.csproj
|
||||
BOOTSTRAP_SOURCE: scripts/Bootstrap/Install-AutopilotMonitor.ps1
|
||||
AGENT_OUTPUT: src/Agent/AutopilotMonitor.Agent.V2/bin/Release/net48
|
||||
@@ -57,18 +60,28 @@ env:
|
||||
VERSIONED_ZIP: AutopilotMonitor-Agent-v2.zip
|
||||
STABLE_ZIP: AutopilotMonitor-Agent.zip
|
||||
STABLE_MANIFEST: version.json
|
||||
# Authenticode via Azure Key Vault; identity and vault come from repository variables.
|
||||
# Authenticode via Azure Key Vault; identity and vault come from repository variables, the
|
||||
# timestamp server (CODE_SIGN_TIMESTAMP_URL) from the endpoint registry like every host here.
|
||||
# CODE_SIGN_EXPECTED_SUBJECT is optional and pins the publisher in the verify step.
|
||||
CODE_SIGN_CLIENT_ID: ${{ vars.CODE_SIGN_CLIENT_ID }}
|
||||
CODE_SIGN_CERTIFICATE_NAME: ${{ vars.CODE_SIGN_CERTIFICATE_NAME }}
|
||||
CODE_SIGN_KEY_VAULT_URL: ${{ vars.CODE_SIGN_KEY_VAULT_URL }}
|
||||
CODE_SIGN_EXPECTED_SUBJECT: ${{ vars.CODE_SIGN_EXPECTED_SUBJECT }}
|
||||
CODE_SIGN_TIMESTAMP_URL: http://timestamp.acs.microsoft.com
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: windows-latest
|
||||
name: Build Agent
|
||||
# Runs only this repository's scripts. The deploy identity trusts the `production`
|
||||
# environment, which admits main only.
|
||||
reserve:
|
||||
name: Reserve build number
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # OIDC federated login for the counter write
|
||||
outputs:
|
||||
build_number: ${{ steps.version.outputs.build_number }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@@ -76,6 +89,11 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Every host in this workflow comes from the one endpoint registry.
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
# Every released build is signed; a missing variable stops the run before a build
|
||||
# number is burnt.
|
||||
- name: Check code-signing configuration
|
||||
@@ -88,33 +106,6 @@ jobs:
|
||||
}
|
||||
Write-Host "Code-signing configuration present (certificate '$env:CODE_SIGN_CERTIFICATE_NAME')"
|
||||
|
||||
# Logs in FIRST and only to fetch the Key Vault token: azure/login sets the default az
|
||||
# account, so the storage login below has to be the last one. Both federated credentials
|
||||
# are bound to the ref, so this job must never be given an `environment:` -- that would
|
||||
# change the OIDC subject claim and break both logins.
|
||||
- name: Login to Azure (code-signing identity)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ env.CODE_SIGN_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
allow-no-subscriptions: true
|
||||
|
||||
- name: Acquire Key Vault token for signing
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$token = az account get-access-token --scope https://vault.azure.net/.default --query accessToken --output tsv
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($token)) {
|
||||
throw "Could not acquire a Key Vault token for the code-signing identity."
|
||||
}
|
||||
Write-Host "::add-mask::$token"
|
||||
"CODE_SIGN_AKV_ACCESS_TOKEN=$token" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Every storage write below (counter blob, agent ZIP, manifests, AdminConfiguration)
|
||||
# is authorized by this federated identity -- no SAS secret. One login covers the whole
|
||||
# job: the federated credential is bound to the ref, and a release run takes ~2 minutes,
|
||||
# well inside the token lifetime. Only the legacy mirror still needs a SAS: that account
|
||||
# lives in the pre-migration tenant, where this principal has no identity.
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
@@ -122,18 +113,6 @@ jobs:
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# SDK 10 = single build toolchain (parity with local build.ps1 / global.json).
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
# Version pinned; roll-forward because the tool targets an older runtime than the SDK above.
|
||||
- name: Install AzureSignTool
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: dotnet tool install --no-cache --global AzureSignTool --version 7.0.1
|
||||
|
||||
# Build numbers come from the shared counter blob -- one counter and one
|
||||
# implementation for CI and the local build (.github/scripts/Request-BuildNumber.ps1).
|
||||
# Reserve-before-build with ETag-CAS: concurrent builds can never mint the same
|
||||
@@ -155,7 +134,7 @@ jobs:
|
||||
$override = [int]$env:BUILD_NUMBER_OVERRIDE
|
||||
}
|
||||
|
||||
$counterUrl = "https://autopilotmonitoreu.blob.core.windows.net/agent/buildcounter-v$($env:LINE_MAJOR).txt"
|
||||
$counterUrl = "$env:AGENT_CONTAINER_URL/buildcounter-v$($env:LINE_MAJOR).txt"
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl $counterUrl -Override $override
|
||||
|
||||
$csproj = Get-Content $env:CSPROJ_PATH -Raw
|
||||
@@ -172,22 +151,166 @@ jobs:
|
||||
"version=$version" >> $env:GITHUB_OUTPUT
|
||||
"tag=v$version" >> $env:GITHUB_OUTPUT
|
||||
|
||||
# Build zone: no login, no secret, no id-token. Its only output is the artifact; the
|
||||
# release job treats that as data and never builds.
|
||||
build:
|
||||
name: Build and test
|
||||
needs: reserve
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# SDK 10 = single build toolchain (parity with local build.ps1 / global.json).
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
- name: Restore dependencies
|
||||
run: |
|
||||
dotnet restore ${{ env.AGENT_PROJECT }}
|
||||
dotnet restore ${{ env.DIALOG_PROJECT }}
|
||||
dotnet restore ${{ env.TEST_PROJECT }}
|
||||
dotnet restore ${{ env.DECISIONCORE_TEST_PROJECT }}
|
||||
dotnet restore ${{ env.CORE_TEST_PROJECT }}
|
||||
dotnet restore ${{ env.DIALOG_TEST_PROJECT }}
|
||||
|
||||
# Every agent suite runs before the release build. ORDER AND BUILD_NUMBER ARE LOAD-BEARING:
|
||||
# Agent.V2.Core.Tests references the agent exe, so the test build compiles it. With the
|
||||
# reserved number it stamps the release version and the Build Agent step is up to date; run
|
||||
# after that step or without the number, it would rewrite the binaries with another version.
|
||||
- name: Run agent tests (all suites)
|
||||
shell: pwsh
|
||||
env:
|
||||
BUILD_NUMBER: ${{ needs.reserve.outputs.build_number }}
|
||||
run: |
|
||||
foreach ($project in @($env:DECISIONCORE_TEST_PROJECT, $env:CORE_TEST_PROJECT, $env:DIALOG_TEST_PROJECT)) {
|
||||
dotnet test $project --configuration Release --no-restore --verbosity quiet
|
||||
if ($LASTEXITCODE -ne 0) { throw "Tests failed: $project" }
|
||||
}
|
||||
|
||||
- name: Build Agent
|
||||
run: dotnet build ${{ env.AGENT_PROJECT }} --configuration Release --no-restore
|
||||
env:
|
||||
BUILD_NUMBER: ${{ steps.version.outputs.build_number }}
|
||||
BUILD_NUMBER: ${{ needs.reserve.outputs.build_number }}
|
||||
|
||||
- name: Build SummaryDialog
|
||||
run: dotnet build ${{ env.DIALOG_PROJECT }} --configuration Release --no-restore
|
||||
|
||||
- name: Run Agent unit tests
|
||||
run: dotnet test ${{ env.TEST_PROJECT }} --configuration Release --no-restore --verbosity quiet
|
||||
# One archive per output folder, unsigned. ZipFile rather than Compress-Archive, which
|
||||
# skips hidden files.
|
||||
- name: Package build output
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
||||
$outDir = Join-Path $env:RUNNER_TEMP 'agent-unsigned'
|
||||
New-Item -Path $outDir -ItemType Directory -Force | Out-Null
|
||||
[System.IO.Compression.ZipFile]::CreateFromDirectory((Join-Path $env:GITHUB_WORKSPACE $env:AGENT_OUTPUT), (Join-Path $outDir 'agent.zip'))
|
||||
[System.IO.Compression.ZipFile]::CreateFromDirectory((Join-Path $env:GITHUB_WORKSPACE $env:DIALOG_OUTPUT), (Join-Path $outDir 'dialog.zip'))
|
||||
Get-ChildItem $outDir | ForEach-Object { Write-Host "$($_.Name): $([math]::Round($_.Length / 1MB, 2)) MB" }
|
||||
|
||||
- name: Upload unsigned build output
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: agent-unsigned
|
||||
path: ${{ runner.temp }}/agent-unsigned
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Trust zone: only this repository's scripts, pinned actions and the pinned signing tool
|
||||
# run here -- never a restore or build. The build output arrives as data.
|
||||
#
|
||||
# No `environment:` yet: the code-signing credential is still bound to the ref, and a job
|
||||
# carries exactly one OIDC subject, so an environment here would break the signing login.
|
||||
release:
|
||||
name: Sign and publish
|
||||
needs: [reserve, build]
|
||||
runs-on: windows-latest
|
||||
permissions:
|
||||
contents: write # tag + GitHub Release
|
||||
id-token: write # code-signing and storage logins, keyless Sigstore signing
|
||||
attestations: write # store the provenance attestation
|
||||
env:
|
||||
AGENT_VERSION: ${{ needs.reserve.outputs.version }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Storage, legacy mirror, docs link, timestamp server: all from the endpoint registry.
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
- name: Download unsigned build output
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: agent-unsigned
|
||||
path: ${{ runner.temp }}/agent-unsigned
|
||||
|
||||
# Only the two expected archives, each into its own fresh output folder; the .NET
|
||||
# extractor refuses entries that would land outside it. Never unpack into the checkout
|
||||
# root: an archive could otherwise replace the scripts this job runs below.
|
||||
- name: Unpack build output
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
||||
foreach ($pair in @(@('agent.zip', $env:AGENT_OUTPUT), @('dialog.zip', $env:DIALOG_OUTPUT))) {
|
||||
$archive = Join-Path $env:RUNNER_TEMP "agent-unsigned/$($pair[0])"
|
||||
$target = Join-Path $env:GITHUB_WORKSPACE $pair[1]
|
||||
if (Test-Path $target) { throw "$($pair[1]) already exists -- refusing to unpack over it." }
|
||||
[System.IO.Compression.ZipFile]::ExtractToDirectory($archive, $target)
|
||||
Write-Host "Unpacked $($pair[0]) into $($pair[1])"
|
||||
}
|
||||
|
||||
# Logs in FIRST and only to fetch the Key Vault token: azure/login sets the default az
|
||||
# account, so the storage login below has to be the last one.
|
||||
- name: Login to Azure (code-signing identity)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ env.CODE_SIGN_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
allow-no-subscriptions: true
|
||||
|
||||
- name: Acquire Key Vault token for signing
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$token = az account get-access-token --scope https://vault.azure.net/.default --query accessToken --output tsv
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($token)) {
|
||||
throw "Could not acquire a Key Vault token for the code-signing identity."
|
||||
}
|
||||
Write-Host "::add-mask::$token"
|
||||
"CODE_SIGN_AKV_ACCESS_TOKEN=$token" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Every storage write below (agent ZIP, manifests, AdminConfiguration) is authorized by
|
||||
# this federated identity -- no SAS secret. Only the legacy mirror still needs a SAS:
|
||||
# that account lives in the pre-migration tenant, where this principal has no identity.
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# Runtime for AzureSignTool only; nothing is built in this job.
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
# Version pinned; roll-forward because the tool targets an older runtime than the SDK above.
|
||||
- name: Install AzureSignTool
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: dotnet tool install --no-cache --global AzureSignTool --version 7.0.1
|
||||
|
||||
# Authenticode over everything we author; third-party assemblies keep their own signature.
|
||||
#
|
||||
@@ -223,7 +346,7 @@ jobs:
|
||||
--timestamp-rfc3161 "$env:CODE_SIGN_TIMESTAMP_URL" `
|
||||
--file-digest sha256 `
|
||||
--description "Autopilot Monitor Agent" `
|
||||
--description-url "https://www.autopilotmonitor.com" `
|
||||
--description-url "$env:WWW_BASE_URL" `
|
||||
($targets | ForEach-Object { $_.FullName })
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
@@ -234,7 +357,7 @@ jobs:
|
||||
id: artifact
|
||||
shell: pwsh
|
||||
run: |
|
||||
$version = "${{ steps.version.outputs.version }}"
|
||||
$version = $env:AGENT_VERSION
|
||||
$stagingDir = "staging/agent"
|
||||
|
||||
if (-not (Test-Path "$env:AGENT_OUTPUT/AutopilotMonitor.Agent.exe")) {
|
||||
@@ -368,14 +491,17 @@ jobs:
|
||||
- name: Upload to Azure Blob Storage
|
||||
if: inputs.upload_blob
|
||||
shell: pwsh
|
||||
env:
|
||||
ZIP_SHA256: ${{ steps.artifact.outputs.sha256 }}
|
||||
EXE_SHA256: ${{ steps.artifact.outputs.exe_sha256 }}
|
||||
run: |
|
||||
$containerUrl = "https://autopilotmonitoreu.blob.core.windows.net/agent"
|
||||
$containerUrl = $env:AGENT_CONTAINER_URL
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
# Legacy mirror (transition until customers moved to the download.-alias):
|
||||
# blobs are ALSO pushed to the old account so the bootstrap URL baked into
|
||||
# customers' Intune scripts keeps serving current bits. Fail-soft -- a
|
||||
# legacy hiccup must never fail a release. Counter lives ONLY on eu.
|
||||
$legacyContainerUrl = "https://autopilotmonitor.blob.core.windows.net/agent"
|
||||
$legacyContainerUrl = $env:LEGACY_AGENT_CONTAINER_URL
|
||||
$legacySasToken = "${{ secrets.AZURE_BLOB_AGENT_LEGACY_SAS_TOKEN }}".TrimStart('?')
|
||||
|
||||
function Send-Blob {
|
||||
@@ -426,7 +552,6 @@ jobs:
|
||||
-SigningToken $env:CODE_SIGN_AKV_ACCESS_TOKEN `
|
||||
-KeyVaultUrl $env:CODE_SIGN_KEY_VAULT_URL `
|
||||
-CertificateName $env:CODE_SIGN_CERTIFICATE_NAME `
|
||||
-TimestampUrl $env:CODE_SIGN_TIMESTAMP_URL `
|
||||
-ExpectedPublisher $env:CODE_SIGN_EXPECTED_SUBJECT
|
||||
}
|
||||
|
||||
@@ -437,10 +562,10 @@ jobs:
|
||||
if ('${{ inputs.publish_as_stable }}' -ne 'true') {
|
||||
Write-Host "SKIPPED: AdminConfiguration update (versioned-only run -- stable namespace unchanged)"
|
||||
} else {
|
||||
$sha256 = "${{ steps.artifact.outputs.sha256 }}"
|
||||
$exeSha256 = "${{ steps.artifact.outputs.exe_sha256 }}"
|
||||
$agentVersion = "${{ steps.version.outputs.version }}"
|
||||
$tableUrl = "https://autopilotmonitoreu.table.core.windows.net/AdminConfiguration(PartitionKey='GlobalConfig',RowKey='config')"
|
||||
$sha256 = $env:ZIP_SHA256
|
||||
$exeSha256 = $env:EXE_SHA256
|
||||
$agentVersion = $env:AGENT_VERSION
|
||||
$tableUrl = $env:ADMIN_CONFIG_URL
|
||||
$tableBody = @{
|
||||
LatestAgentV2Version = $agentVersion
|
||||
LatestAgentV2Sha256 = $sha256
|
||||
@@ -474,7 +599,7 @@ jobs:
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: AutopilotMonitor-Agent-v${{ steps.version.outputs.version }}
|
||||
name: AutopilotMonitor-Agent-v${{ needs.reserve.outputs.version }}
|
||||
path: ${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
|
||||
- name: Upload version metadata
|
||||
@@ -497,15 +622,15 @@ jobs:
|
||||
shell: pwsh
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ steps.version.outputs.tag }}
|
||||
RELEASE_NAME: Autopilot Monitor Agent ${{ steps.version.outputs.version }}
|
||||
RELEASE_TAG: ${{ needs.reserve.outputs.tag }}
|
||||
RELEASE_NAME: Autopilot Monitor Agent ${{ needs.reserve.outputs.version }}
|
||||
RELEASE_PRERELEASE: ${{ inputs.prerelease }}
|
||||
RELEASE_ZIP: ${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
RELEASE_NOTES: ${{ inputs.release_notes }}
|
||||
RELEASE_BODY_TAIL: |
|
||||
### Version Info
|
||||
- **Version:** ${{ steps.version.outputs.version }}
|
||||
- **Build Number:** ${{ steps.version.outputs.build_number }}
|
||||
- **Version:** ${{ needs.reserve.outputs.version }}
|
||||
- **Build Number:** ${{ needs.reserve.outputs.build_number }}
|
||||
- **SHA-256:** `${{ steps.artifact.outputs.sha256 }}`
|
||||
- **Built from:** ${{ github.sha }}
|
||||
- **Published as stable:** ${{ inputs.publish_as_stable }}
|
||||
@@ -513,14 +638,14 @@ jobs:
|
||||
### Provenance
|
||||
The executables are Authenticode-signed and timestamped. The ZIP and both executables carry a [GitHub Artifact Attestation](https://github.com/okieselbach/AutopilotMonitor/attestations) (Sigstore keyless). Verify:
|
||||
```
|
||||
gh attestation verify AutopilotMonitor-Agent-v${{ steps.version.outputs.version }}.zip --repo okieselbach/AutopilotMonitor
|
||||
gh attestation verify AutopilotMonitor-Agent-v${{ needs.reserve.outputs.version }}.zip --repo okieselbach/AutopilotMonitor
|
||||
```
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$notes = "$env:RELEASE_NOTES".Trim()
|
||||
if ($notes -eq '') {
|
||||
$notes = "Minor enhancements and bug fixes. See the [Agent Changelog](https://docs.autopilotmonitor.com/changelog/agent-changelog) for details.`n<!-- release-notes: none -->"
|
||||
$notes = "Minor enhancements and bug fixes. See the [Agent Changelog]($env:DOCS_BASE_URL/changelog/agent-changelog) for details.`n<!-- release-notes: none -->"
|
||||
}
|
||||
|
||||
# Previous release for the compare link. Fail-soft: the first release has none,
|
||||
|
||||
+191
-70
@@ -11,32 +11,42 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
upload_blob:
|
||||
description: 'Upload to blob (customer-facing via download.autopilotmonitor.com/agent/)'
|
||||
description: 'Upload to blob (customer-facing through the download alias)'
|
||||
required: true
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
# Every job states its own permissions. This default is what the build job runs with: wix is
|
||||
# third-party code and must never hold more than a read-only token.
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # keyless Sigstore signing (artifact attestation)
|
||||
attestations: write # store the provenance attestation
|
||||
|
||||
env:
|
||||
MSI_PROJECT_DIR: src/Agent/AutopilotMonitor.BootstrapMsi
|
||||
RUNNER_SCRIPT: scripts/Bootstrap/Start-AutopilotMonitor.ps1
|
||||
STABLE_MSI: AutopilotMonitor-Bootstrap.msi
|
||||
# Authenticode via Azure Key Vault; identity and vault come from repository variables.
|
||||
# Authenticode via Azure Key Vault; identity and vault come from repository variables, the
|
||||
# timestamp server (CODE_SIGN_TIMESTAMP_URL) from the endpoint registry like every host here.
|
||||
# CODE_SIGN_EXPECTED_SUBJECT is optional and pins the publisher in the verify steps.
|
||||
CODE_SIGN_CLIENT_ID: ${{ vars.CODE_SIGN_CLIENT_ID }}
|
||||
CODE_SIGN_CERTIFICATE_NAME: ${{ vars.CODE_SIGN_CERTIFICATE_NAME }}
|
||||
CODE_SIGN_KEY_VAULT_URL: ${{ vars.CODE_SIGN_KEY_VAULT_URL }}
|
||||
CODE_SIGN_EXPECTED_SUBJECT: ${{ vars.CODE_SIGN_EXPECTED_SUBJECT }}
|
||||
CODE_SIGN_TIMESTAMP_URL: http://timestamp.acs.microsoft.com
|
||||
|
||||
jobs:
|
||||
build:
|
||||
# Signs the loader BEFORE the build: wix embeds the file as it finds it, so a loader signed
|
||||
# afterwards would leave the copy inside the MSI unsigned. Devices under an AllSigned policy
|
||||
# need the embedded loader to carry a signature of its own.
|
||||
#
|
||||
# Trust zone: only this repository's scripts, pinned actions and the pinned signing tool run
|
||||
# here. No `environment:` yet: the code-signing credential is still bound to the ref, and a
|
||||
# job carries exactly one OIDC subject.
|
||||
sign-loader:
|
||||
name: Sign bootstrap loader
|
||||
runs-on: windows-latest
|
||||
name: Build Bootstrap MSI
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # code-signing login
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@@ -44,6 +54,11 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Every host in this workflow comes from the one endpoint registry.
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
# The loader executes as SYSTEM on every enrolling device — a parse error or
|
||||
# non-ASCII mojibake (PS 5.1 codepage roulette) must fail the build, not the fleet.
|
||||
# bootstrap-script-gates.yml runs the same checks plus Pester on pushes; this job
|
||||
@@ -60,6 +75,151 @@ jobs:
|
||||
}
|
||||
Write-Host "Runner script parse + ASCII check passed"
|
||||
|
||||
# Every published MSI is signed; a missing variable stops the run before the build.
|
||||
- name: Check code-signing configuration
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($name in 'CODE_SIGN_CLIENT_ID', 'CODE_SIGN_CERTIFICATE_NAME', 'CODE_SIGN_KEY_VAULT_URL') {
|
||||
if ([string]::IsNullOrWhiteSpace([Environment]::GetEnvironmentVariable($name))) {
|
||||
throw "Repository variable $name is not set. Every published MSI must be Authenticode-signed."
|
||||
}
|
||||
}
|
||||
|
||||
- name: Login to Azure (code-signing identity)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ env.CODE_SIGN_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
allow-no-subscriptions: true
|
||||
|
||||
- name: Acquire Key Vault token for signing
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$token = az account get-access-token --scope https://vault.azure.net/.default --query accessToken --output tsv
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($token)) {
|
||||
throw "Could not acquire a Key Vault token for the code-signing identity."
|
||||
}
|
||||
Write-Host "::add-mask::$token"
|
||||
"CODE_SIGN_AKV_ACCESS_TOKEN=$token" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
# Version pinned; roll-forward because the tool targets an older runtime than the SDK.
|
||||
- name: Install AzureSignTool
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: dotnet tool install --no-cache --global AzureSignTool --version 7.0.1
|
||||
|
||||
# A missing timestamp fails the run: without it the signature dies with the certificate.
|
||||
- name: Sign bootstrap loader (Authenticode)
|
||||
shell: pwsh
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
& azuresigntool.exe sign --verbose `
|
||||
--azure-key-vault-url "$env:CODE_SIGN_KEY_VAULT_URL" `
|
||||
--azure-key-vault-accesstoken "$env:CODE_SIGN_AKV_ACCESS_TOKEN" `
|
||||
--azure-key-vault-certificate "$env:CODE_SIGN_CERTIFICATE_NAME" `
|
||||
--timestamp-rfc3161 "$env:CODE_SIGN_TIMESTAMP_URL" `
|
||||
--file-digest sha256 `
|
||||
--description 'Autopilot Monitor bootstrap runner' `
|
||||
--description-url $env:WWW_BASE_URL `
|
||||
$env:RUNNER_SCRIPT
|
||||
if ($LASTEXITCODE -ne 0) { throw "azuresigntool failed with exit code $LASTEXITCODE" }
|
||||
|
||||
$sig = Get-AuthenticodeSignature -FilePath $env:RUNNER_SCRIPT
|
||||
if ($sig.Status -ne 'Valid') { throw "Signature on $env:RUNNER_SCRIPT is '$($sig.Status)', expected 'Valid'." }
|
||||
if (-not $sig.TimeStamperCertificate) { throw "Signature on $env:RUNNER_SCRIPT carries no timestamp." }
|
||||
if ($env:CODE_SIGN_EXPECTED_SUBJECT -and $sig.SignerCertificate.Subject -notlike $env:CODE_SIGN_EXPECTED_SUBJECT) {
|
||||
throw "$env:RUNNER_SCRIPT was signed by '$($sig.SignerCertificate.Subject)', expected '$env:CODE_SIGN_EXPECTED_SUBJECT'."
|
||||
}
|
||||
Write-Host "$env:RUNNER_SCRIPT : Valid, timestamped, signer $($sig.SignerCertificate.Subject.Split(',')[0])"
|
||||
|
||||
- name: Upload signed loader
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: bootstrap-loader-signed
|
||||
path: ${{ env.RUNNER_SCRIPT }}
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Build zone: no login, no secret, no id-token. Its only output is the artifact; the
|
||||
# release job treats that as data and never builds.
|
||||
build:
|
||||
name: Build MSI
|
||||
needs: sign-loader
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download signed loader
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: bootstrap-loader-signed
|
||||
path: ${{ runner.temp }}/loader
|
||||
|
||||
# The signed copy replaces the checkout's unsigned one before wix embeds it.
|
||||
- name: Put the signed loader in place
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Copy-Item -Path (Join-Path $env:RUNNER_TEMP 'loader/Start-AutopilotMonitor.ps1') -Destination $env:RUNNER_SCRIPT -Force
|
||||
$sig = Get-AuthenticodeSignature -FilePath $env:RUNNER_SCRIPT
|
||||
if ($sig.Status -ne 'Valid') { throw "Loader to embed is '$($sig.Status)', expected 'Valid'." }
|
||||
Write-Host "Embedding the signed loader ($((Get-Item $env:RUNNER_SCRIPT).Length) bytes)"
|
||||
|
||||
# Exact version, not a "5.*" range: wix writes the bytes of the MSI that customers
|
||||
# install, so a compromised or simply broken point release must not flow into a build
|
||||
# unnoticed. Bumping is a reviewed commit.
|
||||
- name: Install WiX toolset
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
|
||||
- name: Build MSI
|
||||
shell: pwsh
|
||||
run: |
|
||||
# wix resolves File/@Source relative to the working directory, not the wxs —
|
||||
# build from inside the project dir (same as the local build documented in the wxs).
|
||||
$out = Join-Path (Get-Location) $env:STABLE_MSI
|
||||
Push-Location $env:MSI_PROJECT_DIR
|
||||
wix build Package.wxs -arch x64 -o $out
|
||||
Pop-Location
|
||||
if (-not (Test-Path $env:STABLE_MSI)) { Write-Error "wix build produced no MSI"; exit 1 }
|
||||
Write-Host "Built $env:STABLE_MSI ($([math]::Round((Get-Item $env:STABLE_MSI).Length / 1KB)) KB)"
|
||||
|
||||
- name: Upload unsigned MSI
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: bootstrap-msi-unsigned
|
||||
path: ${{ env.STABLE_MSI }}
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Trust zone: signs, verifies and publishes the MSI as data; never builds. No
|
||||
# `environment:` yet, for the same reason as sign-loader.
|
||||
release:
|
||||
name: Sign and publish MSI
|
||||
needs: build
|
||||
runs-on: windows-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # code-signing and storage logins, keyless Sigstore signing
|
||||
attestations: write # store the provenance attestation
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
# Read from this job's own checkout, never handed over by the build job.
|
||||
- name: Read MSI version from Package.wxs
|
||||
id: version
|
||||
shell: pwsh
|
||||
@@ -73,20 +233,19 @@ jobs:
|
||||
Write-Host "MSI version (from Package.wxs): $version"
|
||||
"version=$version" >> $env:GITHUB_OUTPUT
|
||||
|
||||
# Every published MSI is signed; a missing variable stops the run before the build.
|
||||
- name: Check code-signing configuration
|
||||
- name: Download unsigned MSI
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: bootstrap-msi-unsigned
|
||||
path: ${{ runner.temp }}/msi
|
||||
|
||||
# Only the expected file leaves the download folder.
|
||||
- name: Take the MSI from the build
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($name in 'CODE_SIGN_CLIENT_ID', 'CODE_SIGN_CERTIFICATE_NAME', 'CODE_SIGN_KEY_VAULT_URL') {
|
||||
if ([string]::IsNullOrWhiteSpace([Environment]::GetEnvironmentVariable($name))) {
|
||||
throw "Repository variable $name is not set. Every published MSI must be Authenticode-signed."
|
||||
}
|
||||
}
|
||||
run: Copy-Item -Path (Join-Path $env:RUNNER_TEMP "msi/$env:STABLE_MSI") -Destination $env:STABLE_MSI -ErrorAction Stop
|
||||
|
||||
# Logs in FIRST and only to fetch the Key Vault token: azure/login sets the default az
|
||||
# account, so the storage login below has to be the last one. Both federated credentials
|
||||
# are bound to the ref, so this job must never be given an `environment:` -- that would
|
||||
# change the OIDC subject claim and break both logins.
|
||||
# account, so the storage login below has to be the last one.
|
||||
- name: Login to Azure (code-signing identity)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
@@ -113,50 +272,12 @@ jobs:
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# Exact version, not a "5.*" range: wix writes the bytes of the MSI that customers
|
||||
# install, so a compromised or simply broken point release must not flow into a build
|
||||
# unnoticed. Bumping is a reviewed commit.
|
||||
- name: Install WiX toolset
|
||||
run: dotnet tool install --global wix --version 5.0.2
|
||||
|
||||
# Version pinned; roll-forward because the tool targets an older runtime than the SDK.
|
||||
- name: Install AzureSignTool
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: dotnet tool install --no-cache --global AzureSignTool --version 7.0.1
|
||||
|
||||
# BEFORE the build: wix embeds the file as it finds it, so a loader signed afterwards
|
||||
# would leave the copy inside the MSI unsigned. Devices under an AllSigned policy need
|
||||
# the embedded loader to carry a signature of its own.
|
||||
- name: Sign bootstrap loader (Authenticode)
|
||||
shell: pwsh
|
||||
env:
|
||||
DOTNET_ROLL_FORWARD: LatestMajor
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
& azuresigntool.exe sign --verbose `
|
||||
--azure-key-vault-url "$env:CODE_SIGN_KEY_VAULT_URL" `
|
||||
--azure-key-vault-accesstoken "$env:CODE_SIGN_AKV_ACCESS_TOKEN" `
|
||||
--azure-key-vault-certificate "$env:CODE_SIGN_CERTIFICATE_NAME" `
|
||||
--timestamp-rfc3161 "$env:CODE_SIGN_TIMESTAMP_URL" `
|
||||
--file-digest sha256 `
|
||||
--description 'Autopilot Monitor bootstrap runner' `
|
||||
--description-url 'https://www.autopilotmonitor.com' `
|
||||
$env:RUNNER_SCRIPT
|
||||
if ($LASTEXITCODE -ne 0) { throw "azuresigntool failed with exit code $LASTEXITCODE" }
|
||||
|
||||
- name: Build MSI
|
||||
shell: pwsh
|
||||
run: |
|
||||
# wix resolves File/@Source relative to the working directory, not the wxs —
|
||||
# build from inside the project dir (same as the local build documented in the wxs).
|
||||
$out = Join-Path (Get-Location) $env:STABLE_MSI
|
||||
Push-Location $env:MSI_PROJECT_DIR
|
||||
wix build Package.wxs -arch x64 -o $out
|
||||
Pop-Location
|
||||
if (-not (Test-Path $env:STABLE_MSI)) { Write-Error "wix build produced no MSI"; exit 1 }
|
||||
Write-Host "Built $env:STABLE_MSI ($([math]::Round((Get-Item $env:STABLE_MSI).Length / 1KB)) KB)"
|
||||
|
||||
# Signing rewrites the MSI, so it runs before the hash and the upload. A missing
|
||||
# timestamp fails the run: without it the signature dies with the certificate.
|
||||
- name: Sign MSI (Authenticode)
|
||||
@@ -172,30 +293,30 @@ jobs:
|
||||
--timestamp-rfc3161 "$env:CODE_SIGN_TIMESTAMP_URL" `
|
||||
--file-digest sha256 `
|
||||
--description 'Autopilot Monitor Bootstrap' `
|
||||
--description-url 'https://www.autopilotmonitor.com' `
|
||||
--description-url $env:WWW_BASE_URL `
|
||||
$env:STABLE_MSI
|
||||
if ($LASTEXITCODE -ne 0) { throw "azuresigntool failed with exit code $LASTEXITCODE" }
|
||||
|
||||
# The MSI and the loader it carries -- verified here, before anything is published.
|
||||
foreach ($file in @($env:STABLE_MSI, $env:RUNNER_SCRIPT)) {
|
||||
$sig = Get-AuthenticodeSignature -FilePath $file
|
||||
if ($sig.Status -ne 'Valid') { throw "Signature on $file is '$($sig.Status)', expected 'Valid'." }
|
||||
if (-not $sig.TimeStamperCertificate) { throw "Signature on $file carries no timestamp." }
|
||||
if ($env:CODE_SIGN_EXPECTED_SUBJECT -and $sig.SignerCertificate.Subject -notlike $env:CODE_SIGN_EXPECTED_SUBJECT) {
|
||||
throw "$file was signed by '$($sig.SignerCertificate.Subject)', expected '$env:CODE_SIGN_EXPECTED_SUBJECT'."
|
||||
}
|
||||
Write-Host "$file : Valid, timestamped, signer $($sig.SignerCertificate.Subject.Split(',')[0])"
|
||||
# The loader was verified in sign-loader; here the MSI, before anything is published.
|
||||
$sig = Get-AuthenticodeSignature -FilePath $env:STABLE_MSI
|
||||
if ($sig.Status -ne 'Valid') { throw "Signature on $env:STABLE_MSI is '$($sig.Status)', expected 'Valid'." }
|
||||
if (-not $sig.TimeStamperCertificate) { throw "Signature on $env:STABLE_MSI carries no timestamp." }
|
||||
if ($env:CODE_SIGN_EXPECTED_SUBJECT -and $sig.SignerCertificate.Subject -notlike $env:CODE_SIGN_EXPECTED_SUBJECT) {
|
||||
throw "$env:STABLE_MSI was signed by '$($sig.SignerCertificate.Subject)', expected '$env:CODE_SIGN_EXPECTED_SUBJECT'."
|
||||
}
|
||||
Write-Host "$env:STABLE_MSI : Valid, timestamped, signer $($sig.SignerCertificate.Subject.Split(',')[0])"
|
||||
|
||||
Write-Host "SHA-256: $((Get-FileHash $env:STABLE_MSI -Algorithm SHA256).Hash.ToLowerInvariant())"
|
||||
|
||||
- name: Upload to Azure Blob Storage
|
||||
if: inputs.upload_blob
|
||||
shell: pwsh
|
||||
env:
|
||||
MSI_VERSION: ${{ steps.version.outputs.version }}
|
||||
run: |
|
||||
$containerUrl = "https://autopilotmonitoreu.blob.core.windows.net/agent"
|
||||
$containerUrl = $env:AGENT_CONTAINER_URL
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
$version = "${{ steps.version.outputs.version }}"
|
||||
$version = $env:MSI_VERSION
|
||||
|
||||
# Same Send-Blob contract as build-agent.yml: no-cache on every blob because the
|
||||
# stable name rotates in place behind the download alias. No legacy mirror — the
|
||||
@@ -226,14 +347,14 @@ jobs:
|
||||
shell: pwsh
|
||||
run: |
|
||||
$localSize = (Get-Item $env:STABLE_MSI).Length
|
||||
$resp = Invoke-WebRequest -Uri "https://download.autopilotmonitor.com/agent/$env:STABLE_MSI" -Method Head -UseBasicParsing
|
||||
$resp = Invoke-WebRequest -Uri "$env:DOWNLOAD_ALIAS_URL/$env:STABLE_MSI" -Method Head -UseBasicParsing
|
||||
if ($resp.StatusCode -ne 200) { Write-Error "Alias HEAD returned $($resp.StatusCode)"; exit 1 }
|
||||
$remoteSize = [long]$resp.Headers['Content-Length'][0]
|
||||
if ($remoteSize -ne $localSize) {
|
||||
Write-Error "Size mismatch: alias serves $remoteSize bytes, local MSI is $localSize bytes"
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Alias verified: https://download.autopilotmonitor.com/agent/$env:STABLE_MSI ($remoteSize bytes)"
|
||||
Write-Host "Alias verified: $env:DOWNLOAD_ALIAS_URL/$env:STABLE_MSI ($remoteSize bytes)"
|
||||
|
||||
# Sigstore keyless provenance, same as the agent ZIP; covers the signed bytes. Skipped
|
||||
# on runs that publish nothing -- no log entry for bytes nobody receives. Verify:
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
name: CI
|
||||
|
||||
# Tests, typechecks and drift-checks every push to main and every pull request:
|
||||
# the .NET suites, the web and MCP suites, and a guard that the generated rule
|
||||
# files still match rules/guardrails.json.
|
||||
# the .NET suites, the web and MCP suites, a guard that the generated rule
|
||||
# files still match rules/guardrails.json, and the duplicate-code check.
|
||||
#
|
||||
# The deploy workflows build but never test, so this is the only place the suites
|
||||
# actually run. It does not gate anything — main is deliberately unprotected, so a
|
||||
@@ -46,7 +46,6 @@ on:
|
||||
- '.github/workflows/deploy-mcp.yml'
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
- '.github/workflows/indexnow-notify.yml'
|
||||
- '.github/workflows/oidc-login-test.yml'
|
||||
- '.github/workflows/pester.yml'
|
||||
- '.github/workflows/publish-scripts.yml'
|
||||
pull_request:
|
||||
@@ -80,7 +79,6 @@ on:
|
||||
- '.github/workflows/deploy-mcp.yml'
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
- '.github/workflows/indexnow-notify.yml'
|
||||
- '.github/workflows/oidc-login-test.yml'
|
||||
- '.github/workflows/pester.yml'
|
||||
- '.github/workflows/publish-scripts.yml'
|
||||
workflow_dispatch:
|
||||
@@ -160,6 +158,12 @@ jobs:
|
||||
- name: Lint
|
||||
run: npm run lint -- --max-warnings 0
|
||||
|
||||
# Unused files, exports and dependencies (knip.json). Its own script on
|
||||
# purpose: chained into `lint`, the `-- --max-warnings 0` above would be
|
||||
# handed to knip and ESLint would lose its cap.
|
||||
- name: Unused code
|
||||
run: npm run knip
|
||||
|
||||
- name: Tests
|
||||
run: npm test
|
||||
|
||||
@@ -182,9 +186,19 @@ jobs:
|
||||
|
||||
- run: npm ci
|
||||
|
||||
# tsconfig.json covers sources, tests and scripts; the build emits via
|
||||
# tsconfig.build.json.
|
||||
- name: Typecheck
|
||||
run: npx tsc --noEmit
|
||||
|
||||
# Same rule as the web job: any warning fails — fix new findings, never
|
||||
# raise the cap.
|
||||
- name: Lint
|
||||
run: npm run lint -- --max-warnings 0
|
||||
|
||||
- name: Unused code
|
||||
run: npm run knip
|
||||
|
||||
# The vector-search tests tokenize with the real all-MiniLM-L6-v2 model
|
||||
# (~23 MB tokenizer + ONNX weights). transformers.js caches under
|
||||
# node_modules by default, which npm ci recreates — so every CI run
|
||||
@@ -247,8 +261,33 @@ jobs:
|
||||
|
||||
- name: Fail if the generated files drifted
|
||||
run: |
|
||||
if ! git diff --quiet -- src/Web/autopilot-monitor-web/utils/shared-manifests.generated.ts src/Web/autopilot-monitor-web/utils/wire-types.generated.ts src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts; then
|
||||
if ! git diff --quiet -- src/Web/autopilot-monitor-web/lib/generated/shared-manifests.generated.ts src/Web/autopilot-monitor-web/lib/generated/wire-types.generated.ts src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts; then
|
||||
echo "::error::Generated wire types are out of date. Run 'npm run generate:manifests' in src/Web/autopilot-monitor-web and commit the result."
|
||||
git diff --stat
|
||||
exit 1
|
||||
fi
|
||||
|
||||
duplicates:
|
||||
name: Duplicate code
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: tools/quality
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: tools/quality/package-lock.json
|
||||
|
||||
- run: npm ci --ignore-scripts
|
||||
|
||||
# jscpd over the C# and TypeScript sources (.jscpd.json). A clone missing from
|
||||
# jscpd-baseline.json fails, and so does a baseline entry whose clone is gone,
|
||||
# so the baseline only shrinks. Locally: npm run dup in tools/quality.
|
||||
- name: Duplicate check
|
||||
run: npm run dup
|
||||
|
||||
@@ -28,6 +28,7 @@ on:
|
||||
# Every other workflow: none of them changes the C# or TypeScript CodeQL analyzes.
|
||||
# paths-ignore cannot negate, so they are listed one by one; codeql.yml itself and
|
||||
# .github/codeql/ stay out of the list. A new workflow file is opt-in — add it here too.
|
||||
- '.github/workflows/action-pins.yml'
|
||||
- '.github/workflows/bootstrap-script-gates.yml'
|
||||
- '.github/workflows/build-agent.yml'
|
||||
- '.github/workflows/build-msi.yml'
|
||||
@@ -37,7 +38,6 @@ on:
|
||||
- '.github/workflows/deploy-mcp.yml'
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
- '.github/workflows/indexnow-notify.yml'
|
||||
- '.github/workflows/oidc-login-test.yml'
|
||||
- '.github/workflows/pester.yml'
|
||||
- '.github/workflows/publish-scripts.yml'
|
||||
pull_request:
|
||||
@@ -60,6 +60,7 @@ on:
|
||||
# Every other workflow: none of them changes the C# or TypeScript CodeQL analyzes.
|
||||
# paths-ignore cannot negate, so they are listed one by one; codeql.yml itself and
|
||||
# .github/codeql/ stay out of the list. A new workflow file is opt-in — add it here too.
|
||||
- '.github/workflows/action-pins.yml'
|
||||
- '.github/workflows/bootstrap-script-gates.yml'
|
||||
- '.github/workflows/build-agent.yml'
|
||||
- '.github/workflows/build-msi.yml'
|
||||
@@ -69,7 +70,6 @@ on:
|
||||
- '.github/workflows/deploy-mcp.yml'
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
- '.github/workflows/indexnow-notify.yml'
|
||||
- '.github/workflows/oidc-login-test.yml'
|
||||
- '.github/workflows/pester.yml'
|
||||
- '.github/workflows/publish-scripts.yml'
|
||||
schedule:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
name: Validate & Combine Rules
|
||||
on:
|
||||
push:
|
||||
branches: [main, dev]
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'rules/gather/**'
|
||||
- 'rules/analyze/**'
|
||||
@@ -9,6 +9,7 @@ on:
|
||||
- 'rules/schema/**'
|
||||
- 'rules/scripts/**'
|
||||
- 'rules/guardrails.json'
|
||||
- '.github/workflows/combine-rules.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'rules/gather/**'
|
||||
@@ -17,24 +18,27 @@ on:
|
||||
- 'rules/schema/**'
|
||||
- 'rules/scripts/**'
|
||||
- 'rules/guardrails.json'
|
||||
- '.github/workflows/combine-rules.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate-and-combine:
|
||||
# Pull requests and pushes alike; read-only, because the validator is third-party code.
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
# Lockfile and no lifecycle scripts: this job holds contents:write and pushes to
|
||||
# main, so an install-time script from a hijacked package would run with a token that
|
||||
# can rewrite the repository. rules/scripts/package-lock.json pins ajv, ajv-formats
|
||||
# and every transitive package (Dependabot keeps it current); ajv is pure JS and
|
||||
# needs no install hooks.
|
||||
# Lockfile and no lifecycle scripts: rules/scripts/package-lock.json pins ajv,
|
||||
# ajv-formats and every transitive package (Dependabot keeps it current); ajv is pure
|
||||
# JS and needs no install hooks.
|
||||
- name: Install the rules validator from the lockfile
|
||||
working-directory: rules/scripts
|
||||
run: npm ci --ignore-scripts
|
||||
@@ -48,19 +52,37 @@ jobs:
|
||||
- name: Combine rules into dist/
|
||||
run: node rules/scripts/combine.js
|
||||
|
||||
# Plain git instead of a third-party action: this step runs with contents:write on
|
||||
# main, so no code we do not control gets to push. Checkout leaves the job's
|
||||
# GITHUB_TOKEN in the remote credential helper, which is all `git push` needs.
|
||||
# The one job that can push to main, and it installs nothing: combine.js needs only node's
|
||||
# built-ins, so no package code ever runs next to the write token. Keep it that way -- a
|
||||
# dependency in combine.js would need its own job.
|
||||
publish:
|
||||
needs: validate
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
# Checkout leaves the job's GITHUB_TOKEN in the remote credential helper, which is all
|
||||
# `git push` needs.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
- name: Combine rules into dist/
|
||||
run: node rules/scripts/combine.js
|
||||
|
||||
# Plain git instead of a third-party action, so no code we do not control gets to push.
|
||||
# NOTE: combine.js writes THREE outputs; committing only dist/ would silently discard
|
||||
# the two generated TS modules and leave the guardrails-in-sync CI check red.
|
||||
- name: Commit generated outputs
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add -- rules/dist/ \
|
||||
src/Web/autopilot-monitor-web/utils/guardrails.generated.ts \
|
||||
src/Web/autopilot-monitor-web/lib/generated/guardrails.generated.ts \
|
||||
src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts
|
||||
if git diff --cached --quiet; then
|
||||
echo "Generated outputs unchanged - nothing to commit"
|
||||
|
||||
@@ -3,26 +3,41 @@ name: Deploy Backend (Azure Functions)
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
# Every job states its own permissions. This default is what the build job runs with: restore
|
||||
# and build execute third-party code (NuGet MSBuild targets, the Functions worker SDK) and must
|
||||
# never hold more than a read-only token.
|
||||
permissions:
|
||||
id-token: write # OIDC federated login (secret-less)
|
||||
contents: read
|
||||
|
||||
# Hosts (API, versions container) come from .github/scripts/Get-Endpoints.ps1.
|
||||
env:
|
||||
FUNCTION_APP: autopilotmonitor-api-eu
|
||||
HEALTH_URL: https://autopilotmonitor-api-eu.azurewebsites.net/api/health
|
||||
VERSIONS_CONTAINER: https://autopilotmonitoreu.blob.core.windows.net/versions
|
||||
CSPROJ_PATH: src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
# Runs only this repository's scripts. The deploy identity trusts the `production`
|
||||
# environment, which admits main only.
|
||||
reserve:
|
||||
name: Reserve build number
|
||||
runs-on: ubuntu-latest
|
||||
name: Build and Deploy Backend
|
||||
environment: production
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # OIDC federated login (secret-less)
|
||||
outputs:
|
||||
build_number: ${{ steps.version.outputs.build_number }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
build_utc: ${{ steps.version.outputs.build_utc }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
@@ -30,11 +45,6 @@ jobs:
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Setup .NET 10
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
# Same counter mechanism as the agent line: reserve-before-build with ETag-CAS
|
||||
# against a per-component counter blob. BuildTimestampUtc is pinned here rather
|
||||
# than left to MSBuild's default so the published manifest and the value baked
|
||||
@@ -43,7 +53,7 @@ jobs:
|
||||
id: version
|
||||
shell: pwsh
|
||||
run: |
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "${{ env.VERSIONS_CONTAINER }}/backend.txt"
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "$env:VERSIONS_CONTAINER_URL/backend.txt"
|
||||
|
||||
$csproj = Get-Content ${{ env.CSPROJ_PATH }} -Raw
|
||||
if ($csproj -notmatch '<VersionPrefix>(\d+\.\d+)</VersionPrefix>') {
|
||||
@@ -57,20 +67,90 @@ jobs:
|
||||
"version=$version" >> $env:GITHUB_OUTPUT
|
||||
"build_utc=$buildUtc" >> $env:GITHUB_OUTPUT
|
||||
|
||||
# Build zone: no login, no secret, no id-token. Its only output is the artifact; the
|
||||
# deploy job treats that as data and never builds.
|
||||
build:
|
||||
name: Build
|
||||
needs: reserve
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup .NET 10
|
||||
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
||||
with:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
- name: Restore dependencies
|
||||
run: dotnet restore ${{ env.CSPROJ_PATH }}
|
||||
|
||||
- name: Build
|
||||
run: dotnet build ${{ env.CSPROJ_PATH }} --configuration Release --no-restore -p:SourceRevisionId=${{ github.sha }} -p:BuildNumber=${{ steps.version.outputs.build_number }} -p:BuildTimestampUtc=${{ steps.version.outputs.build_utc }}
|
||||
run: dotnet build ${{ env.CSPROJ_PATH }} --configuration Release --no-restore -p:SourceRevisionId=${{ github.sha }} -p:BuildNumber=${{ needs.reserve.outputs.build_number }} -p:BuildTimestampUtc=${{ needs.reserve.outputs.build_utc }}
|
||||
|
||||
- name: Publish
|
||||
run: dotnet publish ${{ env.CSPROJ_PATH }} --configuration Release --output ./publish --no-build -p:SourceRevisionId=${{ github.sha }} -p:BuildNumber=${{ steps.version.outputs.build_number }} -p:BuildTimestampUtc=${{ steps.version.outputs.build_utc }}
|
||||
run: dotnet publish ${{ env.CSPROJ_PATH }} --configuration Release --output ./publish --no-build -p:SourceRevisionId=${{ github.sha }} -p:BuildNumber=${{ needs.reserve.outputs.build_number }} -p:BuildTimestampUtc=${{ needs.reserve.outputs.build_utc }}
|
||||
|
||||
# One archive, not a folder upload: upload-artifact skips hidden files, and the
|
||||
# isolated worker cannot start without the publish output's .azurefunctions/ folder.
|
||||
- name: Package publish output
|
||||
run: tar -czf "$RUNNER_TEMP/backend-publish.tgz" -C publish .
|
||||
|
||||
- name: Upload publish output
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: backend-publish
|
||||
path: ${{ runner.temp }}/backend-publish.tgz
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Trust zone: only this repository's scripts and pinned actions run here, never a restore
|
||||
# or build. The package is unpacked outside the checkout, so nothing in it can replace the
|
||||
# scripts this job runs.
|
||||
deploy:
|
||||
name: Deploy and verify
|
||||
needs: [reserve, build]
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # OIDC federated login (secret-less)
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
- name: Download publish output
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: backend-publish
|
||||
path: ${{ runner.temp }}/backend-artifact
|
||||
|
||||
# GNU tar refuses members with '..' and strips leading '/'.
|
||||
- name: Unpack publish output
|
||||
run: |
|
||||
mkdir "$RUNNER_TEMP/publish"
|
||||
tar -xzf "$RUNNER_TEMP/backend-artifact/backend-publish.tgz" -C "$RUNNER_TEMP/publish"
|
||||
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Deploy to Azure Functions
|
||||
uses: Azure/functions-action@c5060b3b8bb1ebbcb531abd00c822ecbaa8ea656 # v1.5.7
|
||||
with:
|
||||
app-name: ${{ env.FUNCTION_APP }}
|
||||
package: ./publish
|
||||
package: ${{ runner.temp }}/publish
|
||||
|
||||
# A green deploy step only means the package was accepted. This gate makes the
|
||||
# workflow assert what is actually being served, so the manifest published below
|
||||
@@ -78,16 +158,18 @@ jobs:
|
||||
# /api/health is IP rate-limited at 30/min; one poll per 10s stays well under.
|
||||
- name: Verify deployed build is live
|
||||
shell: pwsh
|
||||
env:
|
||||
EXPECTED_VERSION: ${{ needs.reserve.outputs.version }}
|
||||
run: |
|
||||
$expectedCommit = '${{ github.sha }}'.Substring(0, 7)
|
||||
$expectedVersion = '${{ steps.version.outputs.version }}'
|
||||
$expectedVersion = $env:EXPECTED_VERSION
|
||||
$deadline = [DateTime]::UtcNow.AddMinutes(3)
|
||||
$attempt = 0
|
||||
|
||||
while ([DateTime]::UtcNow -lt $deadline) {
|
||||
$attempt++
|
||||
try {
|
||||
$health = Invoke-RestMethod -Uri '${{ env.HEALTH_URL }}' -Method Get -TimeoutSec 30
|
||||
$health = Invoke-RestMethod -Uri "$env:API_BASE_URL/api/health" -Method Get -TimeoutSec 30
|
||||
if ($health.commitHash -eq $expectedCommit -and $health.version -eq $expectedVersion) {
|
||||
Write-Host "Live after $attempt attempt(s): $($health.version) @ $($health.commitHash), built $($health.buildUtc)"
|
||||
exit 0
|
||||
@@ -107,18 +189,21 @@ jobs:
|
||||
# (rollback, stale instance) rather than a half-finished pipeline.
|
||||
- name: Publish version manifest
|
||||
shell: pwsh
|
||||
env:
|
||||
BACKEND_VERSION: ${{ needs.reserve.outputs.version }}
|
||||
BACKEND_BUILD_UTC: ${{ needs.reserve.outputs.build_utc }}
|
||||
run: |
|
||||
$manifest = [ordered]@{
|
||||
component = 'backend'
|
||||
version = '${{ steps.version.outputs.version }}'
|
||||
version = $env:BACKEND_VERSION
|
||||
commit = '${{ github.sha }}'.Substring(0, 7)
|
||||
buildUtc = '${{ steps.version.outputs.build_utc }}'
|
||||
buildUtc = $env:BACKEND_BUILD_UTC
|
||||
deployedUtc = [DateTime]::UtcNow.ToString('o')
|
||||
runId = '${{ github.run_id }}'
|
||||
} | ConvertTo-Json
|
||||
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/backend.json"
|
||||
$url = "$env:VERSIONS_CONTAINER_URL/backend.json"
|
||||
# no-cache: rotating pointer, must never be served stale by any fronting cache.
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers ($authHeaders + @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' }) -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Write-Host "Published versions/backend.json"
|
||||
|
||||
@@ -16,20 +16,29 @@ env:
|
||||
# reference — with a sha-only tag `az containerapp update` saw the same image
|
||||
# and silently created no revision (2026-08-28).
|
||||
IMAGE_TAG: ${{ github.sha }}-${{ github.run_number }}
|
||||
HEALTH_URL: https://mcp.autopilotmonitor.com/health
|
||||
VERSIONS_CONTAINER: https://autopilotmonitoreu.blob.core.windows.net/versions
|
||||
PACKAGE_JSON: src/McpServer/autopilot-monitor-mcp/package.json
|
||||
# Hosts (MCP, versions container) come from .github/scripts/Get-Endpoints.ps1.
|
||||
|
||||
jobs:
|
||||
# One job on purpose, unlike the other deploys: all third-party code (npm ci, the build, the
|
||||
# embedding precompute) runs inside `docker build`, whose containers see neither the az
|
||||
# login nor the registry credentials nor this job's OIDC request token. That container is
|
||||
# the boundary -- never move npm onto the runner here, or the build gets the job's
|
||||
# credentials. The deploy identity trusts the `production` environment, which admits main only.
|
||||
build-and-deploy:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build and Deploy MCP Server
|
||||
environment: production
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
# The published customer documentation lives in its own PUBLIC repo and is
|
||||
# baked into the image as the search_docs corpus (see the Dockerfile). No
|
||||
# token is needed. Checking it out here rather than vendoring it keeps the
|
||||
@@ -62,7 +71,7 @@ jobs:
|
||||
id: version
|
||||
shell: pwsh
|
||||
run: |
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "${{ env.VERSIONS_CONTAINER }}/mcp.txt"
|
||||
$buildNumber = & ./.github/scripts/Request-BuildNumber.ps1 -CounterUrl "$env:VERSIONS_CONTAINER_URL/mcp.txt"
|
||||
|
||||
$pkgVersion = (Get-Content ${{ env.PACKAGE_JSON }} -Raw | ConvertFrom-Json).version
|
||||
$prefix = ($pkgVersion -split '\.')[0..1] -join '.'
|
||||
@@ -117,7 +126,7 @@ jobs:
|
||||
while ([DateTime]::UtcNow -lt $deadline) {
|
||||
$attempt++
|
||||
try {
|
||||
$health = Invoke-RestMethod -Uri '${{ env.HEALTH_URL }}' -Method Get -TimeoutSec 60
|
||||
$health = Invoke-RestMethod -Uri "$env:MCP_BASE_URL/health" -Method Get -TimeoutSec 60
|
||||
if ($health.commit -eq $expectedCommit -and $health.version -eq $expectedVersion) {
|
||||
Write-Host "Live after $attempt attempt(s): $($health.version) @ $($health.commit), docs $($health.docs.commit) ($($health.docs.chunks) chunks)"
|
||||
exit 0
|
||||
@@ -148,7 +157,7 @@ jobs:
|
||||
} | ConvertTo-Json
|
||||
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/mcp.json"
|
||||
$url = "$env:VERSIONS_CONTAINER_URL/mcp.json"
|
||||
# no-cache: rotating pointer, must never be served stale by any fronting cache.
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers ($authHeaders + @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' }) -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Write-Host "Published versions/mcp.json"
|
||||
|
||||
@@ -5,6 +5,11 @@ name: Deploy Web (Static Web App)
|
||||
# path ci.yml tests (setup-node 22 + npm ci + next build) instead of Oryx guessing inside the
|
||||
# action's container, and the exported site is attested like the agent ZIP and the MSI.
|
||||
#
|
||||
# Three jobs, each holding only what it needs: build runs all third-party code without any
|
||||
# credential, deploy holds the SWA token alone, record attests, verifies and writes the
|
||||
# manifest. A failed record job is re-run on its own; it writes only once the live site
|
||||
# serves this commit.
|
||||
#
|
||||
# PR preview environments were dropped with the action: their per-PR URLs are never
|
||||
# registered as redirect URIs, so nobody could log in on them anyway.
|
||||
|
||||
@@ -17,23 +22,29 @@ on:
|
||||
- '.github/workflows/deploy-web.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
# Every job states its own permissions. This default is what the build job runs with: npm ci
|
||||
# (install scripts included) and next build execute third-party code and must never hold more
|
||||
# than a read-only token.
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # keyless Sigstore signing (artifact attestation)
|
||||
attestations: write # store the provenance attestation
|
||||
|
||||
# Hosts (API, portal, storage) come from .github/scripts/Get-Endpoints.ps1, never from a
|
||||
# repository variable.
|
||||
env:
|
||||
PORTAL_VERSION_URL: https://portal.autopilotmonitor.com/version.json
|
||||
VERSIONS_CONTAINER: https://autopilotmonitoreu.blob.core.windows.net/versions
|
||||
WEB_DIR: src/Web/autopilot-monitor-web
|
||||
# Exact version: `npx --package` resolves from the registry at run time, so the pin is the
|
||||
# only thing that keeps the deploy step's code fixed. Bump deliberately.
|
||||
SWA_CLI_VERSION: 2.0.10
|
||||
|
||||
jobs:
|
||||
build_and_deploy_job:
|
||||
# Build zone: no login, no id-token. The one secret here is the Application Insights
|
||||
# connection string, which ships inside the public bundle anyway.
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
name: Build and Deploy Job
|
||||
outputs:
|
||||
docs_sha: ${{ steps.docs.outputs.sha }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
@@ -41,14 +52,9 @@ jobs:
|
||||
lfs: false
|
||||
fetch-depth: 0 # Full history needed for git-based sitemap lastmod dates
|
||||
|
||||
# Only the version manifest at the very end needs Azure; logging in here rather than
|
||||
# next to that step keeps a credential problem from surfacing after a live deploy.
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
# The customer docs repo is the single source of the "What's new" panel: the prebuild
|
||||
# step (scripts/whats-new/build-whats-new.js) turns changelog/*.md into
|
||||
@@ -105,12 +111,6 @@ jobs:
|
||||
# and the default app for browsers without a stored selection (legacy|primary).
|
||||
NEXT_PUBLIC_ENTRA_LEGACY_CLIENT_ID: ${{ vars.NEXT_PUBLIC_ENTRA_LEGACY_CLIENT_ID }}
|
||||
NEXT_PUBLIC_ENTRA_DEFAULT_APP: ${{ vars.NEXT_PUBLIC_ENTRA_DEFAULT_APP }}
|
||||
NEXT_PUBLIC_ENTRA_REDIRECT_URI: ${{ vars.NEXT_PUBLIC_ENTRA_REDIRECT_URI }}
|
||||
NEXT_PUBLIC_ENTRA_POST_LOGOUT_REDIRECT_URI: ${{ vars.NEXT_PUBLIC_ENTRA_POST_LOGOUT_REDIRECT_URI }}
|
||||
NEXT_PUBLIC_API_BASE_URL: ${{ vars.NEXT_PUBLIC_API_BASE_URL }}
|
||||
# Browsers load the live stats through it, and the build reads it once to bake the
|
||||
# numbers of this moment into the landing page's static HTML (lib/platformStats.ts).
|
||||
NEXT_PUBLIC_PLATFORM_STATS_MANIFEST_URL: ${{ vars.NEXT_PUBLIC_PLATFORM_STATS_MANIFEST_URL }}
|
||||
NEXT_PUBLIC_APPINSIGHTS_CONNECTION_STRING: ${{ secrets.NEXT_PUBLIC_APPINSIGHTS_CONNECTION_STRING }}
|
||||
# Build stamp for public/version.json (scripts/generate-version.js in prebuild);
|
||||
# the verify step below polls the deployed file for exactly this commit.
|
||||
@@ -120,6 +120,18 @@ jobs:
|
||||
# Star counter from the step above; empty when the lookup failed.
|
||||
NEXT_PUBLIC_GITHUB_STARS: ${{ steps.stars.outputs.count }}
|
||||
run: |
|
||||
# All from the endpoint registry (Load endpoints). Empty, the portal would silently call
|
||||
# localhost:7071. The stats manifest is what browsers load the live numbers from, and the
|
||||
# build reads it once to bake the numbers of this moment into the landing page's static
|
||||
# HTML (lib/platformStats.ts). The redirect URIs are only the fallback for the static
|
||||
# render; in the browser lib/msalConfig.ts uses window.location.origin.
|
||||
export NEXT_PUBLIC_API_BASE_URL="$API_BASE_URL"
|
||||
export NEXT_PUBLIC_PLATFORM_STATS_MANIFEST_URL="$STATS_CONTAINER_URL"
|
||||
export NEXT_PUBLIC_ENTRA_REDIRECT_URI="$WWW_BASE_URL"
|
||||
export NEXT_PUBLIC_ENTRA_POST_LOGOUT_REDIRECT_URI="$WWW_BASE_URL"
|
||||
test -n "$NEXT_PUBLIC_API_BASE_URL" || { echo "::error::API_BASE_URL is empty - the portal would call localhost"; exit 1; }
|
||||
test -n "$NEXT_PUBLIC_PLATFORM_STATS_MANIFEST_URL" || { echo "::error::STATS_CONTAINER_URL is empty"; exit 1; }
|
||||
test -n "$NEXT_PUBLIC_ENTRA_REDIRECT_URI" || { echo "::error::WWW_BASE_URL is empty"; exit 1; }
|
||||
npm run build
|
||||
test -f out/version.json || { echo "out/version.json missing - export did not run"; exit 1; }
|
||||
test -f out/whats-new.json || { echo "out/whats-new.json missing - whats-new prebuild did not run"; exit 1; }
|
||||
@@ -136,19 +148,37 @@ jobs:
|
||||
echo "archive=$archive" >> "$GITHUB_OUTPUT"
|
||||
sha256sum "$archive"
|
||||
|
||||
# Sigstore keyless provenance: binds the export digest to this workflow run (repo, ref,
|
||||
# commit, workflow file) in a public transparency log.
|
||||
# Verify: gh attestation verify portal-<sha>.tar.gz --repo okieselbach/AutopilotMonitor
|
||||
- name: Attest build provenance (portal export)
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
|
||||
with:
|
||||
subject-path: ${{ steps.package.outputs.archive }}
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: portal-${{ github.sha }}
|
||||
path: ${{ steps.package.outputs.archive }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Holds the SWA deployment token and nothing else -- deliberately no id-token: the SWA CLI
|
||||
# is third-party npm code whose dependencies resolve at run time.
|
||||
deploy:
|
||||
name: Deploy to Static Web App
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
|
||||
steps:
|
||||
# For staticwebapp.config.json only.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: portal-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/portal
|
||||
|
||||
- name: Unpack export
|
||||
run: |
|
||||
mkdir "$WEB_DIR/out"
|
||||
tar -xzf "$RUNNER_TEMP/portal/portal-${GITHUB_SHA::7}.tar.gz" -C "$WEB_DIR/out"
|
||||
|
||||
# Upload only -- out/ is already built. --swa-config-location points at the app root,
|
||||
# where staticwebapp.config.json lives (next export does not copy it into out/).
|
||||
@@ -163,6 +193,48 @@ jobs:
|
||||
--env production \
|
||||
--no-use-keychain
|
||||
|
||||
# Trust zone: only this repository's scripts and pinned actions run here. The deploy
|
||||
# identity trusts the `production` environment, which admits main only.
|
||||
record:
|
||||
name: Attest, verify and record
|
||||
needs: [build, deploy]
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # keyless Sigstore signing, OIDC login for the manifest
|
||||
attestations: write # store the provenance attestation
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
- name: Download build artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: portal-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/portal
|
||||
|
||||
- name: Locate the export archive
|
||||
id: archive
|
||||
run: |
|
||||
path="$RUNNER_TEMP/portal/portal-${GITHUB_SHA::7}.tar.gz"
|
||||
test -f "$path" || { echo "::error::Export archive for this commit is missing"; exit 1; }
|
||||
echo "path=$path" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Sigstore keyless provenance: binds the export digest to this workflow run (repo, ref,
|
||||
# commit, workflow file) in a public transparency log.
|
||||
# Verify: gh attestation verify portal-<sha>.tar.gz --repo okieselbach/AutopilotMonitor
|
||||
- name: Attest build provenance (portal export)
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
|
||||
with:
|
||||
subject-path: ${{ steps.archive.outputs.path }}
|
||||
|
||||
# The CLI reporting success proves the upload, not that the production host serves the
|
||||
# new build -- verify against the live /version.json (Cache-Control: no-store via
|
||||
# staticwebapp.config.json) before publishing the version manifest.
|
||||
@@ -176,7 +248,7 @@ jobs:
|
||||
while ([DateTime]::UtcNow -lt $deadline) {
|
||||
$attempt++
|
||||
try {
|
||||
$v = Invoke-RestMethod -Uri '${{ env.PORTAL_VERSION_URL }}' -Method Get -TimeoutSec 30 -Headers @{ 'Cache-Control' = 'no-cache' }
|
||||
$v = Invoke-RestMethod -Uri "$env:PORTAL_BASE_URL/version.json" -Method Get -TimeoutSec 30 -Headers @{ 'Cache-Control' = 'no-cache' }
|
||||
if ($v.commit -eq $expectedCommit) {
|
||||
Write-Host "Live after $attempt attempt(s): web @ $($v.commit), built $($v.buildUtc)"
|
||||
exit 0
|
||||
@@ -190,22 +262,36 @@ jobs:
|
||||
|
||||
throw "Portal did not serve commit $expectedCommit within 5 minutes -- deployment not verified, version manifest NOT published"
|
||||
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
# Cross-component deploy record, written last and only on a verified deploy --
|
||||
# same contract as versions/backend.json and versions/mcp.json. docsCommit says which
|
||||
# docs revision the live What's new panel was generated from (as for the MCP corpus).
|
||||
# The docs commit comes from the build job, so it arrives as data and is checked
|
||||
# against the shape of a commit id before it is used.
|
||||
- name: Publish version manifest
|
||||
shell: pwsh
|
||||
env:
|
||||
DOCS_SHA: ${{ needs.build.outputs.docs_sha }}
|
||||
run: |
|
||||
if ($env:DOCS_SHA -notmatch '^[0-9a-f]{40}$') {
|
||||
throw "The build job handed over no valid docs commit -- version manifest NOT published."
|
||||
}
|
||||
$manifest = [ordered]@{
|
||||
component = 'web'
|
||||
commit = '${{ github.sha }}'.Substring(0, 7)
|
||||
deployedUtc = [DateTime]::UtcNow.ToString('o')
|
||||
runId = '${{ github.run_id }}'
|
||||
docsCommit = '${{ steps.docs.outputs.sha }}'.Substring(0, 7)
|
||||
docsCommit = $env:DOCS_SHA.Substring(0, 7)
|
||||
} | ConvertTo-Json
|
||||
|
||||
$authHeaders = & ./.github/scripts/Get-StorageAuthHeaders.ps1
|
||||
$url = "${{ env.VERSIONS_CONTAINER }}/web.json"
|
||||
$url = "$env:VERSIONS_CONTAINER_URL/web.json"
|
||||
# no-cache: rotating pointer, must never be served stale by any fronting cache.
|
||||
Invoke-RestMethod -Uri $url -Method Put -Headers ($authHeaders + @{ 'x-ms-blob-type' = 'BlockBlob'; 'x-ms-blob-cache-control' = 'no-cache' }) -Body $manifest -ContentType 'application/json' | Out-Null
|
||||
Write-Host "Published versions/web.json"
|
||||
|
||||
@@ -3,24 +3,34 @@ name: IndexNow Notify (Bing/Yandex)
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
# The job only talks to public endpoints; it never needs a repository token.
|
||||
permissions: {}
|
||||
# Read-only, for the checkout of the endpoint registry; beyond that the job only talks to
|
||||
# public endpoints.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
name: Submit URLs to IndexNow
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Load endpoints
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
- name: Notify IndexNow
|
||||
run: |
|
||||
URLS=$(curl -s https://www.autopilotmonitor.com/sitemap.xml | grep -oP '<loc>\K[^<]+')
|
||||
URLS=$(curl -s "$WWW_BASE_URL/sitemap.xml" | grep -oP '<loc>\K[^<]+')
|
||||
URL_JSON=$(echo "$URLS" | jq -R -s 'split("\n") | map(select(length > 0))')
|
||||
RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -X POST "https://api.indexnow.org/indexnow" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"host\": \"www.autopilotmonitor.com\",
|
||||
\"host\": \"${WWW_BASE_URL#https://}\",
|
||||
\"key\": \"8f1710b5-f14b-477e-938b-6c01ef851661\",
|
||||
\"keyLocation\": \"https://www.autopilotmonitor.com/IndexNow.txt\",
|
||||
\"keyLocation\": \"$WWW_BASE_URL/IndexNow.txt\",
|
||||
\"urlList\": $URL_JSON
|
||||
}")
|
||||
echo "IndexNow response: HTTP $RESPONSE"
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
# TEMPORARY: proves the secret-less OIDC federated login works end-to-end.
|
||||
# Delete after the cross-tenant migration is verified (or keep as a cheap smoke test).
|
||||
name: OIDC Login Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
login-test:
|
||||
runs-on: ubuntu-latest
|
||||
name: Federated login + read check
|
||||
steps:
|
||||
- name: Login to Azure (OIDC)
|
||||
uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
|
||||
|
||||
- name: Prove identity and RBAC scope
|
||||
run: |
|
||||
az account show --query "{subscription:name, subscriptionId:id, tenantId:tenantId, user:user.name}" -o json
|
||||
echo "--- function app visible with Contributor scope? ---"
|
||||
az functionapp list --query "[].name" -o json
|
||||
@@ -36,13 +36,13 @@ concurrency:
|
||||
|
||||
# Authenticode for the customer scripts. Same identity and same repository variables as
|
||||
# build-agent.yml -- the loader on every device pins this publisher, so the two must not
|
||||
# drift apart.
|
||||
# drift apart. The publisher takes every host, the timestamp server included, from
|
||||
# .github/scripts/Get-Endpoints.ps1.
|
||||
env:
|
||||
CODE_SIGN_CLIENT_ID: ${{ vars.CODE_SIGN_CLIENT_ID }}
|
||||
CODE_SIGN_CERTIFICATE_NAME: ${{ vars.CODE_SIGN_CERTIFICATE_NAME }}
|
||||
CODE_SIGN_KEY_VAULT_URL: ${{ vars.CODE_SIGN_KEY_VAULT_URL }}
|
||||
CODE_SIGN_EXPECTED_SUBJECT: ${{ vars.CODE_SIGN_EXPECTED_SUBJECT }}
|
||||
CODE_SIGN_TIMESTAMP_URL: http://timestamp.acs.microsoft.com
|
||||
|
||||
jobs:
|
||||
# $/ is the self-repository form: this repository at the running commit, never a
|
||||
@@ -124,7 +124,6 @@ jobs:
|
||||
-SigningToken $env:CODE_SIGN_AKV_ACCESS_TOKEN `
|
||||
-KeyVaultUrl $env:CODE_SIGN_KEY_VAULT_URL `
|
||||
-CertificateName $env:CODE_SIGN_CERTIFICATE_NAME `
|
||||
-TimestampUrl $env:CODE_SIGN_TIMESTAMP_URL `
|
||||
-ExpectedPublisher $env:CODE_SIGN_EXPECTED_SUBJECT `
|
||||
-SignedOutputDir "$env:RUNNER_TEMP/signed-scripts" `
|
||||
-DryRun:$dryRun
|
||||
|
||||
@@ -8,5 +8,8 @@
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<MSBuildTreatWarningsAsErrors>true</MSBuildTreatWarningsAsErrors>
|
||||
<WarningsNotAsErrors>NU1901;NU1902;NU1903;NU1904</WarningsNotAsErrors>
|
||||
<!-- Run the IDE code-style analyzers in the build too, so the rules .editorconfig raises
|
||||
to warning fail it like any compiler warning. -->
|
||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
|
||||
+1
-1
Submodule infra updated: f93e475b78...0af4689368
+1
-1
Submodule internal updated: 54d7076308...5fb01d6e4d
@@ -86,12 +86,12 @@ for (const config of configs) {
|
||||
// ── Guardrails generation ───────────────────────────────────────────────────
|
||||
|
||||
const guardrailsPath = path.join(rulesRoot, 'guardrails.json');
|
||||
// NOTE: consumers import from "@/utils/guardrails.generated" (see guardValidation.ts
|
||||
// NOTE: consumers import from "@/lib/generated/guardrails.generated" (see guardValidation.ts
|
||||
// and SectionGatherRules.tsx). The web re-org (commit a3e8e044) moved this file to
|
||||
// utils/ but left this output path pointing at the now-orphaned lib/ copy, which
|
||||
// silently froze the live allowlist. Keep this in sync with the import path.
|
||||
const guardrailsOutput = path.resolve(
|
||||
rulesRoot, '..', 'src', 'Web', 'autopilot-monitor-web', 'utils', 'guardrails.generated.ts'
|
||||
rulesRoot, '..', 'src', 'Web', 'autopilot-monitor-web', 'lib', 'generated', 'guardrails.generated.ts'
|
||||
);
|
||||
|
||||
if (fs.existsSync(guardrailsPath)) {
|
||||
|
||||
+1
@@ -50,6 +50,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
nameof(AgentConfigResponse.MaxBatchSize),
|
||||
nameof(AgentConfigResponse.MaxAuthFailures),
|
||||
nameof(AgentConfigResponse.AuthFailureTimeoutMinutes),
|
||||
nameof(AgentConfigResponse.InterceptedRegistrationRetryBoots),
|
||||
// Merge reads two Collectors knobs (AgentMaxLifetimeMinutes, HelloWaitTimeoutSeconds);
|
||||
// the remaining CollectorConfiguration flags flow untouched to DefaultComponentFactory.
|
||||
nameof(AgentConfigResponse.Collectors),
|
||||
|
||||
+46
@@ -55,6 +55,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
UnrestrictedMode = true,
|
||||
MaxAuthFailures = 7,
|
||||
AuthFailureTimeoutMinutes = 15,
|
||||
InterceptedRegistrationRetryBoots = 2,
|
||||
EnableImeMatchLog = true,
|
||||
Collectors = new CollectorConfiguration
|
||||
{
|
||||
@@ -98,6 +99,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(45, agent.HelloWaitTimeoutSeconds);
|
||||
Assert.Equal(7, agent.MaxAuthFailures);
|
||||
Assert.Equal(15, agent.AuthFailureTimeoutMinutes);
|
||||
Assert.Equal(2, agent.InterceptedRegistrationRetryBoots);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.True(result.UnrestrictedModeChanged);
|
||||
@@ -468,6 +470,50 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(0, agent.AgentMaxLifetimeMinutes);
|
||||
}
|
||||
|
||||
// ============================================================= Intercepted-registration buffer
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)] // clean up at the first intercepted registration
|
||||
[InlineData(3)]
|
||||
public void Merge_copies_intercepted_registration_retry_boots(int remoteValue)
|
||||
{
|
||||
var agent = NewAgentConfig();
|
||||
var remote = FullRemote();
|
||||
remote.InterceptedRegistrationRetryBoots = remoteValue;
|
||||
|
||||
RemoteConfigMerger.Merge(agent, remote);
|
||||
|
||||
Assert.Equal(remoteValue, agent.InterceptedRegistrationRetryBoots);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Merge_keeps_the_agent_default_when_the_backend_sends_no_intercepted_registration_retry_boots()
|
||||
{
|
||||
var agent = NewAgentConfig();
|
||||
var remote = FullRemote();
|
||||
remote.InterceptedRegistrationRetryBoots = null; // backend that predates the field
|
||||
|
||||
RemoteConfigMerger.Merge(agent, remote);
|
||||
|
||||
Assert.Equal(1, agent.InterceptedRegistrationRetryBoots);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(-1, 0)]
|
||||
[InlineData(int.MinValue, 0)]
|
||||
[InlineData(6, 5)]
|
||||
[InlineData(int.MaxValue, 5)]
|
||||
public void Merge_clamps_intercepted_registration_retry_boots_to_zero_through_five(int remoteValue, int expected)
|
||||
{
|
||||
var agent = NewAgentConfig();
|
||||
var remote = FullRemote();
|
||||
remote.InterceptedRegistrationRetryBoots = remoteValue;
|
||||
|
||||
RemoteConfigMerger.Merge(agent, remote);
|
||||
|
||||
Assert.Equal(expected, agent.InterceptedRegistrationRetryBoots);
|
||||
}
|
||||
|
||||
// ============================================================= Guard
|
||||
|
||||
[Fact]
|
||||
|
||||
+17
@@ -214,6 +214,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Null(loaded.LatestAgentSha256);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(3)]
|
||||
[InlineData(null)] // backend that predates the field — the agent keeps its default
|
||||
public void RoundTrip_preserves_intercepted_registration_retry_boots(int? value)
|
||||
{
|
||||
// An intercepted start gets no live config, so the registration buffer it decides on
|
||||
// is the cached value: it must survive the write + read strip untouched.
|
||||
var probe = NewProbe();
|
||||
|
||||
probe.DoCache(new AgentConfigResponse { InterceptedRegistrationRetryBoots = value });
|
||||
var loaded = probe.DoLoad();
|
||||
|
||||
Assert.NotNull(loaded);
|
||||
Assert.Equal(value, loaded.InterceptedRegistrationRetryBoots);
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
private CacheProbe NewProbe()
|
||||
|
||||
+31
-4
@@ -7,6 +7,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
@@ -101,6 +102,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(401, svc.LastFetchAuthStatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchConfig_InterceptedAnswer_FallsBack_WithoutCounting_AndReportsOnce()
|
||||
{
|
||||
// A 403 something on the network path answered (TLS inspection): same fallback as an
|
||||
// auth failure, but the tracker does not count it — it sends ClientCertificateNotReceived.
|
||||
var distress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0,
|
||||
AutopilotMonitor.DecisionCore.Engine.SystemClock.Instance, _logger, distress);
|
||||
var svc = NewService(BackendBehaviour.AlwaysIntercepted403, tracker);
|
||||
|
||||
var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
|
||||
Assert.Equal(1, svc.LastFetchAttempts);
|
||||
Assert.Equal(403, svc.LastFetchAuthStatusCode);
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures);
|
||||
var call = Assert.Single(distress.Calls);
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType);
|
||||
Assert.StartsWith("agent/config http 403 client_certificate_required", call.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchConfig_RetryDisabled_OneAttemptOnly()
|
||||
{
|
||||
@@ -252,10 +275,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
/// <summary>All 4 initial attempts fail; the NEXT call (the recovery fetch) succeeds.</summary>
|
||||
FailAllThenSucceed,
|
||||
AlwaysAuthFailure401,
|
||||
AlwaysIntercepted403,
|
||||
AlwaysReturnNull,
|
||||
}
|
||||
|
||||
private FakeRemoteConfigService NewService(BackendBehaviour behaviour)
|
||||
private FakeRemoteConfigService NewService(BackendBehaviour behaviour, AuthFailureTracker? tracker = null)
|
||||
{
|
||||
// Pass a throwaway BackendApiClient — the fake overrides CallBackendAsync
|
||||
// before it ever hits the wire.
|
||||
@@ -269,7 +293,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
bootstrapToken: null,
|
||||
agentVersion: "0.0.0",
|
||||
logger: _logger);
|
||||
return new FakeRemoteConfigService(apiClient, _logger, behaviour);
|
||||
return new FakeRemoteConfigService(apiClient, _logger, behaviour, tracker);
|
||||
}
|
||||
|
||||
private sealed class FakeRemoteConfigService : RemoteConfigService
|
||||
@@ -277,8 +301,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
private readonly BackendBehaviour _behaviour;
|
||||
private int _calls;
|
||||
|
||||
public FakeRemoteConfigService(BackendApiClient apiClient, AgentLogger logger, BackendBehaviour behaviour)
|
||||
: base(apiClient, "00000000-0000-0000-0000-000000000001", logger)
|
||||
public FakeRemoteConfigService(BackendApiClient apiClient, AgentLogger logger, BackendBehaviour behaviour, AuthFailureTracker? tracker)
|
||||
: base(apiClient, "00000000-0000-0000-0000-000000000001", logger, authFailureTracker: tracker)
|
||||
{
|
||||
_behaviour = behaviour;
|
||||
}
|
||||
@@ -305,6 +329,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
throw new HttpRequestException("simulated network failure");
|
||||
case BackendBehaviour.AlwaysAuthFailure401:
|
||||
throw new BackendAuthException("Unauthorized", 401);
|
||||
case BackendBehaviour.AlwaysIntercepted403:
|
||||
throw new BackendAuthException("The device certificate did not reach the backend.", 403,
|
||||
classification: InterceptionFixtures.CertificateRequired());
|
||||
case BackendBehaviour.AlwaysReturnNull:
|
||||
return Task.FromResult<AgentConfigResponse>(null!);
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Harness
|
||||
{
|
||||
/// <summary>
|
||||
/// Records every distress dispatch instead of sending it. The real anti-flood rules of
|
||||
/// <see cref="DistressReporter"/> are bypassed on purpose: tests assert what the caller dispatched.
|
||||
/// </summary>
|
||||
internal sealed class FakeDistressReporter : DistressReporter
|
||||
{
|
||||
public sealed class Call
|
||||
{
|
||||
public DistressErrorType ErrorType { get; set; }
|
||||
public string Message { get; set; } = default!;
|
||||
public int? HttpStatusCode { get; set; }
|
||||
}
|
||||
|
||||
public List<Call> Calls { get; } = new List<Call>();
|
||||
|
||||
/// <summary>
|
||||
/// The send each dispatch returns; null = already completed. Lets a test keep a send in
|
||||
/// flight to prove a caller waits for it (or stops waiting at its bound).
|
||||
/// </summary>
|
||||
public Func<Task>? Send { get; set; }
|
||||
|
||||
public FakeDistressReporter()
|
||||
: base(baseUrl: "https://example.test", tenantId: "t", manufacturer: "m", model: "d", serialNumber: "s", agentVersion: "0.0.0", logger: null!)
|
||||
{
|
||||
}
|
||||
|
||||
public override Task TrySendAsync(DistressErrorType errorType, string message, int? httpStatusCode = null)
|
||||
{
|
||||
Calls.Add(new Call { ErrorType = errorType, Message = message, HttpStatusCode = httpStatusCode });
|
||||
return Send?.Invoke() ?? Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// The local network fingerprint a failed registration is stored with: SHA-256 over the
|
||||
/// connection type and gateway <see cref="NetworkChangeDetector"/> captures, cut to 16 hex
|
||||
/// characters; unknown (null) whenever either input is missing. Gateways are documentation
|
||||
/// addresses (RFC 5737).
|
||||
/// </summary>
|
||||
public sealed class NetworkFingerprintTests
|
||||
{
|
||||
private static NetworkChangeDetector.NetworkStateSnapshot State(string? connectionType, string? gateway, bool hasNetwork = true) =>
|
||||
new NetworkChangeDetector.NetworkStateSnapshot
|
||||
{
|
||||
HasNetwork = hasNetwork,
|
||||
ConnectionType = connectionType,
|
||||
Gateway = gateway,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Same_connection_type_and_gateway_give_the_same_short_hex_fingerprint()
|
||||
{
|
||||
var first = NetworkFingerprint.Of(State("WiFi", "192.0.2.1"));
|
||||
var second = NetworkFingerprint.Of(State("WiFi", "192.0.2.1"));
|
||||
|
||||
Assert.Equal(first, second);
|
||||
Assert.Equal(NetworkFingerprint.Length, first!.Length);
|
||||
Assert.True(first.All(c => (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')), first);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("WiFi", "198.51.100.1")] // another gateway
|
||||
[InlineData("Ethernet", "192.0.2.1")] // another connection type
|
||||
public void Another_gateway_or_connection_type_is_another_network(string connectionType, string gateway)
|
||||
{
|
||||
Assert.NotEqual(
|
||||
NetworkFingerprint.Of(State("WiFi", "192.0.2.1")),
|
||||
NetworkFingerprint.Of(State(connectionType, gateway)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_network_has_no_fingerprint()
|
||||
{
|
||||
Assert.Null(NetworkFingerprint.Of(null));
|
||||
Assert.Null(NetworkFingerprint.Of(State("WiFi", "192.0.2.1", hasNetwork: false)));
|
||||
Assert.Null(NetworkFingerprint.Of(State("WiFi", null)));
|
||||
Assert.Null(NetworkFingerprint.Of(State(null, "192.0.2.1")));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("aaaa", "aaaa", false)]
|
||||
[InlineData("aaaa", "bbbb", true)]
|
||||
public void Differs_compares_known_fingerprints(string before, string after, bool expected)
|
||||
{
|
||||
Assert.Equal(expected, NetworkFingerprint.Differs(before, after));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, "aaaa")]
|
||||
[InlineData("aaaa", null)]
|
||||
[InlineData(null, null)]
|
||||
public void Differs_is_unknown_when_either_side_is(string? before, string? after)
|
||||
{
|
||||
Assert.Null(NetworkFingerprint.Differs(before, after));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TryCapture_reads_this_machine_without_throwing()
|
||||
{
|
||||
// Machine-dependent: either no active interface (null) or a well-formed fingerprint
|
||||
// that matches the detector's own capture.
|
||||
var captured = NetworkFingerprint.TryCapture();
|
||||
|
||||
if (captured != null)
|
||||
{
|
||||
Assert.Equal(NetworkFingerprint.Length, captured.Length);
|
||||
Assert.Equal(NetworkFingerprint.Of(NetworkChangeDetector.CaptureInterfaceState()), captured);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-5
@@ -64,7 +64,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
private void Buffer(string? rebootUri, long? recordId = null, bool isBackfill = false, DateTime? timeCreatedUtc = null, string? description = null)
|
||||
{
|
||||
_tracker.ProcessEvent(
|
||||
eventId: MdmRebootPolicyTracker.EventId_PolicyRebootRequired,
|
||||
recordId: recordId ?? _nextRecordId++,
|
||||
timeCreatedUtc: timeCreatedUtc ?? At,
|
||||
rebootUri: rebootUri,
|
||||
@@ -226,13 +225,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
// The coalesced reboot restarts the agent; the post-reboot backfill re-reads the same
|
||||
// records — a FLUSHED record must be skipped by the next instance.
|
||||
var first = MakeTracker(stateDirectory: _tmp.Path);
|
||||
first.ProcessEvent(MdmRebootPolicyTracker.EventId_PolicyRebootRequired, recordId: 500,
|
||||
first.ProcessEvent(recordId: 500,
|
||||
timeCreatedUtc: At, rebootUri: UriSvchost, formattedDescription: null, isBackfill: false);
|
||||
first.FlushPending();
|
||||
|
||||
var second = MakeTracker(stateDirectory: _tmp.Path);
|
||||
second.LoadWatermark();
|
||||
second.ProcessEvent(MdmRebootPolicyTracker.EventId_PolicyRebootRequired, recordId: 500,
|
||||
second.ProcessEvent(recordId: 500,
|
||||
timeCreatedUtc: At, rebootUri: UriSvchost, formattedDescription: null, isBackfill: true);
|
||||
second.FlushPending();
|
||||
|
||||
@@ -245,13 +244,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
// Process killed by the very reboot the records announce: nothing was flushed, so the
|
||||
// next instance must re-read and emit the records.
|
||||
var first = MakeTracker(stateDirectory: _tmp.Path);
|
||||
first.ProcessEvent(MdmRebootPolicyTracker.EventId_PolicyRebootRequired, recordId: 500,
|
||||
first.ProcessEvent(recordId: 500,
|
||||
timeCreatedUtc: At, rebootUri: UriSvchost, formattedDescription: null, isBackfill: false);
|
||||
// no FlushPending — simulated reboot kill
|
||||
|
||||
var second = MakeTracker(stateDirectory: _tmp.Path);
|
||||
second.LoadWatermark();
|
||||
second.ProcessEvent(MdmRebootPolicyTracker.EventId_PolicyRebootRequired, recordId: 500,
|
||||
second.ProcessEvent(recordId: 500,
|
||||
timeCreatedUtc: At, rebootUri: UriSvchost, formattedDescription: null, isBackfill: true);
|
||||
second.FlushPending();
|
||||
|
||||
|
||||
+83
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
@@ -49,6 +50,74 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
private static EmergencyReporter Reporter(FakeApiClient client)
|
||||
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null);
|
||||
|
||||
private static EmergencyReporter Reporter(FakeApiClient client, VirtualClock clock)
|
||||
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null, clock: clock);
|
||||
|
||||
private static VirtualClock Clock() => new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc));
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_is_not_suppressed_by_the_cooldown_of_an_earlier_report()
|
||||
{
|
||||
// Terminal report, sent at most once per process right before the termination — a
|
||||
// routine report one minute earlier must not cost it. Every other type keeps the cooldown.
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: 503);
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(2, client.Calls);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, client.Reports[1].ErrorType);
|
||||
|
||||
// Control: a third report of another type inside the window is still held back.
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Second_session_unknown_report_within_the_window_is_still_suppressed()
|
||||
{
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(1, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_still_counts_against_the_per_session_maximum()
|
||||
{
|
||||
// Five delivered reports (ten minutes apart, distinct keys) exhaust the budget; the
|
||||
// exemption waives only the cooldown, not the maximum.
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
for (var i = 0; i < 6; i++) client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
foreach (var status in new[] { 500, 502, 503, 504, 429 })
|
||||
{
|
||||
await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: status);
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
}
|
||||
Assert.Equal(5, client.Calls);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(5, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failed_send_rolls_back_reservation_so_a_later_call_retries()
|
||||
{
|
||||
@@ -142,6 +211,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_carries_error_type_and_http_410()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
var report = Assert.Single(client.Reports);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType);
|
||||
Assert.Equal(410, report.HttpStatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_age_is_carried_in_the_report_and_absent_by_default()
|
||||
{
|
||||
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// The orchestrator side of an intercepted upload: one <c>telemetry_upload_blocked</c> with the
|
||||
/// evidence (queued behind the intercepted batch, never uploaded while paused), and the one
|
||||
/// terminal attempt — requested by <c>DrainSpool</c> or made by <see cref="EnrollmentOrchestrator.Stop"/>.
|
||||
/// </summary>
|
||||
[Collection("SerialThreading")]
|
||||
public sealed class EnrollmentOrchestratorInterceptionTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static TelemetryItemDraft Draft() =>
|
||||
new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "T1_S1",
|
||||
rowKey: "0000000001",
|
||||
payloadJson: "{}",
|
||||
isSessionScoped: true,
|
||||
requiresImmediateFlush: false);
|
||||
|
||||
// The signal and the decision transition of the same post mention the event type too.
|
||||
private static bool IsBlockedEvent(TelemetryItem item) =>
|
||||
item.Kind == TelemetryItemKind.Event
|
||||
&& item.PayloadJson.Contains(AutopilotMonitor.Shared.Constants.EventTypes.TelemetryUploadBlocked);
|
||||
|
||||
private static TelemetryItem? FindBlockedEvent(EnrollmentOrchestrator sut) =>
|
||||
sut.SpoolForTests!.Peek(1000).FirstOrDefault(IsBlockedEvent);
|
||||
|
||||
// Dates stay strings: the wire format of firstBlockedAtUtc is what is under test.
|
||||
private static JObject Parse(string json)
|
||||
{
|
||||
using var reader = new JsonTextReader(new StringReader(json)) { DateParseHandling = DateParseHandling.None };
|
||||
return JObject.Load(reader);
|
||||
}
|
||||
|
||||
/// <summary>Starts the orchestrator and drives the first upload into an interception.</summary>
|
||||
private static async Task<EnrollmentOrchestrator> StartInterceptedAsync(EnrollmentOrchestratorRig rig)
|
||||
{
|
||||
rig.Uploader.QueueIntercepted(5);
|
||||
var sut = rig.Build();
|
||||
sut.Start();
|
||||
sut.SpoolForTests!.Enqueue(Draft());
|
||||
await sut.DrainAsync();
|
||||
Assert.True(sut.Transport.IsInterceptionPaused);
|
||||
return sut;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Interception_queues_one_telemetry_upload_blocked_with_the_evidence_and_wakeups_stay_offline()
|
||||
{
|
||||
using var rig = new EnrollmentOrchestratorRig();
|
||||
var sut = await StartInterceptedAsync(rig);
|
||||
try
|
||||
{
|
||||
TelemetryItem? blocked = null;
|
||||
Assert.True(SpinWait.SpinUntil(() => (blocked = FindBlockedEvent(sut)) != null, WaitMs),
|
||||
"telemetry_upload_blocked never reached the spool.");
|
||||
var calls = rig.Uploader.CallCount;
|
||||
|
||||
var data = (JObject)Parse(blocked!.PayloadJson)["Data"]!;
|
||||
Assert.Equal(
|
||||
new[] { "firstBlockedAtUtc", "operation", "pageTitle", "reason", "serverCertIssuer", "statusCode" },
|
||||
data.Properties().Select(p => p.Name).OrderBy(n => n, StringComparer.Ordinal));
|
||||
Assert.Equal("client_certificate_required", (string?)data["reason"]);
|
||||
Assert.Equal("agent/telemetry", (string?)data["operation"]);
|
||||
Assert.Equal(JTokenType.Integer, data["statusCode"]!.Type);
|
||||
Assert.Equal(403, (int)data["statusCode"]!);
|
||||
Assert.Equal(InterceptionFixtures.PlatformPageTitle, (string?)data["pageTitle"]);
|
||||
Assert.Equal(InterceptionFixtures.InspectionIssuer, (string?)data["serverCertIssuer"]);
|
||||
Assert.Equal(
|
||||
EnrollmentOrchestratorRig.DefaultClockStart.ToString("o", CultureInfo.InvariantCulture),
|
||||
(string?)data["firstBlockedAtUtc"]);
|
||||
|
||||
// The marker was an immediate-upload item; neither its wakeup nor a further drain
|
||||
// touches the network while the pause holds.
|
||||
var drain = await sut.DrainAsync();
|
||||
Assert.Contains("upload intercepted", drain.LastErrorReason);
|
||||
Assert.Equal(calls, rig.Uploader.CallCount);
|
||||
Assert.Single(sut.SpoolForTests!.Peek(1000), IsBlockedEvent);
|
||||
}
|
||||
finally
|
||||
{
|
||||
sut.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Terminal_attempt_requested_by_the_termination_runs_once_and_Stop_adds_none()
|
||||
{
|
||||
using var rig = new EnrollmentOrchestratorRig();
|
||||
var sut = await StartInterceptedAsync(rig);
|
||||
Assert.True(SpinWait.SpinUntil(() => FindBlockedEvent(sut) != null, WaitMs));
|
||||
var calls = rig.Uploader.CallCount;
|
||||
|
||||
sut.RequestTerminalUploadAttempt();
|
||||
Assert.True(SpinWait.SpinUntil(() => rig.Uploader.CallCount == calls + 1, WaitMs),
|
||||
"the woken drain loop did not make the terminal attempt.");
|
||||
|
||||
sut.Stop();
|
||||
Assert.Equal(calls + 1, rig.Uploader.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Stop_makes_the_one_terminal_attempt_when_the_termination_did_not()
|
||||
{
|
||||
using var rig = new EnrollmentOrchestratorRig();
|
||||
var sut = await StartInterceptedAsync(rig);
|
||||
var calls = rig.Uploader.CallCount;
|
||||
|
||||
sut.Stop();
|
||||
|
||||
Assert.Equal(calls + 1, rig.Uploader.CallCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
+32
-1
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Security;
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
@@ -10,6 +11,7 @@ using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
@@ -96,6 +98,35 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
Assert.Equal(TimeSpan.FromSeconds(5), client.Timeout);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateHandler_without_observer_keeps_the_default_validation()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
using var cert = CreateSelfSignedCert();
|
||||
|
||||
using var handler = MtlsHttpClientFactory.CreateHandler(new FakeCertificateResolver(cert), logger);
|
||||
|
||||
Assert.Null(handler.ServerCertificateCustomValidationCallback);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateHandler_with_observer_installs_an_observing_callback_that_never_weakens_validation()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
using var cert = CreateSelfSignedCert();
|
||||
var observer = new ServerCertificateObserver();
|
||||
|
||||
using var handler = MtlsHttpClientFactory.CreateHandler(new FakeCertificateResolver(cert), logger, observer);
|
||||
var callback = handler.ServerCertificateCustomValidationCallback;
|
||||
|
||||
Assert.NotNull(callback);
|
||||
Assert.False(callback!(null!, cert, null!, SslPolicyErrors.RemoteCertificateChainErrors));
|
||||
Assert.True(callback(null!, cert, null!, SslPolicyErrors.None));
|
||||
Assert.Equal(cert.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateHandler_throws_when_resolver_returns_null()
|
||||
{
|
||||
@@ -161,7 +192,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
/// we scan the type hierarchy for the first <see cref="HttpMessageHandler"/>-typed
|
||||
/// instance field to stay version-tolerant.
|
||||
/// </summary>
|
||||
private static HttpMessageHandler GetPipelineHead(HttpClient client)
|
||||
internal static HttpMessageHandler GetPipelineHead(HttpClient client)
|
||||
{
|
||||
const BindingFlags flags = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public;
|
||||
for (var t = (Type?)client.GetType(); t != null; t = t.BaseType)
|
||||
|
||||
+9
-9
@@ -77,7 +77,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
|
||||
ing.Start();
|
||||
host.Start();
|
||||
@@ -98,7 +98,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
|
||||
ing.Start();
|
||||
host.Start();
|
||||
@@ -127,7 +127,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var progress = false;
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller,
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller,
|
||||
espTimeoutProvider: () => null, accountSetupProgressProbe: () => Volatile.Read(ref progress));
|
||||
|
||||
ing.Start();
|
||||
@@ -156,7 +156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var progress = false;
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller,
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller,
|
||||
espTimeoutProvider: () => null, accountSetupProgressProbe: () => Volatile.Read(ref progress));
|
||||
|
||||
ing.Start();
|
||||
@@ -186,7 +186,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller,
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller,
|
||||
espTimeoutProvider: () => null, accountSetupProgressProbe: () => true);
|
||||
|
||||
ing.Start();
|
||||
@@ -212,7 +212,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller,
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller,
|
||||
espTimeoutProvider: () => null, accountSetupProgressProbe: () => true);
|
||||
|
||||
ing.Start();
|
||||
@@ -236,7 +236,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
|
||||
ing.Start();
|
||||
host.Start();
|
||||
@@ -259,7 +259,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
|
||||
ing.Start();
|
||||
host.Start();
|
||||
@@ -282,7 +282,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost(
|
||||
"S1", "T1", ing, clock, NewLogger(),
|
||||
ing, clock, NewLogger(),
|
||||
controller: controller,
|
||||
safetyCapOverride: TimeSpan.FromMilliseconds(75),
|
||||
espTimeoutProvider: () => null);
|
||||
|
||||
@@ -44,7 +44,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
// from a real backend). We exercise the bundle-shape contract here.
|
||||
var rcs = new AutopilotMonitor.Agent.V2.Core.Configuration.RemoteConfigService(
|
||||
auth.BackendApiClient, "tenant-x", logger,
|
||||
auth.EmergencyReporter, auth.DistressReporter, auth.AuthFailureTracker);
|
||||
auth.EmergencyReporter, auth.AuthFailureTracker);
|
||||
var rc = new AgentConfigResponse();
|
||||
var merge = new RemoteConfigMergeResult();
|
||||
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Net.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Runtime;
|
||||
using Xunit;
|
||||
|
||||
@@ -68,6 +74,43 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.False(bundle.HasClientCertificate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildAuthClients_carries_the_server_certificate_observer_for_both_stacks()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var bundle = BackendClientFactory.BuildAuthClients(MinimalBootstrapTokenConfig(), agentVersion: "1.2.3.4", logger: NewLogger(tmp.Path));
|
||||
|
||||
Assert.NotNull(bundle.ServerCertificates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildBackendApiHttpClient_installs_the_observing_callback_on_the_inner_handler()
|
||||
{
|
||||
var observer = new ServerCertificateObserver();
|
||||
using var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA");
|
||||
|
||||
using var client = BackendClientFactory.BuildBackendApiHttpClient(new NetworkMetrics(), clientCertificate: null, agentVersion: "1.2.3.4", observer);
|
||||
var inner = InnerHandler(client);
|
||||
var callback = inner.ServerCertificateCustomValidationCallback;
|
||||
|
||||
Assert.NotNull(callback);
|
||||
Assert.False(callback!(null!, cert, null!, SslPolicyErrors.RemoteCertificateNameMismatch));
|
||||
Assert.True(callback(null!, cert, null!, SslPolicyErrors.None));
|
||||
Assert.Equal(cert.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildBackendApiHttpClient_without_observer_keeps_the_default_validation()
|
||||
{
|
||||
using var client = BackendClientFactory.BuildBackendApiHttpClient(new NetworkMetrics(), clientCertificate: null, agentVersion: "1.2.3.4");
|
||||
|
||||
Assert.Null(InnerHandler(client).ServerCertificateCustomValidationCallback);
|
||||
}
|
||||
|
||||
// Pipeline: NetworkMetricsRecordingHandler → HttpClientHandler.
|
||||
private static HttpClientHandler InnerHandler(HttpClient client) =>
|
||||
(HttpClientHandler)((DelegatingHandler)MtlsHttpClientFactoryTests.GetPipelineHead(client)).InnerHandler;
|
||||
|
||||
// =========================================================== TelemetryClientResult
|
||||
|
||||
[Fact]
|
||||
|
||||
+14
-12
@@ -122,20 +122,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
// ============================================================ failed-registration record
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_records_a_failed_run_with_its_window_and_bounded_error()
|
||||
public void RecordFailedRun_records_a_failed_run_with_its_window_and_bounded_error()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var persistence = new SessionIdPersistence(tmp.Path);
|
||||
var started = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
var returned = BackendSessionRegistration.RecordFailedRun(
|
||||
persistence, SessionRegistrationResult.Failed(new string('e', 1000)),
|
||||
started, started.AddSeconds(1204.46), linkUpAtStart: false, linkUpAtEnd: true,
|
||||
configFetchOutcome: "UsedDefaults", logger: NewLogger(tmp.Path));
|
||||
configFetchOutcome: "UsedDefaults", networkFingerprint: null, logger: NewLogger(tmp.Path));
|
||||
|
||||
var record = persistence.LoadRegistrationFailure();
|
||||
Assert.NotNull(record);
|
||||
Assert.Equal(1, record!.FailedRuns);
|
||||
Assert.Equal(0, record.InterceptedRuns);
|
||||
Assert.Equal("Failed", record.Outcome);
|
||||
Assert.Equal(BackendSessionRegistration.MaxRecordedErrorLength, record.LastError!.Length);
|
||||
Assert.False(record.NetworkLinkUpAtStart);
|
||||
@@ -143,33 +144,34 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.Equal(1204.5, record.AttemptWindowSeconds);
|
||||
Assert.Equal("UsedDefaults", record.ConfigFetchOutcome);
|
||||
Assert.Equal(started.AddSeconds(1204.46), record.LastFailedAtUtc);
|
||||
// The caller decides on the record as written, including this run.
|
||||
Assert.Equal(1, returned!.FailedRuns);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_records_auth_failures_and_clears_on_success()
|
||||
public void RecordFailedRun_records_auth_failures_and_a_success_clears_them()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var persistence = new SessionIdPersistence(tmp.Path);
|
||||
var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
BackendSessionRegistration.RecordFailedRun(
|
||||
persistence, SessionRegistrationResult.AuthFailed(401, "unauthorized"),
|
||||
at, at.AddSeconds(3), true, true, "Succeeded", NewLogger(tmp.Path));
|
||||
at, at.AddSeconds(3), true, true, "Succeeded", null, NewLogger(tmp.Path));
|
||||
Assert.Equal("AuthFailed", persistence.LoadRegistrationFailure()!.Outcome);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
persistence, SessionRegistrationResult.Succeeded(new RegisterSessionResponse { Success = true }),
|
||||
at, at.AddSeconds(1), true, true, "Succeeded", NewLogger(tmp.Path));
|
||||
BackendSessionRegistration.ClearAfterSuccess(persistence, () => null, NewLogger(tmp.Path));
|
||||
Assert.Null(persistence.LoadRegistrationFailure());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_without_persistence_is_a_no_op()
|
||||
public void Recording_without_persistence_is_a_no_op()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
Assert.Null(BackendSessionRegistration.RecordFailedRun(
|
||||
null, SessionRegistrationResult.Failed("x"), DateTime.UtcNow, DateTime.UtcNow,
|
||||
null, null, null, NewLogger(tmp.Path));
|
||||
null, null, null, null, NewLogger(tmp.Path)));
|
||||
Assert.Null(BackendSessionRegistration.ClearAfterSuccess(null, () => null, NewLogger(tmp.Path)));
|
||||
}
|
||||
|
||||
// ============================================================ emergency-break message
|
||||
|
||||
+477
@@ -0,0 +1,477 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Runtime;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
using Xunit;
|
||||
using ProgramAlias = AutopilotMonitor.Agent.V2.Program;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
{
|
||||
/// <summary>
|
||||
/// A registration that something on the network path answered at process start (client
|
||||
/// certificate interception plan, K4/K5): a buffer of further boots, then the emergency break's
|
||||
/// cleanup; the pending ClientCertificateNotReceived distress leaves the machine first; a later
|
||||
/// successful registration reports the failed starts on <c>agent_started</c>. Each start is
|
||||
/// ended the way <see cref="BackendSessionRegistration.Register"/> ends it: record the failed
|
||||
/// run, then conclude it.
|
||||
/// </summary>
|
||||
public sealed class InterceptedRegistrationTests : IDisposable
|
||||
{
|
||||
private static readonly DateTime At = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly SessionIdPersistence _persistence;
|
||||
private readonly string _stateDir;
|
||||
private int _cleanupCalls;
|
||||
|
||||
public InterceptedRegistrationTests()
|
||||
{
|
||||
_logger = new AgentLogger(Path.Combine(_tmp.Path, "logs"), AgentLogLevel.Info);
|
||||
_persistence = new SessionIdPersistence(_tmp.Path);
|
||||
_persistence.GetOrCreate(_logger);
|
||||
_stateDir = Path.Combine(_tmp.Path, "State");
|
||||
}
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
// ============================================================ K4: buffer, then clean up
|
||||
|
||||
[Fact]
|
||||
public void Agent_default_buffer_allows_one_further_boot_then_cleans_up()
|
||||
{
|
||||
var config = new AgentConfiguration();
|
||||
Assert.Equal(1, config.InterceptedRegistrationRetryBoots);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
Assert.False(GaveUp);
|
||||
Assert.True(_persistence.SessionExists());
|
||||
Assert.Equal(0, _cleanupCalls);
|
||||
var record = _persistence.LoadRegistrationFailure()!;
|
||||
Assert.Equal(1, record.FailedRuns);
|
||||
Assert.Equal(1, record.InterceptedRuns);
|
||||
Assert.Equal("AuthFailed", record.Outcome);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config)); // the give-up keeps exit 6
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)] // clean up at the first intercepted start
|
||||
[InlineData(2)]
|
||||
[InlineData(5)]
|
||||
public void Gives_up_on_the_first_intercepted_start_beyond_the_buffer(int buffer)
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = buffer };
|
||||
|
||||
for (var start = 1; start <= buffer; start++)
|
||||
{
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
Assert.False(GaveUp, $"gave up on start {start} with a buffer of {buffer}");
|
||||
}
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, 1)]
|
||||
[InlineData(false, 0)]
|
||||
public void Give_up_runs_the_cleanup_only_with_SelfDestructOnComplete_like_the_emergency_break(
|
||||
bool selfDestructOnComplete, int expectedCleanupCalls)
|
||||
{
|
||||
var config = new AgentConfiguration
|
||||
{
|
||||
InterceptedRegistrationRetryBoots = 0,
|
||||
SelfDestructOnComplete = selfDestructOnComplete,
|
||||
};
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_intercepted_start_takes_the_buffer_from_the_cached_config()
|
||||
{
|
||||
// The config fetch of an intercepted start is intercepted too: it falls back to the
|
||||
// cache, and the registration decides on the cached buffer, not the agent default.
|
||||
var config = new AgentConfiguration();
|
||||
var service = new InterceptedConfigService(_logger, cached: new AgentConfigResponse
|
||||
{
|
||||
InterceptedRegistrationRetryBoots = 0,
|
||||
SelfDestructOnComplete = true,
|
||||
});
|
||||
|
||||
var fetched = await service.FetchConfigAsync(retryOnTransientErrors: true);
|
||||
Assert.Equal(RemoteConfigFetchOutcome.FromCache, service.LastFetchOutcome);
|
||||
RemoteConfigMerger.Merge(config, fetched, _logger);
|
||||
Assert.Equal(0, config.InterceptedRegistrationRetryBoots);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Backend_rejection_and_network_failure_keep_their_exit_codes_and_never_give_up()
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 };
|
||||
|
||||
Assert.Equal(6, Boot(BackendRejected(), config));
|
||||
Assert.Equal(7, Boot(NetworkFailure(), config));
|
||||
|
||||
Assert.False(GaveUp);
|
||||
Assert.True(_persistence.SessionExists());
|
||||
Assert.Equal(0, _cleanupCalls);
|
||||
var record = _persistence.LoadRegistrationFailure()!;
|
||||
Assert.Equal(2, record.FailedRuns);
|
||||
Assert.Equal(0, record.InterceptedRuns);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Only_an_intercepted_start_gives_up_even_when_the_count_already_exceeds_the_buffer()
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 5 };
|
||||
Boot(Intercepted(), config);
|
||||
Boot(Intercepted(), config);
|
||||
config.InterceptedRegistrationRetryBoots = 0; // e.g. a later cached config lowered it
|
||||
|
||||
Assert.Equal(6, Boot(BackendRejected(), config));
|
||||
Assert.Equal(7, Boot(NetworkFailure(), config));
|
||||
|
||||
Assert.False(GaveUp);
|
||||
Assert.Equal(0, _cleanupCalls);
|
||||
var record = _persistence.LoadRegistrationFailure()!;
|
||||
Assert.Equal(4, record.FailedRuns);
|
||||
Assert.Equal(2, record.InterceptedRuns); // unchanged by the other failures
|
||||
Assert.Equal("Failed", record.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failures_of_another_kind_between_interceptions_do_not_reset_the_count()
|
||||
{
|
||||
var config = new AgentConfiguration(); // buffer 1
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
Assert.Equal(7, Boot(NetworkFailure(), config));
|
||||
Assert.Equal(6, Boot(BackendRejected(), config));
|
||||
Assert.False(GaveUp);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config)); // second interception since the last success
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Only_a_successful_registration_resets_the_count()
|
||||
{
|
||||
var config = new AgentConfiguration(); // buffer 1
|
||||
Boot(Intercepted(), config);
|
||||
BackendSessionRegistration.ClearAfterSuccess(_persistence, () => "fp-office", _logger);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
Assert.False(GaveUp);
|
||||
Assert.Equal(1, _persistence.LoadRegistrationFailure()!.InterceptedRuns);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Without_a_written_record_the_buffer_cannot_be_enforced_and_the_next_boot_retries()
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 };
|
||||
|
||||
var exit = BackendSessionRegistration.ConcludeFailedRun(
|
||||
Intercepted(), record: null, config, authFailureTracker: null,
|
||||
_persistence, _stateDir, CleanupFactory, _logger);
|
||||
|
||||
Assert.Equal(6, exit);
|
||||
Assert.False(GaveUp);
|
||||
Assert.True(_persistence.SessionExists());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public async Task The_registration_answer_decides_whether_the_start_counts_as_intercepted(bool foreign)
|
||||
{
|
||||
// Through the real client: no correlation echo + the platform's "Client Certificate
|
||||
// Required" page is an interception; a JSON 403 that echoes the id is the backend's verdict.
|
||||
using var handler = new RecordingHttpMessageHandler();
|
||||
if (foreign)
|
||||
handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
else
|
||||
handler.QueueStatus(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}");
|
||||
using var http = new HttpClient(handler);
|
||||
var apiClient = new BackendApiClient(
|
||||
http, "https://backend.test", manufacturer: "m", model: "d", serialNumber: "s",
|
||||
useBootstrapTokenAuth: false, bootstrapToken: null, agentVersion: "0.0.0", logger: _logger);
|
||||
var distress = new FakeDistressReporter();
|
||||
var config = new AgentConfiguration
|
||||
{
|
||||
ApiBaseUrl = "https://backend.test",
|
||||
TenantId = "00000000-0000-0000-0000-000000000001",
|
||||
SessionId = _persistence.GetOrCreate(_logger),
|
||||
InterceptedRegistrationRetryBoots = 0,
|
||||
};
|
||||
|
||||
var result = await SessionRegistrationHelper.RegisterWithRetryAsync(
|
||||
apiClient, config, agentVersion: "0.0.0", logger: _logger,
|
||||
authFailureTracker: NewTracker(distress),
|
||||
deviceHardware: ("m", "d", "s"),
|
||||
networkLinkWait: _ => Task.CompletedTask);
|
||||
|
||||
Assert.Equal(SessionRegistrationOutcome.AuthFailed, result.Outcome);
|
||||
Assert.Equal(foreign, result.IsIntercepted);
|
||||
Assert.Equal(
|
||||
foreign ? DistressErrorType.ClientCertificateNotReceived : DistressErrorType.DeviceNotRegistered,
|
||||
Assert.Single(distress.Calls).ErrorType);
|
||||
|
||||
Assert.Equal(6, Boot(result, config));
|
||||
Assert.Equal(foreign, GaveUp);
|
||||
}
|
||||
|
||||
// ============================================================ the distress leaves first
|
||||
|
||||
[Fact]
|
||||
public void The_exit_waits_for_the_pending_ClientCertificateNotReceived_send()
|
||||
{
|
||||
var record = RecordFailed(Intercepted(), "fp-office");
|
||||
var (tracker, inFlight) = DispatchSlowCertificateDistress();
|
||||
|
||||
var exit = BackendSessionRegistration.ConcludeFailedRun(
|
||||
Intercepted(), record, new AgentConfiguration(), tracker,
|
||||
_persistence, _stateDir, CleanupFactory, _logger);
|
||||
|
||||
Assert.Equal(6, exit);
|
||||
Assert.True(inFlight.IsCompleted);
|
||||
Assert.False(GaveUp);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_give_up_waits_for_the_pending_send_before_it_cleans_up()
|
||||
{
|
||||
var record = RecordFailed(Intercepted(), "fp-office");
|
||||
var (tracker, inFlight) = DispatchSlowCertificateDistress();
|
||||
bool? sentBeforeCleanup = null;
|
||||
|
||||
var exit = BackendSessionRegistration.ConcludeFailedRun(
|
||||
Intercepted(), record, new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 }, tracker,
|
||||
_persistence, _stateDir,
|
||||
() =>
|
||||
{
|
||||
sentBeforeCleanup = inFlight.IsCompleted;
|
||||
throw new InvalidOperationException("test-harness: cleanup factory must not spawn real PowerShell in unit tests");
|
||||
},
|
||||
_logger);
|
||||
|
||||
Assert.Equal(6, exit);
|
||||
Assert.True(sentBeforeCleanup);
|
||||
Assert.True(GaveUp);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_wait_for_the_distress_is_bounded_by_its_own_http_timeout()
|
||||
{
|
||||
Assert.True(BackendSessionRegistration.PendingDistressBound <= TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ============================================================ K5: measurement point
|
||||
|
||||
[Theory]
|
||||
[InlineData("fp-office", "fp-office", false)]
|
||||
[InlineData("fp-office", "fp-home", true)]
|
||||
public void A_success_after_failed_starts_reports_the_record_it_clears(
|
||||
string failedOn, string registeredOn, bool networkChanged)
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 5 };
|
||||
Boot(NetworkFailure(), config, fingerprint: "fp-elsewhere");
|
||||
Boot(Intercepted(), config, fingerprint: failedOn); // the latest failed start's network counts
|
||||
|
||||
var prior = BackendSessionRegistration.ClearAfterSuccess(_persistence, () => registeredOn, _logger)!;
|
||||
|
||||
Assert.Equal(2, prior.FailedRuns);
|
||||
Assert.Equal(1, prior.InterceptedRuns);
|
||||
Assert.Equal("AuthFailed", prior.LastOutcome);
|
||||
Assert.Equal(networkChanged, prior.NetworkChanged);
|
||||
Assert.Null(_persistence.LoadRegistrationFailureRecord());
|
||||
|
||||
var data = EmitAgentStarted(prior);
|
||||
Assert.Equal(2, data["priorRegistrationFailedRuns"]);
|
||||
Assert.Equal(1, data["priorRegistrationInterceptedRuns"]);
|
||||
Assert.Equal("AuthFailed", data["priorRegistrationLastOutcome"]);
|
||||
Assert.Equal(networkChanged, data["priorRegistrationNetworkChanged"]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, "fp-home")] // the failed start's network was unknown
|
||||
[InlineData("fp-office", null)] // this start's network is unknown
|
||||
public void Network_change_is_omitted_when_either_fingerprint_is_unknown(string? failedOn, string? registeredOn)
|
||||
{
|
||||
Boot(Intercepted(), new AgentConfiguration(), fingerprint: failedOn);
|
||||
|
||||
var prior = BackendSessionRegistration.ClearAfterSuccess(_persistence, () => registeredOn!, _logger)!;
|
||||
|
||||
Assert.Null(prior.NetworkChanged);
|
||||
var data = EmitAgentStarted(prior);
|
||||
Assert.Equal(1, data["priorRegistrationFailedRuns"]);
|
||||
Assert.False(data.ContainsKey("priorRegistrationNetworkChanged"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_success_without_failed_starts_reports_nothing_and_never_reads_the_network()
|
||||
{
|
||||
var captures = 0;
|
||||
|
||||
var prior = BackendSessionRegistration.ClearAfterSuccess(
|
||||
_persistence, () => { captures++; return "fp-office"; }, _logger);
|
||||
|
||||
Assert.Null(prior);
|
||||
Assert.Equal(0, captures);
|
||||
Assert.DoesNotContain(EmitAgentStarted(null).Keys, k => k.StartsWith("priorRegistration", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_network_fingerprint_stays_on_the_device()
|
||||
{
|
||||
Boot(Intercepted(), new AgentConfiguration(), fingerprint: "0123456789abcdef");
|
||||
|
||||
// Kept locally beside the record ...
|
||||
Assert.Contains("0123456789abcdef", File.ReadAllText(Path.Combine(_tmp.Path, "registration-failure.json")));
|
||||
Assert.Equal("0123456789abcdef", _persistence.LoadRegistrationFailureRecord()!.NetworkFingerprint);
|
||||
|
||||
// ... while the summary the emergency break sends cannot carry it.
|
||||
var wire = JsonConvert.SerializeObject(new AgentErrorReport
|
||||
{
|
||||
PriorRegistrationFailure = _persistence.LoadRegistrationFailure(),
|
||||
});
|
||||
Assert.DoesNotContain("0123456789abcdef", wire);
|
||||
Assert.DoesNotContain("Fingerprint", wire);
|
||||
Assert.Contains("\"InterceptedRuns\":1", wire);
|
||||
}
|
||||
|
||||
// ============================================================ helpers
|
||||
|
||||
private static SessionRegistrationResult Intercepted() =>
|
||||
SessionRegistrationResult.AuthFailed(403, "The device certificate did not reach the backend.", intercepted: true);
|
||||
|
||||
private static SessionRegistrationResult BackendRejected() =>
|
||||
SessionRegistrationResult.AuthFailed(403, "Backend returned 403 Forbidden. The device is not authorized.");
|
||||
|
||||
private static SessionRegistrationResult NetworkFailure() =>
|
||||
SessionRegistrationResult.Failed("No such host is known.");
|
||||
|
||||
private bool GaveUp => File.Exists(Path.Combine(_stateDir, "enrollment-complete.marker"));
|
||||
|
||||
private AuthFailureTracker NewTracker(FakeDistressReporter distress) =>
|
||||
new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance, _logger, distress);
|
||||
|
||||
/// <summary>
|
||||
/// What the registration helper does when the answer did not come from the backend: the
|
||||
/// tracker dispatches ClientCertificateNotReceived — here a send that stays in flight for a
|
||||
/// second, started right before the conclusion so nothing else can use the time up.
|
||||
/// </summary>
|
||||
private (AuthFailureTracker Tracker, Task InFlight) DispatchSlowCertificateDistress()
|
||||
{
|
||||
Task? inFlight = null;
|
||||
var distress = new FakeDistressReporter { Send = () => inFlight = Task.Delay(TimeSpan.FromSeconds(1)) };
|
||||
var tracker = NewTracker(distress);
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired());
|
||||
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, Assert.Single(distress.Calls).ErrorType);
|
||||
Assert.False(inFlight!.IsCompleted);
|
||||
return (tracker, inFlight);
|
||||
}
|
||||
|
||||
private RegistrationFailureSummary? RecordFailed(SessionRegistrationResult result, string? fingerprint) =>
|
||||
BackendSessionRegistration.RecordFailedRun(
|
||||
_persistence, result, At, At.AddSeconds(2), linkUpAtStart: true, linkUpAtEnd: true,
|
||||
configFetchOutcome: "FromCache", networkFingerprint: fingerprint, logger: _logger);
|
||||
|
||||
/// <summary>One start whose registration failed with <paramref name="result"/>, ended the way Register ends it.</summary>
|
||||
private int Boot(SessionRegistrationResult result, AgentConfiguration config, string? fingerprint = "fp-office")
|
||||
{
|
||||
var record = RecordFailed(result, fingerprint);
|
||||
return BackendSessionRegistration.ConcludeFailedRun(
|
||||
result, record, config, authFailureTracker: null, _persistence, _stateDir, CleanupFactory, _logger);
|
||||
}
|
||||
|
||||
private CleanupService CleanupFactory()
|
||||
{
|
||||
_cleanupCalls++;
|
||||
// The real factory spawns a PowerShell cleanup script; throwing makes the cleanup retry
|
||||
// swallow and log it, so the call is counted without a side effect (as in ProgramGuardsTests).
|
||||
throw new InvalidOperationException("test-harness: cleanup factory must not spawn real PowerShell in unit tests");
|
||||
}
|
||||
|
||||
/// <summary>The emergency break's end state (ProgramGuardsTests): marker, session files gone, cleanup per SelfDestructOnComplete.</summary>
|
||||
private void AssertCleanedUpLikeTheEmergencyBreak(int expectedCleanupCalls)
|
||||
{
|
||||
Assert.True(GaveUp);
|
||||
Assert.False(_persistence.SessionExists());
|
||||
Assert.Null(_persistence.LoadRegistrationFailure());
|
||||
Assert.Equal(expectedCleanupCalls, _cleanupCalls);
|
||||
// The next start ends at the bootstrap guard, exactly as after the break.
|
||||
Assert.True(ProgramAlias.CheckEnrollmentCompleteMarker(
|
||||
_stateDir, selfDestructOnComplete: false, cleanupServiceFactory: null, _logger, consoleMode: false));
|
||||
}
|
||||
|
||||
private Dictionary<string, object> EmitAgentStarted(PriorRegistrationFailures? prior)
|
||||
{
|
||||
var sink = new FakeSignalIngressSink();
|
||||
LifecycleEmitters.EmitAgentStarted(
|
||||
new InformationalEventPost(sink, SystemClock.Instance, _logger),
|
||||
new AgentConfiguration { SessionId = "S1", TenantId = "00000000-0000-0000-0000-000000000001", CommandLineArgs = string.Empty },
|
||||
new ProgramAlias.PreviousExitSummary { ExitType = "reboot_kill" },
|
||||
"2.0.test",
|
||||
remoteConfigService: null,
|
||||
_logger,
|
||||
priorRegistrationFailures: prior);
|
||||
return (Dictionary<string, object>)Assert.Single(sink.Posted).TypedPayload!;
|
||||
}
|
||||
|
||||
/// <summary>A start whose config fetch is answered by the interception: it falls back to the cache.</summary>
|
||||
private sealed class InterceptedConfigService : RemoteConfigService
|
||||
{
|
||||
private readonly AgentConfigResponse _cached;
|
||||
|
||||
public InterceptedConfigService(AgentLogger logger, AgentConfigResponse cached)
|
||||
: base(
|
||||
new BackendApiClient(
|
||||
httpClient: new HttpClient(), baseUrl: "http://localhost", manufacturer: string.Empty,
|
||||
model: string.Empty, serialNumber: string.Empty, useBootstrapTokenAuth: false,
|
||||
bootstrapToken: null, agentVersion: "0.0.0", logger: logger),
|
||||
"00000000-0000-0000-0000-000000000001",
|
||||
logger)
|
||||
{
|
||||
_cached = cached;
|
||||
}
|
||||
|
||||
protected override Task<AgentConfigResponse> CallBackendAsync() =>
|
||||
throw new BackendAuthException("The device certificate did not reach the backend.", 403,
|
||||
classification: InterceptionFixtures.CertificateRequired());
|
||||
|
||||
protected override AgentConfigResponse LoadCachedConfig() => _cached;
|
||||
|
||||
protected override Task WaitForNetworkLinkAsync() => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
{
|
||||
/// <summary>
|
||||
/// Host wiring for <c>TelemetryUploadOrchestrator.SessionGone</c>
|
||||
/// (HTTP 410 on the current session's telemetry): exactly one
|
||||
/// <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/> emergency report carrying the
|
||||
/// status, then a synthetic forced <c>terminate_session</c> tagged <c>origin=session_gone</c>
|
||||
/// — and nothing at all once a shutdown is already in progress.
|
||||
/// </summary>
|
||||
public sealed class ServerControlPlaneSessionGoneTests
|
||||
{
|
||||
private sealed class RecordingEmergencyReporter : EmergencyReporter
|
||||
{
|
||||
private readonly List<string> _sequence;
|
||||
public List<(AgentErrorType ErrorType, string Message, int? HttpStatusCode)> Reports { get; } =
|
||||
new List<(AgentErrorType, string, int?)>();
|
||||
|
||||
public RecordingEmergencyReporter(List<string> sequence)
|
||||
: base(apiClient: null!, sessionId: "S1", tenantId: "T1", agentVersion: "0.0.0", logger: null!)
|
||||
{
|
||||
_sequence = sequence;
|
||||
}
|
||||
|
||||
public override Task TrySendAsync(
|
||||
AgentErrorType errorType,
|
||||
string message,
|
||||
int? httpStatusCode = null,
|
||||
long? sequenceNumber = null,
|
||||
int attempts = 1,
|
||||
TimeSpan? perAttemptTimeout = null,
|
||||
TimeSpan? retryDelay = null,
|
||||
double? sessionAgeHours = null,
|
||||
RegistrationFailureSummary? priorRegistrationFailure = null)
|
||||
{
|
||||
Reports.Add((errorType, message, httpStatusCode));
|
||||
_sequence.Add("report");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class RecordingDispatcher : ServerActionDispatcher
|
||||
{
|
||||
private readonly List<string> _sequence;
|
||||
public List<ServerAction> Dispatched { get; } = new List<ServerAction>();
|
||||
|
||||
public RecordingDispatcher(AgentConfiguration config, AgentLogger logger, InformationalEventPost post, List<string> sequence)
|
||||
: base(
|
||||
configuration: config,
|
||||
logger: logger,
|
||||
rotateConfigAsync: () => Task.FromResult(true),
|
||||
uploadDiagnosticsAsync: _ => Task.FromResult<DiagnosticsUploadResult>(null!),
|
||||
onTerminateRequested: _ => Task.CompletedTask,
|
||||
post: post)
|
||||
{
|
||||
_sequence = sequence;
|
||||
}
|
||||
|
||||
public override Task DispatchAsync(List<ServerAction> actions)
|
||||
{
|
||||
Dispatched.AddRange(actions);
|
||||
_sequence.Add("dispatch");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Rig : IDisposable
|
||||
{
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public AgentLogger Logger { get; }
|
||||
public List<string> Sequence { get; } = new List<string>();
|
||||
public RecordingEmergencyReporter Reporter { get; }
|
||||
public RecordingDispatcher Dispatcher { get; }
|
||||
public ManualResetEventSlim Shutdown { get; } = new ManualResetEventSlim(false);
|
||||
public ManualResetEventSlim ShutdownComplete { get; } = new ManualResetEventSlim(false);
|
||||
|
||||
public Rig()
|
||||
{
|
||||
Logger = new AgentLogger(Path.Combine(Tmp.Path, "logs"), AgentLogLevel.Info);
|
||||
var config = new AgentConfiguration { SessionId = "S1", TenantId = "T1", ApiBaseUrl = "http://localhost" };
|
||||
var post = new InformationalEventPost(
|
||||
new FakeSignalIngressSink(),
|
||||
new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc)));
|
||||
Reporter = new RecordingEmergencyReporter(Sequence);
|
||||
Dispatcher = new RecordingDispatcher(config, Logger, post, Sequence);
|
||||
}
|
||||
|
||||
public void Handle(string reason = "session gone: http 410") =>
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.HandleSessionGone(
|
||||
reason, Reporter, Dispatcher, Shutdown, ShutdownComplete, Logger);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Shutdown.Dispose();
|
||||
ShutdownComplete.Dispose();
|
||||
Tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reports_once_with_410_then_dispatches_forced_terminate_tagged_session_gone()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
|
||||
rig.Handle();
|
||||
|
||||
Assert.Equal(new[] { "report", "dispatch" }, rig.Sequence);
|
||||
|
||||
var report = Assert.Single(rig.Reporter.Reports);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType);
|
||||
Assert.Equal(410, report.HttpStatusCode);
|
||||
Assert.Contains("410", report.Message);
|
||||
|
||||
var action = Assert.Single(rig.Dispatcher.Dispatched);
|
||||
Assert.Equal(ServerActionTypes.TerminateSession, action.Type);
|
||||
Assert.Contains("410", action.Reason);
|
||||
Assert.NotNull(action.Params);
|
||||
var parameters = action.Params!;
|
||||
Assert.Equal("true", parameters["forceSelfDestruct"]);
|
||||
Assert.Equal("0", parameters["gracePeriodSeconds"]);
|
||||
Assert.Equal(TerminationOrigins.SessionGone, parameters["origin"]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, false)]
|
||||
[InlineData(false, true)]
|
||||
public void Does_nothing_when_a_shutdown_is_already_in_progress(bool shutdownSet, bool shutdownCompleteSet)
|
||||
{
|
||||
using var rig = new Rig();
|
||||
if (shutdownSet) rig.Shutdown.Set();
|
||||
if (shutdownCompleteSet) rig.ShutdownComplete.Set();
|
||||
|
||||
rig.Handle();
|
||||
|
||||
Assert.Empty(rig.Reporter.Reports);
|
||||
Assert.Empty(rig.Dispatcher.Dispatched);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReadOrigin_returns_the_origin_param_or_null()
|
||||
{
|
||||
Assert.Equal(TerminationOrigins.SessionGone,
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = TerminationOrigins.SessionGone }));
|
||||
Assert.Equal(TerminationOrigins.KillSignal,
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = TerminationOrigins.KillSignal, ["forceSelfDestruct"] = "true" }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["adminOutcome"] = "Failed" }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = string.Empty }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(null!));
|
||||
}
|
||||
}
|
||||
}
|
||||
+145
-34
@@ -1,11 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
@@ -261,8 +262,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_endpoint_unavailable_suppresses_distress_but_still_counts()
|
||||
public void RecordFailure_only_the_stopped_app_page_counts_without_distress()
|
||||
{
|
||||
// Rewritten deliberately for the client-certificate interception plan: the former
|
||||
// "endpoint unavailable" flag covered every HTML platform page, so the platform's
|
||||
// "Client Certificate Required" page (TLS inspection on the path) shut healthy agents
|
||||
// down. Now only the stopped-app page keeps that semantics — it counts toward the
|
||||
// shutdown (a dead endpoint must not retry forever) without a distress report that
|
||||
// could not reach it — while the certificate page never counts (tests below).
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var fakeDistress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 2, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
@@ -270,19 +277,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
// Platform-403s from a stopped endpoint: a distress report cannot reach it and the
|
||||
// DeviceNotRegistered classification would be wrong — nothing must be dispatched.
|
||||
tracker.RecordFailure(403, "agent/config", endpointUnavailable: true);
|
||||
tracker.RecordFailure(403, "agent/register-session", endpointUnavailable: true);
|
||||
tracker.RecordFailure(403, "agent/config", InterceptionFixtures.CertificateRequired());
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures);
|
||||
|
||||
tracker.RecordFailure(403, "agent/config", InterceptionFixtures.StoppedApp());
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.StoppedApp());
|
||||
|
||||
Assert.Empty(fakeDistress.Calls);
|
||||
// The shutdown thresholds still apply — a dead endpoint must not retry forever.
|
||||
Assert.Equal(2, tracker.ConsecutiveFailures);
|
||||
Assert.Equal(1, events);
|
||||
Assert.DoesNotContain(fakeDistress.Calls, c => c.ErrorType == DistressErrorType.DeviceNotRegistered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_endpoint_unavailable_does_not_consume_streak_report_slot()
|
||||
public void RecordFailure_stopped_app_page_does_not_consume_streak_report_slot()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var fakeDistress = new FakeDistressReporter();
|
||||
@@ -290,7 +297,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
|
||||
// The streak opens with a platform-403 (no dispatch), then a live backend answers
|
||||
// with a genuine 403 — that one must still produce the streak's single report.
|
||||
tracker.RecordFailure(403, "agent/config", endpointUnavailable: true);
|
||||
tracker.RecordFailure(403, "agent/config", InterceptionFixtures.StoppedApp());
|
||||
tracker.RecordFailure(403, "agent/config");
|
||||
tracker.RecordFailure(403, "agent/config");
|
||||
|
||||
@@ -298,6 +305,133 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.Equal(DistressErrorType.DeviceNotRegistered, fakeDistress.Calls[0].ErrorType);
|
||||
}
|
||||
|
||||
// ============================================================= Foreign answers (interception)
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_foreign_never_reaches_the_threshold()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 10, clock, NewLogger(), new FakeDistressReporter());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired());
|
||||
clock.Now = clock.Now.AddMinutes(3); // also past the time window
|
||||
}
|
||||
|
||||
Assert.Equal(0, events);
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_foreign_dispatches_one_ClientCertificateNotReceived_per_process()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var fakeDistress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired());
|
||||
tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired());
|
||||
tracker.RecordSuccess(); // a success re-arms the streak report, never the per-process one
|
||||
tracker.RecordFailure(401, "agent/telemetry", InterceptionFixtures.CertificateRequired(401));
|
||||
|
||||
var call = Assert.Single(fakeDistress.Calls);
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType);
|
||||
Assert.Equal(403, call.HttpStatusCode);
|
||||
// Operation, status, sub-reason, page title, issuer — in that order, within the 256 budget.
|
||||
Assert.StartsWith("agent/register-session http 403 client_certificate_required;", call.Message);
|
||||
Assert.True(
|
||||
call.Message.IndexOf(InterceptionFixtures.PlatformPageTitle, StringComparison.Ordinal) <
|
||||
call.Message.IndexOf(InterceptionFixtures.InspectionIssuer, StringComparison.Ordinal));
|
||||
Assert.True(call.Message.Length <= 256, $"distress message is {call.Message.Length} characters");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_foreign_within_a_backend_streak_neither_resets_nor_advances_it()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var fakeDistress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
tracker.RecordFailure(403, "agent/telemetry");
|
||||
tracker.RecordFailure(403, "agent/telemetry");
|
||||
tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired());
|
||||
tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired());
|
||||
|
||||
Assert.Equal(2, tracker.ConsecutiveFailures);
|
||||
Assert.Equal(0, events);
|
||||
|
||||
tracker.RecordFailure(403, "agent/telemetry"); // the third backend rejection trips it
|
||||
|
||||
Assert.Equal(3, tracker.ConsecutiveFailures);
|
||||
Assert.Equal(1, events);
|
||||
Assert.Equal(1, fakeDistress.Calls.Count(c => c.ErrorType == DistressErrorType.DeviceNotRegistered));
|
||||
}
|
||||
|
||||
// ============================================================= Pending foreign distress
|
||||
|
||||
[Fact]
|
||||
public void WaitForPendingForeignDistress_returns_at_once_when_nothing_was_dispatched()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), new FakeDistressReporter());
|
||||
|
||||
Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.Zero));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WaitForPendingForeignDistress_waits_for_the_send_to_finish()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
Task? inFlight = null;
|
||||
var fakeDistress = new FakeDistressReporter { Send = () => inFlight = Task.Delay(200) };
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired());
|
||||
|
||||
Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.FromSeconds(10)));
|
||||
Assert.True(inFlight!.IsCompleted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WaitForPendingForeignDistress_stops_at_the_bound()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var neverSent = new TaskCompletionSource<bool>();
|
||||
var fakeDistress = new FakeDistressReporter { Send = () => neverSent.Task };
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired());
|
||||
|
||||
var watch = Stopwatch.StartNew();
|
||||
var finished = tracker.WaitForPendingForeignDistress(TimeSpan.FromMilliseconds(100));
|
||||
watch.Stop();
|
||||
|
||||
Assert.False(finished);
|
||||
Assert.True(watch.Elapsed < TimeSpan.FromSeconds(5), $"waited {watch.Elapsed}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WaitForPendingForeignDistress_never_waits_for_a_backend_streak_report()
|
||||
{
|
||||
// Only the ClientCertificateNotReceived send is kept: a failed start must not block
|
||||
// on anything else.
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var neverSent = new TaskCompletionSource<bool>();
|
||||
var fakeDistress = new FakeDistressReporter { Send = () => neverSent.Task };
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
|
||||
tracker.RecordFailure(403, "agent/register-session");
|
||||
|
||||
Assert.Equal(DistressErrorType.DeviceNotRegistered, Assert.Single(fakeDistress.Calls).ErrorType);
|
||||
Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.Zero));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordSuccess_rearms_distress_dispatch_for_the_next_streak()
|
||||
{
|
||||
@@ -329,29 +463,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.Equal(1, events);
|
||||
}
|
||||
|
||||
private sealed class FakeDistressReporter : DistressReporter
|
||||
{
|
||||
public sealed class Call
|
||||
{
|
||||
public DistressErrorType ErrorType { get; set; }
|
||||
public string Message { get; set; } = default!;
|
||||
public int? HttpStatusCode { get; set; }
|
||||
}
|
||||
|
||||
public List<Call> Calls { get; } = new List<Call>();
|
||||
|
||||
public FakeDistressReporter()
|
||||
: base(baseUrl: "https://example.test", tenantId: "t", manufacturer: "m", model: "d", serialNumber: "s", agentVersion: "0.0.0", logger: null!)
|
||||
{
|
||||
}
|
||||
|
||||
public override Task TrySendAsync(DistressErrorType errorType, string message, int? httpStatusCode = null)
|
||||
{
|
||||
Calls.Add(new Call { ErrorType = errorType, Message = message, HttpStatusCode = httpStatusCode });
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeClock : AutopilotMonitor.DecisionCore.Engine.IClock
|
||||
{
|
||||
public DateTime Now { get; set; }
|
||||
|
||||
@@ -298,9 +298,50 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.False(File.Exists(Path.Combine(tmp.Path, "registration-failure.json")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationFailure_counts_intercepted_runs_and_keeps_the_latest_network_fingerprint()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var sut = new SessionIdPersistence(tmp.Path);
|
||||
var at = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc);
|
||||
var intercepted = FailedRun(at, outcome: "AuthFailed");
|
||||
intercepted.InterceptedRuns = 1;
|
||||
|
||||
sut.RecordRegistrationFailure(intercepted, networkFingerprint: "fp-office");
|
||||
sut.RecordRegistrationFailure(FailedRun(at.AddHours(1)), networkFingerprint: null);
|
||||
var merged = sut.RecordRegistrationFailure(intercepted, networkFingerprint: "fp-home");
|
||||
|
||||
Assert.Equal(3, merged!.FailedRuns);
|
||||
Assert.Equal(2, merged.InterceptedRuns);
|
||||
var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailureRecord();
|
||||
Assert.Equal(3, record!.Summary.FailedRuns);
|
||||
Assert.Equal(2, record.Summary.InterceptedRuns);
|
||||
Assert.Equal("fp-home", record.NetworkFingerprint);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_record_from_an_agent_before_the_interception_buffer_still_loads()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
File.WriteAllText(Path.Combine(tmp.Path, "registration-failure.json"),
|
||||
"{\"FailedRuns\":2,\"FirstFailedAtUtc\":\"2026-09-01T08:00:00Z\",\"LastFailedAtUtc\":\"2026-09-02T08:00:00Z\"," +
|
||||
"\"Outcome\":\"AuthFailed\",\"LastError\":\"x\",\"NetworkLinkUpAtStart\":true,\"NetworkLinkUpAtEnd\":true," +
|
||||
"\"AttemptWindowSeconds\":3.0,\"ConfigFetchOutcome\":\"FromCache\"}");
|
||||
|
||||
var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailureRecord();
|
||||
|
||||
Assert.Equal(2, record!.Summary.FailedRuns);
|
||||
Assert.Equal(0, record.Summary.InterceptedRuns);
|
||||
Assert.Null(record.NetworkFingerprint);
|
||||
Assert.Equal(new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), record.Summary.FirstFailedAtUtc);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not json")]
|
||||
[InlineData("{\"FailedRuns\":0}")]
|
||||
[InlineData("null")]
|
||||
[InlineData("[]")]
|
||||
[InlineData("")]
|
||||
public void LoadRegistrationFailure_reports_nothing_for_a_corrupt_or_empty_record(string content)
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
|
||||
+31
@@ -6,6 +6,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
@@ -96,6 +97,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.Equal(1, tracker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Intercepted_registration_ends_auth_failed_without_counting_and_reports_once()
|
||||
{
|
||||
// Something on the network path answered (TLS inspection): the run still ends
|
||||
// AuthFailed (exit 6, unchanged until the interception buffer exists), but the tracker
|
||||
// does not count it and sends ClientCertificateNotReceived instead of DeviceNotRegistered.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
var config = NewConfig();
|
||||
|
||||
var apiClient = new FakeApiClient(_ =>
|
||||
throw new BackendAuthException("The device certificate did not reach the backend.", statusCode: 403,
|
||||
classification: InterceptionFixtures.CertificateRequired()));
|
||||
var distress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0,
|
||||
clock: SystemClock.Instance, logger: logger, distressReporter: distress);
|
||||
|
||||
var result = await SessionRegistrationHelper.RegisterWithRetryAsync(
|
||||
apiClient, config, agentVersion: "0.0.0", logger: logger, authFailureTracker: tracker,
|
||||
networkLinkWait: _ => Task.CompletedTask);
|
||||
|
||||
Assert.Equal(SessionRegistrationOutcome.AuthFailed, result.Outcome);
|
||||
Assert.Equal(403, result.HttpStatusCode);
|
||||
Assert.Equal(1, apiClient.CallCount);
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures);
|
||||
var call = Assert.Single(distress.Calls);
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType);
|
||||
Assert.StartsWith("agent/register-session http 403 client_certificate_required", call.Message);
|
||||
}
|
||||
|
||||
// ── SESSION-OWNER-BINDING ────────────────────────────────────────────
|
||||
|
||||
private static BackendAuthException OwnerMismatch()
|
||||
|
||||
+3
-3
@@ -479,14 +479,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
public HashSet<string> ExistingKeys { get; } =
|
||||
new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public IReadOnlyList<string> Enumerate(RegistryHive hive, string packagesPath)
|
||||
public IReadOnlyList<string> Enumerate(RegistryHive _, string packagesPath)
|
||||
{
|
||||
return PackagesByPath.TryGetValue(packagesPath, out var ids)
|
||||
? (IReadOnlyList<string>)ids
|
||||
: Array.Empty<string>();
|
||||
}
|
||||
|
||||
public bool TryRead(RegistryHive hive, string packagesPath, string packageId, out RealmJoinPackageSnapshot snapshot)
|
||||
public bool TryRead(RegistryHive _, string _1, string packageId, out RealmJoinPackageSnapshot snapshot)
|
||||
{
|
||||
snapshot = new RealmJoinPackageSnapshot(
|
||||
packageId: packageId,
|
||||
@@ -497,7 +497,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
return true;
|
||||
}
|
||||
|
||||
public bool KeyExists(RegistryHive hive, string subPath)
|
||||
public bool KeyExists(RegistryHive _, string subPath)
|
||||
=> ExistingKeys.Contains(subPath);
|
||||
}
|
||||
|
||||
|
||||
+113
@@ -165,6 +165,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
public IReadOnlyCollection<string> StarvedAlreadyReportedOverride { get; set; } = Array.Empty<string>();
|
||||
public List<string> TerminationActionLog { get; } = new List<string>();
|
||||
public TimeSpan SpoolDrainPeriodOverride { get; set; } = TimeSpan.Zero;
|
||||
public int TerminalUploadAttemptRequests;
|
||||
// Session bc803955 — DevicePrep app-wave settle wait. Zero (default) disables the
|
||||
// wait so the existing timing-sensitive tests stay untouched; the dedicated settle
|
||||
// tests opt in with a real budget.
|
||||
@@ -267,6 +268,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
public long IngressPendingSignalCount => _rig.IngressPendingSignalCountAccessor!();
|
||||
public bool CanObserveSpool => _rig.PendingItemCountAccessor != null;
|
||||
public int SpoolPendingItemCount => _rig.PendingItemCountAccessor!();
|
||||
|
||||
public void RequestTerminalUploadAttempt()
|
||||
{
|
||||
Interlocked.Increment(ref _rig.TerminalUploadAttemptRequests);
|
||||
_rig.TerminationActionLog.Add("terminalUploadAttempt");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>ARCH-F2 — recording shutdown gate (no real shutdown.exe in tests).</summary>
|
||||
@@ -310,6 +317,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
new EnrollmentTerminatedEventArgs(
|
||||
reason, outcome, stage.ToString(), at ?? EndUtc);
|
||||
|
||||
/// <summary>
|
||||
/// The args <c>ServerControlPlane.OnTerminateRequested</c> synthesises for a
|
||||
/// <c>terminate_session</c> action tagged with <paramref name="origin"/>.
|
||||
/// </summary>
|
||||
private static EnrollmentTerminatedEventArgs ServerArgs(string origin) =>
|
||||
new EnrollmentTerminatedEventArgs(
|
||||
EnrollmentTerminationReason.DecisionTerminalStage,
|
||||
EnrollmentTerminationOutcome.Failed,
|
||||
SessionStage.Failed.ToString(),
|
||||
EndUtc,
|
||||
details: "Server-requested termination",
|
||||
origin: origin);
|
||||
|
||||
[Fact]
|
||||
public void Handle_writes_final_status_json_in_state_directory()
|
||||
{
|
||||
@@ -458,6 +478,99 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.Equal(0, rig.CleanupService.Invocations);
|
||||
}
|
||||
|
||||
// ============================================================= origin=session_gone (HTTP 410)
|
||||
|
||||
[Fact]
|
||||
public void Handle_session_gone_skips_spool_drain_and_diagnostics_but_still_cleans_up()
|
||||
{
|
||||
// The backend no longer knows the session: every drain would only burn its budget on
|
||||
// 410s and a diagnostics package would describe a deleted session. Marker, cleanup
|
||||
// and the (forced) self-destruct still run.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build();
|
||||
rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5);
|
||||
var spoolPolls = 0;
|
||||
rig.PendingItemCountAccessor = () => { spoolPolls++; return 5; }; // never drains — a poll would burn the budget
|
||||
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true);
|
||||
|
||||
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.SessionGone));
|
||||
|
||||
Assert.Equal(0, spoolPolls);
|
||||
Assert.Equal(0, rig.TerminalUploadAttemptRequests);
|
||||
Assert.Equal(0, rig.DiagnosticsUploads);
|
||||
Assert.DoesNotContain(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
|
||||
Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes);
|
||||
Assert.True(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
|
||||
Assert.Equal(1, rig.CleanupService.Invocations);
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
}
|
||||
|
||||
public enum TerminalExit { SelfDestruct, StandaloneReboot, WhiteGlovePart1 }
|
||||
|
||||
[Theory]
|
||||
[InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed, TerminalExit.SelfDestruct)]
|
||||
[InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Failed, SessionStage.Failed, TerminalExit.SelfDestruct)]
|
||||
[InlineData(EnrollmentTerminationReason.StoppedWaiting, EnrollmentTerminationOutcome.TimedOut, SessionStage.AwaitingHello, TerminalExit.SelfDestruct)]
|
||||
[InlineData(EnrollmentTerminationReason.MaxLifetimeExceeded, EnrollmentTerminationOutcome.TimedOut, SessionStage.AwaitingHello, TerminalExit.SelfDestruct)]
|
||||
[InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed, TerminalExit.StandaloneReboot)]
|
||||
[InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed, TerminalExit.WhiteGlovePart1)]
|
||||
public void Handle_while_uploads_are_intercepted_makes_one_terminal_attempt_then_exits_as_usual(
|
||||
EnrollmentTerminationReason reason, EnrollmentTerminationOutcome outcome, SessionStage stage, TerminalExit exit)
|
||||
{
|
||||
// Enrollment end during an interception pause: no deferral. DrainSpool asks for the one
|
||||
// terminal upload attempt, waits its bounded budget (the spool never empties — every
|
||||
// upload is answered on the network path), then the exit runs exactly as for any
|
||||
// session. Unsent telemetry is lost, as today.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = stage }.Build();
|
||||
rig.SpoolDrainPeriodOverride = TimeSpan.FromMilliseconds(200);
|
||||
rig.PendingItemCountAccessor = () => 7;
|
||||
rig.DiagnosticsResult = new DiagnosticsUploadResult
|
||||
{
|
||||
ErrorCode = "The device certificate did not reach the backend (TLS inspection or a proxy on the network path): 403 Forbidden ...",
|
||||
};
|
||||
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: exit != TerminalExit.StandaloneReboot);
|
||||
cfg.RebootOnComplete = exit == TerminalExit.StandaloneReboot;
|
||||
|
||||
rig.Build(cfg).Handle(sender: null!, Args(reason, outcome, stage));
|
||||
|
||||
Assert.Equal(1, rig.TerminalUploadAttemptRequests);
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
Assert.True(rig.TerminationActionLog.IndexOf("terminalUploadAttempt") < rig.TerminationActionLog.IndexOf("signalShutdown"));
|
||||
if (exit == TerminalExit.WhiteGlovePart1)
|
||||
{
|
||||
Assert.True(rig.SessionPersistence.IsWhiteGloveResume());
|
||||
Assert.Equal(0, rig.CleanupService.Invocations);
|
||||
return;
|
||||
}
|
||||
|
||||
Assert.Equal(1, rig.DiagnosticsUploads);
|
||||
Assert.Contains(Constants.EventTypes.DiagnosticsUploadFailed, rig.EmittedEventTypes);
|
||||
Assert.True(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
|
||||
Assert.Equal(exit == TerminalExit.SelfDestruct ? 1 : 0, rig.CleanupService.Invocations);
|
||||
Assert.Equal(exit == TerminalExit.StandaloneReboot ? 1 : 0, rig.RebootInvocations);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_kill_signal_origin_still_drains_spool_and_uploads_diagnostics()
|
||||
{
|
||||
// The administrator kill signal keeps its full sequence — only session_gone skips.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build();
|
||||
rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5);
|
||||
var spoolPolls = 0;
|
||||
rig.PendingItemCountAccessor = () => { spoolPolls++; return 0; };
|
||||
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true);
|
||||
|
||||
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.KillSignal));
|
||||
|
||||
Assert.True(spoolPolls > 0);
|
||||
Assert.Equal(1, rig.DiagnosticsUploads);
|
||||
Assert.Contains(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
|
||||
Assert.Equal(1, rig.CleanupService.Invocations);
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_signals_shutdown_exactly_once_even_on_error()
|
||||
{
|
||||
|
||||
+148
-18
@@ -6,16 +6,19 @@ using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// 401/403 classification in <see cref="BackendApiClient"/>: a backend JSON 403 is a real
|
||||
/// device-authorization verdict, while an HTML body means the response came from the Azure
|
||||
/// platform / an edge proxy (stopped or retired app) and must NOT be reported as
|
||||
/// "device is not authorized". Field case 2026-08-09: a pre-cutover agent hit
|
||||
/// the stopped legacy Function App and support chased a non-existent authorization problem.
|
||||
/// 401/403 classification in <see cref="BackendApiClient"/>: only a response that echoes the
|
||||
/// request's correlation id is the backend's device-authorization verdict. The platform page of
|
||||
/// a stopped app (field case 2026-08-09: a pre-cutover agent hit the stopped legacy Function App
|
||||
/// and support chased a non-existent authorization problem) and anything answered on the network
|
||||
/// path (TLS inspection: "Client Certificate Required", proxy pages) must NOT be reported as
|
||||
/// "device is not authorized".
|
||||
/// </summary>
|
||||
public sealed class BackendApiClientAuthClassificationTests
|
||||
{
|
||||
@@ -24,17 +27,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
private readonly Func<HttpRequestMessage, HttpResponseMessage> _responder;
|
||||
public StubHandler(Func<HttpRequestMessage, HttpResponseMessage> responder) => _responder = responder;
|
||||
|
||||
public int Calls { get; private set; }
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(_responder(request));
|
||||
{
|
||||
Calls++;
|
||||
var response = _responder(request);
|
||||
response.RequestMessage = request; // as HttpClientHandler does
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
}
|
||||
|
||||
private static BackendApiClient BuildClient(HttpStatusCode status, string body, string mediaType)
|
||||
{
|
||||
var handler = new StubHandler(_ => new HttpResponseMessage(status)
|
||||
/// <param name="fromBackend">
|
||||
/// True: the response echoes the request's X-Correlation-ID like the backend's middleware.
|
||||
/// False: a platform or proxy answer without the echo.
|
||||
/// </param>
|
||||
private static StubHandler Answer(HttpStatusCode status, string body, string mediaType, bool fromBackend) =>
|
||||
new StubHandler(request =>
|
||||
{
|
||||
Content = new StringContent(body, Encoding.UTF8, mediaType)
|
||||
var response = new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, mediaType) };
|
||||
if (fromBackend && request.Headers.TryGetValues(ResponseOriginClassifier.CorrelationIdHeader, out var ids))
|
||||
response.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, ids);
|
||||
return response;
|
||||
});
|
||||
return new BackendApiClient(
|
||||
|
||||
private static BackendApiClient BuildClient(HttpMessageHandler handler, ServerCertificateObserver? serverCertificates = null) =>
|
||||
new BackendApiClient(
|
||||
new HttpClient(handler),
|
||||
baseUrl: "https://backend.example",
|
||||
manufacturer: "Contoso",
|
||||
@@ -43,8 +61,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
useBootstrapTokenAuth: false,
|
||||
bootstrapToken: null,
|
||||
agentVersion: "2.0.0",
|
||||
logger: null);
|
||||
}
|
||||
logger: null,
|
||||
serverCertificates: serverCertificates);
|
||||
|
||||
private static BackendApiClient BuildClient(HttpStatusCode status, string body, string mediaType, bool fromBackend = true) =>
|
||||
BuildClient(Answer(status, body, mediaType, fromBackend));
|
||||
|
||||
[Fact]
|
||||
public async Task Backend_json_403_is_reported_as_device_not_authorized()
|
||||
@@ -134,16 +155,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
public async Task Platform_html_403_is_reported_as_endpoint_unavailable_with_page_title()
|
||||
{
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden,
|
||||
"<!DOCTYPE html><html><head><title>Web App - Unavailable</title></head><body>stopped</body></html>",
|
||||
"text/html");
|
||||
HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage, "text/html", fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||
|
||||
Assert.Equal(403, ex.StatusCode);
|
||||
Assert.True(ex.EndpointUnavailable);
|
||||
Assert.Contains("Web App - Unavailable", ex.Message);
|
||||
Assert.False(ex.IsForeignResponse);
|
||||
Assert.Equal(ResponseOrigin.PlatformStopped, ex.Origin);
|
||||
Assert.Contains("HTML platform error page (\"Web App - Unavailable\")", ex.Message);
|
||||
Assert.Contains("NOT a device-authorization failure", ex.Message);
|
||||
Assert.Contains("stale ApiBaseUrl", ex.Message);
|
||||
Assert.DoesNotContain("Check client certificate", ex.Message);
|
||||
}
|
||||
|
||||
@@ -153,7 +175,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden,
|
||||
" <html><body>Error 403 - This web app is stopped.</body></html>",
|
||||
"text/plain");
|
||||
"text/plain",
|
||||
fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||
|
||||
@@ -161,6 +184,113 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Contains("HTML platform error page", ex.Message);
|
||||
}
|
||||
|
||||
// ── answered on the network path (TLS inspection, proxy) ─────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Certificate_required_page_says_the_certificate_did_not_reach_the_backend()
|
||||
{
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||
|
||||
Assert.Equal(403, ex.StatusCode);
|
||||
Assert.True(ex.IsForeignResponse);
|
||||
Assert.False(ex.EndpointUnavailable);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, ex.Classification.ForeignReason);
|
||||
Assert.Null(ex.ErrorCode);
|
||||
// The cause leads: RegistrationFailureSummary.LastError keeps 256 characters.
|
||||
Assert.StartsWith("The device certificate did not reach the backend (TLS inspection or a proxy on the network path)", ex.Message);
|
||||
Assert.Contains("HTML platform error page (\"Web App - Unavailable\")", ex.Message);
|
||||
Assert.True(ex.Message.IndexOf("Web App - Unavailable", StringComparison.Ordinal) < 256);
|
||||
Assert.Contains("NOT a device-authorization failure", ex.Message);
|
||||
Assert.DoesNotContain("stale ApiBaseUrl", ex.Message);
|
||||
Assert.DoesNotContain("Check client certificate", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Proxy_page_keeps_the_phrase_and_title_ANALYZE_NET_001_matches()
|
||||
{
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage, "text/html", fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(
|
||||
() => client.GetDiagnosticsUploadUrlAsync("tenant-1", "session-1", "package.zip"));
|
||||
|
||||
Assert.True(ex.IsForeignResponse);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, ex.Classification.ForeignReason);
|
||||
Assert.Contains("HTML platform error page (\"Example Gateway - Notification\")", ex.Message);
|
||||
Assert.Contains("403", ex.Message);
|
||||
Assert.DoesNotContain("stale ApiBaseUrl", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Json_403_without_the_echo_is_foreign_and_its_errorCode_is_ignored()
|
||||
{
|
||||
// An errorCode in a body the backend did not author must not drive a session rotation.
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden,
|
||||
"{\"success\":false,\"errorCode\":\"session_owner_mismatch\"}",
|
||||
"application/json",
|
||||
fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(
|
||||
() => client.RegisterSessionAsync(new AutopilotMonitor.Shared.Models.SessionRegistration()));
|
||||
|
||||
Assert.True(ex.IsForeignResponse);
|
||||
Assert.Null(ex.ErrorCode);
|
||||
Assert.Contains("a response without the backend's correlation echo", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Foreign_answer_carries_the_observed_server_certificate_issuer()
|
||||
{
|
||||
var observer = new ServerCertificateObserver();
|
||||
using (var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA"))
|
||||
{
|
||||
observer.Validate(cert, System.Net.Security.SslPolicyErrors.None);
|
||||
}
|
||||
using var client = BuildClient(
|
||||
Answer(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false),
|
||||
observer);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||
|
||||
Assert.Equal("CN=Example Inspection CA", ex.Classification.ServerCertIssuer);
|
||||
}
|
||||
|
||||
// ── agent/error: a report the backend never received is not delivered ──
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.OK, true, true)]
|
||||
[InlineData(HttpStatusCode.InternalServerError, true, true)] // unchanged: only the round-trip counts
|
||||
[InlineData(HttpStatusCode.Forbidden, true, true)] // the backend answered
|
||||
[InlineData(HttpStatusCode.Forbidden, false, false)] // answered on the network path
|
||||
[InlineData(HttpStatusCode.Unauthorized, false, false)]
|
||||
public async Task ReportAgentError_counts_as_delivered_unless_the_backend_never_answered(
|
||||
HttpStatusCode status, bool fromBackend, bool expectedDelivered)
|
||||
{
|
||||
using var client = BuildClient(status, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend);
|
||||
|
||||
var delivered = await client.ReportAgentErrorAsync(new AgentErrorReport { TenantId = "tenant-1", SessionId = "session-1" });
|
||||
|
||||
Assert.Equal(expectedDelivered, delivered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Intercepted_emergency_report_rolls_its_reservation_back()
|
||||
{
|
||||
var handler = Answer(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false);
|
||||
using var client = BuildClient(handler);
|
||||
var reporter = new EmergencyReporter(client, "session-1", "tenant-1", "2.0.0", logger: null);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "first");
|
||||
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "second");
|
||||
|
||||
// Not delivered → no dedup, no cooldown: the same key is attempted again.
|
||||
Assert.Equal(2, handler.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Backend_401_keeps_authorization_message_and_status()
|
||||
{
|
||||
|
||||
+32
@@ -156,6 +156,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Contains("BackendTelemetryUploader: ingest PERMANENT", logContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UploadBatch_logs_the_whole_interception_finding_at_warning()
|
||||
{
|
||||
// The old PERMANENT line carried the first 200 body characters, which ended before the
|
||||
// platform's "Client Certificate Required" text. The INTERCEPTED line names the finding.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var sut = new BackendTelemetryUploader(
|
||||
httpClient: http,
|
||||
baseUrl: "https://backend.test",
|
||||
tenantId: "T1",
|
||||
logger: logger);
|
||||
|
||||
var result = await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
Assert.True(result.IsIntercepted);
|
||||
|
||||
Assert.True(handler.Captured[0].TryGetHeader("X-Correlation-ID", out var corr));
|
||||
var logContent = ReadLogs(tmp.Path);
|
||||
Assert.Contains("BackendTelemetryUploader: ingest INTERCEPTED", logContent);
|
||||
Assert.Contains($"corr={corr}", logContent);
|
||||
Assert.Contains("status=403", logContent);
|
||||
Assert.Contains("reason=client_certificate_required", logContent);
|
||||
Assert.Contains($"pageTitle=\"{InterceptionFixtures.PlatformPageTitle}\"", logContent);
|
||||
Assert.Contains("serverCertIssuer=\"(unknown)\"", logContent);
|
||||
Assert.DoesNotContain("ingest PERMANENT", logContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UploadBatch_empty_batch_no_log_no_network()
|
||||
{
|
||||
|
||||
+229
-2
@@ -4,7 +4,16 @@ using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Xunit;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -51,7 +60,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
string? model = "Latitude",
|
||||
string? serial = "ABC123",
|
||||
string? bootstrap = null,
|
||||
string? version = "11.0.0")
|
||||
string? version = "11.0.0",
|
||||
AuthFailureTracker? tracker = null,
|
||||
ServerCertificateObserver? serverCertificates = null,
|
||||
NetworkMetrics? networkMetrics = null)
|
||||
{
|
||||
Http = new HttpClient(Handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
Sut = new BackendTelemetryUploader(
|
||||
@@ -62,7 +74,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
model: model,
|
||||
serialNumber: serial,
|
||||
bootstrapToken: bootstrap,
|
||||
agentVersion: version);
|
||||
agentVersion: version,
|
||||
authFailureTracker: tracker,
|
||||
serverCertificates: serverCertificates,
|
||||
networkMetrics: networkMetrics);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
@@ -245,6 +260,85 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Null(result.ErrorReason);
|
||||
}
|
||||
|
||||
// ============================================================ 2xx without the correlation echo
|
||||
|
||||
[Fact]
|
||||
public async Task Ok_with_the_echo_is_a_success_and_not_counted()
|
||||
{
|
||||
var metrics = new NetworkMetrics();
|
||||
using var rig = new Rig(networkMetrics: metrics);
|
||||
rig.Handler.QueueStatus(HttpStatusCode.OK);
|
||||
|
||||
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal(0, metrics.OkWithoutEchoCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ok_without_the_echo_is_still_a_success_and_only_counted()
|
||||
{
|
||||
// A 200 that did not come from the backend (e.g. a block page served with 200): the
|
||||
// upload path behaves exactly as before — control signals are still read — and the
|
||||
// only difference is the counter.
|
||||
var metrics = new NetworkMetrics();
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance,
|
||||
new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Info));
|
||||
tracker.RecordFailure(403, "agent/telemetry");
|
||||
using var rig = new Rig(tracker: tracker, networkMetrics: metrics);
|
||||
rig.Handler.QueueForeign(HttpStatusCode.OK, "<html><title>Example</title></html>");
|
||||
rig.Handler.QueueForeign(HttpStatusCode.OK, "{\"deviceKillSignal\":true}", mediaType: "application/json");
|
||||
|
||||
var page = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
var signals = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(2) }, CancellationToken.None);
|
||||
|
||||
Assert.True(page.Success);
|
||||
Assert.Null(page.ErrorReason);
|
||||
Assert.True(signals.Success);
|
||||
Assert.True(signals.DeviceKillSignal);
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures); // a 2xx still resets the auth streak
|
||||
Assert.Equal(2, metrics.OkWithoutEchoCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Answers_other_than_2xx_never_count_as_ok_without_echo()
|
||||
{
|
||||
var metrics = new NetworkMetrics();
|
||||
using var rig = new Rig(networkMetrics: metrics);
|
||||
rig.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
rig.Handler.QueueForeign(HttpStatusCode.ServiceUnavailable, "<html>busy</html>");
|
||||
|
||||
await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, metrics.OkWithoutEchoCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_count_reaches_the_agent_metrics_snapshot()
|
||||
{
|
||||
var metrics = new NetworkMetrics();
|
||||
using var rig = new Rig(networkMetrics: metrics);
|
||||
rig.Handler.QueueForeign(HttpStatusCode.OK, "<html><title>Example</title></html>");
|
||||
await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
var sink = new FakeSignalIngressSink();
|
||||
using var tmp = new TempDirectory();
|
||||
var collector = new AgentSelfMetricsCollector(
|
||||
sessionId: "S1",
|
||||
tenantId: "T1",
|
||||
post: new InformationalEventPost(sink, SystemClock.Instance),
|
||||
networkMetrics: metrics,
|
||||
logger: new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Info),
|
||||
agentVersion: "2.0.test");
|
||||
collector.CollectSafe();
|
||||
|
||||
var snapshot = (IDictionary<string, object>)Assert.Single(sink.Posted, p =>
|
||||
p.TypedPayload is IDictionary<string, object> d && d.ContainsKey("agent_version")).TypedPayload!;
|
||||
Assert.Equal(1L, snapshot["telemetry_upload_ok_without_echo"]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.Unauthorized)]
|
||||
[InlineData(HttpStatusCode.Forbidden)]
|
||||
@@ -261,6 +355,104 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Contains(((int)status).ToString(), result.ErrorReason);
|
||||
}
|
||||
|
||||
// ============================================================ 401/403 origin
|
||||
|
||||
private sealed class OriginRig : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
|
||||
public FakeDistressReporter Distress { get; } = new FakeDistressReporter();
|
||||
public AuthFailureTracker Tracker { get; }
|
||||
public Rig Upload { get; }
|
||||
|
||||
public OriginRig()
|
||||
{
|
||||
Tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance,
|
||||
new AgentLogger(_tmp.Path, AgentLogLevel.Info), Distress);
|
||||
var serverCertificates = new ServerCertificateObserver();
|
||||
using (var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA"))
|
||||
{
|
||||
serverCertificates.Validate(cert, System.Net.Security.SslPolicyErrors.None);
|
||||
}
|
||||
Upload = new Rig(tracker: Tracker, serverCertificates: serverCertificates);
|
||||
}
|
||||
|
||||
public Task<UploadResult> UploadAsync() => Upload.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Upload.Dispose();
|
||||
_tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Backend_403_is_an_auth_failure_that_counts_and_reports_DeviceNotRegistered()
|
||||
{
|
||||
using var rig = new OriginRig();
|
||||
rig.Upload.Handler.QueueStatus(HttpStatusCode.Forbidden, body: "{\"error\":\"Device validation failed\"}");
|
||||
|
||||
var result = await rig.UploadAsync();
|
||||
|
||||
Assert.True(result.IsAuthFailure);
|
||||
Assert.False(result.IsIntercepted);
|
||||
Assert.Equal(1, rig.Tracker.ConsecutiveFailures);
|
||||
Assert.Equal(DistressErrorType.DeviceNotRegistered, Assert.Single(rig.Distress.Calls).ErrorType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Stopped_app_403_counts_like_an_auth_failure_without_distress()
|
||||
{
|
||||
using var rig = new OriginRig();
|
||||
rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage);
|
||||
|
||||
var result = await rig.UploadAsync();
|
||||
|
||||
Assert.True(result.IsAuthFailure);
|
||||
Assert.False(result.IsIntercepted);
|
||||
Assert.Equal(1, rig.Tracker.ConsecutiveFailures);
|
||||
Assert.Empty(rig.Distress.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Certificate_required_403_is_intercepted_never_counted_and_reported_once()
|
||||
{
|
||||
using var rig = new OriginRig();
|
||||
rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
|
||||
var result = await rig.UploadAsync();
|
||||
await rig.UploadAsync();
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.True(result.IsIntercepted);
|
||||
Assert.False(result.IsAuthFailure);
|
||||
Assert.False(result.IsTransient);
|
||||
Assert.Equal(403, result.Interception!.StatusCode);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, result.Interception.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.PlatformPageTitle, result.Interception.PageTitle);
|
||||
Assert.Equal("CN=Example Inspection CA", result.Interception.ServerCertIssuer);
|
||||
Assert.Equal(0, rig.Tracker.ConsecutiveFailures);
|
||||
var distress = Assert.Single(rig.Distress.Calls);
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, distress.ErrorType);
|
||||
Assert.Contains("agent/telemetry http 403 client_certificate_required", distress.Message);
|
||||
Assert.Contains("CN=Example Inspection CA", distress.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Proxy_page_403_is_intercepted_as_foreign_response()
|
||||
{
|
||||
using var rig = new OriginRig();
|
||||
rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage);
|
||||
|
||||
var result = await rig.UploadAsync();
|
||||
|
||||
Assert.True(result.IsIntercepted);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, result.Interception!.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.ProxyPageTitle, result.Interception.PageTitle);
|
||||
Assert.Equal(0, rig.Tracker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.BadRequest)] // 400
|
||||
[InlineData(HttpStatusCode.NotFound)] // 404
|
||||
@@ -275,9 +467,44 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.False(result.IsTransient);
|
||||
Assert.False(result.IsSessionGone);
|
||||
Assert.Contains(((int)status).ToString(), result.ErrorReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Gone_410_maps_to_SessionGone()
|
||||
{
|
||||
// 410 = the backend no longer knows this session. Not transient (no retry), not auth,
|
||||
// not poison, not a size problem — a distinct terminal kind the drain reacts to.
|
||||
using var rig = new Rig();
|
||||
rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "session unknown");
|
||||
|
||||
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.True(result.IsSessionGone);
|
||||
Assert.False(result.IsTransient);
|
||||
Assert.False(result.IsAuthFailure);
|
||||
Assert.False(result.IsPoison);
|
||||
Assert.False(result.RequiresSplit);
|
||||
Assert.Contains("410", result.ErrorReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Gone_410_with_poison_body_is_still_SessionGone_never_poison()
|
||||
{
|
||||
// A poison body on 410 must not be honoured: dropping items + emitting
|
||||
// telemetry_upload_poisoned would only produce the next 410 (emit/reject loop).
|
||||
using var rig = new Rig();
|
||||
rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "{\"poison\":true,\"rejectedRowKeys\":[\"rk1\"],\"reason\":\"x\"}");
|
||||
|
||||
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.True(result.IsSessionGone);
|
||||
Assert.False(result.IsPoison);
|
||||
Assert.Null(result.PoisonRowKeys);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.RequestTimeout)] // 408
|
||||
[InlineData((HttpStatusCode)429)] // TooManyRequests
|
||||
|
||||
+17
@@ -68,6 +68,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeBackendTelemetryUploader QueueSessionGone(string reason = "session gone: http 410")
|
||||
{
|
||||
_script.Enqueue(_ => UploadResult.SessionGone(reason));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>Queues a 403 the platform's "Client Certificate Required" page answered (an interception).</summary>
|
||||
public FakeBackendTelemetryUploader QueueIntercepted(int times = 1)
|
||||
{
|
||||
for (int i = 0; i < times; i++)
|
||||
{
|
||||
_script.Enqueue(_ => UploadResult.Intercepted(
|
||||
"intercepted: http 403 (client_certificate_required)", InterceptionFixtures.CertificateRequired()));
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeBackendTelemetryUploader QueueThrow(Exception ex)
|
||||
{
|
||||
_script.Enqueue(_ => throw ex);
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// Synthetic copies of the pages an agent gets instead of a backend answer: the Azure platform
|
||||
/// page when no client certificate arrived, the platform page of a stopped app, and a proxy's
|
||||
/// block page. Rebuilt from the observed structure and wording; none of them is a captured
|
||||
/// artifact. The style block keeps the platform marker text past the first 200 characters, where
|
||||
/// the old upload log excerpt ended.
|
||||
/// </summary>
|
||||
internal static class InterceptionFixtures
|
||||
{
|
||||
public const string PlatformPageTitle = "Web App - Unavailable";
|
||||
public const string ProxyPageTitle = "Example Gateway - Notification";
|
||||
public const string InspectionIssuer = "CN=Example Inspection CA, O=Example Security Ltd, C=US";
|
||||
|
||||
private const string PlatformPageHead =
|
||||
"<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\r\n" +
|
||||
"<html xmlns=\"http://www.w3.org/1999/xhtml\">\r\n<head>\r\n<title>Web App - Unavailable</title>\r\n" +
|
||||
"<style type=\"text/css\">\r\n html { height: 100%; width: 100%; }\r\n" +
|
||||
" #content { margin: 0 auto; padding: 40px 0 0; font-family: \"Segoe UI\", Arial, sans-serif; }\r\n" +
|
||||
" h3 { color: #3a3a3a; font-size: 26px; font-weight: 300; }\r\n</style>\r\n</head>\r\n<body>\r\n<div id=\"content\">\r\n";
|
||||
|
||||
public const string CertificateRequiredPage =
|
||||
PlatformPageHead +
|
||||
"<h3>Error 403 - Forbidden: Client Certificate Required.</h3>\r\n" +
|
||||
"<p>The web app you have attempted to reach requires a client certificate and does not accept any requests without one.</p>\r\n" +
|
||||
"</div>\r\n</body>\r\n</html>\r\n";
|
||||
|
||||
public const string StoppedAppPage =
|
||||
PlatformPageHead +
|
||||
"<h3>Error 403 - This web app is stopped.</h3>\r\n" +
|
||||
"<p>The web app you have attempted to reach is currently stopped and does not accept any requests.</p>\r\n" +
|
||||
"</div>\r\n</body>\r\n</html>\r\n";
|
||||
|
||||
public const string ProxyPage =
|
||||
"<!DOCTYPE html>\r\n<html><head><meta charset=\"utf-8\"><title>Example Gateway - Notification</title></head>\r\n" +
|
||||
"<body><h1>Access blocked</h1><p>The request was blocked by the security policy of your organization.</p></body></html>\r\n";
|
||||
|
||||
/// <summary>The platform's "Client Certificate Required" page seen through an inspecting proxy.</summary>
|
||||
public static ResponseClassification CertificateRequired(int statusCode = 403) =>
|
||||
ResponseClassification.ForForeign(statusCode, ResponseClassification.ClientCertificateRequired, PlatformPageTitle, isHtml: true, InspectionIssuer);
|
||||
|
||||
public static ResponseClassification StoppedApp(int statusCode = 403) =>
|
||||
ResponseClassification.ForPlatformStopped(statusCode, PlatformPageTitle, isHtml: true, serverCertIssuer: null);
|
||||
}
|
||||
}
|
||||
+18
-1
@@ -7,6 +7,7 @@ using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
@@ -23,17 +24,33 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
|
||||
public IReadOnlyList<CapturedRequest> Captured => _captured;
|
||||
|
||||
/// <summary>
|
||||
/// Queues a response from the backend: like the backend's correlation middleware, it echoes
|
||||
/// the request's <c>X-Correlation-ID</c> — the proof of origin a 401/403 is classified by.
|
||||
/// </summary>
|
||||
public RecordingHttpMessageHandler QueueStatus(HttpStatusCode status, string? body = null)
|
||||
{
|
||||
_script.Enqueue(_ =>
|
||||
_script.Enqueue(request =>
|
||||
{
|
||||
var resp = new HttpResponseMessage(status);
|
||||
if (body != null) resp.Content = new StringContent(body, Encoding.UTF8, "application/json");
|
||||
if (request.Headers.TryGetValues(ResponseOriginClassifier.CorrelationIdHeader, out var ids))
|
||||
resp.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, ids);
|
||||
return resp;
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a response that something on the network path produced (platform page, proxy
|
||||
/// page): no correlation echo.
|
||||
/// </summary>
|
||||
public RecordingHttpMessageHandler QueueForeign(HttpStatusCode status, string body, string mediaType = "text/html")
|
||||
{
|
||||
_script.Enqueue(_ => new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, mediaType) });
|
||||
return this;
|
||||
}
|
||||
|
||||
public RecordingHttpMessageHandler QueueThrow(Exception ex)
|
||||
{
|
||||
_script.Enqueue(_ => throw ex);
|
||||
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
#nullable enable
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="ResponseOriginClassifier"/>: only the echo of the request's correlation id proves
|
||||
/// that a 401/403 came from the backend. Fixtures are synthetic copies of the platform and proxy
|
||||
/// pages (<see cref="InterceptionFixtures"/>).
|
||||
/// </summary>
|
||||
public sealed class ResponseOriginClassifierTests
|
||||
{
|
||||
private const string SentId = "0123456789abcdef0123456789abcdef";
|
||||
|
||||
private static HttpResponseMessage Response(HttpStatusCode status, string? body, string mediaType, string? echo = null)
|
||||
{
|
||||
var response = new HttpResponseMessage(status);
|
||||
if (body != null) response.Content = new StringContent(body, Encoding.UTF8, mediaType);
|
||||
if (echo != null) response.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, echo);
|
||||
return response;
|
||||
}
|
||||
|
||||
private static ResponseClassification Classify(HttpResponseMessage response, string? issuer = null)
|
||||
{
|
||||
var body = ResponseOriginClassifier.ReadBodyPrefixAsync(response).GetAwaiter().GetResult();
|
||||
return ResponseOriginClassifier.Classify(response, SentId, body, issuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Platform_certificate_required_page_is_foreign_client_certificate_required()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html");
|
||||
|
||||
var c = Classify(response, InterceptionFixtures.InspectionIssuer);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, c.Origin);
|
||||
Assert.True(c.IsForeign);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, c.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.PlatformPageTitle, c.PageTitle);
|
||||
Assert.True(c.IsHtml);
|
||||
Assert.Equal(403, c.StatusCode);
|
||||
Assert.Equal(InterceptionFixtures.InspectionIssuer, c.ServerCertIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Platform_stopped_app_page_is_platform_stopped()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage, "text/html");
|
||||
|
||||
var c = Classify(response);
|
||||
|
||||
Assert.Equal(ResponseOrigin.PlatformStopped, c.Origin);
|
||||
Assert.False(c.IsForeign);
|
||||
Assert.Null(c.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.PlatformPageTitle, c.PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Proxy_page_is_foreign_response_with_its_title()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage, "text/html");
|
||||
|
||||
var c = Classify(response);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, c.Origin);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, c.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.ProxyPageTitle, c.PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Json_403_with_echoed_correlation_id_is_backend()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}", "application/json", echo: SentId);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Backend, Classify(response).Origin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Json_403_without_echo_is_foreign_response()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}", "application/json");
|
||||
|
||||
var c = Classify(response);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, c.Origin);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, c.ForeignReason);
|
||||
Assert.False(c.IsHtml);
|
||||
Assert.Null(c.PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Html_page_with_echo_is_backend_because_the_header_wins()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", echo: SentId);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Backend, Classify(response).Origin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Echo_of_another_request_id_is_not_proof()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Unauthorized, "{}", "application/json", echo: "ffffffffffffffffffffffffffffffff");
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, Classify(response).Origin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_sent_id_can_never_be_echoed()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, "{}", "application/json", echo: SentId);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, ResponseOriginClassifier.Classify(response, sentCorrelationId: null, bodyPrefix: "{}").Origin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Markers_match_case_insensitively_and_in_plain_text()
|
||||
{
|
||||
using var stopped = Response(HttpStatusCode.Forbidden, "error 403 - this web app is stopped.", "text/plain");
|
||||
using var certificate = Response(HttpStatusCode.Forbidden, "Forbidden: CLIENT CERTIFICATE REQUIRED", "text/plain");
|
||||
|
||||
Assert.Equal(ResponseOrigin.PlatformStopped, Classify(stopped).Origin);
|
||||
var c = Classify(certificate);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, c.ForeignReason);
|
||||
Assert.False(c.IsHtml);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Html_served_with_generic_content_type_is_sniffed()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, " <html><head><title>Blocked</title></head></html>", "text/plain");
|
||||
|
||||
var c = Classify(response);
|
||||
|
||||
Assert.True(c.IsHtml);
|
||||
Assert.Equal("Blocked", c.PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Page_title_is_capped_at_80_characters()
|
||||
{
|
||||
var longTitle = new string('x', 120);
|
||||
using var response = Response(HttpStatusCode.Forbidden, $"<html><head><title>{longTitle}</title></head></html>", "text/html");
|
||||
|
||||
Assert.Equal(new string('x', 80), Classify(response).PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Body_prefix_is_capped_and_a_marker_beyond_it_is_not_seen()
|
||||
{
|
||||
var body = new string(' ', ResponseOriginClassifier.MaxBodyChars) + "Client Certificate Required";
|
||||
using var response = Response(HttpStatusCode.Forbidden, body, "text/plain");
|
||||
|
||||
var prefix = await ResponseOriginClassifier.ReadBodyPrefixAsync(response);
|
||||
|
||||
Assert.Equal(ResponseOriginClassifier.MaxBodyChars, prefix!.Length);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, ResponseOriginClassifier.Classify(response, SentId, prefix).ForeignReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Missing_body_reads_as_null_and_classifies_as_foreign_response()
|
||||
{
|
||||
using var response = new HttpResponseMessage(HttpStatusCode.Forbidden);
|
||||
|
||||
Assert.Null(await ResponseOriginClassifier.ReadBodyPrefixAsync(response));
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, Classify(response).ForeignReason);
|
||||
}
|
||||
}
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Security;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="ServerCertificateObserver"/> must never weaken TLS validation: the installed
|
||||
/// callback returns exactly the verdict .NET Framework applies without one
|
||||
/// (<c>sslPolicyErrors == None</c>) and only records the issuer as evidence.
|
||||
/// </summary>
|
||||
public sealed class ServerCertificateObserverTests
|
||||
{
|
||||
internal static X509Certificate2 SelfSigned(string subject)
|
||||
{
|
||||
using (var rsa = RSA.Create(2048))
|
||||
{
|
||||
var request = new CertificateRequest(subject, rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
|
||||
return request.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1));
|
||||
}
|
||||
}
|
||||
|
||||
private static Func<HttpRequestMessage, X509Certificate2, X509Chain, SslPolicyErrors, bool> AttachedCallback(ServerCertificateObserver observer)
|
||||
{
|
||||
var handler = new HttpClientHandler();
|
||||
observer.Attach(handler);
|
||||
return handler.ServerCertificateCustomValidationCallback!;
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(SslPolicyErrors.None, true)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNotAvailable, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNameMismatch, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateChainErrors, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateNameMismatch, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateChainErrors, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNameMismatch | SslPolicyErrors.RemoteCertificateChainErrors, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateNameMismatch | SslPolicyErrors.RemoteCertificateChainErrors, false)]
|
||||
public void Callback_accepts_only_a_certificate_without_policy_errors(SslPolicyErrors errors, bool expected)
|
||||
{
|
||||
using var certificate = SelfSigned("CN=Example Inspection CA");
|
||||
var observer = new ServerCertificateObserver();
|
||||
var callback = AttachedCallback(observer);
|
||||
|
||||
Assert.Equal(expected, callback(null!, certificate, null!, errors));
|
||||
// Recorded whatever the verdict: a rejected inspection certificate is evidence too.
|
||||
Assert.Equal(certificate.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Last_validated_connection_wins()
|
||||
{
|
||||
using var first = SelfSigned("CN=Example Backend CA");
|
||||
using var second = SelfSigned("CN=Example Inspection CA");
|
||||
var observer = new ServerCertificateObserver();
|
||||
var callback = AttachedCallback(observer);
|
||||
|
||||
Assert.Null(observer.LastIssuer);
|
||||
callback(null!, first, null!, SslPolicyErrors.None);
|
||||
callback(null!, second, null!, SslPolicyErrors.None);
|
||||
|
||||
Assert.Equal(second.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Missing_certificate_is_rejected_and_keeps_the_last_issuer()
|
||||
{
|
||||
using var certificate = SelfSigned("CN=Example Backend CA");
|
||||
var observer = new ServerCertificateObserver();
|
||||
var callback = AttachedCallback(observer);
|
||||
callback(null!, certificate, null!, SslPolicyErrors.None);
|
||||
|
||||
Assert.False(callback(null!, null!, null!, SslPolicyErrors.RemoteCertificateNotAvailable));
|
||||
Assert.Equal(certificate.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Long_issuer_is_capped()
|
||||
{
|
||||
using var certificate = SelfSigned($"CN=Example Inspection CA, O={new string('o', 200)}");
|
||||
var observer = new ServerCertificateObserver();
|
||||
|
||||
observer.Validate(certificate, SslPolicyErrors.None);
|
||||
|
||||
Assert.Equal(ServerCertificateObserver.MaxIssuerLength, observer.LastIssuer!.Length);
|
||||
Assert.StartsWith(observer.LastIssuer, certificate.Issuer, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,10 +10,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
public sealed class TelemetrySpoolTests
|
||||
{
|
||||
private static TelemetryItemDraft EventDraft(string rowKey = "row-1") =>
|
||||
private const string OwnKey = "tenant_session";
|
||||
private const string ForeignKey = "tenant_old-session";
|
||||
|
||||
private static TelemetryItemDraft EventDraft(string rowKey = "row-1") => EventDraft(rowKey, OwnKey);
|
||||
|
||||
private static TelemetryItemDraft EventDraft(string rowKey, string partitionKey) =>
|
||||
new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "tenant_session",
|
||||
partitionKey: partitionKey,
|
||||
rowKey: rowKey,
|
||||
payloadJson: "{\"EventType\":\"agent_started\"}",
|
||||
isSessionScoped: true);
|
||||
@@ -195,6 +200,86 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Equal(1, next.TelemetryItemId);
|
||||
}
|
||||
|
||||
// ============================================================ Startup partition-key filter
|
||||
|
||||
[Fact]
|
||||
public void Ctor_drops_leading_pending_items_of_another_session_and_persists_cursor()
|
||||
{
|
||||
// Same spool directory, new session id: the previous session's unsent items would only
|
||||
// draw a 410 that must not end the new session. They are dropped on startup; the own
|
||||
// item behind them stays pending.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
s1.Enqueue(EventDraft("own", OwnKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(1, s2.LastUploadedItemId);
|
||||
Assert.Equal(2, s2.LastAssignedItemId);
|
||||
Assert.Equal(1, s2.PendingItemCount);
|
||||
Assert.Equal("own", Assert.Single(s2.Peek(10)).RowKey);
|
||||
|
||||
// Cursor persisted: a later start without the filter sees the same tail with the same bytes.
|
||||
var s3 = new TelemetrySpool(tmp.Path, Clock());
|
||||
Assert.Equal(1, s3.LastUploadedItemId);
|
||||
Assert.Equal(1, s3.PendingItemCount);
|
||||
Assert.Equal(s3.PendingBytes, s2.PendingBytes);
|
||||
|
||||
var next = s2.Enqueue(EventDraft("next", OwnKey));
|
||||
Assert.Equal(3, next.TelemetryItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_drops_nothing_when_a_foreign_item_follows_an_own_item()
|
||||
{
|
||||
// Impossible by construction (one process = one session); if it happens the order is
|
||||
// not trustworthy, so the filter keeps everything instead of guessing.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("own", OwnKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(3, s2.PendingItemCount);
|
||||
Assert.Equal(s1.PendingBytes, s2.PendingBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_keeps_pending_items_that_carry_the_expected_key()
|
||||
{
|
||||
// WhiteGlove Part 2 reuses the Part-1 session id: its leftovers carry the expected key.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("a", OwnKey));
|
||||
s1.Enqueue(EventDraft("b", OwnKey));
|
||||
s1.Enqueue(EventDraft("c", OwnKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(3, s2.PendingItemCount);
|
||||
Assert.Equal(s1.PendingBytes, s2.PendingBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_without_expected_key_keeps_foreign_items()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock());
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(2, s2.PendingItemCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Enqueue_with_RequiresImmediateFlush_raises_ImmediateFlushRequested()
|
||||
{
|
||||
|
||||
+282
@@ -0,0 +1,282 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// The transport-local pause after an intercepted upload (a 401/403 something on the network
|
||||
/// path answered): no HTTP while it holds, backoff 30 s → 15 min, a success ends it, the
|
||||
/// terminal phase gets exactly one attempt despite it, and it is never an auth failure.
|
||||
/// Driven by the virtual clock — the pause is a "not before", never a timer.
|
||||
/// </summary>
|
||||
public sealed class TelemetryUploadOrchestratorInterceptionTests
|
||||
{
|
||||
private static readonly DateTime Start = new DateTime(2026, 10, 12, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private sealed class Rig : IDisposable
|
||||
{
|
||||
private int _rowKey;
|
||||
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public VirtualClock Clock { get; } = new VirtualClock(Start);
|
||||
public FakeBackendTelemetryUploader Uploader { get; } = new FakeBackendTelemetryUploader();
|
||||
public TelemetryUploadOrchestrator Sut { get; }
|
||||
|
||||
public Rig()
|
||||
{
|
||||
var spool = new TelemetrySpool(Tmp.Path, Clock);
|
||||
Sut = new TelemetryUploadOrchestrator(spool, Uploader, Clock, batchSize: 10, retryBackoffs: new[] { TimeSpan.Zero });
|
||||
}
|
||||
|
||||
public void Enqueue(int count = 1)
|
||||
{
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
var rowKey = (++_rowKey).ToString("D10");
|
||||
Sut.Enqueue(new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "T1_S1",
|
||||
rowKey: rowKey,
|
||||
payloadJson: "{}",
|
||||
isSessionScoped: true,
|
||||
requiresImmediateFlush: false));
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() { Sut.Dispose(); Tmp.Dispose(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Intercepted_upload_retains_the_batch_and_later_drains_make_no_http_call()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue(2);
|
||||
rig.Uploader.QueueIntercepted();
|
||||
|
||||
var first = await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, rig.Uploader.CallCount);
|
||||
Assert.Equal(1, first.FailedBatches);
|
||||
Assert.Equal(0, first.UploadedItems);
|
||||
Assert.Contains("intercepted", first.LastErrorReason);
|
||||
Assert.Equal(-1, rig.Sut.LastUploadedItemId);
|
||||
Assert.True(rig.Sut.IsInterceptionPaused);
|
||||
Assert.False(rig.Sut.IsPaused);
|
||||
|
||||
// Periodic drains and immediate-flush wakeups (a new item, another drain) during the
|
||||
// pause: no HTTP, and an expected state rather than another failed batch.
|
||||
rig.Enqueue();
|
||||
var paused = await rig.Sut.DrainAllAsync();
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(29));
|
||||
await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, rig.Uploader.CallCount);
|
||||
Assert.Equal(0, paused.FailedBatches);
|
||||
Assert.Contains("upload intercepted, next attempt not before", paused.LastErrorReason);
|
||||
Assert.Equal(-1, rig.Sut.LastUploadedItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Backoff_runs_30s_1m_2m_5m_10m_then_15m_capped()
|
||||
{
|
||||
var expected = new[] { 30, 60, 120, 300, 600, 900, 900 }.Select(s => TimeSpan.FromSeconds(s)).ToArray();
|
||||
Assert.Equal(expected.Take(6), TelemetryUploadOrchestrator.InterceptionBackoffs);
|
||||
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueIntercepted(expected.Length);
|
||||
|
||||
foreach (var delay in expected)
|
||||
{
|
||||
var calls = rig.Uploader.CallCount;
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(calls + 1, rig.Uploader.CallCount);
|
||||
|
||||
rig.Clock.Advance(delay - TimeSpan.FromSeconds(1));
|
||||
Assert.True(rig.Sut.IsInterceptionPaused);
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(calls + 1, rig.Uploader.CallCount);
|
||||
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(1));
|
||||
Assert.False(rig.Sut.IsInterceptionPaused);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A clock that can be set back, as NTP does to a device that booted with a future RTC time.</summary>
|
||||
private sealed class SettableClock : AutopilotMonitor.DecisionCore.Engine.IClock
|
||||
{
|
||||
public SettableClock(DateTime utcNow) { UtcNow = utcNow; }
|
||||
public DateTime UtcNow { get; set; }
|
||||
public Task Delay(TimeSpan delay, System.Threading.CancellationToken cancellationToken = default) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Clock_set_back_past_the_pause_start_ends_the_pause_instead_of_stretching_it()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var clock = new SettableClock(Start);
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueIntercepted(2);
|
||||
using var sut = new TelemetryUploadOrchestrator(new TelemetrySpool(tmp.Path, clock), uploader, clock, retryBackoffs: new[] { TimeSpan.Zero });
|
||||
sut.Enqueue(new TelemetryItemDraft(TelemetryItemKind.Event, "T1_S1", "0000000001", "{}", isSessionScoped: true));
|
||||
|
||||
await sut.DrainAllAsync();
|
||||
clock.UtcNow = Start.AddHours(-2);
|
||||
|
||||
Assert.False(sut.IsInterceptionPaused);
|
||||
await sut.DrainAllAsync();
|
||||
Assert.Equal(2, uploader.CallCount);
|
||||
Assert.True(sut.IsInterceptionPaused); // re-armed from the corrected clock
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Successful_upload_ends_the_pause_and_resets_the_backoff()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue(3);
|
||||
rig.Uploader.QueueIntercepted(2).QueueOk().QueueIntercepted();
|
||||
|
||||
await rig.Sut.DrainAllAsync(); // intercepted, 30 s
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||
await rig.Sut.DrainAllAsync(); // intercepted, 1 min
|
||||
rig.Clock.Advance(TimeSpan.FromMinutes(1));
|
||||
var cleared = await rig.Sut.DrainAllAsync(); // the path is clear
|
||||
|
||||
Assert.Equal(3, cleared.UploadedItems);
|
||||
Assert.False(rig.Sut.IsInterceptionPaused);
|
||||
|
||||
rig.Enqueue();
|
||||
await rig.Sut.DrainAllAsync(); // intercepted again: back to 30 s
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(29));
|
||||
Assert.True(rig.Sut.IsInterceptionPaused);
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(1));
|
||||
Assert.False(rig.Sut.IsInterceptionPaused);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Terminal_drain_makes_exactly_one_attempt_despite_the_pause()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueIntercepted(3);
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(1, rig.Uploader.CallCount);
|
||||
|
||||
Assert.True(rig.Sut.GrantTerminalAttempt());
|
||||
var terminal = await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(2, rig.Uploader.CallCount);
|
||||
Assert.Equal(1, terminal.FailedBatches);
|
||||
Assert.True(rig.Sut.IsInterceptionPaused);
|
||||
|
||||
// One attempt per process: a later terminal drain (orchestrator.Stop) honours the pause.
|
||||
rig.Sut.GrantTerminalAttempt();
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(2, rig.Uploader.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Terminal_attempt_that_gets_through_drains_everything()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue(25);
|
||||
rig.Uploader.QueueIntercepted().QueueOk(3);
|
||||
await rig.Sut.DrainAllAsync();
|
||||
|
||||
rig.Sut.GrantTerminalAttempt();
|
||||
var terminal = await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(25, terminal.UploadedItems);
|
||||
Assert.Equal(4, rig.Uploader.CallCount);
|
||||
Assert.Equal(24, rig.Sut.LastUploadedItemId);
|
||||
Assert.False(rig.Sut.IsInterceptionPaused);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Grant_without_a_pause_changes_nothing_until_an_interception_meets_it()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
Assert.False(rig.Sut.GrantTerminalAttempt());
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueOk().QueueIntercepted(3);
|
||||
|
||||
var normal = await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(1, normal.UploadedItems);
|
||||
|
||||
rig.Enqueue();
|
||||
await rig.Sut.DrainAllAsync(); // intercepted during the terminal phase
|
||||
await rig.Sut.DrainAllAsync(); // the granted attempt
|
||||
await rig.Sut.DrainAllAsync(); // pause honoured again
|
||||
|
||||
Assert.Equal(3, rig.Uploader.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeviceBlocked_pause_still_skips_terminal_drains()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueOkWithSignals(deviceBlocked: true, unblockAt: Start.AddMinutes(10));
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.True(rig.Sut.IsPaused);
|
||||
|
||||
rig.Enqueue();
|
||||
Assert.False(rig.Sut.GrantTerminalAttempt());
|
||||
var terminal = await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, rig.Uploader.CallCount);
|
||||
Assert.Contains("device blocked", terminal.LastErrorReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task First_interception_is_reported_once_and_never_as_a_blocked_or_auth_failure()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueIntercepted(2);
|
||||
var reports = new List<InterceptionReport>();
|
||||
var blocked = 0;
|
||||
rig.Sut.UploadIntercepted += (_, r) => reports.Add(r);
|
||||
rig.Sut.UploadBlocked += (_, __) => blocked++;
|
||||
|
||||
await rig.Sut.DrainAllAsync();
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||
await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(2, rig.Uploader.CallCount);
|
||||
var report = Assert.Single(reports);
|
||||
Assert.Equal(Start, report.FirstBlockedAtUtc);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, report.Interception.ForeignReason);
|
||||
Assert.Equal(403, report.Interception.StatusCode);
|
||||
Assert.Equal(InterceptionFixtures.InspectionIssuer, report.Interception.ServerCertIssuer);
|
||||
Assert.Equal(0, blocked);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Permanent_block_report_keeps_its_own_once_per_process_gate()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueIntercepted().QueuePermanent("http 400: contract").QueuePermanent("http 400: contract");
|
||||
int intercepted = 0, blocked = 0;
|
||||
rig.Sut.UploadIntercepted += (_, __) => intercepted++;
|
||||
rig.Sut.UploadBlocked += (_, __) => blocked++;
|
||||
|
||||
await rig.Sut.DrainAllAsync();
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||
await rig.Sut.DrainAllAsync();
|
||||
await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(3, rig.Uploader.CallCount);
|
||||
Assert.Equal(1, intercepted);
|
||||
Assert.Equal(1, blocked);
|
||||
}
|
||||
}
|
||||
}
|
||||
+118
-2
@@ -10,10 +10,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
public sealed class TelemetryUploadOrchestratorTests
|
||||
{
|
||||
private static TelemetryItemDraft Draft(string rowKey) =>
|
||||
private const string OwnKey = "tenant_session";
|
||||
|
||||
private static TelemetryItemDraft Draft(string rowKey) => Draft(rowKey, OwnKey);
|
||||
|
||||
private static TelemetryItemDraft Draft(string rowKey, string partitionKey) =>
|
||||
new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "tenant_session",
|
||||
partitionKey: partitionKey,
|
||||
rowKey: rowKey,
|
||||
payloadJson: $"{{\"rk\":\"{rowKey}\"}}",
|
||||
isSessionScoped: true);
|
||||
@@ -157,6 +161,118 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Equal(1, blockedCount);
|
||||
}
|
||||
|
||||
// ============================================================ HTTP 410 — session gone
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_for_current_session_stops_drain_retains_cursor_and_raises_once()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("a"));
|
||||
spool.Enqueue(Draft("b"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
var gone = new System.Collections.Generic.List<string>();
|
||||
int blockedCount = 0;
|
||||
PoisonReport? poison = null;
|
||||
sut.SessionGone += (_, reason) => gone.Add(reason);
|
||||
sut.UploadBlocked += (_, __) => blockedCount++;
|
||||
sut.BatchPoisoned += (_, p) => poison = p;
|
||||
|
||||
var first = await sut.DrainAllAsync();
|
||||
|
||||
Assert.False(first.Success);
|
||||
Assert.Equal(1, first.FailedBatches);
|
||||
Assert.Equal(0, first.UploadedItems);
|
||||
Assert.Equal(1, uploader.CallCount); // no retries — nothing can make a 410 succeed
|
||||
Assert.Equal(-1, sut.LastUploadedItemId); // retained — items stay on disk for the diag ZIP
|
||||
Assert.Single(gone);
|
||||
Assert.Contains("410", gone[0]);
|
||||
Assert.Equal(0, blockedCount); // not a "blocked" pipeline
|
||||
Assert.Null(poison);
|
||||
|
||||
// One-shot: the next drain hits the 410 again but does not raise again.
|
||||
var second = await sut.DrainAllAsync();
|
||||
Assert.False(second.Success);
|
||||
Assert.Equal(2, uploader.CallCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
Assert.Single(gone);
|
||||
Assert.Equal(0, blockedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_for_leading_foreign_items_drops_only_them_and_continues()
|
||||
{
|
||||
// Leftovers of a previous session id at the head of the spool (defence in depth — the
|
||||
// spool filters these at startup). A 410 for them must not end the current session:
|
||||
// drop exactly the leading foreign run, then carry on with the current session's items.
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("old-a", "tenant_old-session"));
|
||||
spool.Enqueue(Draft("old-b", "tenant_old-session"));
|
||||
spool.Enqueue(Draft("new", OwnKey));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueOk();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
int goneCount = 0, blockedCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
sut.UploadBlocked += (_, __) => blockedCount++;
|
||||
|
||||
var result = await sut.DrainAllAsync();
|
||||
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal(0, result.FailedBatches);
|
||||
Assert.Equal(1, result.UploadedItems);
|
||||
Assert.Equal(2, uploader.CallCount);
|
||||
Assert.Equal(3, uploader.Received[0].Count); // first batch carried all three
|
||||
Assert.Equal("new", Assert.Single(uploader.Received[1]).RowKey); // retry carried only the own item
|
||||
Assert.Equal(2, sut.LastUploadedItemId);
|
||||
Assert.Equal(0, goneCount);
|
||||
Assert.Equal(0, blockedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_is_not_latched_before_a_handler_is_subscribed()
|
||||
{
|
||||
// The host subscribes after EnrollmentOrchestrator.Start; a 410 on the very first drain
|
||||
// must still reach it on the next drain instead of being swallowed by the one-shot guard.
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("only"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
|
||||
await sut.DrainAllAsync(); // nobody listening yet
|
||||
|
||||
int goneCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
await sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, goneCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_without_a_current_key_treats_every_batch_as_own()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("x", "tenant_whatever"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock());
|
||||
int goneCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
|
||||
var result = await sut.DrainAllAsync();
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.Equal(1, goneCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TooLarge_splits_batch_and_uploads_everything()
|
||||
{
|
||||
|
||||
@@ -195,6 +195,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// </summary>
|
||||
public int AbsoluteMaxSessionHours { get; set; } = 48;
|
||||
|
||||
/// <summary>
|
||||
/// Further boots the agent may try after a session registration was answered by a
|
||||
/// 401/403 the backend did not author, before it cleans up like the emergency break.
|
||||
/// Merged from remote config (also from the cache, since an intercepted start gets no
|
||||
/// live config), clamped to 0..5. Default: 1.
|
||||
/// </summary>
|
||||
public int InterceptedRegistrationRetryBoots { get; set; } = 1;
|
||||
|
||||
/// <summary>
|
||||
/// Bootstrap token for pre-MDM auth during OOBE.
|
||||
/// Embedded by the OOBE bootstrapper script via --bootstrap-token CLI arg.
|
||||
|
||||
@@ -28,6 +28,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// </summary>
|
||||
public static class RemoteConfigMerger
|
||||
{
|
||||
/// <summary>
|
||||
/// Upper bound for <see cref="AgentConfiguration.InterceptedRegistrationRetryBoots"/>: each
|
||||
/// extra boot keeps an intercepted device's agent installed until its next boot.
|
||||
/// </summary>
|
||||
private const int MaxInterceptedRegistrationRetryBoots = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Overwrites the relevant <paramref name="agentConfig"/> fields with values from
|
||||
/// <paramref name="remoteConfig"/>. Remote wins unconditionally for every field that has
|
||||
@@ -174,6 +180,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
agentConfig.MaxAuthFailures = remoteConfig.MaxAuthFailures;
|
||||
agentConfig.AuthFailureTimeoutMinutes = remoteConfig.AuthFailureTimeoutMinutes;
|
||||
|
||||
// ---------------- Intercepted-registration buffer — null (a backend that predates the
|
||||
// field) keeps the current value; anything else is clamped to
|
||||
// 0..MaxInterceptedRegistrationRetryBoots.
|
||||
if (remoteConfig.InterceptedRegistrationRetryBoots.HasValue)
|
||||
{
|
||||
var requested = remoteConfig.InterceptedRegistrationRetryBoots.Value;
|
||||
var bounded = Math.Max(0, Math.Min(MaxInterceptedRegistrationRetryBoots, requested));
|
||||
if (bounded != requested)
|
||||
logger?.Warning($"Remote config: InterceptedRegistrationRetryBoots {requested} is outside 0..{MaxInterceptedRegistrationRetryBoots} — using {bounded}.");
|
||||
agentConfig.InterceptedRegistrationRetryBoots = bounded;
|
||||
}
|
||||
|
||||
// ---------------- Nested CollectorConfiguration — only the two knobs that have a
|
||||
// V2 consumer on AgentConfiguration itself. The rest flow through
|
||||
// the untouched remoteConfig.Collectors to DefaultComponentFactory.
|
||||
|
||||
@@ -21,7 +21,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly string _cacheFilePath;
|
||||
private readonly EmergencyReporter _emergencyReporter;
|
||||
private readonly DistressReporter _distressReporter;
|
||||
private readonly AuthFailureTracker _authFailureTracker;
|
||||
|
||||
private AgentConfigResponse _currentConfig;
|
||||
@@ -92,13 +91,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// </summary>
|
||||
public bool RecoveredAfterFetchFailure { get; private set; }
|
||||
|
||||
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null)
|
||||
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, AuthFailureTracker authFailureTracker = null)
|
||||
{
|
||||
_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_emergencyReporter = emergencyReporter;
|
||||
_distressReporter = distressReporter;
|
||||
_authFailureTracker = authFailureTracker;
|
||||
|
||||
var cacheDir = Environment.ExpandEnvironmentVariables(@"%ProgramData%\AutopilotMonitor\Config");
|
||||
@@ -180,10 +178,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
{
|
||||
// Auth failures are NOT retryable — 401/403 won't go away with another
|
||||
// attempt; record + bail straight to cache/defaults so an operator can
|
||||
// see the AuthCertificateRejected/DeviceNotRegistered distress signal
|
||||
// without 100 s of needless retry delay.
|
||||
_logger.Warning($"Config fetch authentication failed: {ex.Message}");
|
||||
_authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config", ex.EndpointUnavailable);
|
||||
// see the distress signal without 100 s of needless retry delay. A 401/403
|
||||
// from something on the network path is not counted by the tracker; it
|
||||
// reports ClientCertificateNotReceived instead.
|
||||
_logger.Warning(ex.IsForeignResponse
|
||||
? $"Config fetch intercepted on the network path: {ex.Message}"
|
||||
: $"Config fetch authentication failed: {ex.Message}");
|
||||
_authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config", ex.Classification);
|
||||
lastException = ex;
|
||||
LastFetchAuthStatusCode = ex.StatusCode;
|
||||
break;
|
||||
|
||||
+4
-4
@@ -990,7 +990,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// for each detection. When HS-NEW-RESULT later fires with the full payload, the UI
|
||||
/// reducer's dataCompleteness scoring keeps the more complete entry.
|
||||
/// </summary>
|
||||
private void HandleHealthScriptDetectionResult(Match match, Dictionary<string, string> parameters)
|
||||
private void HandleHealthScriptDetectionResult(Match match)
|
||||
{
|
||||
var id = match.Groups["id"]?.Value;
|
||||
var compliance = match.Groups["compliance"]?.Value;
|
||||
@@ -1112,7 +1112,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
sample,
|
||||
@"\[HS\] the (?<part>pre|post)-remdiation detection script compliance result for (?<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}) is (?<compliance>True|False)",
|
||||
System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||||
HandleHealthScriptDetectionResult(match, new Dictionary<string, string>());
|
||||
HandleHealthScriptDetectionResult(match);
|
||||
}
|
||||
|
||||
/// <summary>Test seam: seed the per-policy health-script cycle start timestamp as if an
|
||||
@@ -1125,7 +1125,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
_healthScriptStartTimes[policyId] = (startedAtUtc, provenance);
|
||||
}
|
||||
|
||||
private void HandleHealthScriptResult(Match match, Dictionary<string, string> parameters)
|
||||
private void HandleHealthScriptResult(Match match)
|
||||
{
|
||||
var json = match.Groups["json"]?.Value;
|
||||
if (string.IsNullOrEmpty(json))
|
||||
@@ -1141,7 +1141,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// <summary>
|
||||
/// Test seam that bypasses the regex-match plumbing and feeds a raw <c>[HS] new result</c>
|
||||
/// JSON payload directly into the parser. Production code path runs through
|
||||
/// <see cref="HandleHealthScriptResult(Match, Dictionary{string, string})"/>; this method
|
||||
/// <see cref="HandleHealthScriptResult(Match)"/>; this method
|
||||
/// exists so unit tests can exercise the JSON parser + 1-3 phase emit logic without
|
||||
/// having to write CMTrace log files into a temp directory.
|
||||
/// </summary>
|
||||
|
||||
+2
-2
@@ -882,11 +882,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
break;
|
||||
|
||||
case "healthscriptresult":
|
||||
HandleHealthScriptResult(match, pattern.Parameters);
|
||||
HandleHealthScriptResult(match);
|
||||
break;
|
||||
|
||||
case "healthscriptdetectionresult":
|
||||
HandleHealthScriptDetectionResult(match, pattern.Parameters);
|
||||
HandleHealthScriptDetectionResult(match);
|
||||
break;
|
||||
|
||||
default:
|
||||
|
||||
-2
@@ -237,7 +237,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
var rebootUri = ExtractRebootUri(ParseEventData(xml), description);
|
||||
|
||||
ProcessEvent(
|
||||
eventId: record.Id,
|
||||
recordId: recordId,
|
||||
timeCreatedUtc: record.TimeCreated?.ToUniversalTime(),
|
||||
rebootUri: rebootUri,
|
||||
@@ -251,7 +250,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// — after the debounce quiet-period, at backfill completion, or at <see cref="Stop"/>.
|
||||
/// </summary>
|
||||
internal void ProcessEvent(
|
||||
int eventId,
|
||||
long recordId,
|
||||
DateTime? timeCreatedUtc,
|
||||
string rebootUri,
|
||||
|
||||
+2
-2
@@ -1382,7 +1382,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// window (user clicks "Try again" / IME re-sync flips the subcategory back
|
||||
// to inProgress) — firing the stale args would terminate the agent on a
|
||||
// failure Windows no longer reports, and the retry outcome is lost.
|
||||
recovered = IsSettledFailureRecovered(categoryName, args);
|
||||
recovered = IsSettledFailureRecovered(categoryName);
|
||||
if (recovered)
|
||||
{
|
||||
if (args.FailedSubcategory != null
|
||||
@@ -1447,7 +1447,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// and must fire (with the latched args — statusText/HRESULT of the new subcategory
|
||||
/// were never captured). Caller holds <see cref="_stateLock"/>.
|
||||
/// </summary>
|
||||
private bool IsSettledFailureRecovered(string categoryName, EspFailureDetectedEventArgs args)
|
||||
private bool IsSettledFailureRecovered(string categoryName)
|
||||
{
|
||||
if (_lastCategorySucceeded != null
|
||||
&& _lastCategorySucceeded.TryGetValue(categoryName, out var succeeded)
|
||||
|
||||
+2
-2
@@ -241,7 +241,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
if (alreadyFired)
|
||||
continue;
|
||||
|
||||
RunProbe(probeIndex, thresholdMinutes, idleMinutes, genAtStart);
|
||||
RunProbe(probeIndex, idleMinutes, genAtStart);
|
||||
|
||||
lock (_stateLock)
|
||||
{
|
||||
@@ -253,7 +253,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
}
|
||||
}
|
||||
|
||||
private void RunProbe(int probeIndex, int thresholdMinutes, double idleMinutes, int generation)
|
||||
private void RunProbe(int probeIndex, double idleMinutes, int generation)
|
||||
{
|
||||
var started = DateTime.UtcNow;
|
||||
var result = new ProbeResult { ProbeIndex = probeIndex, IdleMinutes = Math.Round(idleMinutes, 1) };
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Persistence;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Short local fingerprint of the network a start ran on: the first <see cref="Length"/> hex
|
||||
/// characters of SHA-256 over the active interface's connection type and default gateway, as
|
||||
/// <see cref="NetworkChangeDetector"/> captures them. Stored beside the failed-registration
|
||||
/// record and compared by the next successful registration; only that comparison (changed or
|
||||
/// not) leaves the device, never a fingerprint.
|
||||
/// </summary>
|
||||
internal static class NetworkFingerprint
|
||||
{
|
||||
internal const int Length = 16;
|
||||
|
||||
/// <summary>Fingerprint of the current network; null when it is unknown. Never throws.</summary>
|
||||
public static string? TryCapture()
|
||||
{
|
||||
try
|
||||
{
|
||||
return Of(NetworkChangeDetector.CaptureInterfaceState());
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fingerprint of a captured state; null without an active interface, type or gateway.</summary>
|
||||
internal static string? Of(NetworkChangeDetector.NetworkStateSnapshot? state)
|
||||
{
|
||||
if (state == null || !state.HasNetwork
|
||||
|| string.IsNullOrEmpty(state.ConnectionType) || string.IsNullOrEmpty(state.Gateway))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return StartupEventGate.HashString(state.ConnectionType + "|" + state.Gateway).Substring(0, Length);
|
||||
}
|
||||
|
||||
/// <summary>True/false when both fingerprints are known, null when either is unknown.</summary>
|
||||
public static bool? Differs(string? before, string? after) =>
|
||||
before == null || after == null ? (bool?)null : !string.Equals(before, after, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -1052,7 +1052,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
if (!IsAllowedBlobUploadUrl(blobUploadUrl))
|
||||
{
|
||||
_logger.Warning(
|
||||
$"Blob upload rejected: URL is not an allowed Azure Blob Storage endpoint " +
|
||||
$"Blob upload of {blobName} rejected: URL is not an allowed Azure Blob Storage endpoint " +
|
||||
$"(expected https://*.blob.core.windows.net or sovereign-cloud equivalent). " +
|
||||
$"URL prefix: {BuildSasUrlPrefix(blobUploadUrl)}");
|
||||
return (false, "url_host_rejected");
|
||||
@@ -1086,7 +1086,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
// Auth errors (401/403) are permanent — SAS token invalid or expired, retrying won't help
|
||||
if (statusCode == 401 || statusCode == 403)
|
||||
{
|
||||
_logger.Warning($"Blob upload auth error (not retryable): {lastErrorCode} - {responseBody}");
|
||||
_logger.Warning($"Blob upload of {blobName}: auth error (not retryable): {lastErrorCode} - {responseBody}");
|
||||
return (false, lastErrorCode);
|
||||
}
|
||||
|
||||
@@ -1106,7 +1106,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
_logger.Warning($"Diagnostics package upload failed after {maxRetries} attempts");
|
||||
_logger.Warning($"Diagnostics package upload failed after {maxRetries} attempts: {blobName}");
|
||||
return (false, lastErrorCode);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,7 +145,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
/// On any failure, returns normally so the current version continues.
|
||||
/// </summary>
|
||||
/// <param name="forceUpdate">
|
||||
/// When true, bypasses the IsNewerVersion check — used by the runtime-hash-mismatch trigger
|
||||
/// When true, bypasses the version comparison — used by the runtime-hash-mismatch trigger
|
||||
/// to cover hotfixes that reuse the same version number.
|
||||
/// </param>
|
||||
/// <param name="triggerReason">
|
||||
@@ -517,16 +517,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares two version strings. Returns true if latest is newer than current.
|
||||
/// Strips SemVer suffixes (+metadata, -prerelease) before parsing because
|
||||
/// System.Version cannot handle them (e.g. "1.0.386+b7f8d3c..." would fail).
|
||||
/// </summary>
|
||||
private static bool IsNewerVersion(string current, string latest)
|
||||
{
|
||||
return CompareVersions(current, latest) > 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares two version strings. Returns negative when latest is older than current,
|
||||
/// zero when equal, positive when latest is newer than current. Returns
|
||||
|
||||
-5
@@ -827,11 +827,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather
|
||||
sb.Append(value);
|
||||
}
|
||||
|
||||
private EventSeverity ParseSeverity(string severity)
|
||||
{
|
||||
return GatherRuleContext.ParseSeverity(severity);
|
||||
}
|
||||
|
||||
private void StopAllTimers()
|
||||
{
|
||||
foreach (var timer in _intervalTimers.Values)
|
||||
|
||||
+2
-5
@@ -104,7 +104,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
private bool _preinstalledReported; // office_preinstalled_detected already emitted (emit-once guard)
|
||||
private DateTime? _startedAtUtc;
|
||||
private string? _startedTrigger;
|
||||
private OfficeDoSample? _lastDo;
|
||||
private OfficeDoSample? _peakDo; // highest-bytes sample seen — basis for the completed doSummary
|
||||
|
||||
public OfficeInstallDetector(
|
||||
@@ -252,7 +251,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
// _peakDo). The seed is persisted directly (it is the lifecycle's first DO sample).
|
||||
if (_state == DetectorState.Active)
|
||||
{
|
||||
_lastDo = sample;
|
||||
_peakDo = sample;
|
||||
PersistState(OfficeInstallStateData.StateActive);
|
||||
}
|
||||
@@ -261,7 +259,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
|
||||
// Active: no progress event — the sample only updates the DO summary data. A grown
|
||||
// peak is re-persisted (throttled) so a mid-install agent restart keeps the doSummary.
|
||||
_lastDo = sample;
|
||||
var peakGrew = _peakDo == null || sample.TotalBytesDownloaded > _peakDo.TotalBytesDownloaded;
|
||||
if (peakGrew) _peakDo = sample;
|
||||
if (peakGrew && _statePersistence != null
|
||||
@@ -380,7 +377,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
// wrapper that re-runs C2R) the watcher's initial scan completes the lifecycle synchronously.
|
||||
// Emitting started first guarantees the correct order (started → completed); otherwise a
|
||||
// pre-installed Office produced completed-before-started (field session a7525e97).
|
||||
EmitLifecycle(Constants.EventTypes.OfficeInstallStarted, snap, EventSeverity.Info, PhaseOf(snap), isTerminal: false);
|
||||
EmitLifecycle(Constants.EventTypes.OfficeInstallStarted, snap, EventSeverity.Info, InFlightPhase, isTerminal: false);
|
||||
// Persist Active BEFORE ObserveInstallationPath: when Office is already on disk the
|
||||
// host's binary watcher completes the lifecycle synchronously from inside that callback
|
||||
// (and persists Completed) — persisting Active afterwards would overwrite the terminal.
|
||||
@@ -508,7 +505,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
|
||||
// StreamingFinished does not exist in the C2R registry, so the in-flight phase is reported as a
|
||||
// single "Installing"; the terminal completion uses "Completed".
|
||||
private static string PhaseOf(OfficeC2RSnapshot snap) => "Installing";
|
||||
private const string InFlightPhase = "Installing";
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Payload
|
||||
|
||||
+9
-6
@@ -85,12 +85,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
|
||||
protected override void Collect()
|
||||
{
|
||||
// Full path writes 38 keys: agent_version + process metrics (5: cpu, ws, private,
|
||||
// threads, handles) + spool stats (5) + network delta (9: requests, failures,
|
||||
// bytes_up/down, avg_latency, total_up/down/requests/latency) + IME tracker health
|
||||
// (9) + signal pipeline (9: 5 counters, stage, applied ordinal, wg level+score).
|
||||
// cap=38 → HashHelpers.GetPrime(38)=41 buckets → no resize on the last key.
|
||||
var data = new Dictionary<string, object>(capacity: 38, StringComparer.Ordinal)
|
||||
// Full path writes 48 keys: agent_version + process metrics (5: cpu, ws, private,
|
||||
// threads, handles) + spool stats (5) + network (10: requests, failures,
|
||||
// bytes_up/down, avg_latency, total_up/down/requests/latency, upload 2xx without
|
||||
// echo) + IME tracker health (18) + signal pipeline (9: 5 counters, stage, applied
|
||||
// ordinal, wg level+score). cap=48 → no resize on the last key.
|
||||
var data = new Dictionary<string, object>(capacity: 48, StringComparer.Ordinal)
|
||||
{
|
||||
{ "agent_version", _agentVersion }
|
||||
};
|
||||
@@ -160,6 +160,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
}
|
||||
|
||||
// --- Network delta ---
|
||||
// Cumulative and independent of the delta baseline: telemetry uploads answered 2xx
|
||||
// without the backend's correlation echo (counted only — see BackendTelemetryUploader).
|
||||
data["telemetry_upload_ok_without_echo"] = _networkMetrics.OkWithoutEchoCount;
|
||||
try
|
||||
{
|
||||
var currentNet = _networkMetrics.GetSnapshot();
|
||||
|
||||
+33
-30
@@ -26,7 +26,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
private readonly string _tenantId;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly string _apiBaseUrl;
|
||||
|
||||
private NetworkStateSnapshot _lastKnownState;
|
||||
private Timer _debounceTimer;
|
||||
@@ -38,6 +37,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
private const int ConnectivityCheckDelayMs = 5000;
|
||||
private const int ConnectivityCheckTimeoutMs = 8000;
|
||||
private const string Source = "NetworkChangeDetector";
|
||||
private const string WifiConnectionType = "WiFi";
|
||||
|
||||
/// <summary>
|
||||
/// Well-known MDM/Entra endpoints that must be reachable for enrollment to succeed.
|
||||
@@ -54,14 +54,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
InformationalEventPost post,
|
||||
AgentLogger logger,
|
||||
string apiBaseUrl = null)
|
||||
AgentLogger logger)
|
||||
{
|
||||
_sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
_post = post ?? throw new ArgumentNullException(nameof(post));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_apiBaseUrl = apiBaseUrl;
|
||||
}
|
||||
|
||||
public void Start()
|
||||
@@ -225,15 +223,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
{
|
||||
if (_disposed || _connectivityCheckClient == null) return;
|
||||
|
||||
// Build endpoint list: static MDM endpoints + dynamic backend URL
|
||||
// Static MDM endpoints only. The backend API stays out on purpose: enrollment does not
|
||||
// need it, so its outage must not raise a connectivity alarm.
|
||||
var endpoints = new List<(string Name, string Url)>(MdmEndpoints);
|
||||
// Optionally add the Autopilot Monitor API endpoint, but we don't want to cause false alarms if it's down or unreachable,
|
||||
// since it's not critical for enrollment success and may be available only after certain network changes (e.g. if it's behind a captive portal).
|
||||
// For now, we can omit it from the critical checks.
|
||||
// if (!string.IsNullOrEmpty(_apiBaseUrl))
|
||||
// {
|
||||
// endpoints.Add(("Autopilot Monitor API", _apiBaseUrl));
|
||||
// }
|
||||
|
||||
// Run all checks in parallel
|
||||
var tasks = endpoints.Select(ep => CheckEndpointAsync(ep.Name, ep.Url)).ToArray();
|
||||
@@ -321,6 +313,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
private NetworkStateSnapshot CaptureCurrentState()
|
||||
{
|
||||
var snapshot = CaptureInterfaceState(_logger);
|
||||
|
||||
// Collect WiFi SSID synchronously (needed for before/after comparison)
|
||||
if (snapshot.ConnectionType == WifiConnectionType)
|
||||
{
|
||||
Guid? nicGuid = Guid.TryParse(snapshot.InterfaceId, out var parsedGuid) ? parsedGuid : (Guid?)null;
|
||||
var wifiInfo = CollectWiFiInfo(nicGuid);
|
||||
snapshot.WifiSsid = wifiInfo.ssid;
|
||||
snapshot.WifiSignalPercent = wifiInfo.signal;
|
||||
snapshot.WifiRadioType = wifiInfo.radioType;
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Interface-level state of the active network (<see cref="NetworkInterfaceLocator"/>):
|
||||
/// connection type, adapter, first IPv4 address, first gateway — no WiFi details. Shared
|
||||
/// with <see cref="NetworkFingerprint"/> so the registration record fingerprints the network
|
||||
/// exactly as this detector sees it. Never throws: a failed read is "no network".
|
||||
/// </summary>
|
||||
internal static NetworkStateSnapshot CaptureInterfaceState(AgentLogger logger = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -355,13 +370,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
}
|
||||
}
|
||||
|
||||
var snapshot = new NetworkStateSnapshot
|
||||
return new NetworkStateSnapshot
|
||||
{
|
||||
HasNetwork = true,
|
||||
AdapterName = activeNic.Name,
|
||||
AdapterDescription = activeNic.Description,
|
||||
InterfaceType = activeNic.NetworkInterfaceType.ToString(),
|
||||
ConnectionType = isWifi ? "WiFi" : "Ethernet",
|
||||
ConnectionType = isWifi ? WifiConnectionType : "Ethernet",
|
||||
MacAddress = activeNic.GetPhysicalAddress().ToString(),
|
||||
InterfaceId = activeNic.Id,
|
||||
LinkSpeedMbps = activeNic.Speed / 1_000_000,
|
||||
@@ -369,22 +384,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
Gateway = gateway,
|
||||
CapturedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Collect WiFi SSID synchronously (needed for before/after comparison)
|
||||
if (isWifi)
|
||||
{
|
||||
Guid? nicGuid = Guid.TryParse(activeNic.Id, out var parsedGuid) ? parsedGuid : (Guid?)null;
|
||||
var wifiInfo = CollectWiFiInfo(nicGuid);
|
||||
snapshot.WifiSsid = wifiInfo.ssid;
|
||||
snapshot.WifiSignalPercent = wifiInfo.signal;
|
||||
snapshot.WifiRadioType = wifiInfo.radioType;
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"NetworkChangeDetector: state capture failed: {ex.Message}");
|
||||
logger?.Debug($"NetworkChangeDetector: state capture failed: {ex.Message}");
|
||||
return new NetworkStateSnapshot { HasNetwork = false, CapturedAt = DateTime.UtcNow };
|
||||
}
|
||||
}
|
||||
@@ -443,7 +446,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
if (snapshot == null || !snapshot.HasNetwork)
|
||||
return "None";
|
||||
|
||||
if (snapshot.ConnectionType == "WiFi" && !string.IsNullOrEmpty(snapshot.WifiSsid))
|
||||
if (snapshot.ConnectionType == WifiConnectionType && !string.IsNullOrEmpty(snapshot.WifiSsid))
|
||||
return $"WiFi '{snapshot.WifiSsid}'";
|
||||
|
||||
return $"{snapshot.ConnectionType} ({snapshot.IpAddress ?? "no IP"})";
|
||||
@@ -474,7 +477,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
if (before.ConnectionType != after.ConnectionType)
|
||||
return "type_change";
|
||||
|
||||
if (before.ConnectionType == "WiFi" && after.ConnectionType == "WiFi" &&
|
||||
if (before.ConnectionType == WifiConnectionType && after.ConnectionType == WifiConnectionType &&
|
||||
before.WifiSsid != after.WifiSsid)
|
||||
return "ssid_change";
|
||||
|
||||
@@ -488,7 +491,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
// Inner type: network state snapshot
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
private class NetworkStateSnapshot
|
||||
internal sealed class NetworkStateSnapshot
|
||||
{
|
||||
public bool HasNetwork { get; set; }
|
||||
public string AdapterName { get; set; }
|
||||
|
||||
+1
-4
@@ -69,20 +69,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Provisioning
|
||||
private readonly string _tenantId;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly IClock? _clock;
|
||||
|
||||
public ProvisioningPackageCollector(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
InformationalEventPost post,
|
||||
AgentLogger logger,
|
||||
IClock? clock = null)
|
||||
AgentLogger logger)
|
||||
{
|
||||
_sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
_post = post ?? throw new ArgumentNullException(nameof(post));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_clock = clock;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+114
-105
@@ -1,9 +1,11 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
@@ -40,6 +42,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
private readonly string _bootstrapToken;
|
||||
private readonly string _agentVersion;
|
||||
private readonly Logging.AgentLogger _logger;
|
||||
private readonly ServerCertificateObserver _serverCertificates;
|
||||
|
||||
/// <summary>
|
||||
/// Protected ctor for testability — <c>SessionRegistrationHelperTests.FakeApiClient</c>
|
||||
@@ -48,6 +51,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// </summary>
|
||||
protected BackendApiClient() { }
|
||||
|
||||
/// <param name="serverCertificates">
|
||||
/// Observer installed on the <paramref name="httpClient"/>'s handler; its issuer becomes
|
||||
/// evidence on a 401/403 that did not come from the backend. Null = no evidence.
|
||||
/// </param>
|
||||
public BackendApiClient(
|
||||
HttpClient httpClient,
|
||||
string baseUrl,
|
||||
@@ -57,7 +64,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
bool useBootstrapTokenAuth,
|
||||
string bootstrapToken,
|
||||
string agentVersion,
|
||||
Logging.AgentLogger logger)
|
||||
Logging.AgentLogger logger,
|
||||
ServerCertificateObserver serverCertificates = null)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("baseUrl is mandatory.", nameof(baseUrl));
|
||||
@@ -69,6 +77,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
_bootstrapToken = bootstrapToken;
|
||||
_agentVersion = agentVersion;
|
||||
_logger = logger;
|
||||
_serverCertificates = serverCertificates;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -107,7 +116,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
{
|
||||
using (var httpRequest = new HttpRequestMessage(HttpMethod.Get, url))
|
||||
{
|
||||
AddSecurityHeaders(httpRequest);
|
||||
var correlationId = AddSecurityHeaders(httpRequest);
|
||||
_logger?.Debug($"GetAgentConfigAsync: GET {url} (attempt {attempt}/{maxAttempts})");
|
||||
|
||||
HttpResponseMessage response;
|
||||
@@ -137,7 +146,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
continue;
|
||||
}
|
||||
|
||||
await ThrowOnAuthFailureAsync(response).ConfigureAwait(false);
|
||||
await ThrowOnAuthFailureAsync(response, correlationId).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
@@ -173,9 +182,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// emergency-break path passes a longer <paramref name="timeout"/> because its first
|
||||
/// request of a freshly booted process pays cold TLS + DNS + a possible backend
|
||||
/// cold start (observed 503s take well over 5 s).
|
||||
/// Returns <c>true</c> when the HTTP round-trip completed (the endpoint always
|
||||
/// answers 200), <c>false</c> on timeout/transport failure — the caller-side
|
||||
/// anti-flood layer uses this to decide whether the report counts as delivered.
|
||||
/// Returns <c>true</c> when the HTTP round-trip completed (the endpoint answers 200 for
|
||||
/// every report it receives), <c>false</c> on timeout/transport failure or when a
|
||||
/// 401/403 came from something other than the backend (the report never arrived) — the
|
||||
/// caller-side anti-flood layer uses this to decide whether the report counts as delivered.
|
||||
/// Virtual for tests.
|
||||
/// </summary>
|
||||
public virtual async Task<bool> ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null)
|
||||
@@ -195,13 +205,25 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
{
|
||||
// Backend security validation requires tenant id before parsing the body.
|
||||
httpRequest.Headers.Add("X-Tenant-Id", report.TenantId);
|
||||
AddSecurityHeaders(httpRequest);
|
||||
var correlationId = AddSecurityHeaders(httpRequest);
|
||||
|
||||
using (var cts = new CancellationTokenSource(timeout ?? TimeSpan.FromSeconds(5)))
|
||||
using (var response = await _httpClient.SendAsync(httpRequest, cts.Token).ConfigureAwait(false))
|
||||
{
|
||||
// Response status is deliberately ignored — endpoint always returns 200.
|
||||
return true;
|
||||
// Every status counts as delivered (unchanged) — except a 401/403 without the
|
||||
// backend's correlation echo: then the report never reached the backend.
|
||||
if (!IsAuthFailureStatus(response.StatusCode))
|
||||
return true;
|
||||
|
||||
var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false);
|
||||
var classification = ResponseOriginClassifier.Classify(response, correlationId, body);
|
||||
if (!classification.IsForeign)
|
||||
return true;
|
||||
|
||||
_logger?.Debug(
|
||||
$"ReportAgentErrorAsync: http {(int)response.StatusCode} did not come from the backend " +
|
||||
$"({classification.ForeignReason}) — report not delivered.");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -221,7 +243,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json")
|
||||
})
|
||||
{
|
||||
AddSecurityHeaders(httpRequest);
|
||||
var correlationId = AddSecurityHeaders(httpRequest);
|
||||
|
||||
HttpResponseMessage response;
|
||||
try
|
||||
@@ -236,7 +258,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
|
||||
using (response)
|
||||
{
|
||||
await ThrowOnAuthFailureAsync(response).ConfigureAwait(false);
|
||||
await ThrowOnAuthFailureAsync(response, correlationId).ConfigureAwait(false);
|
||||
await ThrowOnCodedConflictAsync(response).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
@@ -284,61 +306,76 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// Throws <see cref="BackendAuthException"/> for 401/403 responses so callers can
|
||||
/// distinguish authentication failures from transient server errors.
|
||||
/// <para>
|
||||
/// Field case (2026-08-09): a pre-cutover agent build pointed at a
|
||||
/// retired Function App; Azure answers for a stopped app with a platform-level 403
|
||||
/// carrying an HTML error page ("Web App - Unavailable"). Our backend never returns
|
||||
/// HTML on 401/403 — an HTML body therefore means the response did not come from the
|
||||
/// backend's auth layer at all (stopped/retired app, edge proxy, WAF). Reporting that
|
||||
/// as "device is not authorized" sends troubleshooting down the wrong path, so the
|
||||
/// two cases get distinct messages and the exception carries a flag.
|
||||
/// Only a response that echoes this request's <c>X-Correlation-ID</c> is the backend's
|
||||
/// verdict (<see cref="ResponseOriginClassifier"/>). Anything else never came from the
|
||||
/// backend's auth layer: the platform page of a stopped app (field case 2026-08-09: a
|
||||
/// pre-cutover build pointed at a retired Function App), or the platform's "Client
|
||||
/// Certificate Required" page / a proxy page when TLS inspection on the network path
|
||||
/// strips the device certificate. Reporting those as "device is not authorized" sends
|
||||
/// troubleshooting down the wrong path, so each origin gets its own message and the
|
||||
/// exception carries the classification.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private static async Task ThrowOnAuthFailureAsync(HttpResponseMessage response)
|
||||
private async Task ThrowOnAuthFailureAsync(HttpResponseMessage response, string correlationId)
|
||||
{
|
||||
if (response.StatusCode != System.Net.HttpStatusCode.Unauthorized &&
|
||||
response.StatusCode != System.Net.HttpStatusCode.Forbidden)
|
||||
{
|
||||
if (!IsAuthFailureStatus(response.StatusCode))
|
||||
return;
|
||||
}
|
||||
|
||||
var statusText = $"{(int)response.StatusCode} {response.StatusCode}";
|
||||
var statusCode = (int)response.StatusCode;
|
||||
var statusText = $"{statusCode} {response.StatusCode}";
|
||||
var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false);
|
||||
var classification = ResponseOriginClassifier.Classify(
|
||||
response, correlationId, body, _serverCertificates?.LastIssuer);
|
||||
|
||||
string body = null;
|
||||
try
|
||||
if (classification.Origin != ResponseOrigin.Backend)
|
||||
{
|
||||
body = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Body is diagnostic-only; classification below treats null as non-HTML.
|
||||
}
|
||||
|
||||
if (LooksLikeHtmlErrorPage(response, body))
|
||||
{
|
||||
var title = TryExtractHtmlTitle(body);
|
||||
var titleText = title != null ? $" (\"{title}\")" : string.Empty;
|
||||
var host = response.RequestMessage?.RequestUri?.Host ?? "(unknown host)";
|
||||
throw new BackendAuthException(
|
||||
$"Backend returned {statusText} with an HTML platform error page{titleText} from '{host}'. " +
|
||||
"This response did not come from the Autopilot Monitor backend — the endpoint appears stopped, " +
|
||||
"retired, or intercepted by a proxy. This is NOT a device-authorization failure; an outdated " +
|
||||
"agent build pointing at a stale ApiBaseUrl is the most likely cause.",
|
||||
(int)response.StatusCode,
|
||||
endpointUnavailable: true);
|
||||
DescribeNonBackendAnswer(response, statusText, classification),
|
||||
statusCode,
|
||||
classification: classification);
|
||||
}
|
||||
|
||||
// Backend JSON bodies may carry a machine-readable errorCode (Shared
|
||||
// Constants.AgentErrorCodes) that tells the caller how to react — e.g.
|
||||
// session_owner_mismatch → rotate the session id instead of treating this as a
|
||||
// certificate rejection. Absent/unparseable → null, classic auth failure.
|
||||
var errorCode = TryExtractErrorCode(body);
|
||||
throw new BackendAuthException(
|
||||
$"Backend returned {statusText}. " +
|
||||
"The device is not authorized. Check client certificate and Autopilot device validation.",
|
||||
(int)response.StatusCode,
|
||||
errorCode: errorCode);
|
||||
statusCode,
|
||||
errorCode: TryExtractErrorCode(body),
|
||||
classification: classification);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Message for a 401/403 the backend did not author. The cause leads because
|
||||
/// <c>RegistrationFailureSummary.LastError</c> keeps only 256 characters; HTML answers
|
||||
/// keep the phrase "HTML platform error page" plus the page title (ANALYZE-NET-001 matches both).
|
||||
/// </summary>
|
||||
private static string DescribeNonBackendAnswer(HttpResponseMessage response, string statusText, ResponseClassification classification)
|
||||
{
|
||||
var host = response.RequestMessage?.RequestUri?.Host ?? "(unknown host)";
|
||||
var titleText = classification.PageTitle != null ? $" (\"{classification.PageTitle}\")" : string.Empty;
|
||||
var htmlPage = $"an HTML platform error page{titleText}";
|
||||
|
||||
if (classification.Origin == ResponseOrigin.PlatformStopped)
|
||||
{
|
||||
return $"Backend returned {statusText} with {(classification.IsHtml ? htmlPage : "a platform error page")} from '{host}'. " +
|
||||
"This response did not come from the Autopilot Monitor backend — the endpoint appears stopped or retired. " +
|
||||
"This is NOT a device-authorization failure; an outdated agent build pointing at a stale ApiBaseUrl " +
|
||||
"is the most likely cause.";
|
||||
}
|
||||
|
||||
var cause = classification.ForeignReason == ResponseClassification.ClientCertificateRequired
|
||||
? "The device certificate did not reach the backend (TLS inspection or a proxy on the network path)"
|
||||
: "Something on the network path answered instead of the backend (TLS inspection, a proxy or a security gateway)";
|
||||
var answer = classification.IsHtml ? htmlPage : "a response without the backend's correlation echo";
|
||||
return $"{cause}: {statusText} from '{host}' was {answer}. This is NOT a device-authorization failure.";
|
||||
}
|
||||
|
||||
private static bool IsAuthFailureStatus(HttpStatusCode statusCode) =>
|
||||
statusCode == HttpStatusCode.Unauthorized || statusCode == HttpStatusCode.Forbidden;
|
||||
|
||||
/// <summary>
|
||||
/// Throws <see cref="BackendRefusalException"/> for a 409 whose JSON body carries a
|
||||
/// machine-readable <c>errorCode</c> — register-session answers
|
||||
@@ -395,62 +432,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when a 401/403 response is an HTML page rather than a backend JSON/plain-text
|
||||
/// reply. Checks the declared Content-Type first, then falls back to sniffing the body
|
||||
/// prefix so a platform page served with a generic Content-Type is still recognized.
|
||||
/// Adds device / version / bootstrap-token security headers and returns the request's
|
||||
/// correlation id. Client certificate is sent at the TLS layer via the externally-supplied
|
||||
/// HttpClient's HttpClientHandler.
|
||||
/// </summary>
|
||||
private static bool LooksLikeHtmlErrorPage(HttpResponseMessage response, string body)
|
||||
{
|
||||
var mediaType = response.Content?.Headers?.ContentType?.MediaType;
|
||||
if (string.Equals(mediaType, "text/html", StringComparison.OrdinalIgnoreCase))
|
||||
return true;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
return false;
|
||||
|
||||
var head = body.TrimStart();
|
||||
return head.StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase) ||
|
||||
head.StartsWith("<html", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Best-effort extraction of the HTML <title> (e.g. "Web App - Unavailable") so the
|
||||
/// log line names the platform page without dumping markup. Null when absent/oversized.
|
||||
/// </summary>
|
||||
private static string TryExtractHtmlTitle(string body)
|
||||
{
|
||||
if (string.IsNullOrEmpty(body)) return null;
|
||||
try
|
||||
{
|
||||
var match = System.Text.RegularExpressions.Regex.Match(
|
||||
body, @"<title[^>]*>\s*(.*?)\s*</title>",
|
||||
System.Text.RegularExpressions.RegexOptions.IgnoreCase |
|
||||
System.Text.RegularExpressions.RegexOptions.Singleline,
|
||||
TimeSpan.FromMilliseconds(250));
|
||||
if (!match.Success) return null;
|
||||
|
||||
var title = match.Groups[1].Value;
|
||||
return title.Length == 0 ? null : (title.Length > 80 ? title.Substring(0, 80) : title);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds device / version / bootstrap-token security headers. Client certificate is sent
|
||||
/// at the TLS layer via the externally-supplied HttpClient's HttpClientHandler.
|
||||
/// </summary>
|
||||
private void AddSecurityHeaders(HttpRequestMessage request)
|
||||
private string AddSecurityHeaders(HttpRequestMessage request)
|
||||
{
|
||||
// Per-request correlation id for end-to-end tracing (mirrors BackendTelemetryUploader).
|
||||
// The backend's CorrelationIdMiddleware reads X-Correlation-ID; without it the backend
|
||||
// mints a fresh id per inbound request and agent↔backend calls cannot be tied together.
|
||||
// Generated per request so each attempt (e.g. config-fetch 503 retries) is independently
|
||||
// traceable.
|
||||
// traceable — and so the echo on a 401/403 proves which request the backend answered.
|
||||
var correlationId = Guid.NewGuid().ToString("N");
|
||||
request.Headers.Add("X-Correlation-ID", correlationId);
|
||||
request.Headers.Add(ResponseOriginClassifier.CorrelationIdHeader, correlationId);
|
||||
_logger?.Debug($"BackendApiClient: {request.Method} {request.RequestUri?.AbsolutePath} (corr={correlationId})");
|
||||
|
||||
if (_useBootstrapTokenAuth && !string.IsNullOrEmpty(_bootstrapToken))
|
||||
@@ -464,6 +458,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
request.Headers.Add("X-Device-SerialNumber", _serialNumber);
|
||||
if (!string.IsNullOrEmpty(_agentVersion))
|
||||
request.Headers.Add("X-Agent-Version", _agentVersion);
|
||||
|
||||
return correlationId;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
@@ -473,23 +469,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thrown when the backend returns 401 or 403, indicating the device is not authorized —
|
||||
/// or, when <see cref="EndpointUnavailable"/> is set, that the 401/403 was a platform-level
|
||||
/// HTML response (stopped/retired app, edge proxy) rather than a backend auth verdict.
|
||||
/// Thrown for a 401 or 403. <see cref="Origin"/> says who answered: the backend (the device is
|
||||
/// not authorized), the platform page of a stopped app (<see cref="EndpointUnavailable"/>), or
|
||||
/// something on the network path (<see cref="IsForeignResponse"/>) — the last two say nothing
|
||||
/// about this device's authorization.
|
||||
/// </summary>
|
||||
public class BackendAuthException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// HTTP status code returned by the backend (401 or 403).
|
||||
/// HTTP status code of the response (401 or 403).
|
||||
/// Used by the DistressReporter to classify the failure type.
|
||||
/// </summary>
|
||||
public int StatusCode { get; }
|
||||
|
||||
/// <summary>Classification and evidence of the response; <see cref="ResponseOrigin.Backend"/> when constructed without one.</summary>
|
||||
public ResponseClassification Classification { get; }
|
||||
|
||||
public ResponseOrigin Origin => Classification.Origin;
|
||||
|
||||
/// <summary>
|
||||
/// True when the 401/403 carried an HTML platform error page — the endpoint itself is
|
||||
/// stopped/retired/proxied and the status says nothing about this device's authorization.
|
||||
/// True only for the platform page of a stopped app: the endpoint itself is down. Counts
|
||||
/// toward the auth shutdown like a backend rejection, without a distress report.
|
||||
/// </summary>
|
||||
public bool EndpointUnavailable { get; }
|
||||
public bool EndpointUnavailable => Origin == ResponseOrigin.PlatformStopped;
|
||||
|
||||
/// <summary>
|
||||
/// True when something other than the backend answered (the platform's "Client Certificate
|
||||
/// Required" page, a proxy page): the device certificate did not reach the backend. Never
|
||||
/// counts as an authentication failure.
|
||||
/// </summary>
|
||||
public bool IsForeignResponse => Origin == ResponseOrigin.Foreign;
|
||||
|
||||
/// <summary>
|
||||
/// Machine-readable code from the backend's JSON error body (see Shared
|
||||
@@ -497,11 +506,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// </summary>
|
||||
public string ErrorCode { get; }
|
||||
|
||||
public BackendAuthException(string message, int statusCode = 0, bool endpointUnavailable = false, string errorCode = null) : base(message)
|
||||
public BackendAuthException(string message, int statusCode = 0, string errorCode = null, ResponseClassification classification = null) : base(message)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
EndpointUnavailable = endpointUnavailable;
|
||||
ErrorCode = errorCode;
|
||||
Classification = classification ?? ResponseClassification.ForBackend(statusCode);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
@@ -14,6 +15,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// - Same error key (ErrorType + HTTP status code) is only ever sent once per session
|
||||
/// - At most <see cref="MaxReportsPerSession"/> emergency reports total per session
|
||||
/// - At most one report every <see cref="MinIntervalMinutes"/> minutes regardless of error type
|
||||
/// (the terminal <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/> report is exempt)
|
||||
///
|
||||
/// A failure in this reporter is silently swallowed — it must never cascade into the main loop.
|
||||
/// </summary>
|
||||
@@ -32,6 +34,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
private readonly string _tenantId;
|
||||
private readonly string _agentVersion;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly IClock _clock;
|
||||
|
||||
private readonly HashSet<string> _reportedKeys = new HashSet<string>();
|
||||
private readonly object _antiFloodLock = new object();
|
||||
@@ -43,13 +46,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
string agentVersion,
|
||||
AgentLogger logger)
|
||||
AgentLogger logger,
|
||||
IClock clock = null)
|
||||
{
|
||||
_apiClient = apiClient;
|
||||
_sessionId = sessionId;
|
||||
_tenantId = tenantId;
|
||||
_agentVersion = agentVersion;
|
||||
_logger = logger;
|
||||
_clock = clock ?? SystemClock.Instance;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -100,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
// All anti-flood checks are protected by a lock to prevent race conditions when
|
||||
// TrySendAsync is called concurrently (fire-and-forget from multiple threads).
|
||||
var key = $"{errorType}:{httpStatusCode}";
|
||||
var now = _clock.UtcNow;
|
||||
int currentReportCount;
|
||||
DateTime? previousLastReportTime;
|
||||
|
||||
@@ -117,8 +123,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
return;
|
||||
}
|
||||
|
||||
if (_lastReportTime != null &&
|
||||
(DateTime.UtcNow - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
|
||||
// The session-unknown report is terminal and sent at most once per process (the
|
||||
// termination follows immediately): a routine report minutes earlier must not cost
|
||||
// the last word about the session. Only the cooldown is waived; key dedup and the
|
||||
// per-session maximum still apply.
|
||||
var exemptFromCooldown = errorType == AgentErrorType.TelemetryRejectedSessionUnknown;
|
||||
if (!exemptFromCooldown && _lastReportTime != null &&
|
||||
(now - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
|
||||
{
|
||||
_logger?.Debug($"[EmergencyChannel] Suppressed (cooldown active, last report at {_lastReportTime.Value:HH:mm:ss}): {key}");
|
||||
return;
|
||||
@@ -127,7 +138,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
previousLastReportTime = _lastReportTime;
|
||||
_reportedKeys.Add(key);
|
||||
_reportCount++;
|
||||
_lastReportTime = DateTime.UtcNow;
|
||||
_lastReportTime = now;
|
||||
currentReportCount = _reportCount;
|
||||
}
|
||||
|
||||
@@ -145,7 +156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
SessionAgeHours = sessionAgeHours,
|
||||
PriorRegistrationFailure = priorRegistrationFailure,
|
||||
AgentVersion = _agentVersion,
|
||||
Timestamp = DateTime.UtcNow
|
||||
Timestamp = now
|
||||
};
|
||||
|
||||
var delivered = false;
|
||||
|
||||
@@ -14,6 +14,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
private long _totalBytesUp;
|
||||
private long _totalBytesDown;
|
||||
private long _totalLatencyMs;
|
||||
private long _okWithoutEchoCount;
|
||||
|
||||
public void RecordRequest(long bytesUp, long bytesDown, long latencyMs, bool failed)
|
||||
{
|
||||
@@ -25,6 +26,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
Interlocked.Increment(ref _failureCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts a 2xx telemetry upload whose response did not echo the request's
|
||||
/// <c>X-Correlation-ID</c>: something other than the backend may have answered (a block page
|
||||
/// served with 200). Observed only — the upload still counts as a success.
|
||||
/// </summary>
|
||||
public void RecordOkWithoutEcho() => Interlocked.Increment(ref _okWithoutEchoCount);
|
||||
|
||||
/// <summary>Cumulative <see cref="RecordOkWithoutEcho"/> count since the process started.</summary>
|
||||
public long OkWithoutEchoCount => Interlocked.Read(ref _okWithoutEchoCount);
|
||||
|
||||
/// <summary>
|
||||
/// Returns an immutable snapshot of all counters for delta calculation.
|
||||
/// </summary>
|
||||
|
||||
@@ -377,8 +377,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
if (analyzers.KeepAwakeDuringUserEsp)
|
||||
{
|
||||
hosts.Add(new UserEspKeepAwakeHost(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
ingress: ingress,
|
||||
clock: clock,
|
||||
logger: logger,
|
||||
@@ -648,8 +646,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
tenantId: tenantId,
|
||||
ingress: ingress,
|
||||
clock: clock,
|
||||
logger: logger,
|
||||
apiBaseUrl: _agentConfig.ApiBaseUrl));
|
||||
logger: logger));
|
||||
|
||||
// Live AC/battery watcher — WMI Win32_PowerManagementEvent push (no polling), emits
|
||||
// power_state_change on AC↔battery transitions and 50/30/15 % downward threshold
|
||||
|
||||
@@ -10,6 +10,7 @@ using AutopilotMonitor.Agent.V2.Core.Persistence;
|
||||
using AutopilotMonitor.Agent.V2.Core.Telemetry.Events;
|
||||
using AutopilotMonitor.Agent.V2.Core.Telemetry.Signals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Telemetry.Transitions;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.DecisionCore.Classifiers;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
@@ -315,6 +316,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// </summary>
|
||||
public long IngressPendingSignalCount => _ingress?.PendingSignalCount ?? 0L;
|
||||
|
||||
/// <summary>
|
||||
/// Terminal phase: grants the transport its one upload attempt despite an active
|
||||
/// interception pause (<see cref="TelemetryUploadOrchestrator.GrantTerminalAttempt"/>) and,
|
||||
/// only while that pause holds, wakes the drain loop so the attempt runs inside the caller's
|
||||
/// bounded drain window. Without an interception pause nothing changes. No-op before
|
||||
/// <see cref="Start"/>.
|
||||
/// </summary>
|
||||
public void RequestTerminalUploadAttempt()
|
||||
{
|
||||
var transport = _transport;
|
||||
if (transport == null) return;
|
||||
if (transport.GrantTerminalAttempt())
|
||||
OnImmediateFlushRequested(this, EventArgs.Empty);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Typed read-model surfaces returned by <see cref="IComponentFactory.CreateCollectorHosts"/>
|
||||
/// (ARCH-F4). Set during <see cref="Start"/> step 14; <c>null</c> before Start and when
|
||||
@@ -371,14 +387,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
// 3) Telemetry-Transport. Batch size flows from AgentConfiguration.MaxBatchSize
|
||||
// via Program.cs (P1 fix: previously the remote-config knob was merged but
|
||||
// never applied, so tenants saw drainInterval/MaxBatchSize have no effect).
|
||||
_spool = new TelemetrySpool(_transportDirectory, _clock, _logger);
|
||||
_transport = new TelemetryUploadOrchestrator(_spool, _uploader, _clock, batchSize: _uploadBatchSize);
|
||||
// The partition key names the registered session: the spool drops leftovers of a
|
||||
// previous session id on startup, and the orchestrator tells a 410 for this session
|
||||
// (SessionGone → host terminates) from a 410 for such leftovers (dropped).
|
||||
var partitionKey = TelemetryPartitionKey.ForSession(_tenantId, _sessionId);
|
||||
_spool = new TelemetrySpool(_transportDirectory, _clock, _logger, expectedPartitionKey: partitionKey);
|
||||
_transport = new TelemetryUploadOrchestrator(
|
||||
_spool, _uploader, _clock, batchSize: _uploadBatchSize, currentPartitionKey: partitionKey, logger: _logger);
|
||||
|
||||
// TRACE-H1 — surface skipped oversized items + blocked-pipeline conditions as timeline
|
||||
// events. Raised after the drain guard releases, so posting back through the ingress
|
||||
// here cannot deadlock the drain.
|
||||
_transport.BatchPoisoned += OnBatchPoisoned;
|
||||
_transport.UploadBlocked += OnUploadBlocked;
|
||||
_transport.UploadIntercepted += OnUploadIntercepted;
|
||||
|
||||
// The spool reference reaches peripheral collectors (AgentSelfMetricsCollector via
|
||||
// PeriodicCollectorLifecycleHost) as a CreateCollectorHosts parameter at step 14,
|
||||
@@ -999,11 +1021,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
catch (AggregateException) { /* wrapped OperationCanceledException */ }
|
||||
catch (Exception ex) { _logger.Warning($"EnrollmentOrchestrator: drain loop join failed: {ex.Message}"); }
|
||||
|
||||
// 5) Terminaler Drain — finale Batches.
|
||||
// 5) Terminaler Drain — finale Batches. An active interception pause still lets the
|
||||
// process make its one terminal attempt here, unless DrainSpool already used it.
|
||||
try
|
||||
{
|
||||
using (var timeoutCts = new CancellationTokenSource(_terminalDrainTimeout))
|
||||
{
|
||||
_transport?.GrantTerminalAttempt();
|
||||
_transport?.DrainAllAsync(timeoutCts.Token).GetAwaiter().GetResult();
|
||||
}
|
||||
}
|
||||
@@ -1150,6 +1174,56 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the first intercepted upload of the process as <c>telemetry_upload_blocked</c>
|
||||
/// with the interception evidence. Gated once per process by the transport, independently
|
||||
/// of <see cref="OnUploadBlocked"/>. Queued behind the intercepted batch like every other
|
||||
/// item: it reaches the backend only if the path clears in this run, and its arrival time
|
||||
/// marks the end of the interception.
|
||||
/// </summary>
|
||||
private void OnUploadIntercepted(object? sender, InterceptionReport report)
|
||||
{
|
||||
var interception = report.Interception;
|
||||
_logger.Warning(
|
||||
$"EnrollmentOrchestrator: telemetry upload intercepted on the network path ({interception.ForeignReason}, " +
|
||||
$"http {interception.StatusCode}, pageTitle=\"{interception.PageTitle ?? "(none)"}\", " +
|
||||
$"serverCertIssuer=\"{interception.ServerCertIssuer ?? "(unknown)"}\") — uploads pause with backoff; " +
|
||||
"the agent keeps observing.");
|
||||
|
||||
var ingress = _ingress;
|
||||
if (ingress == null) return;
|
||||
try
|
||||
{
|
||||
new InformationalEventPost(ingress, _clock).Emit(new Shared.Models.EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.TelemetryUploadBlocked,
|
||||
Severity = Shared.Models.EventSeverity.Warning,
|
||||
Source = "TelemetryUploadOrchestrator",
|
||||
Phase = Shared.Models.EnrollmentPhase.Unknown,
|
||||
Message = "Telemetry upload intercepted on the network path: the answer did not come from the "
|
||||
+ "backend, so the device certificate did not reach it. Uploads pause with backoff and "
|
||||
+ "resume once the path clears; this marker is queued behind the intercepted batch.",
|
||||
Timestamp = _clock.UtcNow,
|
||||
ImmediateUpload = true,
|
||||
Data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
["reason"] = interception.ForeignReason ?? ResponseClassification.ForeignResponse,
|
||||
["operation"] = "agent/telemetry",
|
||||
["statusCode"] = interception.StatusCode,
|
||||
["pageTitle"] = interception.PageTitle ?? string.Empty,
|
||||
["serverCertIssuer"] = interception.ServerCertIssuer ?? string.Empty,
|
||||
["firstBlockedAtUtc"] = report.FirstBlockedAtUtc.ToString("o", CultureInfo.InvariantCulture),
|
||||
},
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error("EnrollmentOrchestrator: failed to post telemetry_upload_blocked (interception).", ex);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Test helpers
|
||||
|
||||
/// <summary>
|
||||
|
||||
+25
-1
@@ -73,13 +73,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// </summary>
|
||||
public string? StopReason { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <c>origin</c> param of a server-requested <c>terminate_session</c> action
|
||||
/// (<see cref="TerminationOrigins"/>); null for every termination the agent decided
|
||||
/// itself. <see cref="TerminationOrigins.SessionGone"/> makes the termination handler
|
||||
/// skip the spool drain and the diagnostics upload — the backend no longer knows the
|
||||
/// session, so both could only fail.
|
||||
/// </summary>
|
||||
public string? Origin { get; }
|
||||
|
||||
public EnrollmentTerminatedEventArgs(
|
||||
EnrollmentTerminationReason reason,
|
||||
EnrollmentTerminationOutcome outcome,
|
||||
string? stageName,
|
||||
DateTime terminatedAtUtc,
|
||||
string? details = null,
|
||||
string? stopReason = null)
|
||||
string? stopReason = null,
|
||||
string? origin = null)
|
||||
{
|
||||
Reason = reason;
|
||||
Outcome = outcome;
|
||||
@@ -87,6 +97,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
TerminatedAtUtc = terminatedAtUtc;
|
||||
Details = details;
|
||||
StopReason = stopReason;
|
||||
Origin = origin;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Values of the <c>origin</c> param on a synthesised <c>terminate_session</c>
|
||||
/// <c>ServerAction</c>, carried into <see cref="EnrollmentTerminatedEventArgs.Origin"/>.
|
||||
/// </summary>
|
||||
public static class TerminationOrigins
|
||||
{
|
||||
/// <summary>Administrator kill signal (<c>DeviceKillSignal</c> on a 2xx telemetry response).</summary>
|
||||
public const string KillSignal = "kill_signal";
|
||||
|
||||
/// <summary>The backend answered HTTP 410 for the current session's telemetry.</summary>
|
||||
public const string SessionGone = "session_gone";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
|
||||
private readonly DesktopArrivalDetector _detector;
|
||||
private readonly DesktopArrivalDetectorAdapter _adapter;
|
||||
private readonly AgentLogger _logger;
|
||||
private int _disposed;
|
||||
|
||||
public DesktopArrivalHost(
|
||||
@@ -30,7 +29,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
string? tenantId = null,
|
||||
Action<string>? onRealUserOwnerObserved = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes);
|
||||
_adapter = new DesktopArrivalDetectorAdapter(_detector, ingress, clock);
|
||||
|
||||
|
||||
@@ -26,14 +26,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
string tenantId,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
AgentLogger logger,
|
||||
string? apiBaseUrl)
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
_logger = logger;
|
||||
var post = new InformationalEventPost(ingress, clock);
|
||||
_detector = new NetworkChangeDetector(sessionId, tenantId, post, logger, apiBaseUrl);
|
||||
_detector = new NetworkChangeDetector(sessionId, tenantId, post, logger);
|
||||
}
|
||||
|
||||
public void Start()
|
||||
|
||||
+1
-1
@@ -60,7 +60,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
var post = new InformationalEventPost(ingress, clock, logger);
|
||||
_collector = new ProvisioningPackageCollector(sessionId, tenantId, post, logger, clock);
|
||||
_collector = new ProvisioningPackageCollector(sessionId, tenantId, post, logger);
|
||||
_observableIngress = ingress as SignalIngress;
|
||||
}
|
||||
|
||||
|
||||
@@ -91,8 +91,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
|
||||
private static readonly string AccountSetupPhaseName = EnrollmentPhase.AccountSetup.ToString();
|
||||
|
||||
private readonly string _sessionId;
|
||||
private readonly string _tenantId;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly KeepAwakeController _controller;
|
||||
@@ -112,8 +110,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
private int _disposed;
|
||||
|
||||
public UserEspKeepAwakeHost(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
AgentLogger logger,
|
||||
@@ -125,8 +121,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
_sessionId = sessionId ?? string.Empty;
|
||||
_tenantId = tenantId ?? string.Empty;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_post = new InformationalEventPost(ingress, clock, logger);
|
||||
_controller = controller ?? new KeepAwakeController(logger);
|
||||
|
||||
@@ -5,6 +5,7 @@ using System.Net.Http;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
@@ -40,10 +41,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// (<see cref="HttpClient"/>'s <c>_handler</c>-Feld ist internal und in net48
|
||||
/// reflection-unzuverlässig).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <paramref name="serverCertificates"/>, when supplied, observes the server certificate of
|
||||
/// every new connection without changing the validation verdict.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static HttpClientHandler CreateHandler(
|
||||
ICertificateResolver resolver,
|
||||
AgentLogger logger)
|
||||
AgentLogger logger,
|
||||
ServerCertificateObserver? serverCertificates = null)
|
||||
{
|
||||
if (resolver == null) throw new ArgumentNullException(nameof(resolver));
|
||||
if (logger == null) throw new ArgumentNullException(nameof(logger));
|
||||
@@ -62,6 +68,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
|
||||
};
|
||||
handler.ClientCertificates.Add(cert);
|
||||
serverCertificates?.Attach(handler);
|
||||
|
||||
logger.Info(
|
||||
$"MtlsHttpClientFactory: handler ready (cert thumbprint={cert.Thumbprint}).");
|
||||
@@ -85,9 +92,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
ICertificateResolver resolver,
|
||||
AgentLogger logger,
|
||||
TimeSpan? timeout = null,
|
||||
NetworkMetrics? metrics = null)
|
||||
NetworkMetrics? metrics = null,
|
||||
ServerCertificateObserver? serverCertificates = null)
|
||||
{
|
||||
var innerHandler = CreateHandler(resolver, logger);
|
||||
var innerHandler = CreateHandler(resolver, logger, serverCertificates);
|
||||
HttpMessageHandler pipelineHead = innerHandler;
|
||||
if (metrics != null)
|
||||
{
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
@@ -29,6 +31,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// twice about a certificate it already rejected.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only a 401/403 the backend authored counts (<see cref="ResponseOriginClassifier"/>). One
|
||||
/// from something on the network path (TLS inspection, a proxy page) never counts and never
|
||||
/// touches the streak — it says nothing about this device's authorization — and is reported
|
||||
/// once per process as <see cref="DistressErrorType.ClientCertificateNotReceived"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Thread-safety: consecutive-count uses <see cref="Interlocked"/>; the first-failure timestamp
|
||||
/// is protected by a single monitor lock. <see cref="ThresholdExceeded"/> is raised outside the
|
||||
/// lock to prevent handler re-entrancy deadlocks.
|
||||
@@ -47,6 +55,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
private DateTime? _firstFailureUtc;
|
||||
private int _thresholdFired; // 0/1 — Interlocked exchange makes the event single-shot
|
||||
private int _distressDispatched; // 0/1 per failure streak — reset by RecordSuccess
|
||||
private int _foreignDistressDispatched; // 0/1 per process — never reset
|
||||
private Task _foreignDistressSend; // the per-process ClientCertificateNotReceived send; null until dispatched
|
||||
|
||||
public AuthFailureTracker(
|
||||
int maxFailures,
|
||||
@@ -90,16 +100,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// dispatch further distress signals. Emits <see cref="ThresholdExceeded"/> the first
|
||||
/// time either ceiling is crossed; subsequent calls after termination is armed are no-ops.
|
||||
/// </summary>
|
||||
/// <param name="endpointUnavailable">
|
||||
/// True when the 401/403 was a platform-level HTML response (stopped/retired app, edge
|
||||
/// proxy — see <see cref="BackendAuthException.EndpointUnavailable"/>). Suppresses the
|
||||
/// distress dispatch: the report would go to the very endpoint that just answered with a
|
||||
/// platform error page, and its DeviceNotRegistered classification would be wrong anyway.
|
||||
/// Counting and the shutdown thresholds are unaffected. Should a later failure in the
|
||||
/// same streak come from a live backend, that one still dispatches the streak's report.
|
||||
/// <param name="classification">
|
||||
/// Who answered (<see cref="ResponseOriginClassifier"/>); null = the backend.
|
||||
/// <see cref="ResponseOrigin.PlatformStopped"/> counts but suppresses the distress dispatch:
|
||||
/// the report would go to the very endpoint that just answered with its stopped page, and
|
||||
/// DeviceNotRegistered would be wrong anyway; a later failure in the same streak from a live
|
||||
/// backend still dispatches the streak's report. <see cref="ResponseOrigin.Foreign"/> never
|
||||
/// counts — see <see cref="RecordForeignResponse"/>.
|
||||
/// </param>
|
||||
public void RecordFailure(int statusCode, string operation, bool endpointUnavailable = false)
|
||||
public void RecordFailure(int statusCode, string operation, ResponseClassification classification = null)
|
||||
{
|
||||
if (classification?.IsForeign == true)
|
||||
{
|
||||
RecordForeignResponse(statusCode, operation, classification);
|
||||
return;
|
||||
}
|
||||
|
||||
if (Volatile.Read(ref _thresholdFired) == 1) return;
|
||||
|
||||
var count = Interlocked.Increment(ref _consecutiveFailures);
|
||||
@@ -114,9 +130,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
|
||||
// V1 parity — only one distress report per failure streak. Repeat hits use the local
|
||||
// log only; the backend already knows from the first report that the device is
|
||||
// unauthorized. Endpoint-unavailable failures never dispatch (nothing to send to) but
|
||||
// unauthorized. Stopped-endpoint failures never dispatch (nothing to send to) but
|
||||
// also never consume the streak's report slot.
|
||||
if (endpointUnavailable)
|
||||
if (classification?.Origin == ResponseOrigin.PlatformStopped)
|
||||
{
|
||||
if (_distressReporter != null && Volatile.Read(ref _distressDispatched) == 0)
|
||||
{
|
||||
@@ -180,6 +196,60 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
_logger.Warning($"AuthFailureTracker: ThresholdExceeded handler threw: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A 401/403 that something on the network path authored. Never counts and never touches
|
||||
/// the streak — an interception says nothing about this device's authorization, and five
|
||||
/// of them must not shut a healthy agent down. The first one per process dispatches
|
||||
/// <see cref="DistressErrorType.ClientCertificateNotReceived"/>: the distress channel needs
|
||||
/// no client certificate, so it gets through where the request did not. Message order is
|
||||
/// operation, status, sub-reason, page title, server-certificate issuer (the reporter cuts
|
||||
/// at 256 characters).
|
||||
/// </summary>
|
||||
private void RecordForeignResponse(int statusCode, string operation, ResponseClassification classification)
|
||||
{
|
||||
var finding =
|
||||
$"{operation} http {statusCode} {classification.ForeignReason}; " +
|
||||
$"page \"{classification.PageTitle ?? "(none)"}\"; " +
|
||||
$"server cert issuer \"{classification.ServerCertIssuer ?? "(unknown)"}\"";
|
||||
_logger.Info($"AuthFailureTracker: answer did not come from the backend — not an authentication failure, not counted ({finding}).");
|
||||
|
||||
if (_distressReporter == null || Interlocked.CompareExchange(ref _foreignDistressDispatched, 1, 0) != 0)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
// Fire-and-forget like every other dispatch here, but kept: a caller about to end
|
||||
// the process waits for it (WaitForPendingForeignDistress).
|
||||
Volatile.Write(ref _foreignDistressSend, _distressReporter.TrySendAsync(
|
||||
DistressErrorType.ClientCertificateNotReceived, finding, httpStatusCode: statusCode));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"AuthFailureTracker: distress dispatch threw: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gives this process's <see cref="DistressErrorType.ClientCertificateNotReceived"/> send up
|
||||
/// to <paramref name="bound"/> to leave the machine. The dispatch is fire-and-forget; a caller
|
||||
/// about to end the process (a start whose registration failed exits or cleans up right away)
|
||||
/// waits here so the report is not abandoned mid-flight. Returns true when nothing is pending
|
||||
/// or the send finished within the bound. Never throws.
|
||||
/// </summary>
|
||||
public bool WaitForPendingForeignDistress(TimeSpan bound)
|
||||
{
|
||||
var send = Volatile.Read(ref _foreignDistressSend);
|
||||
if (send == null || send.IsCompleted) return true;
|
||||
try
|
||||
{
|
||||
return send.Wait(bound);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return true; // faulted or cancelled: finished, nothing left to wait for
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class AuthFailureThresholdEventArgs : EventArgs
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
@@ -27,6 +29,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
private readonly string _registrationFailurePath;
|
||||
private readonly object _lockObject = new object();
|
||||
|
||||
// Agent-local key in registration-failure.json; RegistrationFailureSummary has no such member.
|
||||
private const string LocalNetworkFingerprintKey = "NetworkFingerprint";
|
||||
|
||||
public SessionIdPersistence(string dataDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(dataDirectory))
|
||||
@@ -170,22 +175,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
|
||||
/// <summary>
|
||||
/// Folds one failed registration run into <c>registration-failure.json</c>: the run
|
||||
/// counter grows, the first failure time is kept, everything else describes the latest
|
||||
/// run. A run that never registers exits and leaves nothing else behind — the emergency
|
||||
/// break reads this file so the backend learns why the session never appeared.
|
||||
/// Best-effort; never throws.
|
||||
/// counters grow (an intercepted run — <c>InterceptedRuns</c> > 0 on
|
||||
/// <paramref name="latestRun"/> — also counts toward
|
||||
/// <see cref="RegistrationFailureSummary.InterceptedRuns"/>), the first failure time is kept,
|
||||
/// everything else describes the latest run. A run that never registers exits and leaves
|
||||
/// nothing else behind — the emergency break reads this file so the backend learns why the
|
||||
/// session never appeared. <paramref name="networkFingerprint"/> (the latest run's network)
|
||||
/// is stored beside the summary, never in it, so it cannot reach the wire. Written through to
|
||||
/// disk (L.12): the process exits right after and the next boot decides on the record.
|
||||
/// Returns the merged summary, null when it could not be written. Never throws.
|
||||
/// </summary>
|
||||
public void RecordRegistrationFailure(RegistrationFailureSummary latestRun, AgentLogger logger = null)
|
||||
public RegistrationFailureSummary RecordRegistrationFailure(
|
||||
RegistrationFailureSummary latestRun, AgentLogger logger = null, string networkFingerprint = null)
|
||||
{
|
||||
if (latestRun == null) return;
|
||||
if (latestRun == null) return null;
|
||||
lock (_lockObject)
|
||||
{
|
||||
try
|
||||
{
|
||||
var previous = LoadRegistrationFailureInternal();
|
||||
var previous = LoadRegistrationFailureInternal()?.Summary;
|
||||
var merged = new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = (previous?.FailedRuns ?? 0) + 1,
|
||||
InterceptedRuns = (previous?.InterceptedRuns ?? 0) + (latestRun.InterceptedRuns > 0 ? 1 : 0),
|
||||
FirstFailedAtUtc = previous?.FirstFailedAtUtc ?? latestRun.LastFailedAtUtc,
|
||||
LastFailedAtUtc = latestRun.LastFailedAtUtc,
|
||||
Outcome = latestRun.Outcome,
|
||||
@@ -195,18 +207,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
AttemptWindowSeconds = latestRun.AttemptWindowSeconds,
|
||||
ConfigFetchOutcome = latestRun.ConfigFetchOutcome,
|
||||
};
|
||||
File.WriteAllText(_registrationFailurePath, JsonConvert.SerializeObject(merged));
|
||||
logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns}.");
|
||||
WriteRegistrationFailure(merged, networkFingerprint);
|
||||
logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns} ({merged.InterceptedRuns} intercepted).");
|
||||
return merged;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"SessionIdPersistence: RecordRegistrationFailure failed: {ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The failed-registration record since the last success, or <c>null</c> (none, unreadable).</summary>
|
||||
/// <summary>
|
||||
/// The failed-registration record since the last success, or <c>null</c> (none, unreadable).
|
||||
/// The summary only — what the emergency break may send.
|
||||
/// </summary>
|
||||
public RegistrationFailureSummary LoadRegistrationFailure()
|
||||
{
|
||||
lock (_lockObject) return LoadRegistrationFailureInternal()?.Summary;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The failed-registration record plus the agent-local network fingerprint of its latest
|
||||
/// run, or <c>null</c> (none, unreadable). For the next successful start's comparison only.
|
||||
/// </summary>
|
||||
public RegistrationFailureRecord LoadRegistrationFailureRecord()
|
||||
{
|
||||
lock (_lockObject) return LoadRegistrationFailureInternal();
|
||||
}
|
||||
@@ -252,13 +278,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
}
|
||||
|
||||
private RegistrationFailureSummary LoadRegistrationFailureInternal()
|
||||
private RegistrationFailureRecord LoadRegistrationFailureInternal()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(_registrationFailurePath)) return null;
|
||||
var record = JsonConvert.DeserializeObject<RegistrationFailureSummary>(File.ReadAllText(_registrationFailurePath));
|
||||
return record != null && record.FailedRuns > 0 ? record : null;
|
||||
var json = JObject.Parse(File.ReadAllText(_registrationFailurePath));
|
||||
var summary = json.ToObject<RegistrationFailureSummary>();
|
||||
return summary != null && summary.FailedRuns > 0
|
||||
? new RegistrationFailureRecord(summary, (string)json[LocalNetworkFingerprintKey])
|
||||
: null;
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -267,6 +296,26 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The summary's own properties plus one agent-local key: the fingerprint sits next to the
|
||||
/// wire type, never inside it (the summary deserializes without it).
|
||||
/// </summary>
|
||||
private void WriteRegistrationFailure(RegistrationFailureSummary summary, string networkFingerprint)
|
||||
{
|
||||
var json = JObject.FromObject(summary);
|
||||
if (!string.IsNullOrEmpty(networkFingerprint))
|
||||
json[LocalNetworkFingerprintKey] = networkFingerprint;
|
||||
|
||||
var bytes = Encoding.UTF8.GetBytes(json.ToString(Formatting.None));
|
||||
using (var stream = new FileStream(
|
||||
_registrationFailurePath, FileMode.Create, FileAccess.Write, FileShare.None,
|
||||
bufferSize: 4096, options: FileOptions.WriteThrough))
|
||||
{
|
||||
stream.Write(bytes, 0, bytes.Length);
|
||||
stream.Flush(flushToDisk: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path, AgentLogger logger)
|
||||
{
|
||||
try { if (File.Exists(path)) File.Delete(path); }
|
||||
@@ -305,4 +354,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>registration-failure.json</c> as the agent keeps it: the <see cref="RegistrationFailureSummary"/>
|
||||
/// the emergency break may send, plus the network fingerprint of the latest failed run, which
|
||||
/// stays on the device — only the next successful start's comparison (changed or not) is reported.
|
||||
/// </summary>
|
||||
public sealed class RegistrationFailureRecord
|
||||
{
|
||||
public RegistrationFailureRecord(RegistrationFailureSummary summary, string networkFingerprint)
|
||||
{
|
||||
Summary = summary ?? throw new ArgumentNullException(nameof(summary));
|
||||
NetworkFingerprint = networkFingerprint;
|
||||
}
|
||||
|
||||
public RegistrationFailureSummary Summary { get; }
|
||||
|
||||
/// <summary>Fingerprint of the latest failed run's network; null when it was unknown.</summary>
|
||||
public string NetworkFingerprint { get; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// <item>On <see cref="BackendAuthException"/> (401/403) → report to the
|
||||
/// <see cref="AuthFailureTracker"/> (which fires the first-failure distress + may trip
|
||||
/// the shutdown threshold) and return <see cref="SessionRegistrationOutcome.AuthFailed"/>
|
||||
/// without retrying — the backend has definitively rejected the device cert.</item>
|
||||
/// without retrying — the backend has definitively rejected the device cert. A 401/403
|
||||
/// something on the network path answered ends the same way, marked
|
||||
/// <see cref="SessionRegistrationResult.IsIntercepted"/>.</item>
|
||||
/// <item>Exception to the above — a 403 or 409 carrying error code
|
||||
/// <c>session_owner_mismatch</c> (SESSION-OWNER-BINDING): the persisted SessionId names a
|
||||
/// session that is not this device's — bound to another device identity (403, typically an
|
||||
@@ -135,8 +137,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
logger.Error($"Session owner mismatch reported by backend but session rotation failed: {rotateEx.Message}", rotateEx);
|
||||
if (ex is BackendAuthException authEx)
|
||||
{
|
||||
authFailureTracker?.RecordFailure(authEx.StatusCode, "agent/register-session", authEx.EndpointUnavailable);
|
||||
return SessionRegistrationResult.AuthFailed(authEx.StatusCode, authEx.Message);
|
||||
authFailureTracker?.RecordFailure(authEx.StatusCode, "agent/register-session", authEx.Classification);
|
||||
return SessionRegistrationResult.AuthFailed(authEx.StatusCode, authEx.Message, authEx.IsForeignResponse);
|
||||
}
|
||||
return SessionRegistrationResult.Failed(ex.Message);
|
||||
}
|
||||
@@ -154,12 +156,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
catch (BackendAuthException ex)
|
||||
{
|
||||
logger.Error($"Session registration authentication failed ({ex.StatusCode}): {ex.Message}");
|
||||
logger.Error(ex.IsForeignResponse
|
||||
? $"Session registration intercepted on the network path ({ex.StatusCode}): {ex.Message}"
|
||||
: $"Session registration authentication failed ({ex.StatusCode}): {ex.Message}");
|
||||
// Feed the central auth-failure tracker so the first-failure distress dispatches
|
||||
// and the shutdown threshold advances. V1 parity: no retry on auth-failure —
|
||||
// backend has definitively rejected the cert/token.
|
||||
authFailureTracker?.RecordFailure(ex.StatusCode, "agent/register-session", ex.EndpointUnavailable);
|
||||
return SessionRegistrationResult.AuthFailed(ex.StatusCode, ex.Message);
|
||||
// backend has definitively rejected the cert/token. A 401/403 from something on
|
||||
// the network path is not counted; the tracker reports it once per process as
|
||||
// ClientCertificateNotReceived, and the run ends AuthFailed marked intercepted
|
||||
// (the caller's buffer decides between a further boot and cleaning up).
|
||||
authFailureTracker?.RecordFailure(ex.StatusCode, "agent/register-session", ex.Classification);
|
||||
return SessionRegistrationResult.AuthFailed(ex.StatusCode, ex.Message, ex.IsForeignResponse);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -274,7 +281,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// <summary>Backend responded with <c>Success=true</c>. The session row is live.</summary>
|
||||
Succeeded = 0,
|
||||
|
||||
/// <summary>401/403 from the backend — cert/token rejected. No retry was attempted.</summary>
|
||||
/// <summary>
|
||||
/// 401/403 — cert/token rejected by the backend, or answered by something on the network
|
||||
/// path (<see cref="SessionRegistrationResult.IsIntercepted"/>). No retry was attempted.
|
||||
/// </summary>
|
||||
AuthFailed = 1,
|
||||
|
||||
/// <summary>All attempts failed with non-auth errors (network, 5xx, malformed response).</summary>
|
||||
@@ -299,25 +309,34 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// </summary>
|
||||
public string RotatedFromSessionId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// True when the run ended on a 401/403 that something on the network path answered (TLS
|
||||
/// inspection, a proxy page — <see cref="BackendAuthException.IsForeignResponse"/>), not
|
||||
/// the backend. Such starts count toward the intercepted-registration buffer.
|
||||
/// </summary>
|
||||
public bool IsIntercepted { get; }
|
||||
|
||||
private SessionRegistrationResult(
|
||||
SessionRegistrationOutcome outcome,
|
||||
RegisterSessionResponse response,
|
||||
int httpStatusCode,
|
||||
string errorMessage,
|
||||
string rotatedFromSessionId = null)
|
||||
string rotatedFromSessionId = null,
|
||||
bool isIntercepted = false)
|
||||
{
|
||||
Outcome = outcome;
|
||||
Response = response;
|
||||
HttpStatusCode = httpStatusCode;
|
||||
ErrorMessage = errorMessage;
|
||||
RotatedFromSessionId = rotatedFromSessionId;
|
||||
IsIntercepted = isIntercepted;
|
||||
}
|
||||
|
||||
public static SessionRegistrationResult Succeeded(RegisterSessionResponse response, string rotatedFromSessionId = null)
|
||||
=> new SessionRegistrationResult(SessionRegistrationOutcome.Succeeded, response, 200, null, rotatedFromSessionId);
|
||||
|
||||
public static SessionRegistrationResult AuthFailed(int statusCode, string message)
|
||||
=> new SessionRegistrationResult(SessionRegistrationOutcome.AuthFailed, null, statusCode, message);
|
||||
public static SessionRegistrationResult AuthFailed(int statusCode, string message, bool intercepted = false)
|
||||
=> new SessionRegistrationResult(SessionRegistrationOutcome.AuthFailed, null, statusCode, message, isIntercepted: intercepted);
|
||||
|
||||
public static SessionRegistrationResult Failed(string message)
|
||||
=> new SessionRegistrationResult(SessionRegistrationOutcome.Failed, null, 0, message);
|
||||
|
||||
@@ -45,7 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Events
|
||||
_sequenceCounter = sequenceCounter ?? throw new ArgumentNullException(nameof(sequenceCounter));
|
||||
_sessionId = sessionId;
|
||||
_tenantId = tenantId;
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
_traceEventsEnabled = traceEventsEnabled;
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Signals
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId));
|
||||
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Transitions
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId));
|
||||
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+35
-6
@@ -306,7 +306,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
});
|
||||
|
||||
DelayLateEventGrace();
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
// Option 2 (WG Part 1 graceful-exit hardening, 2026-04-30): write the
|
||||
// clean-exit marker BEFORE _signalShutdown returns control to the main
|
||||
@@ -329,7 +329,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
|
||||
RunUploadDiagnosticsWithEvents(args);
|
||||
WriteEnrollmentCompleteMarker(args);
|
||||
RunStandaloneRebootIfRequested();
|
||||
RunStandaloneRebootIfRequested(args);
|
||||
RunSelfDestructIfAppropriate(args);
|
||||
// Option 2 (same hardening) — covers the standard Completed / Failed terminal
|
||||
// path too. Cleanup PowerShell does not touch this marker, so writing it before
|
||||
@@ -979,6 +979,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// </param>
|
||||
private void RunUploadDiagnosticsWithEvents(EnrollmentTerminatedEventArgs args, string suffixOverride = null)
|
||||
{
|
||||
if (IsSessionGone(args))
|
||||
{
|
||||
// The backend answered 410 for this session: the package would describe a session
|
||||
// that no longer exists, and its diagnostics_* events could never be accepted.
|
||||
_logger.Info("EnrollmentTerminationHandler: diagnostics upload skipped — backend no longer knows this session.");
|
||||
return;
|
||||
}
|
||||
|
||||
var mode = _configuration.DiagnosticsUploadMode ?? "Off";
|
||||
if (!_configuration.DiagnosticsUploadEnabled || string.Equals(mode, "Off", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
@@ -1124,7 +1132,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// but enables <c>RebootOnComplete</c>, the agent's final act is <c>shutdown.exe /r</c>
|
||||
/// with the configured delay, giving the user a visible countdown.
|
||||
/// </summary>
|
||||
private void RunStandaloneRebootIfRequested()
|
||||
private void RunStandaloneRebootIfRequested(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
if (_configuration.SelfDestructOnComplete) return;
|
||||
if (!_configuration.RebootOnComplete) return;
|
||||
@@ -1148,7 +1156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
ImmediateUpload = true,
|
||||
});
|
||||
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -1185,7 +1193,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
// just above. The WG-Part-1 and standalone-reboot paths already drain first; this
|
||||
// closes the gap on the self-destruct path. Bounded + best-effort (never blocks
|
||||
// termination on a wedged backend).
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
var service = _cleanupServiceFactory();
|
||||
service.ExecuteSelfDestruct();
|
||||
@@ -1318,8 +1326,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
}
|
||||
}
|
||||
|
||||
private void DrainSpool()
|
||||
/// <summary>
|
||||
/// <see cref="TerminationOrigins.SessionGone"/>: the backend no longer knows the session.
|
||||
/// Every drain would only collect more 410s against a bounded budget, and a diagnostics
|
||||
/// package would describe a deleted session — both are skipped; marker, cleanup and
|
||||
/// self-destruct run as usual.
|
||||
/// </summary>
|
||||
private static bool IsSessionGone(EnrollmentTerminatedEventArgs args) =>
|
||||
string.Equals(args.Origin, TerminationOrigins.SessionGone, StringComparison.Ordinal);
|
||||
|
||||
private void DrainSpool(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
if (IsSessionGone(args))
|
||||
{
|
||||
_logger.Info("EnrollmentTerminationHandler: spool drain skipped — backend no longer knows this session.");
|
||||
return;
|
||||
}
|
||||
|
||||
// An interception pause (uploads answered by something on the network path) would
|
||||
// otherwise skip this drain entirely: the terminal phase gets exactly one attempt, then
|
||||
// termination goes on as usual — whatever did not go up is lost, as with any session.
|
||||
try { _drain.RequestTerminalUploadAttempt(); }
|
||||
catch (Exception ex) { _logger.Warning($"EnrollmentTerminationHandler: terminal upload attempt request threw: {ex.Message}"); }
|
||||
|
||||
// Block briefly so pending events can land before the next destructive step
|
||||
// (shutdown.exe, self-destruct). Two phases share the same bounded budget:
|
||||
//
|
||||
|
||||
@@ -54,5 +54,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// <see cref="CanObserveSpool"/> is true.
|
||||
/// </summary>
|
||||
int SpoolPendingItemCount { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Called once at the start of every <c>DrainSpool</c>: lets the transport make its one
|
||||
/// terminal upload attempt even while an interception pause holds uploads back (production:
|
||||
/// <see cref="Orchestration.EnrollmentOrchestrator.RequestTerminalUploadAttempt"/>).
|
||||
/// Without an interception pause it changes nothing.
|
||||
/// </summary>
|
||||
void RequestTerminalUploadAttempt();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,5 +31,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
public bool CanObserveSpool => true;
|
||||
|
||||
public int SpoolPendingItemCount => _orchestrator.PendingItemCount;
|
||||
|
||||
public void RequestTerminalUploadAttempt() => _orchestrator.RequestTerminalUploadAttempt();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport
|
||||
{
|
||||
/// <summary>Who answered a 401/403 — see <see cref="ResponseOriginClassifier"/>.</summary>
|
||||
public enum ResponseOrigin
|
||||
{
|
||||
/// <summary>The backend: the response echoes the request's <c>X-Correlation-ID</c>. A real authorization verdict.</summary>
|
||||
Backend = 0,
|
||||
|
||||
/// <summary>
|
||||
/// The platform page of a stopped web app (no echo, "This web app is stopped"). The endpoint
|
||||
/// itself is down; counts toward the auth shutdown so a dead endpoint does not keep the agent alive.
|
||||
/// </summary>
|
||||
PlatformStopped = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Anything else without the echo: the platform's "Client Certificate Required" page (the device
|
||||
/// certificate did not reach the backend) or a page of a proxy on the network path. Never an
|
||||
/// authorization verdict.
|
||||
/// </summary>
|
||||
Foreign = 2,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A classified 401/403 plus the evidence the agent reports about it (log, distress,
|
||||
/// <c>telemetry_upload_blocked</c>). Built by <see cref="ResponseOriginClassifier.Classify"/>.
|
||||
/// </summary>
|
||||
public sealed class ResponseClassification
|
||||
{
|
||||
/// <summary>Foreign sub-reason: the platform answered "Client Certificate Required".</summary>
|
||||
public const string ClientCertificateRequired = "client_certificate_required";
|
||||
|
||||
/// <summary>Foreign sub-reason: any other answer that did not come from the backend (e.g. a proxy page).</summary>
|
||||
public const string ForeignResponse = "foreign_response";
|
||||
|
||||
private ResponseClassification(
|
||||
ResponseOrigin origin,
|
||||
int statusCode,
|
||||
string? foreignReason,
|
||||
string? pageTitle,
|
||||
bool isHtml,
|
||||
string? serverCertIssuer)
|
||||
{
|
||||
Origin = origin;
|
||||
StatusCode = statusCode;
|
||||
ForeignReason = foreignReason;
|
||||
PageTitle = pageTitle;
|
||||
IsHtml = isHtml;
|
||||
ServerCertIssuer = serverCertIssuer;
|
||||
}
|
||||
|
||||
public ResponseOrigin Origin { get; }
|
||||
|
||||
/// <summary>HTTP status of the classified response (401 or 403).</summary>
|
||||
public int StatusCode { get; }
|
||||
|
||||
/// <summary><see cref="ClientCertificateRequired"/> or <see cref="ForeignResponse"/> for <see cref="ResponseOrigin.Foreign"/>; null otherwise.</summary>
|
||||
public string? ForeignReason { get; }
|
||||
|
||||
/// <summary>HTML <c><title></c> of the page (at most 80 characters), null when absent.</summary>
|
||||
public string? PageTitle { get; }
|
||||
|
||||
/// <summary>True when the body is an HTML page (declared <c>text/html</c> or sniffed).</summary>
|
||||
public bool IsHtml { get; }
|
||||
|
||||
/// <summary>Issuer of the server certificate of the last validated backend connection (<see cref="ServerCertificateObserver"/>); null when unknown.</summary>
|
||||
public string? ServerCertIssuer { get; }
|
||||
|
||||
public bool IsForeign => Origin == ResponseOrigin.Foreign;
|
||||
|
||||
internal static ResponseClassification ForBackend(int statusCode) =>
|
||||
new ResponseClassification(ResponseOrigin.Backend, statusCode, null, null, isHtml: false, serverCertIssuer: null);
|
||||
|
||||
internal static ResponseClassification ForPlatformStopped(int statusCode, string? pageTitle, bool isHtml, string? serverCertIssuer) =>
|
||||
new ResponseClassification(ResponseOrigin.PlatformStopped, statusCode, null, pageTitle, isHtml, serverCertIssuer);
|
||||
|
||||
internal static ResponseClassification ForForeign(int statusCode, string foreignReason, string? pageTitle, bool isHtml, string? serverCertIssuer) =>
|
||||
new ResponseClassification(ResponseOrigin.Foreign, statusCode, foreignReason, pageTitle, isHtml, serverCertIssuer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decides whether a 401/403 came from the backend at all. The backend's correlation
|
||||
/// middleware echoes the request's <c>X-Correlation-ID</c> on every response it writes (the
|
||||
/// agent sends a fresh id per request); the Azure platform's own pages and a proxy's pages never
|
||||
/// carry it. The echo is therefore the only proof of origin — body shape, content type and
|
||||
/// title are evidence, not proof. Shared by every agent call site that can receive a 401/403.
|
||||
/// </summary>
|
||||
public static class ResponseOriginClassifier
|
||||
{
|
||||
/// <summary>Request header the backend echoes on every response it writes.</summary>
|
||||
public const string CorrelationIdHeader = "X-Correlation-ID";
|
||||
|
||||
/// <summary>Only the head of a body is inspected; platform and proxy pages are a few KB.</summary>
|
||||
public const int MaxBodyChars = 16 * 1024;
|
||||
|
||||
private const int MaxTitleLength = 80;
|
||||
|
||||
// Text of the platform page of a stopped web app ("Error 403 - This web app is stopped.").
|
||||
private const string StoppedAppMarker = "This web app is stopped";
|
||||
|
||||
// Text of the platform page served when no client certificate arrived
|
||||
// ("Error 403 - Forbidden: Client Certificate Required.").
|
||||
private const string ClientCertificateRequiredMarker = "Client Certificate Required";
|
||||
|
||||
/// <summary>
|
||||
/// Classifies a 401/403. <paramref name="sentCorrelationId"/> is the id this request carried;
|
||||
/// <paramref name="bodyPrefix"/> is the head of the body (<see cref="ReadBodyPrefixAsync"/>);
|
||||
/// <paramref name="serverCertIssuer"/> is passed through as evidence. An echo always wins:
|
||||
/// a backend page is the backend's verdict whatever it looks like.
|
||||
/// </summary>
|
||||
public static ResponseClassification Classify(
|
||||
HttpResponseMessage response,
|
||||
string? sentCorrelationId,
|
||||
string? bodyPrefix,
|
||||
string? serverCertIssuer = null)
|
||||
{
|
||||
if (response == null) throw new ArgumentNullException(nameof(response));
|
||||
|
||||
var statusCode = (int)response.StatusCode;
|
||||
if (EchoesCorrelationId(response, sentCorrelationId))
|
||||
return ResponseClassification.ForBackend(statusCode);
|
||||
|
||||
var body = Cap(bodyPrefix);
|
||||
var isHtml = LooksLikeHtml(response, body);
|
||||
var title = TryExtractHtmlTitle(body);
|
||||
|
||||
if (Contains(body, StoppedAppMarker))
|
||||
return ResponseClassification.ForPlatformStopped(statusCode, title, isHtml, serverCertIssuer);
|
||||
|
||||
var reason = Contains(body, ClientCertificateRequiredMarker)
|
||||
? ResponseClassification.ClientCertificateRequired
|
||||
: ResponseClassification.ForeignResponse;
|
||||
return ResponseClassification.ForForeign(statusCode, reason, title, isHtml, serverCertIssuer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The first <see cref="MaxBodyChars"/> characters of the body, null when there is none or it
|
||||
/// cannot be read. Never throws — the body is evidence only.
|
||||
/// </summary>
|
||||
public static async Task<string?> ReadBodyPrefixAsync(HttpResponseMessage response)
|
||||
{
|
||||
var content = response?.Content;
|
||||
if (content == null) return null;
|
||||
try
|
||||
{
|
||||
return Cap(await content.ReadAsStringAsync().ConfigureAwait(false));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="response"/> carries <paramref name="sentCorrelationId"/> in
|
||||
/// <see cref="CorrelationIdHeader"/> — the backend wrote it. Also used on 2xx answers, where
|
||||
/// a missing echo is only counted (<c>BackendTelemetryUploader</c>), never acted on.
|
||||
/// </summary>
|
||||
public static bool EchoesCorrelationId(HttpResponseMessage response, string? sentCorrelationId)
|
||||
{
|
||||
if (response == null) throw new ArgumentNullException(nameof(response));
|
||||
if (string.IsNullOrEmpty(sentCorrelationId)) return false;
|
||||
if (!response.Headers.TryGetValues(CorrelationIdHeader, out var values)) return false;
|
||||
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (string.Equals(value?.Trim(), sentCorrelationId, StringComparison.Ordinal)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when the body is an HTML page rather than a JSON/plain-text reply. Checks the declared
|
||||
/// Content-Type first, then sniffs the body prefix so a page served with a generic
|
||||
/// Content-Type is still recognized.
|
||||
/// </summary>
|
||||
private static bool LooksLikeHtml(HttpResponseMessage response, string? body)
|
||||
{
|
||||
var mediaType = response.Content?.Headers?.ContentType?.MediaType;
|
||||
if (string.Equals(mediaType, "text/html", StringComparison.OrdinalIgnoreCase))
|
||||
return true;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
return false;
|
||||
|
||||
var head = body!.TrimStart();
|
||||
return head.StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase) ||
|
||||
head.StartsWith("<html", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Best-effort extraction of the HTML <title> (e.g. "Web App - Unavailable") so logs and
|
||||
/// reports name the page without dumping markup. Null when absent, empty or the regex times out.
|
||||
/// </summary>
|
||||
private static string? TryExtractHtmlTitle(string? body)
|
||||
{
|
||||
if (string.IsNullOrEmpty(body)) return null;
|
||||
try
|
||||
{
|
||||
var match = Regex.Match(
|
||||
body, @"<title[^>]*>\s*(.*?)\s*</title>",
|
||||
RegexOptions.IgnoreCase | RegexOptions.Singleline,
|
||||
TimeSpan.FromMilliseconds(250));
|
||||
if (!match.Success) return null;
|
||||
|
||||
var title = match.Groups[1].Value;
|
||||
return title.Length == 0 ? null : (title.Length > MaxTitleLength ? title.Substring(0, MaxTitleLength) : title);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool Contains(string? body, string marker) =>
|
||||
body != null && body.IndexOf(marker, StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
|
||||
private static string? Cap(string? body) =>
|
||||
body != null && body.Length > MaxBodyChars ? body.Substring(0, MaxBodyChars) : body;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Security;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Threading;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// Remembers the issuer of the server certificate of the last TLS connection a backend
|
||||
/// <see cref="HttpClientHandler"/> validated. Evidence for a 401/403 that did not come from the
|
||||
/// backend: a TLS-inspecting proxy presents a certificate from its own CA, which names the
|
||||
/// product; a Microsoft issuer means the device certificate got lost on the device itself.
|
||||
/// <para>
|
||||
/// Observation only: the callback returns exactly <c>sslPolicyErrors == None</c>, the verdict
|
||||
/// .NET Framework applies to a client connection without a callback, so validation is never
|
||||
/// weakened. The callback runs during the handshake of a new connection; a reused keep-alive
|
||||
/// connection is not validated again.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ServerCertificateObserver
|
||||
{
|
||||
/// <summary>Longest issuer kept. The CN leads the distinguished name and survives the cut.</summary>
|
||||
internal const int MaxIssuerLength = 128;
|
||||
|
||||
private string? _lastIssuer;
|
||||
|
||||
/// <summary>Issuer of the last validated server certificate, null before the first handshake.</summary>
|
||||
public string? LastIssuer => Volatile.Read(ref _lastIssuer);
|
||||
|
||||
/// <summary>Installs the observing validation callback on <paramref name="handler"/>.</summary>
|
||||
public void Attach(HttpClientHandler handler)
|
||||
{
|
||||
if (handler == null) throw new ArgumentNullException(nameof(handler));
|
||||
handler.ServerCertificateCustomValidationCallback =
|
||||
(_, certificate, _, sslPolicyErrors) => Validate(certificate, sslPolicyErrors);
|
||||
}
|
||||
|
||||
/// <summary>Records the issuer and returns the unchanged default verdict.</summary>
|
||||
internal bool Validate(X509Certificate2? certificate, SslPolicyErrors sslPolicyErrors)
|
||||
{
|
||||
Record(certificate);
|
||||
return sslPolicyErrors == SslPolicyErrors.None;
|
||||
}
|
||||
|
||||
private void Record(X509Certificate2? certificate)
|
||||
{
|
||||
if (certificate == null) return;
|
||||
try
|
||||
{
|
||||
var issuer = certificate.Issuer;
|
||||
if (string.IsNullOrEmpty(issuer)) return;
|
||||
Volatile.Write(ref _lastIssuer, issuer.Length > MaxIssuerLength ? issuer.Substring(0, MaxIssuerLength) : issuer);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Observation must never fail a handshake.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+58
-22
@@ -9,6 +9,7 @@ using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -49,7 +50,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
private readonly string? _agentVersion;
|
||||
private readonly AuthFailureTracker? _authFailureTracker;
|
||||
private readonly Logging.AgentLogger? _logger;
|
||||
private readonly ServerCertificateObserver? _serverCertificates;
|
||||
private readonly NetworkMetrics? _networkMetrics;
|
||||
|
||||
/// <param name="serverCertificates">
|
||||
/// Observer installed on the <paramref name="httpClient"/>'s handler; its issuer becomes
|
||||
/// evidence on an intercepted upload. Null = no evidence.
|
||||
/// </param>
|
||||
/// <param name="networkMetrics">
|
||||
/// Counts 2xx answers without the correlation echo (<c>agent_metrics_snapshot</c>). Null = not counted.
|
||||
/// </param>
|
||||
public BackendTelemetryUploader(
|
||||
HttpClient httpClient,
|
||||
string baseUrl,
|
||||
@@ -60,7 +70,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
string? bootstrapToken = null,
|
||||
string? agentVersion = null,
|
||||
AuthFailureTracker? authFailureTracker = null,
|
||||
Logging.AgentLogger? logger = null)
|
||||
Logging.AgentLogger? logger = null,
|
||||
ServerCertificateObserver? serverCertificates = null,
|
||||
NetworkMetrics? networkMetrics = null)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("BaseUrl is mandatory.", nameof(baseUrl));
|
||||
@@ -75,6 +87,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
_agentVersion = agentVersion;
|
||||
_authFailureTracker = authFailureTracker;
|
||||
_logger = logger;
|
||||
_serverCertificates = serverCertificates;
|
||||
_networkMetrics = networkMetrics;
|
||||
}
|
||||
|
||||
public async Task<UploadResult> UploadBatchAsync(
|
||||
@@ -117,7 +131,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
|
||||
request.Content.Headers.ContentEncoding.Add("gzip");
|
||||
AddSecurityHeaders(request);
|
||||
request.Headers.Add("X-Correlation-ID", correlationId);
|
||||
request.Headers.Add(ResponseOriginClassifier.CorrelationIdHeader, correlationId);
|
||||
|
||||
// P14: device-clock send time, one value per request. Lets the backend separate
|
||||
// spool delay (SentAt − OccurredUtc, both device frame) from device-vs-server
|
||||
@@ -154,7 +168,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
|
||||
using (response)
|
||||
{
|
||||
var result = await MapResponseAsync(response).ConfigureAwait(false);
|
||||
var result = await MapResponseAsync(response, correlationId).ConfigureAwait(false);
|
||||
sw.Stop();
|
||||
|
||||
if (result.Success)
|
||||
@@ -165,6 +179,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
_logger?.Warning($"BackendTelemetryUploader: ingest TRANSIENT (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
}
|
||||
else if (result.Interception != null)
|
||||
{
|
||||
// The whole page finding instead of a body excerpt: the 200-character
|
||||
// excerpt used to end before the "Client Certificate Required" text.
|
||||
var interception = result.Interception;
|
||||
_logger?.Warning(
|
||||
$"BackendTelemetryUploader: ingest INTERCEPTED — the answer did not come from the backend " +
|
||||
$"(corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={interception.StatusCode}, " +
|
||||
$"reason={interception.ForeignReason}, pageTitle=\"{interception.PageTitle ?? "(none)"}\", " +
|
||||
$"serverCertIssuer=\"{interception.ServerCertIssuer ?? "(unknown)"}\").");
|
||||
}
|
||||
else if (result.IsSessionGone)
|
||||
{
|
||||
_logger?.Error($"BackendTelemetryUploader: ingest rejected — backend no longer knows this session (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger?.Error($"BackendTelemetryUploader: ingest PERMANENT fail (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
@@ -175,7 +204,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<UploadResult> MapResponseAsync(HttpResponseMessage response)
|
||||
private async Task<UploadResult> MapResponseAsync(HttpResponseMessage response, string correlationId)
|
||||
{
|
||||
var statusCode = (int)response.StatusCode;
|
||||
|
||||
@@ -185,22 +214,37 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// backend hiccup does not eventually shut the agent down.
|
||||
_authFailureTracker?.RecordSuccess();
|
||||
|
||||
// A 2xx without the backend's correlation echo may be a block page served with 200.
|
||||
// Counted only (agent_metrics_snapshot) and still a success: whether every real
|
||||
// backend 2xx carries the echo is measured before anything is ever enforced on it.
|
||||
if (!ResponseOriginClassifier.EchoesCorrelationId(response, correlationId))
|
||||
_networkMetrics?.RecordOkWithoutEcho();
|
||||
|
||||
// M4.6.ε — parse backend-to-agent control signals from the 2xx response body.
|
||||
// Body is best-effort JSON; anything unparseable degrades cleanly to plain Ok().
|
||||
return await TryReadControlSignalsAsync(response).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var reason = await TryReadReasonAsync(response).ConfigureAwait(false);
|
||||
var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false);
|
||||
var reason = string.IsNullOrEmpty(body) ? string.Empty : (body!.Length <= 200 ? body : body.Substring(0, 200));
|
||||
var shortReason = $"http {statusCode}{(string.IsNullOrEmpty(reason) ? string.Empty : ": " + reason)}";
|
||||
|
||||
switch (response.StatusCode)
|
||||
{
|
||||
case HttpStatusCode.Unauthorized: // 401
|
||||
case HttpStatusCode.Forbidden: // 403
|
||||
{
|
||||
// Feed the central tracker so the agent can shut down after MaxAuthFailures
|
||||
// instead of retrying a permanently-revoked certificate forever.
|
||||
_authFailureTracker?.RecordFailure(statusCode, "agent/telemetry");
|
||||
return UploadResult.Unauthorized($"unauthorized: {shortReason}");
|
||||
// instead of retrying a permanently-revoked certificate forever. It counts only
|
||||
// what the backend (or the platform page of a stopped endpoint) answered; an
|
||||
// answer from the network path is reported once and never counted.
|
||||
var classification = ResponseOriginClassifier.Classify(
|
||||
response, correlationId, body, _serverCertificates?.LastIssuer);
|
||||
_authFailureTracker?.RecordFailure(statusCode, "agent/telemetry", classification);
|
||||
return classification.IsForeign
|
||||
? UploadResult.Intercepted($"intercepted: http {statusCode} ({classification.ForeignReason})", classification)
|
||||
: UploadResult.Unauthorized($"unauthorized: {shortReason}");
|
||||
}
|
||||
|
||||
case HttpStatusCode.RequestTimeout: // 408
|
||||
case (HttpStatusCode)429: // TooManyRequests — enum available net48 SP1+, cast for safety
|
||||
@@ -208,6 +252,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
|
||||
case HttpStatusCode.RequestEntityTooLarge: // 413 — the data is fine, only the batch
|
||||
return UploadResult.TooLarge(shortReason); // size is wrong → split & retry (P1).
|
||||
|
||||
case HttpStatusCode.Gone: // 410 — the backend no longer knows this session.
|
||||
// Decided BEFORE the poison-body check: a poison body on 410 would make the
|
||||
// drain drop items and emit telemetry_upload_poisoned, which the backend
|
||||
// rejects with 410 again — an emit/reject loop for a session that is over.
|
||||
return UploadResult.SessionGone($"session gone: {shortReason}");
|
||||
}
|
||||
|
||||
if (statusCode >= 500 && statusCode <= 599) return UploadResult.Transient(shortReason);
|
||||
@@ -228,20 +278,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
return UploadResult.Transient(shortReason);
|
||||
}
|
||||
|
||||
private static async Task<string> TryReadReasonAsync(HttpResponseMessage response)
|
||||
{
|
||||
try
|
||||
{
|
||||
var body = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
if (string.IsNullOrEmpty(body)) return string.Empty;
|
||||
return body.Length <= 200 ? body : body.Substring(0, 200);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// M4.6.ε — parse backend-to-agent control signals from a 2xx batch-response body. Fields
|
||||
/// are <b>optional</b>: an empty body or a body without any of these fields is treated as
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
/// <summary>
|
||||
/// The first intercepted upload of the process, carried by
|
||||
/// <see cref="TelemetryUploadOrchestrator.UploadIntercepted"/> so the orchestrator can surface a
|
||||
/// <c>telemetry_upload_blocked</c> event with the evidence.
|
||||
/// </summary>
|
||||
public sealed class InterceptionReport
|
||||
{
|
||||
public InterceptionReport(ResponseClassification interception, DateTime firstBlockedAtUtc)
|
||||
{
|
||||
Interception = interception ?? throw new ArgumentNullException(nameof(interception));
|
||||
FirstBlockedAtUtc = firstBlockedAtUtc;
|
||||
}
|
||||
|
||||
/// <summary>Status, sub-reason, page title and server-certificate issuer of the intercepted upload.</summary>
|
||||
public ResponseClassification Interception { get; }
|
||||
|
||||
/// <summary>When the first upload was intercepted (agent clock, UTC).</summary>
|
||||
public DateTime FirstBlockedAtUtc { get; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
#nullable enable
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
/// <summary>
|
||||
/// The one composition of a session's Azure-Table partition key. The three emitters stamp
|
||||
/// it on every <see cref="TelemetryItem"/>; the spool and the upload orchestrator compare
|
||||
/// against it to tell the current session's items from leftovers of a previous session id.
|
||||
/// A drift between producer and comparer would silently drop or retain the wrong items.
|
||||
/// </summary>
|
||||
public static class TelemetryPartitionKey
|
||||
{
|
||||
public static string ForSession(string tenantId, string sessionId) => $"{tenantId}_{sessionId}";
|
||||
}
|
||||
}
|
||||
@@ -74,7 +74,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
public int ByteLength { get; }
|
||||
}
|
||||
|
||||
public TelemetrySpool(string directoryPath, IClock clock, Logging.AgentLogger? logger = null)
|
||||
/// <param name="expectedPartitionKey">
|
||||
/// Partition key of the session this process serves (<see cref="TelemetryPartitionKey.ForSession"/>).
|
||||
/// When set, pending items of another session id found at the head of the spool are dropped
|
||||
/// on startup — see <see cref="DropLeadingForeignPending"/>. Null disables the filter.
|
||||
/// </param>
|
||||
public TelemetrySpool(
|
||||
string directoryPath,
|
||||
IClock clock,
|
||||
Logging.AgentLogger? logger = null,
|
||||
string? expectedPartitionKey = null)
|
||||
{
|
||||
if (string.IsNullOrEmpty(directoryPath))
|
||||
{
|
||||
@@ -96,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// count as "pending" and need to be rehydrated into _pending.
|
||||
_lastUploadedItemId = _cursor.Load();
|
||||
_lastAssignedItemId = ScanSpoolAndRehydratePending();
|
||||
DropLeadingForeignPending(expectedPartitionKey);
|
||||
_peakPendingCount = _pending.Count;
|
||||
}
|
||||
|
||||
@@ -312,5 +322,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
}
|
||||
return highest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Startup filter: drops the LEADING run of pending items whose partition key is not
|
||||
/// <paramref name="expectedPartitionKey"/>, advancing and persisting the cursor past them.
|
||||
/// Such items are leftovers of a previous session id in the same spool directory; the
|
||||
/// backend answers 410 for them, and that 410 must never end the session this process
|
||||
/// serves. One process serves one session, so a foreign item AFTER an own item cannot
|
||||
/// occur — if it does, the order is not trustworthy and nothing is dropped. WhiteGlove
|
||||
/// Part 2 reuses the Part-1 session id, so its leftovers carry the expected key and stay.
|
||||
/// Called once from the ctor, after rehydration.
|
||||
/// </summary>
|
||||
private void DropLeadingForeignPending(string? expectedPartitionKey)
|
||||
{
|
||||
if (string.IsNullOrEmpty(expectedPartitionKey) || _pending.Count == 0) return;
|
||||
|
||||
var leadingForeign = 0;
|
||||
long lastForeignId = -1;
|
||||
var seenOwn = false;
|
||||
foreach (var entry in _pending)
|
||||
{
|
||||
var isForeign = !string.Equals(entry.Item.PartitionKey, expectedPartitionKey, StringComparison.Ordinal);
|
||||
if (!isForeign)
|
||||
{
|
||||
seenOwn = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!seenOwn)
|
||||
{
|
||||
leadingForeign++;
|
||||
lastForeignId = entry.Item.TelemetryItemId;
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger?.Error(
|
||||
$"TelemetrySpool: pending itemId={entry.Item.TelemetryItemId} carries a foreign partition key after items of the current session — spool order is inconsistent; keeping all {_pending.Count} pending item(s).");
|
||||
return;
|
||||
}
|
||||
|
||||
if (leadingForeign == 0) return;
|
||||
|
||||
try
|
||||
{
|
||||
_cursor.Save(lastForeignId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Keep the items rather than risk a cursor that disagrees with disk; the drain
|
||||
// loop drops the same leading run on its first 410 (defence in depth).
|
||||
_logger?.Warning($"TelemetrySpool: could not persist the cursor past {leadingForeign} foreign pending item(s): {ex.Message}");
|
||||
return;
|
||||
}
|
||||
|
||||
_lastUploadedItemId = lastForeignId;
|
||||
for (var i = 0; i < leadingForeign; i++)
|
||||
{
|
||||
_pendingBytes -= _pending.First!.Value.ByteLength;
|
||||
_pending.RemoveFirst();
|
||||
}
|
||||
|
||||
_logger?.Warning(
|
||||
$"TelemetrySpool: dropped {leadingForeign} pending item(s) of a previous session id (cursor advanced to {lastForeignId}, {_pending.Count} pending item(s) of the current session kept).");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+234
-11
@@ -25,10 +25,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
TimeSpan.FromMilliseconds(1600),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Pause after each consecutive intercepted upload (the last entry repeats). Every attempt
|
||||
/// is a POST of the head batch; the platform or proxy rejects it before the backend sees it.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<TimeSpan> InterceptionBackoffs = new[]
|
||||
{
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromMinutes(1),
|
||||
TimeSpan.FromMinutes(2),
|
||||
TimeSpan.FromMinutes(5),
|
||||
TimeSpan.FromMinutes(10),
|
||||
TimeSpan.FromMinutes(15),
|
||||
};
|
||||
|
||||
private const int TerminalAttemptNotGranted = 0;
|
||||
private const int TerminalAttemptGranted = 1;
|
||||
private const int TerminalAttemptUsed = 2;
|
||||
|
||||
private readonly ITelemetrySpool _spool;
|
||||
private readonly IBackendTelemetryUploader _uploader;
|
||||
private readonly IClock _clock;
|
||||
private readonly int _batchSize;
|
||||
private readonly IReadOnlyList<TimeSpan> _retryBackoffs;
|
||||
private readonly SemaphoreSlim _drainGuard = new SemaphoreSlim(1, 1);
|
||||
private bool _disposed;
|
||||
@@ -36,6 +53,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// M4.6.ε — backend-to-agent control signals.
|
||||
private DateTime? _pausedUntilUtc;
|
||||
|
||||
// Interception pause — a "not before" for the next upload after an intercepted one, modelled
|
||||
// on the DeviceBlocked pause: drains compare it with the clock and return without HTTP, no
|
||||
// timer (L.7). One immutable window so the termination thread reads it atomically; null =
|
||||
// never intercepted or cleared by a success. The backoff step and the one-shot report are
|
||||
// only touched under the drain guard.
|
||||
private InterceptionWindow? _interceptionWindow;
|
||||
private int _interceptionBackoffStep;
|
||||
private bool _interceptionReported;
|
||||
|
||||
// The one upload attempt the terminal phase may make despite the interception pause.
|
||||
private int _terminalAttemptState;
|
||||
|
||||
// P1 — current effective batch size. Shrinks (never grows) when the backend returns 413,
|
||||
// so an oversized batch is split down instead of discarded. Only mutated under the drain
|
||||
// guard. Floor 1.
|
||||
@@ -45,22 +74,39 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// telemetry_upload_blocked exactly once per process rather than on every 30 s drain.
|
||||
private bool _uploadBlockedReported;
|
||||
|
||||
// Partition key of the session this process serves; null = every batch counts as own.
|
||||
private readonly string? _currentPartitionKey;
|
||||
private readonly Logging.AgentLogger? _logger;
|
||||
|
||||
// One-shot guard for SessionGone. Latched only when a handler is subscribed: the host
|
||||
// wires the handler after EnrollmentOrchestrator.Start, and a 410 on the very first
|
||||
// drain must not be swallowed before anyone listens.
|
||||
private bool _sessionGoneReported;
|
||||
|
||||
/// <param name="currentPartitionKey">
|
||||
/// <see cref="TelemetryPartitionKey.ForSession"/> of the registered session. A 410 on a
|
||||
/// batch that starts with this key ends the session (<see cref="SessionGone"/>); a 410 on
|
||||
/// a batch that starts with any other key only drops those leftover items.
|
||||
/// </param>
|
||||
public TelemetryUploadOrchestrator(
|
||||
ITelemetrySpool spool,
|
||||
IBackendTelemetryUploader uploader,
|
||||
IClock clock,
|
||||
int batchSize = 100,
|
||||
IReadOnlyList<TimeSpan>? retryBackoffs = null)
|
||||
IReadOnlyList<TimeSpan>? retryBackoffs = null,
|
||||
string? currentPartitionKey = null,
|
||||
Logging.AgentLogger? logger = null)
|
||||
{
|
||||
_spool = spool ?? throw new ArgumentNullException(nameof(spool));
|
||||
_uploader = uploader ?? throw new ArgumentNullException(nameof(uploader));
|
||||
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
|
||||
if (batchSize <= 0) throw new ArgumentOutOfRangeException(nameof(batchSize), "BatchSize must be positive.");
|
||||
_batchSize = batchSize;
|
||||
_effectiveBatchSize = batchSize;
|
||||
|
||||
_retryBackoffs = retryBackoffs ?? DefaultRetryBackoffs;
|
||||
_currentPartitionKey = currentPartitionKey;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -97,12 +143,61 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// </summary>
|
||||
public event EventHandler<string>? UploadBlocked;
|
||||
|
||||
/// <summary>
|
||||
/// Raised once per process (after the drain guard releases) for the first intercepted upload
|
||||
/// (<see cref="UploadResult.IsIntercepted"/>), independent of <see cref="UploadBlocked"/>.
|
||||
/// Like that event, anything the handler posts queues behind the intercepted batch and
|
||||
/// reaches the backend only once the path clears in the same run.
|
||||
/// </summary>
|
||||
public event EventHandler<InterceptionReport>? UploadIntercepted;
|
||||
|
||||
/// <summary>
|
||||
/// Raised once per process (after the drain guard releases) when the backend answered
|
||||
/// HTTP 410 for a batch of the CURRENT session: the session is over server-side (never
|
||||
/// registered, deleted, or being deleted) and no retry can succeed. The batch is retained
|
||||
/// (cursor stays) and the drain stops; the host reports once over the emergency channel
|
||||
/// and terminates the agent with self-destruct. Carries the backend error reason.
|
||||
/// </summary>
|
||||
public event EventHandler<string>? SessionGone;
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> when the last successful upload returned <see cref="UploadResult.DeviceBlocked"/>
|
||||
/// and the paused-until window has not expired. Drain is a no-op in this state.
|
||||
/// </summary>
|
||||
public bool IsPaused => _pausedUntilUtc.HasValue && _pausedUntilUtc.Value > _clock.UtcNow;
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> while the pause after an intercepted upload has not expired. Drains return
|
||||
/// without HTTP in this state — except the one attempt <see cref="GrantTerminalAttempt"/>
|
||||
/// allows. A successful upload ends the pause and resets the backoff.
|
||||
/// </summary>
|
||||
public bool IsInterceptionPaused
|
||||
{
|
||||
get
|
||||
{
|
||||
var window = Volatile.Read(ref _interceptionWindow);
|
||||
if (window == null) return false;
|
||||
|
||||
// Device clocks jump (RTC, NTP). A clock set back past the pause start ends the
|
||||
// pause instead of stretching it by the jump; the next attempt re-arms it.
|
||||
var now = _clock.UtcNow;
|
||||
return now >= window.FromUtc && now < window.UntilUtc;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Grants the single upload attempt per process that the terminal phase may make despite an
|
||||
/// active interception pause; the first drain that meets the pause afterwards uses it.
|
||||
/// Idempotent. Without an interception pause the grant changes nothing. The DeviceBlocked
|
||||
/// pause is not affected and still skips terminal drains. Returns
|
||||
/// <see cref="IsInterceptionPaused"/>, so the caller knows whether waking the drain pays off.
|
||||
/// </summary>
|
||||
public bool GrantTerminalAttempt()
|
||||
{
|
||||
Interlocked.CompareExchange(ref _terminalAttemptState, TerminalAttemptGranted, TerminalAttemptNotGranted);
|
||||
return IsInterceptionPaused;
|
||||
}
|
||||
|
||||
public long LastUploadedItemId => _spool.LastUploadedItemId;
|
||||
|
||||
public TelemetryItem Enqueue(TelemetryItemDraft draft)
|
||||
@@ -124,6 +219,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
List<UploadResult>? pendingSignals = null;
|
||||
List<PoisonReport>? pendingPoison = null;
|
||||
string? pendingBlockedReason = null;
|
||||
string? pendingSessionGoneReason = null;
|
||||
InterceptionReport? pendingInterception = null;
|
||||
DrainResult result;
|
||||
|
||||
try
|
||||
@@ -132,12 +229,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
int failedBatches = 0;
|
||||
string? lastError = null;
|
||||
|
||||
// P1 — shared success-side control-signal handling. Applied to BOTH the normal
|
||||
// success path and the poison-survivor re-upload, so a DeviceBlocked / DeviceKill /
|
||||
// AdminAction / Actions response on the survivor upload is honoured identically and
|
||||
// never silently dropped. Returns true if the backend just quarantined the device.
|
||||
bool CollectControlSignals(UploadResult o)
|
||||
// P1 — shared success-side handling. Applied to BOTH the normal success path and
|
||||
// the poison-survivor re-upload, so a DeviceBlocked / DeviceKill / AdminAction /
|
||||
// Actions response on the survivor upload is honoured identically and never silently
|
||||
// dropped, and either success ends an interception pause. Returns true if the
|
||||
// backend just quarantined the device.
|
||||
bool HandleSuccessfulUpload(UploadResult o)
|
||||
{
|
||||
EndInterceptionPause();
|
||||
ApplyControlSignals(o); // mutates _pausedUntilUtc synchronously (next IsPaused sees it)
|
||||
if (CarriesSignal(o)) (pendingSignals ??= new List<UploadResult>()).Add(o);
|
||||
return o.DeviceBlocked;
|
||||
@@ -156,6 +255,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
var batch = _spool.Peek(_effectiveBatchSize);
|
||||
if (batch.Count == 0) break;
|
||||
|
||||
// Interception pause: no HTTP before the not-before passes — periodic drains and
|
||||
// immediate-flush wakeups alike — except the one attempt the terminal phase is
|
||||
// granted. Checked after the Peek so only a real attempt uses the grant.
|
||||
if (IsInterceptionPaused && !TryUseTerminalAttempt())
|
||||
{
|
||||
lastError = $"upload intercepted, next attempt not before {_interceptionWindow?.UntilUtc:O}";
|
||||
break;
|
||||
}
|
||||
|
||||
var outcome = await TryUploadWithRetryAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (outcome.Success)
|
||||
@@ -166,7 +274,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// Stop draining immediately if the backend just quarantined the device —
|
||||
// the next loop iteration would see IsPaused and break anyway, but there
|
||||
// is no point in even Peeking another batch.
|
||||
if (CollectControlSignals(outcome)) break;
|
||||
if (HandleSuccessfulUpload(outcome)) break;
|
||||
}
|
||||
else if (outcome.Interception != null)
|
||||
{
|
||||
// Something on the network path answered instead of the backend (TLS
|
||||
// inspection, proxy page). RETAIN (cursor stays), pause with backoff, and
|
||||
// surface the first one once per process — never as an auth failure.
|
||||
failedBatches++;
|
||||
lastError = outcome.ErrorReason;
|
||||
StartInterceptionPause(outcome.Interception);
|
||||
if (!_interceptionReported)
|
||||
{
|
||||
_interceptionReported = true;
|
||||
pendingInterception = new InterceptionReport(outcome.Interception, _clock.UtcNow);
|
||||
}
|
||||
break;
|
||||
}
|
||||
else if (outcome.IsPoison)
|
||||
{
|
||||
@@ -210,7 +333,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// P1 fix: honour any control signals the survivor re-upload carried
|
||||
// (DeviceBlocked / DeviceKill / AdminAction / Actions) — same as the
|
||||
// normal success path; previously these were silently dropped.
|
||||
deviceBlockedBySurvivors = CollectControlSignals(survivorOutcome);
|
||||
deviceBlockedBySurvivors = HandleSuccessfulUpload(survivorOutcome);
|
||||
}
|
||||
|
||||
// Survivors uploaded (or none) — advance past the whole batch; the named items
|
||||
@@ -247,6 +370,35 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
new PoisonReport(1, oversizeId, "oversize: " + outcome.ErrorReason, PoisonKind.Oversize));
|
||||
break;
|
||||
}
|
||||
else if (outcome.IsSessionGone)
|
||||
{
|
||||
// HTTP 410 — the backend no longer knows the session these items belong to.
|
||||
var foreignRun = CountLeadingForeignItems(batch);
|
||||
if (foreignRun > 0)
|
||||
{
|
||||
// Leftovers of a previous session id (the spool drops these at startup;
|
||||
// this is defence in depth). Drop only that leading run and carry on: a
|
||||
// legitimate new session must never be ended because of old items.
|
||||
var throughId = batch[foreignRun - 1].TelemetryItemId;
|
||||
_spool.MarkUploaded(throughId);
|
||||
_logger?.Warning(
|
||||
$"TelemetryUploadOrchestrator: dropped {foreignRun} item(s) of a previous session id the backend rejected with 410 (cursor advanced to {throughId}).");
|
||||
continue;
|
||||
}
|
||||
|
||||
// The current session is over server-side. RETAIN (cursor stays, the items
|
||||
// remain on disk for the diagnostics ZIP), stop this drain, and surface it
|
||||
// once — not as telemetry_upload_blocked, which would only queue another
|
||||
// item behind a batch that can never be accepted.
|
||||
failedBatches++;
|
||||
lastError = outcome.ErrorReason;
|
||||
if (!_sessionGoneReported && SessionGone != null)
|
||||
{
|
||||
_sessionGoneReported = true;
|
||||
pendingSessionGoneReason = outcome.ErrorReason;
|
||||
}
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
// P1 — RETAIN. Transient (retry next drain), auth-permanent (uploader already
|
||||
@@ -261,7 +413,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
|
||||
// One-shot loud marker so a stuck non-transient, non-auth block is visible on
|
||||
// the backend instead of only in the agent log (transient = normal retry;
|
||||
// auth = the session is already shutting down via the auth path).
|
||||
// a backend auth rejection = the auth path shuts the agent down; an
|
||||
// interception has its own one-shot report above).
|
||||
if (!outcome.IsTransient && !outcome.IsAuthFailure && !_uploadBlockedReported)
|
||||
{
|
||||
_uploadBlockedReported = true;
|
||||
@@ -301,9 +454,79 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
if (pendingSessionGoneReason != null)
|
||||
{
|
||||
try { SessionGone?.Invoke(this, pendingSessionGoneReason); }
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
if (pendingInterception != null)
|
||||
{
|
||||
try { UploadIntercepted?.Invoke(this, pendingInterception); }
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Arms the interception pause with the next backoff step (the last step repeats). Called
|
||||
/// under the drain guard.
|
||||
/// </summary>
|
||||
private void StartInterceptionPause(ResponseClassification interception)
|
||||
{
|
||||
var delay = InterceptionBackoffs[Math.Min(_interceptionBackoffStep, InterceptionBackoffs.Count - 1)];
|
||||
_interceptionBackoffStep = Math.Min(_interceptionBackoffStep + 1, InterceptionBackoffs.Count - 1);
|
||||
var now = _clock.UtcNow;
|
||||
Volatile.Write(ref _interceptionWindow, new InterceptionWindow(now, now + delay));
|
||||
|
||||
// The uploader already logged the page finding at Warning; this line adds the schedule.
|
||||
_logger?.Info(
|
||||
$"TelemetryUploadOrchestrator: upload intercepted ({interception.ForeignReason}, http {interception.StatusCode}) — " +
|
||||
$"batch retained, next attempt not before {now + delay:O} ({delay.TotalSeconds:0}s).");
|
||||
}
|
||||
|
||||
/// <summary>A successful upload ends the interception pause and resets the backoff. Called under the drain guard.</summary>
|
||||
private void EndInterceptionPause()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _interceptionWindow, null) == null) return;
|
||||
|
||||
_interceptionBackoffStep = 0;
|
||||
_logger?.Info("TelemetryUploadOrchestrator: upload path clear again after an interception — uploads resume.");
|
||||
}
|
||||
|
||||
private bool TryUseTerminalAttempt() =>
|
||||
Interlocked.CompareExchange(ref _terminalAttemptState, TerminalAttemptUsed, TerminalAttemptGranted) == TerminalAttemptGranted;
|
||||
|
||||
private sealed class InterceptionWindow
|
||||
{
|
||||
public InterceptionWindow(DateTime fromUtc, DateTime untilUtc)
|
||||
{
|
||||
FromUtc = fromUtc;
|
||||
UntilUtc = untilUtc;
|
||||
}
|
||||
|
||||
public DateTime FromUtc { get; }
|
||||
public DateTime UntilUtc { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Number of items at the head of <paramref name="batch"/> whose partition key is not the
|
||||
/// current session's. 0 when no current key is configured (every batch counts as own).
|
||||
/// </summary>
|
||||
private int CountLeadingForeignItems(IReadOnlyList<TelemetryItem> batch)
|
||||
{
|
||||
if (_currentPartitionKey == null) return 0;
|
||||
|
||||
var count = 0;
|
||||
while (count < batch.Count
|
||||
&& !string.Equals(batch[count].PartitionKey, _currentPartitionKey, StringComparison.Ordinal))
|
||||
{
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private static bool CarriesSignal(UploadResult outcome) =>
|
||||
outcome.DeviceBlocked
|
||||
|| outcome.DeviceKillSignal
|
||||
|
||||
@@ -30,7 +30,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
bool isAuthFailure = false,
|
||||
bool requiresSplit = false,
|
||||
bool isPoison = false,
|
||||
IReadOnlyList<string>? poisonRowKeys = null)
|
||||
IReadOnlyList<string>? poisonRowKeys = null,
|
||||
bool isSessionGone = false,
|
||||
ResponseClassification? interception = null)
|
||||
{
|
||||
Success = success;
|
||||
ErrorReason = errorReason;
|
||||
@@ -44,6 +46,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
RequiresSplit = requiresSplit;
|
||||
IsPoison = isPoison;
|
||||
PoisonRowKeys = poisonRowKeys;
|
||||
IsSessionGone = isSessionGone;
|
||||
Interception = interception;
|
||||
}
|
||||
|
||||
public bool Success { get; }
|
||||
@@ -81,6 +85,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// <summary>The RowKeys the backend named as poison (item-level). Non-empty when <see cref="IsPoison"/>.</summary>
|
||||
public IReadOnlyList<string>? PoisonRowKeys { get; }
|
||||
|
||||
/// <summary>
|
||||
/// HTTP 410: the backend no longer knows the session the batch belongs to (never
|
||||
/// registered, deleted, or being deleted). Not transient, not auth, not poison — retrying
|
||||
/// can never succeed. The orchestrator stops the drain (cursor stays) and raises
|
||||
/// <see cref="TelemetryUploadOrchestrator.SessionGone"/> when the batch belongs to the
|
||||
/// current session, or drops the batch's leading items when they are leftovers of a
|
||||
/// previous session id.
|
||||
/// </summary>
|
||||
public bool IsSessionGone { get; }
|
||||
|
||||
/// <summary>
|
||||
/// A 401/403 that did not come from the backend (no <c>X-Correlation-ID</c> echo): the
|
||||
/// platform's "Client Certificate Required" page or a proxy page — TLS inspection on the
|
||||
/// network path. Not an auth failure: never counted by the <c>AuthFailureTracker</c>. The
|
||||
/// orchestrator retains the batch and pauses uploads with backoff.
|
||||
/// </summary>
|
||||
public bool IsIntercepted => Interception != null;
|
||||
|
||||
/// <summary>Status, sub-reason, page title and server-certificate issuer of the interception; null unless <see cref="IsIntercepted"/>.</summary>
|
||||
public ResponseClassification? Interception { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Non-terminal quarantine signalled by the backend. The agent should stop draining
|
||||
/// uploads until <see cref="UnblockAt"/> (if set) elapses — the session stays alive so
|
||||
@@ -165,5 +190,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
isPoison: true, poisonRowKeys: rejectedRowKeys ?? throw new ArgumentNullException(nameof(rejectedRowKeys)));
|
||||
|
||||
/// <summary>
|
||||
/// HTTP 410 — the backend no longer knows the session. Flagged <see cref="IsSessionGone"/>;
|
||||
/// never retried, never split, never treated as poison.
|
||||
/// </summary>
|
||||
public static UploadResult SessionGone(string reason) =>
|
||||
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
isSessionGone: true);
|
||||
|
||||
/// <summary>
|
||||
/// A 401/403 something on the network path answered (<see cref="ResponseOrigin.Foreign"/>).
|
||||
/// Flagged <see cref="IsIntercepted"/>, never <see cref="IsAuthFailure"/>.
|
||||
/// </summary>
|
||||
public static UploadResult Intercepted(string reason, ResponseClassification interception)
|
||||
{
|
||||
if (interception == null) throw new ArgumentNullException(nameof(interception));
|
||||
if (!interception.IsForeign) throw new ArgumentException("Only a foreign response is an interception.", nameof(interception));
|
||||
return new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
interception: interception);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,10 +256,10 @@ namespace AutopilotMonitor.Agent.V2
|
||||
/// <summary>
|
||||
/// <c>true</c> when the file-based <c>enrollment-complete.marker</c> signals that a
|
||||
/// previous session already completed enrollment — written by
|
||||
/// <see cref="CheckSessionAgeEmergencyBreak"/> right before it fires the cleanup retry,
|
||||
/// so that a next-boot observing the marker exits cleanly even if the cleanup retry
|
||||
/// failed to remove the Scheduled Task. Triggers another self-destruct attempt when
|
||||
/// <paramref name="selfDestructOnComplete"/> is set.
|
||||
/// <see cref="EndMonitoringAtStart"/> (emergency break, intercepted-registration give-up,
|
||||
/// config kill signal) right before it fires the cleanup retry, so that a next-boot observing the marker exits
|
||||
/// cleanly even if the cleanup retry failed to remove the Scheduled Task. Triggers another
|
||||
/// self-destruct attempt when <paramref name="selfDestructOnComplete"/> is set.
|
||||
/// <para>
|
||||
/// The <c>HKLM\SOFTWARE\AutopilotMonitor\Deployed</c> registry value is NOT consulted
|
||||
/// here: it is a bootstrap-script reentry lock (read by Install-AutopilotMonitor-v2.ps1
|
||||
@@ -350,23 +350,10 @@ namespace AutopilotMonitor.Agent.V2
|
||||
try { onBreakFired?.Invoke(sessionAgeHours); }
|
||||
catch (Exception cbEx) { logger.Debug($"Emergency-break notify callback failed: {cbEx.Message}"); }
|
||||
|
||||
// Write enrollment-complete.marker so the next start exits cleanly even if
|
||||
// the cleanup retry below fails (Scheduled Task still alive, files locked, …).
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(stateDirectory);
|
||||
var markerPath = Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName);
|
||||
File.WriteAllText(markerPath,
|
||||
$"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)");
|
||||
}
|
||||
catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); }
|
||||
|
||||
if (selfDestructOnComplete)
|
||||
{
|
||||
TryRetryCleanup(cleanupServiceFactory, logger, "emergency_break");
|
||||
}
|
||||
|
||||
persistence.Delete(logger);
|
||||
EndMonitoringAtStart(
|
||||
stateDirectory, persistence, selfDestructOnComplete, cleanupServiceFactory, logger,
|
||||
markerText: $"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)",
|
||||
cleanupReason: "emergency_break");
|
||||
logger.Info("Emergency break: cleanup completed. Agent will exit.");
|
||||
return true;
|
||||
}
|
||||
@@ -377,6 +364,40 @@ namespace AutopilotMonitor.Agent.V2
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends monitoring at process start: writes <c>enrollment-complete.marker</c> first so the
|
||||
/// next start exits cleanly even if the cleanup retry fails (Scheduled Task still alive,
|
||||
/// files locked, …), runs the cleanup retry when <paramref name="runSelfDestruct"/> is set
|
||||
/// (the emergency break and the give-up pass <c>SelfDestructOnComplete</c>, the kill signal
|
||||
/// forces it), and deletes the persisted session files. Shared by the session-age emergency
|
||||
/// break, the intercepted-registration give-up
|
||||
/// (<c>BackendSessionRegistration.ConcludeFailedRun</c>) and the config-channel kill signal
|
||||
/// so all three leave the device in the same state. Never throws.
|
||||
/// </summary>
|
||||
internal static void EndMonitoringAtStart(
|
||||
string stateDirectory,
|
||||
SessionIdPersistence persistence,
|
||||
bool runSelfDestruct,
|
||||
Func<CleanupService> cleanupServiceFactory,
|
||||
AgentLogger logger,
|
||||
string markerText,
|
||||
string cleanupReason)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(stateDirectory);
|
||||
File.WriteAllText(Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName), markerText);
|
||||
}
|
||||
catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); }
|
||||
|
||||
if (runSelfDestruct)
|
||||
{
|
||||
TryRetryCleanup(cleanupServiceFactory, logger, cleanupReason);
|
||||
}
|
||||
|
||||
persistence?.Delete(logger);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Kill-switch on the control channel: honours a <c>DeviceKillSignal</c> served on the
|
||||
/// <c>agent/config</c> response. Config is fetched at every process start (boot via
|
||||
@@ -409,22 +430,15 @@ namespace AutopilotMonitor.Agent.V2
|
||||
if (consoleMode)
|
||||
Console.Out.WriteLine("KILL: administrator issued a remote kill signal. Cleaning up and exiting.");
|
||||
|
||||
// Write enrollment-complete.marker first so the next start exits cleanly even if
|
||||
// the cleanup below fails (Scheduled Task still alive, files locked, …) — same
|
||||
// guarantee-of-termination pattern as the session-age emergency break.
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(stateDirectory);
|
||||
var markerPath = Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName);
|
||||
File.WriteAllText(markerPath, $"Kill signal at {DateTime.UtcNow:O} (config channel)");
|
||||
}
|
||||
catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); }
|
||||
SessionIdPersistence persistence = null;
|
||||
try { persistence = new SessionIdPersistence(dataDirectory); }
|
||||
catch (Exception ex) { logger.Warning($"Failed to open the persisted session after kill: {ex.Message}"); }
|
||||
|
||||
// forceSelfDestruct — no SelfDestructOnComplete gate, matching the telemetry-kill.
|
||||
TryRetryCleanup(cleanupServiceFactory, logger, "kill_signal");
|
||||
|
||||
try { new SessionIdPersistence(dataDirectory).Delete(logger); }
|
||||
catch (Exception ex) { logger.Warning($"Failed to delete persisted session after kill: {ex.Message}"); }
|
||||
EndMonitoringAtStart(
|
||||
stateDirectory, persistence, runSelfDestruct: true, cleanupServiceFactory, logger,
|
||||
markerText: $"Kill signal at {DateTime.UtcNow:O} (config channel)",
|
||||
cleanupReason: "kill_signal");
|
||||
|
||||
logger.Info("Kill signal handled — agent exiting.");
|
||||
return true;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user